From 076834db41ef00257804d68eff02558ec69004fd Mon Sep 17 00:00:00 2001 From: Rohan Date: Wed, 8 Jul 2026 18:43:00 -0400 Subject: [PATCH 01/63] [k/N] wal refactor: reorganize wal buffer so its owned by batch writer (#1882) --- slatedb/src/batch_write.rs | 62 ++- slatedb/src/compactor.rs | 2 +- slatedb/src/db.rs | 238 +++++++--- slatedb/src/db/builder.rs | 16 +- slatedb/src/db_common.rs | 3 +- slatedb/src/db_state.rs | 24 +- slatedb/src/db_stats.rs | 12 - slatedb/src/lib.rs | 2 +- .../src/memtable_flusher/manifest_writer.rs | 14 +- slatedb/src/memtable_flusher/tracker.rs | 12 + slatedb/src/memtable_flusher/uploader.rs | 14 +- slatedb/src/oracle.rs | 6 - slatedb/src/wal_buffer.rs | 435 ++++++++++++------ slatedb/src/wal_id.rs | 3 - .../content/docs/docs/operations/metrics.mdx | 11 +- 15 files changed, 592 insertions(+), 262 deletions(-) delete mode 100644 slatedb/src/wal_id.rs diff --git a/slatedb/src/batch_write.rs b/slatedb/src/batch_write.rs index 69e1f47e4..7560768c6 100644 --- a/slatedb/src/batch_write.rs +++ b/slatedb/src/batch_write.rs @@ -43,6 +43,7 @@ use crate::dispatcher::MessageHandler; use crate::mem_table::KVTable; use crate::types::RowEntry; use crate::utils::WatchableOnceCellReader; +use crate::wal_buffer::WalBufferManager; use crate::{batch::WriteBatch, db::DbInner, db::WriteHandle, error::SlateDBError}; use bytes::Bytes; use parking_lot::RwLockWriteGuard; @@ -108,13 +109,15 @@ impl std::fmt::Debug for BatchWriterMessage { pub(crate) struct WriteBatchEventHandler { db_inner: Arc, is_first_write: bool, + wal_buffer: Arc, } impl WriteBatchEventHandler { - pub(crate) fn new(db_inner: Arc) -> Self { + pub(crate) fn new(db_inner: Arc, wal_buffer: Arc) -> Self { Self { db_inner, is_first_write: true, + wal_buffer, } } } @@ -131,7 +134,7 @@ impl MessageHandler for WriteBatchEventHandler { }) => { let result = self .db_inner - .write_batch(batch, &options, txn.as_ref()) + .write_batch(batch, &options, txn.as_ref(), self.wal_buffer.as_ref()) .await; // if this is the first write and the WAL is disabled, make sure users are flushing // their memtables in a timely manner. @@ -153,7 +156,9 @@ impl MessageHandler for WriteBatchEventHandler { freeze_memtable, done, } = flush_msg; - let result = self.db_inner.flush_batch_writer(freeze_memtable); + let result = self + .db_inner + .flush_batch_writer(freeze_memtable, self.wal_buffer.as_ref()); let _ = done.send(result); Ok(()) } @@ -192,6 +197,7 @@ impl DbInner { batch: WriteBatch, options: &WriteOptions, txn: Option<&DbTransaction>, + wal_buffer: &WalBufferManager, ) -> WriteBatchResult { let _options = options; #[cfg(not(dst))] @@ -249,8 +255,8 @@ impl DbInner { // would violate the guarantee that batches are written atomically. We do // this by appending the entire entry batch in a single call to the WAL buffer, // which holds a write lock during the append. - let wal_watcher = self.wal_buffer.append(&entries)?; - self.wal_buffer.maybe_trigger_flush()?; + let wal_watcher = wal_buffer.append(&entries)?; + wal_buffer.maybe_trigger_flush()?; // TODO: handle sync here, if sync is enabled, we can call `flush` here. let's put this // in another Pull Request. self.write_entries_to_memtable(entries, touched_segments); @@ -265,7 +271,7 @@ impl DbInner { // update the last_applied_seq to wal buffer. if a chunk of WAL entries are applied to the memtable // and flushed to the remote storage, WAL buffer manager will recycle these WAL entries. - self.wal_buffer.track_last_applied_seq(commit_seq); + wal_buffer.track_last_applied_seq(commit_seq); // insert a fail point to make it easier to test the case where the last_committed_seq is not updated. // this is useful for testing the case where the reader is not able to see the writes. @@ -299,15 +305,18 @@ impl DbInner { self.record_memtable_sequence(commit_seq); // maybe freeze the memtable. - self.maybe_freeze_current_memtable()?; + self.maybe_freeze_current_memtable(wal_buffer)?; let write_handle = WriteHandle::new(commit_seq, now); Ok((write_handle, durable_watcher)) } - fn maybe_freeze_current_memtable(&self) -> Result<(), SlateDBError> { - let replay_after_wal_id = self.wal_buffer.recent_flushed_wal_id(); + fn maybe_freeze_current_memtable( + &self, + wal_buffer: &WalBufferManager, + ) -> Result<(), SlateDBError> { + let replay_after_wal_id = wal_buffer.last_flushed_wal_id(); let mut guard = self.state.write(); let meta = guard.memtable().metadata(); @@ -338,9 +347,10 @@ impl DbInner { fn flush_batch_writer( &self, freeze_memtable: bool, + wal_buffer: &WalBufferManager, ) -> Result>, SlateDBError> { let flush_rx = if self.wal_enabled { - self.wal_buffer.flush()? + wal_buffer.flush()? } else { let (flush_tx, flush_rx) = oneshot::channel(); flush_tx @@ -352,7 +362,7 @@ impl DbInner { // Note that this likely won't reflect the result of the above flush call as we don't // block until the flush completes. That's fine, as any earlier wal is still a safe // replay point. - let replay_after_wal_id = self.wal_buffer.recent_flushed_wal_id(); + let replay_after_wal_id = wal_buffer.last_flushed_wal_id(); let mut guard = self.state.write(); self.freeze_current_memtable_with_state_guard(&mut guard, replay_after_wal_id); } @@ -548,8 +558,16 @@ mod tests { ) .await .unwrap(); + let wal_buffer = Arc::new(WalBufferManager::new( + db.inner.status_manager.clone(), + &db.inner.recorder, + 0, + db.inner.table_store.clone(), + 1024, + None, + )); - let mut handler = WriteBatchEventHandler::new(db.inner.clone()); + let mut handler = WriteBatchEventHandler::new(db.inner.clone(), wal_buffer); assert!(handler.is_first_write); let mut batch = WriteBatch::new(); @@ -569,8 +587,16 @@ mod tests { let db = Db::open("/tmp/test_user_defined_seqnum", object_store) .await .unwrap(); + let wal_buffer = Arc::new(WalBufferManager::new( + db.inner.status_manager.clone(), + &db.inner.recorder, + 0, + db.inner.table_store.clone(), + 1024, + None, + )); - let mut handler = WriteBatchEventHandler::new(db.inner.clone()); + let mut handler = WriteBatchEventHandler::new(db.inner.clone(), wal_buffer); // Write with a user-defined seqnum let mut batch = WriteBatch::new(); @@ -604,8 +630,16 @@ mod tests { ) .await .unwrap(); + let wal_buffer = Arc::new(WalBufferManager::new( + db.inner.status_manager.clone(), + &db.inner.recorder, + 0, + db.inner.table_store.clone(), + 1024, + None, + )); - let mut handler = WriteBatchEventHandler::new(db.inner.clone()); + let mut handler = WriteBatchEventHandler::new(db.inner.clone(), wal_buffer); // First, do a normal write to advance the oracle let mut batch = WriteBatch::new(); diff --git a/slatedb/src/compactor.rs b/slatedb/src/compactor.rs index 6e45022a9..39eaf95b1 100644 --- a/slatedb/src/compactor.rs +++ b/slatedb/src/compactor.rs @@ -5878,7 +5878,7 @@ mod tests { let db_state = db.inner.state.read(); let cow_db_state = db_state.state(); ( - db.inner.wal_buffer.is_empty(), + db.inner.wal_observer.status().buffered_wal_entries_count == 0, db_state.memtable().is_empty() && cow_db_state.imm_memtable.is_empty(), db_state.state().core().clone(), ) diff --git a/slatedb/src/db.rs b/slatedb/src/db.rs index a2928efe2..489a1df64 100644 --- a/slatedb/src/db.rs +++ b/slatedb/src/db.rs @@ -70,7 +70,7 @@ use crate::tablestore::TableStore; use crate::transaction_manager::TransactionManager; use crate::types::KeyValue; use crate::utils::{format_bytes_si, SafeSender}; -use crate::wal_buffer::{WalBufferManager, WAL_BUFFER_TASK_NAME}; +use crate::wal_buffer::{WalEvent, WalObserver, WalStatus, WAL_BUFFER_TASK_NAME}; use crate::wal_replay::{WalReplayIterator, WalReplayOptions}; use crate::{DbCacheManagerOps, DbMetadataOps, DbReadOps, DbWriteOps}; use slatedb_common::clock::SystemClock; @@ -106,9 +106,9 @@ pub(crate) struct DbInner { pub(crate) oracle: Arc, pub(crate) flush_merge_operator: Option, pub(crate) reader: Reader, - /// [`wal_buffer`] manages the in-memory WAL buffer, it manages the flushing - /// of the WAL buffer to the remote storage. - pub(crate) wal_buffer: Arc, + /// [`wal_observer`] inspects the status of WAL buffer. The WAL buffer itself is owned by + /// the batch write task. + pub(crate) wal_observer: DbWalObserver, pub(crate) wal_enabled: bool, /// [`txn_manager`] tracks all the live transactions and related metadata. pub(crate) txn_manager: Arc, @@ -130,6 +130,7 @@ impl DbInner { manifest: DirtyObject, memtable_flusher: Arc, write_notifier: SafeSender, + wal_observer: WalObserver, recorder: MetricsRecorderHelper, fp_registry: Arc, merge_operator: Option, @@ -171,20 +172,9 @@ impl DbInner { merge_operator.clone(), ); - let recent_flushed_wal_id = state.read().state().core().replay_after_wal_id; - let wal_buffer = Arc::new(WalBufferManager::new( - state.clone(), - status_manager.clone(), - db_stats.clone(), - recent_flushed_wal_id, - oracle.clone(), - table_store.clone(), - settings.l0_sst_size_bytes, - settings.flush_interval, - )); - let txn_manager = Arc::new(TransactionManager::new(oracle.clone(), rand.clone())); let snapshot_manager = Arc::new(SnapshotManager::new(oracle.clone(), rand.clone())); + let wal_observer = DbWalObserver::new(wal_observer, oracle.clone(), state.clone()); let db_inner = Self { state, @@ -193,7 +183,7 @@ impl DbInner { oracle, wal_enabled, table_store, - wal_buffer, + wal_observer, write_notifier, db_stats, mono_clock, @@ -319,8 +309,8 @@ impl DbInner { pub(crate) async fn maybe_apply_backpressure(&self) -> Result<(), SlateDBError> { loop { self.check_closed()?; - let (wal_size_bytes, imm_memtable_size_bytes) = { - let wal_size_bytes = self.wal_buffer.estimated_bytes()?; + let (wal_status, imm_memtable_size_bytes) = { + let wal_status = self.wal_observer.status(); let imm_memtable_size_bytes = { let guard = self.state.read(); // Exclude active memtable to avoid a write lock. @@ -337,9 +327,9 @@ impl DbInner { }) .sum::() }; - (wal_size_bytes, imm_memtable_size_bytes) + (wal_status, imm_memtable_size_bytes) }; - let total_mem_size_bytes = wal_size_bytes + imm_memtable_size_bytes; + let total_mem_size_bytes = wal_status.estimated_bytes + imm_memtable_size_bytes; self.db_stats .total_mem_size_bytes .set(total_mem_size_bytes as i64); @@ -347,7 +337,7 @@ impl DbInner { trace!( "checking backpressure [total_mem_size_bytes={}, wal_size_bytes={}, imm_memtable_size_bytes={}, max_unflushed_bytes={}]", format_bytes_si(total_mem_size_bytes as u64), - format_bytes_si(wal_size_bytes as u64), + format_bytes_si(wal_status.estimated_bytes as u64), format_bytes_si(imm_memtable_size_bytes as u64), format_bytes_si(self.settings.max_unflushed_bytes as u64), ); @@ -357,7 +347,7 @@ impl DbInner { warn!( "unflushed memtable size exceeds max_unflushed_bytes. applying backpressure. [total_mem_size_bytes={}, wal_size_bytes={}, imm_memtable_size_bytes={}, max_unflushed_bytes={}]", format_bytes_si(total_mem_size_bytes as u64), - format_bytes_si(wal_size_bytes as u64), + format_bytes_si(wal_status.estimated_bytes as u64), format_bytes_si(imm_memtable_size_bytes as u64), format_bytes_si(self.settings.max_unflushed_bytes as u64), ); @@ -367,16 +357,10 @@ impl DbInner { guard.state().imm_memtable.back().cloned() }; - let watcher_for_oldest_unflushed_wal = - self.wal_buffer.watcher_for_oldest_unflushed_wal(); - // There is a window of time after mem_size_bytes is larger than max_unflushed_bytes - // but before we get the memtable and wal table. During that time, if the memtable and/or - // wal table are fully flushed out, we should short circuit since the select! will always - // time out. - if maybe_oldest_unflushed_memtable.is_none() - && watcher_for_oldest_unflushed_wal.is_none() - { + // but before we get the memtable. During that time, if the memtable is fully + // flushed out, we should short circuit to avoid blocking indefinitely. + if maybe_oldest_unflushed_memtable.is_none() && wal_status.estimated_bytes == 0 { continue; } @@ -388,13 +372,9 @@ impl DbInner { } }; - let await_flush_wal = async { - if let Some(mut watcher) = watcher_for_oldest_unflushed_wal { - watcher.await_value().await - } else { - std::future::pending().await - } - }; + let await_flush_wal = self + .wal_observer + .wait_until_wal_released(wal_status.last_purged_wal_id); let timeout_fut = self.system_clock.sleep(Duration::from_secs(30)); let await_closed = async { @@ -473,6 +453,14 @@ impl DbInner { } async fn replay_wal(&self, wal_id_range: Range) -> Result<(), SlateDBError> { + let mut current_memtable_wal_id = self + .state + .read() + .state() + .manifest + .value + .core + .replay_after_wal_id; let writer_epoch = self.state.read().state().manifest.value.writer_epoch; fail_point!( Arc::clone(&self.fp_registry), @@ -556,7 +544,9 @@ impl DbInner { assert!(self.oracle.last_remote_persisted_seq() <= replayed_table.last_seq); self.oracle.advance_durable_seq(replayed_table.last_seq); self.maybe_apply_backpressure().await?; - self.replay_memtable(replayed_table)?; + let replayed_table_last_wal_id = replayed_table.last_wal_id; + self.replay_memtable(current_memtable_wal_id, replayed_table)?; + current_memtable_wal_id = replayed_table_last_wal_id; } let guard = self.state.read(); @@ -2059,6 +2049,58 @@ impl WriteHandle { } } +/// Wraps [`WalObserver`] and injects a [`crate::wal_buffer::WalStatusListener`] +/// that updates the oracle and manifest, and drives cross-task notifications about wal events +/// via a [`tokio::sync::watch`] channel. +#[derive(Clone)] +pub(crate) struct DbWalObserver { + status_rx: tokio::sync::watch::Receiver, + wrapped: WalObserver, +} + +impl DbWalObserver { + fn new(wrapped: WalObserver, oracle: Arc, db_state: Arc>) -> Self { + let (status_tx, status_rx) = tokio::sync::watch::channel(wrapped.status()); + wrapped.subscribe(Arc::new(move |event| { + let status = match event { + WalEvent::WalFlushed(status) => status, + WalEvent::WalFrozen(status) => status, + WalEvent::MemoryReleased(status) => status, + }; + if let Some(seq) = status.last_flushed_seq { + oracle.advance_durable_seq(seq); + } + let mut guard = db_state.write(); + guard.set_next_wal_id(status.next_wal_id); + drop(guard); + let _ = status_tx.send(status); + })); + Self { status_rx, wrapped } + } + + pub(crate) fn status(&self) -> WalStatus { + self.wrapped.status() + } + + async fn wait_on_condition( + &self, + predicate: impl FnMut(&WalStatus) -> bool, + ) -> Result<(), SlateDBError> { + let mut status_rx = self.status_rx.clone(); + status_rx + .wait_for(predicate) + .await + .map_err(|_| SlateDBError::Closed)?; + Ok(()) + } + + /// Waits until the wal a given wal id is released by the wal writer + async fn wait_until_wal_released(&self, last_purged_wal_id: u64) -> Result<(), SlateDBError> { + self.wait_on_condition(|status| status.last_purged_wal_id > last_purged_wal_id) + .await + } +} + #[cfg(test)] mod tests { use super::*; @@ -3028,11 +3070,11 @@ mod tests { .unwrap(); // a sanity check: the wal contains the most recent write - assert_ne!(kv_store.inner.wal_buffer.estimated_bytes().unwrap(), 0); + assert_ne!(kv_store.inner.wal_observer.status().estimated_bytes, 0); // and a flush() should clear it kv_store.flush().await.unwrap(); - assert_eq!(kv_store.inner.wal_buffer.estimated_bytes().unwrap(), 0); + assert_eq!(kv_store.inner.wal_observer.status().estimated_bytes, 0); } #[tokio::test] @@ -3065,18 +3107,40 @@ mod tests { .unwrap(); // Sanity check: WAL has buffered entries before close. - assert_eq!(kv_store.inner.wal_buffer.buffered_wal_entries_count(), 1); assert_eq!( - lookup_metric(&metrics_recorder, crate::db_stats::WAL_BUFFER_FLUSHES).unwrap(), + kv_store + .inner + .wal_observer + .status() + .buffered_wal_entries_count, + 1 + ); + assert_eq!( + lookup_metric( + &metrics_recorder, + crate::wal_buffer::stats::WAL_BUFFER_FLUSHES + ) + .unwrap(), 0 ); kv_store.close().await.unwrap(); // close() should trigger a flush when the db is open. - assert_eq!(kv_store.inner.wal_buffer.buffered_wal_entries_count(), 0); assert_eq!( - lookup_metric(&metrics_recorder, crate::db_stats::WAL_BUFFER_FLUSHES).unwrap(), + kv_store + .inner + .wal_observer + .status() + .buffered_wal_entries_count, + 0 + ); + assert_eq!( + lookup_metric( + &metrics_recorder, + crate::wal_buffer::stats::WAL_BUFFER_FLUSHES + ) + .unwrap(), 1 ); } @@ -3124,9 +3188,13 @@ mod tests { .unwrap(); // Sanity check: WAL has buffered entries before close. - assert_eq!(db.inner.wal_buffer.buffered_wal_entries_count(), 1); + assert_eq!(db.inner.wal_observer.status().buffered_wal_entries_count, 1); assert_eq!( - lookup_metric(&metrics_recorder, crate::db_stats::WAL_BUFFER_FLUSHES).unwrap(), + lookup_metric( + &metrics_recorder, + crate::wal_buffer::stats::WAL_BUFFER_FLUSHES + ) + .unwrap(), 0 ); @@ -3138,9 +3206,13 @@ mod tests { // close() should succeed but not flush when failed. db.close().await.unwrap(); - assert_eq!(db.inner.wal_buffer.buffered_wal_entries_count(), 1); + assert_eq!(db.inner.wal_observer.status().buffered_wal_entries_count, 1); assert_eq!( - lookup_metric(&metrics_recorder, crate::db_stats::WAL_BUFFER_FLUSHES).unwrap(), + lookup_metric( + &metrics_recorder, + crate::wal_buffer::stats::WAL_BUFFER_FLUSHES + ) + .unwrap(), 0 ); let status = db.status(); @@ -3175,17 +3247,25 @@ mod tests { .await .unwrap(); - assert_eq!(db.inner.wal_buffer.buffered_wal_entries_count(), 1); + assert_eq!(db.inner.wal_observer.status().buffered_wal_entries_count, 1); assert_eq!( - lookup_metric(&metrics_recorder, crate::db_stats::WAL_BUFFER_FLUSHES).unwrap(), + lookup_metric( + &metrics_recorder, + crate::wal_buffer::stats::WAL_BUFFER_FLUSHES + ) + .unwrap(), 0 ); db.close().await.unwrap(); - assert_eq!(db.inner.wal_buffer.buffered_wal_entries_count(), 0); + assert_eq!(db.inner.wal_observer.status().buffered_wal_entries_count, 0); assert_eq!( - lookup_metric(&metrics_recorder, crate::db_stats::WAL_BUFFER_FLUSHES).unwrap(), + lookup_metric( + &metrics_recorder, + crate::wal_buffer::stats::WAL_BUFFER_FLUSHES + ) + .unwrap(), 1 ); let status = db.status(); @@ -3355,13 +3435,15 @@ mod tests { .await .unwrap(); assert_eq!( - lookup_metric(&metrics_recorder, crate::db_stats::WAL_FLUSH_BYTES).unwrap_or(0), + lookup_metric(&metrics_recorder, crate::wal_buffer::stats::WAL_FLUSH_BYTES) + .unwrap_or(0), 0, ); db.flush().await.unwrap(); - let wal_bytes = lookup_metric(&metrics_recorder, crate::db_stats::WAL_FLUSH_BYTES).unwrap(); + let wal_bytes = + lookup_metric(&metrics_recorder, crate::wal_buffer::stats::WAL_FLUSH_BYTES).unwrap(); let memtable_bytes = lookup_metric(&metrics_recorder, crate::db_stats::MEMTABLE_WRITE_BYTES).unwrap(); // WAL SST framing/footer makes the encoded payload at least as large as @@ -4213,7 +4295,7 @@ mod tests { } // Verify WALs flushes. - let wal_id = kv_store.inner.wal_buffer.recent_flushed_wal_id(); + let wal_id = kv_store.inner.wal_observer.status().last_flushed_wal_id; assert_eq!(wal_id, MAX_WAL_FLUSHES_BEFORE_L0_FLUSH); // account for the empty WAL written for fencing // Verify no memtable was frozen or L0 flush happened. @@ -4376,7 +4458,7 @@ mod tests { // Verify that the WAL was also flushed since we guarantee // memtable data is persisted in the WAL prior to L0 flush. - let recent_flushed_wal_id = kv_store.inner.wal_buffer.recent_flushed_wal_id(); + let recent_flushed_wal_id = kv_store.inner.wal_observer.status().last_flushed_wal_id; assert_eq!(recent_flushed_wal_id, 2); // Verify that the data is still accessible after flush @@ -4440,7 +4522,14 @@ mod tests { .await .unwrap(); - assert_eq!(kv_store.inner.wal_buffer.buffered_wal_entries_count(), 1); + assert_eq!( + kv_store + .inner + .wal_observer + .status() + .buffered_wal_entries_count, + 1 + ); kv_store .flush_with_options(FlushOptions { @@ -4449,7 +4538,14 @@ mod tests { .await .unwrap(); - assert_eq!(kv_store.inner.wal_buffer.buffered_wal_entries_count(), 0); + assert_eq!( + kv_store + .inner + .wal_observer + .status() + .buffered_wal_entries_count, + 0 + ); let wal_reader = WalReader::new(path, wal_object_store); let wal_files = wal_reader.list(..).await.unwrap(); @@ -4769,7 +4865,7 @@ mod tests { .unwrap(); // Get initial WAL ID to verify flush occurred - let initial_wal_id = kv_store.inner.wal_buffer.recent_flushed_wal_id(); + let initial_wal_id = kv_store.inner.wal_observer.status().last_flushed_wal_id; // Flush WAL using flush_with_options - this should succeed without error let flush_result = kv_store @@ -4790,7 +4886,7 @@ mod tests { // Verify that the WAL buffer is in a consistent state after flush // The recent_flushed_wal_id should be at least as high as before - let final_wal_id = kv_store.inner.wal_buffer.recent_flushed_wal_id(); + let final_wal_id = kv_store.inner.wal_observer.status().last_flushed_wal_id; assert!( final_wal_id >= initial_wal_id, "WAL ID should not decrease after flush" @@ -4909,14 +5005,14 @@ mod tests { .unwrap(); // Wait for put to end up in the WAL buffer - let this_wal_buffer = db.inner.wal_buffer.clone(); + let this_wal_buffer = db.inner.wal_observer.clone(); wait_for(Box::new(move || { - this_wal_buffer.buffered_wal_entries_count() > 0 + this_wal_buffer.status().buffered_wal_entries_count > 0 })) .await; // Verify that there is now 1 WAL entry in memory. - assert_eq!(db.inner.wal_buffer.buffered_wal_entries_count(), 1); + assert_eq!(db.inner.wal_observer.status().buffered_wal_entries_count, 1); // Put another WAL entry, which should trigger backpressure. Do this in a separate // task since the put() is blocked until the WAL is flushed, which isn't happening @@ -4981,7 +5077,7 @@ mod tests { db.put_with_options(b"key1", &large_value, &PutOptions::default(), &write_opts) .await .unwrap(); - assert_eq!(db.inner.wal_buffer.buffered_wal_entries_count(), 1); + assert_eq!(db.inner.wal_observer.status().buffered_wal_entries_count, 1); // Start backpressure on a cloned inner handle. This parks the task on // the same wait path used by writers before they enqueue a batch. @@ -5738,7 +5834,7 @@ mod tests { let value1 = [b'b'; 96]; let result = db.put(&key1, &value1).await; assert!(result.is_ok(), "Failed to write key1"); - assert_eq!(db.inner.wal_buffer.recent_flushed_wal_id(), 2); + assert_eq!(db.inner.wal_observer.status().last_flushed_wal_id, 2); // Let background flush attempts fail while WAL durability preserves recovery. // expect to fail as l0 upload is blocked @@ -9495,7 +9591,7 @@ mod tests { // then: let estimated = lookup_metric( &metrics_recorder, - crate::db_stats::WAL_BUFFER_ESTIMATED_BYTES, + crate::wal_buffer::stats::WAL_BUFFER_ESTIMATED_BYTES, ); assert!( estimated.is_some_and(|v| v > 0), @@ -9651,11 +9747,11 @@ mod tests { .await .unwrap(); if i == 0 { - first_l0_flushed_wal_id = source.inner.wal_buffer.recent_flushed_wal_id(); + first_l0_flushed_wal_id = source.inner.wal_observer.status().last_flushed_wal_id; } l0_flushed_seq = write.seqnum(); } - let l0_flushed_boundary_wal_id = source.inner.wal_buffer.recent_flushed_wal_id(); + let l0_flushed_boundary_wal_id = source.inner.wal_observer.status().last_flushed_wal_id; assert!(l0_flushed_boundary_wal_id > first_l0_flushed_wal_id); // Write several smaller records, each flushed into a separate WAL. On @@ -9679,7 +9775,7 @@ mod tests { .await .unwrap(); } - let final_source_wal_id = source.inner.wal_buffer.recent_flushed_wal_id(); + let final_source_wal_id = source.inner.wal_observer.status().last_flushed_wal_id; assert!(final_source_wal_id >= l0_flushed_boundary_wal_id + 2); // Recover with a much smaller replay target so WAL replay splits into diff --git a/slatedb/src/db/builder.rs b/slatedb/src/db/builder.rs index 24ace6a7c..10c6558a0 100644 --- a/slatedb/src/db/builder.rs +++ b/slatedb/src/db/builder.rs @@ -160,6 +160,7 @@ use crate::retrying_object_store::RetryingObjectStore; use crate::tablestore::{TableStore, TableStoreKind}; use crate::utils::SafeSender; use crate::utils::WatchableOnceCell; +use crate::wal_buffer::WalBufferManager; use slatedb_common::clock::DefaultSystemClock; use slatedb_common::clock::SystemClock; use slatedb_common::metrics::MetricsRecorder; @@ -586,6 +587,16 @@ impl> DbBuilder

{ BTreeSet::new(), ); + let recent_flushed_wal_id = replay_range.end - 1; + let wal_buffer = Arc::new(WalBufferManager::new( + status_manager.clone(), + &recorder, + recent_flushed_wal_id, + table_store.clone(), + self.settings.l0_sst_size_bytes, + self.settings.flush_interval, + )); + // Setup communication channels wired to the shared closed state. let reader = status_manager.result_reader(); let (write_tx, write_rx) = SafeSender::unbounded_channel(reader); @@ -601,6 +612,7 @@ impl> DbBuilder

{ manifest_dirty, Arc::clone(&memtable_flusher), write_tx, + wal_buffer.observer(), recorder.clone(), self.fp_registry.clone(), self.merge_operator.clone(), @@ -617,11 +629,11 @@ impl> DbBuilder

{ system_clock.clone(), )); if inner.wal_enabled { - inner.wal_buffer.init(task_executor.clone()).await?; + wal_buffer.init(task_executor.clone()).await?; }; task_executor.add_handler( WRITE_BATCH_TASK_NAME.to_string(), - Box::new(WriteBatchEventHandler::new(inner.clone())), + Box::new(WriteBatchEventHandler::new(inner.clone(), wal_buffer)), write_rx, &tokio_handle, )?; diff --git a/slatedb/src/db_common.rs b/slatedb/src/db_common.rs index efeaf8e46..80c75dc57 100644 --- a/slatedb/src/db_common.rs +++ b/slatedb/src/db_common.rs @@ -28,9 +28,9 @@ pub(crate) fn extract_segment_prefix( impl DbInner { pub(crate) fn replay_memtable( &self, + current_memtable_wal_id: u64, replayed_memtable: ReplayedMemtable, ) -> Result<(), SlateDBError> { - let current_memtable_wal_id = self.wal_buffer.recent_flushed_wal_id(); let mut guard = self.state.write(); // The active memtable was installed by the previous replay step, so its @@ -55,7 +55,6 @@ impl DbInner { // replace the memtable guard.replace_memtable(replayed_memtable.table); - self.wal_buffer.advance_recent_flushed_wal_id(last_wal); let dirty_manifest = guard.state().manifest.clone(); drop(guard); self.status_manager.report_manifest(dirty_manifest.into()); diff --git a/slatedb/src/db_state.rs b/slatedb/src/db_state.rs index cde95c6c3..08ec6d8d5 100644 --- a/slatedb/src/db_state.rs +++ b/slatedb/src/db_state.rs @@ -4,7 +4,6 @@ use crate::error::SlateDBError; use crate::manifest::{Manifest, ManifestCore}; use crate::mem_table::{ImmutableMemtable, KVTable, WritableKVTable}; use crate::reader::DbStateReader; -use crate::wal_id::WalIdStore; use bytes::Bytes; use serde::Serialize; use slatedb_txn_obj::DirtyObject; @@ -771,6 +770,13 @@ impl DbState { }); } + pub(crate) fn set_next_wal_id(&mut self, next_wal_id: u64) { + self.modify(|modifier| { + assert!(next_wal_id >= modifier.state.manifest.value.core.next_wal_sst_id); + modifier.state.manifest.value.core.next_wal_sst_id = next_wal_id; + }) + } + pub(crate) fn replace_memtable(&mut self, memtable: WritableKVTable) { assert!(self.memtable.is_empty()); let _ = std::mem::replace(&mut self.memtable, memtable); @@ -832,22 +838,6 @@ impl<'a> StateModifier<'a> { } } -impl WalIdStore for parking_lot::RwLock { - /// increment the next wal id, and return the previous value. - fn next_wal_id(&self) -> u64 { - let mut state = self.write(); - - // not sure why, but it doesn't compile without the return - // statement -- probably some generic inference bug - #[allow(clippy::needless_return)] - return state.modify(|modifier| { - let next_wal_id = modifier.state.manifest.value.core.next_wal_sst_id; - modifier.state.manifest.value.core.next_wal_sst_id += 1; - next_wal_id - }); - } -} - #[cfg(test)] mod tests { use crate::bytes_range::BytesRange; diff --git a/slatedb/src/db_stats.rs b/slatedb/src/db_stats.rs index c2c2bbf90..8c4475634 100644 --- a/slatedb/src/db_stats.rs +++ b/slatedb/src/db_stats.rs @@ -23,9 +23,6 @@ pub const L0_STALL_TYPE_LABEL: &str = "type"; pub const L0_STALL_TYPE_NUM_SSTS: &str = "num_ssts"; pub const L0_STALL_TYPE_NUM_SSTS_PER_KEY: &str = "num_ssts_per_key"; pub const IMMUTABLE_MEMTABLE_FLUSHES: &str = db_stat_name!("immutable_memtable_flushes"); -pub const WAL_BUFFER_FLUSHES: &str = db_stat_name!("wal_buffer_flushes"); -pub const WAL_BUFFER_FLUSH_REQUESTS: &str = db_stat_name!("wal_buffer_flush_requests"); -pub const WAL_BUFFER_ESTIMATED_BYTES: &str = db_stat_name!("wal_buffer_estimated_bytes"); pub const TOTAL_MEM_SIZE_BYTES: &str = db_stat_name!("total_mem_size_bytes"); pub const L0_SST_COUNT: &str = db_stat_name!("l0_sst_count"); pub const SEGMENT_MAX_L0_SST_COUNT: &str = db_stat_name!("segment_max_l0_sst_count"); @@ -39,7 +36,6 @@ pub const SST_FILTER_NEGATIVE_COUNT: &str = db_stat_name!("sst_filter_negative_c /// write_amp = (`WAL_FLUSH_BYTES` + `L0_FLUSH_BYTES` + `compactor::stats::BYTES_COMPACTED`) /// / `MEMTABLE_WRITE_BYTES` pub const MEMTABLE_WRITE_BYTES: &str = db_stat_name!("memtable_write_bytes"); -pub const WAL_FLUSH_BYTES: &str = db_stat_name!("wal_flush_bytes"); /// Label key distinguishing filter metrics for point lookups from those for /// prefix scans. Value is one of [`FILTER_KIND_POINT`] or @@ -50,9 +46,6 @@ pub const FILTER_KIND_PREFIX: &str = "prefix"; pub(crate) struct DbStatsInner { pub(crate) immutable_memtable_flushes: Arc, - pub(crate) wal_buffer_estimated_bytes: Arc, - pub(crate) wal_buffer_flushes: Arc, - pub(crate) wal_buffer_flush_requests: Arc, pub(crate) sst_filter_point_false_positives: Arc, pub(crate) sst_filter_point_positives: Arc, pub(crate) sst_filter_point_negatives: Arc, @@ -74,7 +67,6 @@ pub(crate) struct DbStatsInner { pub(crate) merge_operator_read_operands: Arc, pub(crate) merge_operator_flush_operands: Arc, pub(crate) memtable_write_bytes: Arc, - pub(crate) wal_flush_bytes: Arc, } #[derive(Clone)] @@ -95,9 +87,6 @@ impl DbStats { pub(crate) fn new(recorder: &MetricsRecorderHelper) -> DbStats { let inner = DbStatsInner { immutable_memtable_flushes: recorder.counter(IMMUTABLE_MEMTABLE_FLUSHES).register(), - wal_buffer_estimated_bytes: recorder.gauge(WAL_BUFFER_ESTIMATED_BYTES).register(), - wal_buffer_flushes: recorder.counter(WAL_BUFFER_FLUSHES).register(), - wal_buffer_flush_requests: recorder.counter(WAL_BUFFER_FLUSH_REQUESTS).register(), sst_filter_point_false_positives: recorder .counter(SST_FILTER_FALSE_POSITIVE_COUNT) .labels(&[(FILTER_KIND_LABEL, FILTER_KIND_POINT)]) @@ -160,7 +149,6 @@ impl DbStats { .description(MERGE_OPERATOR_OPERANDS_DESCRIPTION) .register(), memtable_write_bytes: recorder.counter(MEMTABLE_WRITE_BYTES).register(), - wal_flush_bytes: recorder.counter(WAL_FLUSH_BYTES).register(), }; DbStats { inner: Arc::new(inner), diff --git a/slatedb/src/lib.rs b/slatedb/src/lib.rs index 4147912d1..89a0027cb 100644 --- a/slatedb/src/lib.rs +++ b/slatedb/src/lib.rs @@ -75,6 +75,7 @@ pub use sst_stats::{BlockStats, SstStats}; pub use transaction_manager::IsolationLevel; pub use types::KeyValue; pub use types::{RowEntry, ValueDeletable}; +pub use wal_buffer::stats as wal_buffer_stats; pub use wal_reader::{WalFile, WalFileIterator, WalReader}; pub mod admin; @@ -171,7 +172,6 @@ mod utils; mod fence; mod wal; mod wal_buffer; -mod wal_id; mod wal_reader; mod wal_replay; diff --git a/slatedb/src/memtable_flusher/manifest_writer.rs b/slatedb/src/memtable_flusher/manifest_writer.rs index 15e31f69b..6758adb2b 100644 --- a/slatedb/src/memtable_flusher/manifest_writer.rs +++ b/slatedb/src/memtable_flusher/manifest_writer.rs @@ -880,6 +880,7 @@ mod tests { use crate::tablestore::{TableStore, TableStoreKind}; use crate::types::RowEntry; use crate::utils::WatchableOnceCell; + use crate::wal_buffer::WalBufferManager; use bytes::Bytes; use fail_parallel::FailPointRegistry; use object_store::memory::InMemory; @@ -887,7 +888,7 @@ mod tests { use object_store::ObjectStore; use slatedb_common::clock::DefaultSystemClock; use slatedb_common::clock::SystemClock; - use slatedb_common::metrics::MetricsRecorderHelper; + use slatedb_common::metrics::{DefaultMetricsRecorder, MetricLevel, MetricsRecorderHelper}; use slatedb_common::DbRand; use std::sync::Arc; use std::time::Duration; @@ -1040,6 +1041,16 @@ mod tests { let status_manager = DbStatusManager::new(0); let (write_tx, _) = crate::utils::SafeSender::unbounded_channel(status_manager.result_reader()); + let recorder = Arc::new(DefaultMetricsRecorder::new()); + let helper = MetricsRecorderHelper::new(recorder, MetricLevel::Info); + let wal_buffer = Arc::new(WalBufferManager::new( + status_manager.clone(), + &helper, + 0, + table_store.clone(), + 1024, + None, + )); let inner = Arc::new( DbInner::new( settings.clone(), @@ -1051,6 +1062,7 @@ mod tests { &WatchableOnceCell::new(), )), write_tx, + wal_buffer.observer(), db_metrics, fp_registry, None, diff --git a/slatedb/src/memtable_flusher/tracker.rs b/slatedb/src/memtable_flusher/tracker.rs index b56ac06f7..360222a65 100644 --- a/slatedb/src/memtable_flusher/tracker.rs +++ b/slatedb/src/memtable_flusher/tracker.rs @@ -539,6 +539,7 @@ mod tests { use crate::tablestore::{TableStore, TableStoreKind}; use crate::types::RowEntry; use crate::utils::{SafeSender, WatchableOnceCell}; + use crate::wal_buffer::WalBufferManager; use bytes::Bytes; use fail_parallel::FailPointRegistry; use object_store::memory::InMemory; @@ -603,6 +604,16 @@ mod tests { let status_manager = DbStatusManager::new(0); let (write_tx, _) = SafeSender::::unbounded_channel(status_manager.result_reader()); + let recorder = Arc::new(DefaultMetricsRecorder::new()); + let helper = MetricsRecorderHelper::new(recorder, MetricLevel::Info); + let wal_buffer = Arc::new(WalBufferManager::new( + status_manager.clone(), + &helper, + 0, + table_store.clone(), + 1024, + None, + )); let inner = Arc::new( DbInner::new( settings, @@ -612,6 +623,7 @@ mod tests { stored_manifest.prepare_dirty().unwrap(), Arc::new(MemtableFlusher::new(&status_manager)), write_tx, + wal_buffer.observer(), db_metrics, fp_registry, None, diff --git a/slatedb/src/memtable_flusher/uploader.rs b/slatedb/src/memtable_flusher/uploader.rs index 243ffac1d..e25b0a27c 100644 --- a/slatedb/src/memtable_flusher/uploader.rs +++ b/slatedb/src/memtable_flusher/uploader.rs @@ -290,13 +290,14 @@ mod tests { use crate::test_utils::FixedThreeBytePrefixExtractor; use crate::types::{RowEntry, ValueDeletable}; use crate::utils::WatchableOnceCell; + use crate::wal_buffer::WalBufferManager; use bytes::Bytes; use fail_parallel::FailPointRegistry; use object_store::memory::InMemory; use object_store::path::Path; use object_store::ObjectStore; use slatedb_common::clock::{DefaultSystemClock, SystemClock}; - use slatedb_common::metrics::MetricsRecorderHelper; + use slatedb_common::metrics::{DefaultMetricsRecorder, MetricLevel, MetricsRecorderHelper}; use slatedb_common::DbRand; use std::sync::Arc; use std::time::Duration; @@ -339,6 +340,16 @@ mod tests { let status_manager = DbStatusManager::new(0); let (write_tx, _) = crate::utils::SafeSender::unbounded_channel(status_manager.result_reader()); + let recorder = Arc::new(DefaultMetricsRecorder::new()); + let helper = MetricsRecorderHelper::new(recorder, MetricLevel::Info); + let wal_buffer = Arc::new(WalBufferManager::new( + status_manager.clone(), + &helper, + 0, + table_store.clone(), + 1024, + None, + )); Arc::new( DbInner::new( settings, @@ -350,6 +361,7 @@ mod tests { &status_manager, )), write_tx, + wal_buffer.observer(), db_metrics, fp_registry, None, diff --git a/slatedb/src/oracle.rs b/slatedb/src/oracle.rs index 907628d78..292ba14a9 100644 --- a/slatedb/src/oracle.rs +++ b/slatedb/src/oracle.rs @@ -61,12 +61,6 @@ impl DbOracle { self.last_durable_seq.fetch_max(seq, SeqCst); self.status_reporter.report_durable_seq(seq); } - - #[cfg(test)] - pub(crate) fn set_durable_seq_unsafe(&self, value: u64) { - self.last_durable_seq.store(value, SeqCst); - self.status_reporter.report_durable_seq(value); - } } impl Oracle for DbOracle { diff --git a/slatedb/src/wal_buffer.rs b/slatedb/src/wal_buffer.rs index 7e3c23b79..2fe8f62df 100644 --- a/slatedb/src/wal_buffer.rs +++ b/slatedb/src/wal_buffer.rs @@ -3,26 +3,25 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::Duration; -use async_trait::async_trait; -use futures::{stream::BoxStream, StreamExt}; -use log::{error, trace}; -use tokio::{runtime::Handle, sync::oneshot}; -use tracing::instrument; - use crate::db_state::SsTableId; -use crate::db_stats::DbStats; use crate::db_status::ClosedResultWriter; use crate::dispatcher::{MessageHandler, MessageHandlerExecutor, MessageTickerDef}; use crate::error::SlateDBError; -use crate::oracle::{DbOracle, Oracle}; use crate::tablestore::TableStore; use crate::types::RowEntry; use crate::utils::SafeSender; use crate::utils::{format_bytes_si, WatchableOnceCell, WatchableOnceCellReader}; -use crate::wal_id::WalIdStore; +use async_trait::async_trait; +use futures::{stream::BoxStream, StreamExt}; +use log::{error, trace}; +use slatedb_common::metrics::MetricsRecorderHelper; +use tokio::{runtime::Handle, sync::oneshot}; +use tracing::instrument; pub(crate) const WAL_BUFFER_TASK_NAME: &str = "wal_writer"; +pub(crate) type WalStatusListener = Arc; + /// [`WalBufferManager`] buffers write operations in memory before flushing them to persistent storage. /// The flush operation only targets Remote storage right now, later we can add an option to flush to local /// storage. @@ -48,9 +47,8 @@ pub(crate) const WAL_BUFFER_TASK_NAME: &str = "wal_writer"; /// operations. The manager becomes unusable after encountering a fatal error. pub(crate) struct WalBufferManager { inner: Arc>, - wal_id_incrementor: Arc, status_manager: crate::db_status::DbStatusManager, - db_stats: DbStats, + stats: stats::WalBufferStats, table_store: Arc, max_wal_bytes_size: usize, max_flush_interval: Option, @@ -72,14 +70,20 @@ struct WalBufferManagerInner { /// Whenever a WAL is applied to Memtable and successfully flushed to remote storage, /// the immutable wal can be recycled in memory. last_applied_seq: Option, + /// The next wal id that will be generated + next_wal_id: u64, /// Monotonically increasing epoch incremented each time the current WAL is /// frozen. Used with `last_flush_requested_epoch` to deduplicate size-triggered /// flush requests. flush_epoch: u64, - /// The flusher will update the recent_flushed_wal_id and last_flushed_seq when the flush is done. - recent_flushed_wal_id: u64, - /// The oracle to track the last flushed sequence number. - oracle: Arc, + /// The flusher will update the last_flushed_wal_id and last_flushed_seq when the flush is done. + last_flushed_wal_id: u64, + /// The last seq that was flushed to the WAL. This value will be None until the first flush. + last_flushed_seq: Option, + /// The last wal id that was deallocated from the buffer + last_purged_wal_id: u64, + /// A listener to which the wal sends events. Currently the only event is a wal file flush. + listener: Option, } /// Stores entries to the write-ahead log (WAL) in memory. @@ -110,11 +114,9 @@ struct WalBufferIterator { impl WalBufferManager { pub(crate) fn new( - wal_id_incrementor: Arc, status_manager: crate::db_status::DbStatusManager, - db_stats: DbStats, - recent_flushed_wal_id: u64, - oracle: Arc, + recorder: &MetricsRecorderHelper, + last_flushed_wal_id: u64, table_store: Arc, max_wal_bytes_size: usize, max_flush_interval: Option, @@ -126,16 +128,18 @@ impl WalBufferManager { immutable_wals, last_applied_seq: None, flush_epoch: 1, - recent_flushed_wal_id, + last_flushed_wal_id, + last_purged_wal_id: last_flushed_wal_id, + next_wal_id: last_flushed_wal_id + 1, + last_flushed_seq: None, flush_tx: None, task_executor: None, - oracle, + listener: None, }; Self { inner: Arc::new(parking_lot::RwLock::new(inner)), - wal_id_incrementor, status_manager, - db_stats, + stats: stats::WalBufferStats::new(recorder), table_store, max_wal_bytes_size, max_flush_interval, @@ -143,6 +147,7 @@ impl WalBufferManager { } } + // todo: consider consolidating with new pub(crate) async fn init( self: &Arc, task_executor: Arc, @@ -171,53 +176,21 @@ impl WalBufferManager { result } - #[cfg(test)] - pub(crate) fn buffered_wal_entries_count(&self) -> usize { - let guard = self.inner.read(); - let flushing_wal_entries_count = guard - .immutable_wals - .iter() - .map(|(_, wal)| wal.len()) - .sum::(); - guard.current_wal.len() + flushing_wal_entries_count - } - - pub(crate) fn recent_flushed_wal_id(&self) -> u64 { + pub(crate) fn last_flushed_wal_id(&self) -> u64 { let inner = self.inner.read(); - inner.recent_flushed_wal_id + inner.last_flushed_wal_id } - /// Advance `recent_flushed_wal_id` to at least `wal_id`. - pub(crate) fn advance_recent_flushed_wal_id(&self, wal_id: u64) { + fn subscribe(&self, listener: WalStatusListener) { + // TODO: consider extending to multiple listeners let mut inner = self.inner.write(); - if wal_id > inner.recent_flushed_wal_id { - inner.recent_flushed_wal_id = wal_id; - } - } - - #[cfg(test)] // used in compactor.rs - pub(crate) fn is_empty(&self) -> bool { - let inner = self.inner.read(); - inner.current_wal.is_empty() && inner.immutable_wals.is_empty() + assert!(inner.listener.is_none()); + inner.listener = Some(listener); } - /// Returns the total size of all unflushed WALs in bytes. - pub(crate) fn estimated_bytes(&self) -> Result { + pub(crate) fn status(&self) -> WalStatus { let inner = self.inner.read(); - let current_wal_size = self - .table_store - .estimate_encoded_size_wal(inner.current_wal.len(), inner.current_wal.size()); - - let imm_wal_size = inner - .immutable_wals - .iter() - .map(|(_, wal)| { - self.table_store - .estimate_encoded_size_wal(wal.len(), wal.size()) - }) - .sum::(); - - Ok(current_wal_size + imm_wal_size) + inner.status(&self.table_store) } /// Append row entries to the current WAL. Returns a watcher for durability notification. @@ -274,27 +247,16 @@ impl WalBufferManager { } } - let estimated_bytes = self.estimated_bytes()?; - self.db_stats - .wal_buffer_estimated_bytes - .set(estimated_bytes as i64); + let status = self.status(); + self.stats + .estimated_bytes + .set(status.estimated_bytes as i64); Ok(durable_watcher) } - /// Returns a watcher to await durability of the oldest unflushed WAL. - /// If there are immutable WALs, it returns a watcher for the oldest immutable WAL. - /// Otherwise, it returns a watcher for the current WAL if it's not empty. - /// Returns None if there are no unflushed WALs. - pub(crate) fn watcher_for_oldest_unflushed_wal( - &self, - ) -> Option>> { - let guard = self.inner.read(); - if let Some((_, wal)) = guard.immutable_wals.front() { - Some(wal.durable_watcher()) - } else if !guard.current_wal.is_empty() { - Some(guard.current_wal.durable_watcher()) - } else { - None + pub(crate) fn observer(self: &Arc) -> WalObserver { + WalObserver { + wal_buffer: self.clone(), } } @@ -303,7 +265,7 @@ impl WalBufferManager { &self, result_tx: Option>>, ) -> Result<(), SlateDBError> { - self.db_stats.wal_buffer_flush_requests.increment(1); + self.stats.flush_requests.increment(1); let flush_tx = self .inner .read() @@ -327,7 +289,7 @@ impl WalBufferManager { let inner = self.inner.read(); let mut flushing_wals = Vec::new(); for (wal_id, wal) in inner.immutable_wals.iter() { - if *wal_id > inner.recent_flushed_wal_id { + if *wal_id > inner.last_flushed_wal_id { flushing_wals.push((*wal_id, wal.clone())); } } @@ -354,24 +316,40 @@ impl WalBufferManager { } // increment the last flushed wal id, and last flushed seq - { + let (status, listener) = { let mut inner = self.inner.write(); - inner.recent_flushed_wal_id = *wal_id; + inner.last_flushed_wal_id = *wal_id; if let Some(seq) = wal.last_seq() { - inner.oracle.advance_durable_seq(seq); + if let Some(last_flushed_seq) = inner.last_flushed_seq { + assert!(seq >= last_flushed_seq); + } + inner.last_flushed_seq = Some(seq); } - } + let status = inner.status(&self.table_store); + let listener = inner.listener.clone(); + (status, listener) + }; + // TODO: we probably want to release immutable wals first for the backpressure check // notify durable only when the flush is successful. + if let Some(l) = listener { + (*l)(WalEvent::WalFlushed(status)) + } wal.notify_durable(result.clone()); } self.maybe_release_immutable_wals(); + let status = self.status(); + let listener = self.inner.read().listener.clone(); + if let Some(l) = listener { + (*l)(WalEvent::MemoryReleased(status)) + } + Ok(()) } async fn do_flush_one_wal(&self, wal_id: u64, wal: Arc) -> Result<(), SlateDBError> { - self.db_stats.wal_buffer_flushes.increment(1); + self.stats.flushes.increment(1); let mut sst_builder = self.table_store.wal_table_builder(); let mut iter = wal.iter(); @@ -384,7 +362,7 @@ impl WalBufferManager { self.table_store .write_sst(&SsTableId::Wal(wal_id), &encoded_sst, false) .await?; - self.db_stats.wal_flush_bytes.increment(written_bytes); + self.stats.flush_bytes.increment(written_bytes); Ok(()) } @@ -394,13 +372,20 @@ impl WalBufferManager { return Ok(()); } - let next_wal_id = self.wal_id_incrementor.next_wal_id(); let mut inner = self.inner.write(); + let next_wal_id = inner.next_wal_id; + inner.next_wal_id += 1; let current_wal = std::mem::replace(&mut inner.current_wal, WalBuffer::new()); inner.flush_epoch += 1; inner .immutable_wals .push_back((next_wal_id, Arc::new(current_wal))); + let status = inner.status(&self.table_store); + let listener = inner.listener.clone(); + drop(inner); + if let Some(l) = listener { + (*l)(WalEvent::WalFrozen(status)) + } Ok(()) } @@ -414,6 +399,7 @@ impl WalBufferManager { inner.last_applied_seq = Some(seq); } self.maybe_release_immutable_wals(); + // don't notify here - notifications should only be issued from the flush task } /// Recycle the immutable WALs that are flushed to the remote storage. @@ -425,13 +411,14 @@ impl WalBufferManager { None => return, }; - let last_flushed_seq = inner.oracle.last_remote_persisted_seq(); + let last_flushed_seq = inner.last_flushed_seq; let mut releaseable_count = 0; for (_, wal) in inner.immutable_wals.iter() { if wal .last_seq() - .map(|seq| seq <= last_applied_seq && seq <= last_flushed_seq) + // TODO: check me (make sure seq starts at 1) + .map(|seq| seq <= last_applied_seq && seq <= last_flushed_seq.unwrap_or(0)) .unwrap_or(false) { releaseable_count += 1; @@ -445,7 +432,14 @@ impl WalBufferManager { "draining immutable wals [releaseable_count={}]", releaseable_count ); - inner.immutable_wals.drain(..releaseable_count); + let last_purged = inner + .immutable_wals + .drain(..releaseable_count) + .map(|(id, _wal)| id) + .max(); + if let Some(last_purged) = last_purged { + inner.last_purged_wal_id = last_purged; + } } } @@ -462,6 +456,37 @@ impl WalBufferManager { } } +impl WalBufferManagerInner { + /// Returns the total size of all unflushed WALs in bytes. + fn estimated_bytes(&self, table_store: &TableStore) -> usize { + let current_wal_size = + table_store.estimate_encoded_size_wal(self.current_wal.len(), self.current_wal.size()); + let imm_wal_size = self + .immutable_wals + .iter() + .map(|(_, wal)| table_store.estimate_encoded_size_wal(wal.len(), wal.size())) + .sum::(); + current_wal_size + imm_wal_size + } + + fn status(&self, table_store: &TableStore) -> WalStatus { + let flushing_wal_entries_count = self + .immutable_wals + .iter() + .map(|(_, wal)| wal.len()) + .sum::(); + let buffered_wal_entries_count = self.current_wal.len() + flushing_wal_entries_count; + WalStatus { + estimated_bytes: self.estimated_bytes(table_store), + next_wal_id: self.next_wal_id, + last_flushed_wal_id: self.last_flushed_wal_id, + last_flushed_seq: self.last_flushed_seq, + last_purged_wal_id: self.last_purged_wal_id, + buffered_wal_entries_count, + } + } +} + impl WalBuffer { /// Creates a new empty `WalBuffer`. fn new() -> Self { @@ -600,6 +625,87 @@ impl MessageHandler for WalFlushHandler { } } +/// Interface for getting information about the current state of the Wal +#[derive(Clone)] +pub(crate) struct WalObserver { + wal_buffer: Arc, +} + +/// Describes the current status of the WAL +#[derive(Debug, Clone)] +pub(crate) struct WalStatus { + /// The estimated in-memory bytes used by the WAL to buffer unflushed writes. + pub(crate) estimated_bytes: usize, + pub(crate) next_wal_id: u64, + /// The id of the last WAL file that was durably flushed + #[allow(dead_code)] + pub(crate) last_flushed_wal_id: u64, + /// The last sequence number that was durably flushed + pub(crate) last_flushed_seq: Option, + /// The last WAL file id whose memory was released + pub(crate) last_purged_wal_id: u64, + /// The number of writes currently buffered + #[allow(dead_code)] + pub(crate) buffered_wal_entries_count: usize, +} + +/// An event emitted by [`WalBufferManager`] to subscribers. +#[derive(Debug, Clone)] +pub(crate) enum WalEvent { + /// Emitted when the current buffer is frozen + WalFrozen(WalStatus), + /// Emitted when a WAL file is durably flushed to storage. On receipt of this event, SlateDB + /// notifies write tasks blocked on [`crate::config::WriteOptions::await_durable`] + WalFlushed(WalStatus), + /// Emitted when `WalBufferManager` releases buffer memory. + MemoryReleased(WalStatus), +} + +impl WalObserver { + /// Gets information about the Wal buffer's current state + pub(crate) fn status(&self) -> WalStatus { + self.wal_buffer.status() + } + + pub(crate) fn subscribe(&self, listener: WalStatusListener) { + self.wal_buffer.subscribe(listener); + } +} + +pub mod stats { + use slatedb_common::metrics::{CounterFn, GaugeFn, MetricsRecorderHelper}; + use std::sync::Arc; + + macro_rules! wal_stat_name { + ($suffix:expr) => { + concat!("slatedb.wal.", $suffix) + }; + } + + pub const WAL_BUFFER_FLUSHES: &str = wal_stat_name!("wal_buffer_flushes"); + pub const WAL_BUFFER_FLUSH_REQUESTS: &str = wal_stat_name!("wal_buffer_flush_requests"); + pub const WAL_BUFFER_ESTIMATED_BYTES: &str = wal_stat_name!("wal_buffer_estimated_bytes"); + pub const WAL_FLUSH_BYTES: &str = wal_stat_name!("wal_flush_bytes"); + + pub(super) struct WalBufferStats { + pub(super) estimated_bytes: Arc, + pub(super) flushes: Arc, + pub(super) flush_requests: Arc, + pub(super) flush_bytes: Arc, + } + + impl WalBufferStats { + pub(super) fn new(recorder: &MetricsRecorderHelper) -> Self { + Self { + estimated_bytes: recorder.gauge(WAL_BUFFER_ESTIMATED_BYTES).register(), + flushes: recorder.counter(WAL_BUFFER_FLUSHES).register(), + flush_requests: recorder.counter(WAL_BUFFER_FLUSH_REQUESTS).register(), + flush_bytes: recorder.counter(WAL_FLUSH_BYTES).register(), + } + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -608,6 +714,7 @@ mod tests { use crate::iter::RowEntryIterator; use crate::manifest::SsTableView; use crate::object_stores::ObjectStores; + use crate::oracle::DbOracle; use crate::sst_iter::{SstIterator, SstIteratorOptions}; use crate::tablestore::{TableStore, TableStoreKind}; use crate::types::{RowEntry, ValueDeletable}; @@ -618,8 +725,7 @@ mod tests { lookup_metric, DefaultMetricsRecorder, MetricLevel, MetricsRecorderHelper, }; use slatedb_common::MockSystemClock; - use std::sync::atomic::{AtomicU64, Ordering}; - use std::sync::Arc; + use std::sync::{Arc, Mutex}; use std::time::Duration; fn make_entry(key: &str, value: &str, seq: u64, create_ts: Option) -> RowEntry { @@ -793,21 +899,10 @@ mod tests { assert!(buffer.size() > 100_000); } - struct MockWalIdStore { - next_id: AtomicU64, - } - - impl WalIdStore for MockWalIdStore { - fn next_wal_id(&self) -> u64 { - self.next_id.fetch_add(1, Ordering::SeqCst) - } - } - async fn setup_wal_buffer() -> ( Arc, Arc, Arc, - DbStats, Arc, ) { setup_wal_buffer_with_flush_interval(Duration::from_millis(10)).await @@ -819,12 +914,20 @@ mod tests { Arc, Arc, Arc, - DbStats, Arc, ) { - let wal_id_store: Arc = Arc::new(MockWalIdStore { - next_id: AtomicU64::new(1), - }); + setup_wal_buffer_with_args(flush_interval, Arc::new(|_status| {})).await + } + + async fn setup_wal_buffer_with_args( + flush_interval: Duration, + listener: WalStatusListener, + ) -> ( + Arc, + Arc, + Arc, + Arc, + ) { let object_store: Arc = Arc::new(InMemory::new()); let table_store = Arc::new(TableStore::new( ObjectStores::new(object_store, None), @@ -839,17 +942,21 @@ mod tests { let oracle = Arc::new(DbOracle::new(0, 0, 0, status_manager.clone())); let recorder = Arc::new(DefaultMetricsRecorder::new()); let helper = MetricsRecorderHelper::new(recorder.clone(), MetricLevel::default()); - let db_stats = DbStats::new(&helper); let wal_buffer = Arc::new(WalBufferManager::new( - wal_id_store, status_manager.clone(), - db_stats.clone(), + &helper, 0, // recent_flushed_wal_id - oracle, table_store.clone(), 1000, // max_wal_bytes_size Some(flush_interval), // max_flush_interval )); + wal_buffer.subscribe(Arc::new(move |status| { + (*listener)(status.clone()); + let WalEvent::WalFlushed(status) = status else { + return; + }; + oracle.advance_durable_seq(status.last_flushed_seq.unwrap_or(0)) + })); let task_executor = Arc::new(MessageHandlerExecutor::new( Arc::new(status_manager), system_clock.clone(), @@ -858,12 +965,12 @@ mod tests { task_executor .monitor_on(&Handle::current()) .expect("failed to monitor executor"); - (wal_buffer, table_store, test_clock, db_stats, recorder) + (wal_buffer, table_store, test_clock, recorder) } #[tokio::test] async fn test_basic_append_and_flush_operations() { - let (wal_buffer, table_store, _, _, _) = setup_wal_buffer().await; + let (wal_buffer, table_store, _, _) = setup_wal_buffer().await; // Append some entries let entry1 = make_entry("key1", "value1", 1, None); @@ -905,11 +1012,11 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_size_based_flush_triggering() { - let (wal_buffer, _, _, _, _) = setup_wal_buffer_with_flush_interval(Duration::MAX).await; + let (wal_buffer, _, _, _) = setup_wal_buffer_with_flush_interval(Duration::MAX).await; // Append entries until we exceed the size threshold let mut seq = 1; - while wal_buffer.estimated_bytes().unwrap() < wal_buffer.max_wal_bytes_size { + while wal_buffer.status().estimated_bytes < wal_buffer.max_wal_bytes_size { let entry = make_entry(&format!("key{}", seq), &format!("value{}", seq), seq, None); wal_buffer.append(&[entry]).unwrap(); seq += 1; @@ -917,12 +1024,12 @@ mod tests { let mut reader = wal_buffer.maybe_trigger_flush().unwrap(); reader.await_value().await.unwrap(); - assert_eq!(wal_buffer.recent_flushed_wal_id(), 1); + assert_eq!(wal_buffer.last_flushed_wal_id(), 1); } #[tokio::test] async fn test_immutable_wal_reclaim() { - let (wal_buffer, _, _, _, _) = setup_wal_buffer().await; + let (wal_buffer, _, _, _) = setup_wal_buffer().await; // Append entries to create multiple WALs for i in 0..100 { @@ -931,7 +1038,7 @@ mod tests { wal_buffer.append(&[entry]).unwrap(); wal_buffer.flush().unwrap().await.unwrap().unwrap(); } - assert_eq!(wal_buffer.recent_flushed_wal_id(), 100); + assert_eq!(wal_buffer.last_flushed_wal_id(), 100); assert_eq!(wal_buffer.inner.read().immutable_wals.len(), 100); wal_buffer.track_last_applied_seq(50); @@ -940,7 +1047,7 @@ mod tests { #[tokio::test] async fn test_immutable_wal_reclaim_with_flush_check() { - let (wal_buffer, _, _, _, _) = setup_wal_buffer().await; + let (wal_buffer, _, _, _) = setup_wal_buffer().await; // Append entries to create multiple WALs for i in 0..100 { @@ -951,12 +1058,12 @@ mod tests { } wal_buffer.track_last_applied_seq(50); assert_eq!(wal_buffer.inner.read().immutable_wals.len(), 50); - assert_eq!(wal_buffer.recent_flushed_wal_id(), 100); + assert_eq!(wal_buffer.last_flushed_wal_id(), 100); // set flush seq to 80, and track last applied seq to 90, it should release 20 wals { - let inner = wal_buffer.inner.write(); - inner.oracle.set_durable_seq_unsafe(80); + let mut inner = wal_buffer.inner.write(); + inner.last_flushed_seq = Some(80); } wal_buffer.track_last_applied_seq(90); assert_eq!(wal_buffer.inner.read().immutable_wals.len(), 20); @@ -964,7 +1071,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_maybe_trigger_flush_spams_flush_requests() { - let (wal_buffer, _, _, _db_stats, recorder) = + let (wal_buffer, _, _, recorder) = setup_wal_buffer_with_flush_interval(Duration::MAX).await; // Simulate many writers each appending a small entry and calling @@ -979,12 +1086,12 @@ mod tests { } let size_triggered_requests = - lookup_metric(&recorder, crate::db_stats::WAL_BUFFER_FLUSH_REQUESTS).unwrap(); + lookup_metric(&recorder, stats::WAL_BUFFER_FLUSH_REQUESTS).unwrap(); // Explicitly flush to drain everything, including any partial current WAL. wal_buffer.flush().unwrap().await.unwrap().unwrap(); - let actual_flushes = lookup_metric(&recorder, crate::db_stats::WAL_BUFFER_FLUSHES).unwrap(); + let actual_flushes = lookup_metric(&recorder, stats::WAL_BUFFER_FLUSHES).unwrap(); // With the flush_requested flag, the number of size-triggered requests // should be bounded by the number of WALs, not by the number of writes. @@ -1001,4 +1108,76 @@ mod tests { actual_flushes, ); } + + fn recording_listener() -> (WalStatusListener, Arc>>) { + let events = Arc::new(std::sync::Mutex::new(Vec::new())); + let recorder = events.clone(); + let listener = Arc::new(move |event| { + recorder.lock().unwrap().push(event); + }); + (listener, events) + } + + #[tokio::test] + async fn test_listener_notified_when_flush_task_flushes_wal() { + // given: + let (listener, events) = recording_listener(); + let (wal_buffer, _, _, _) = setup_wal_buffer_with_args(Duration::MAX, listener).await; + + // when: Append an entry and explicitly flush it, driving the background flush task. + wal_buffer + .append(&[make_entry("key1", "value1", 1, None)]) + .unwrap(); + wal_buffer.flush().unwrap().await.unwrap().unwrap(); + + // then: the listener should have been notified that wal 1 was flushed. + let recorded = events.lock().unwrap().clone(); + let mut flushed: Vec<_> = recorded + .iter() + .filter_map(|e| { + if let WalEvent::WalFlushed(status) = e { + Some(status) + } else { + None + } + }) + .collect(); + assert_eq!(flushed.len(), 1); + let status = flushed.pop().unwrap(); + assert_eq!(status.last_flushed_wal_id, 1); + assert_eq!(status.last_flushed_seq, Some(1)); + } + + #[tokio::test] + async fn test_listener_notified_when_flush_task_releases_wal() { + // given: + let (listener, events) = recording_listener(); + let (wal_buffer, _, _, _) = setup_wal_buffer_with_args(Duration::MAX, listener).await; + wal_buffer.track_last_applied_seq(10); + + // when: + wal_buffer + .append(&[make_entry("key1", "value1", 1, None)]) + .unwrap(); + wal_buffer.flush().unwrap().await.unwrap().unwrap(); + // The flush should have released the immutable wal from memory. + assert_eq!(wal_buffer.inner.read().immutable_wals.len(), 0); + + // The listener should have been notified that wal 1 was purged. + // then: the listener should have been notified that wal 1 was flushed. + let recorded = events.lock().unwrap().clone(); + let mut flushed: Vec<_> = recorded + .iter() + .filter_map(|e| { + if let WalEvent::MemoryReleased(status) = e { + Some(status) + } else { + None + } + }) + .collect(); + assert_eq!(flushed.len(), 1); + let status = flushed.pop().unwrap(); + assert_eq!(status.last_purged_wal_id, 1); + } } diff --git a/slatedb/src/wal_id.rs b/slatedb/src/wal_id.rs deleted file mode 100644 index 6575dc7f6..000000000 --- a/slatedb/src/wal_id.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub(crate) trait WalIdStore: Send + Sync + 'static { - fn next_wal_id(&self) -> u64; -} diff --git a/website/src/content/docs/docs/operations/metrics.mdx b/website/src/content/docs/docs/operations/metrics.mdx index 4b52922d4..dad0e0647 100644 --- a/website/src/content/docs/docs/operations/metrics.mdx +++ b/website/src/content/docs/docs/operations/metrics.mdx @@ -97,9 +97,6 @@ All metric names use dot-separated notation: `slatedb..`. | `slatedb.db.backpressure_count` | counter | | Backpressure events | | `slatedb.db.l0_stall_count` | counter | `type`: num_ssts, num_ssts_per_key | L0 flush dispatch stalls by cause | | `slatedb.db.immutable_memtable_flushes` | counter | | Immutable memtable flushes | -| `slatedb.db.wal_buffer_flushes` | counter | | WAL buffer flushes | -| `slatedb.db.wal_buffer_flush_requests` | counter | | WAL buffer flush requests | -| `slatedb.db.wal_buffer_estimated_bytes` | gauge | | Estimated WAL buffer size | | `slatedb.db.total_mem_size_bytes` | gauge | | Total memory usage | | `slatedb.db.l0_sst_count` | gauge | | L0 SST count summed across all segment trees | | `slatedb.db.segment_max_l0_sst_count` | gauge | | Maximum L0 SST count across all segments (useful for detecting backpressure on specific segments) | @@ -107,6 +104,14 @@ All metric names use dot-separated notation: `slatedb..`. | `slatedb.db.sst_filter_positive_count` | counter | | Bloom filter positives | | `slatedb.db.sst_filter_negative_count` | counter | | Bloom filter negatives | +### Write-Ahead Log (WAL) (`slatedb.wal.*`) + +| Name | Type | Labels | Description | +|------|------|--------|-------------| +| `slatedb.wal.wal_buffer_flushes` | counter | | WAL buffer flushes | +| `slatedb.wal.wal_buffer_flush_requests` | counter | | WAL buffer flush requests | +| `slatedb.wal.wal_buffer_estimated_bytes` | gauge | | Estimated WAL buffer size | + ### Block cache (`slatedb.db_cache.*`) | Name | Type | Labels | Description | From 5cdc57de8219cd2c524d6ed288f89d04c6018d20 Mon Sep 17 00:00:00 2001 From: Rohan Date: Thu, 9 Jul 2026 00:04:25 -0400 Subject: [PATCH 02/63] [k/n] wal refactor: only update last seen wal id with flushed wals (#1885) --- slatedb/src/db.rs | 13 ++++++------- slatedb/src/wal_buffer.rs | 11 ----------- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/slatedb/src/db.rs b/slatedb/src/db.rs index 489a1df64..f5e19390b 100644 --- a/slatedb/src/db.rs +++ b/slatedb/src/db.rs @@ -2064,14 +2064,13 @@ impl DbWalObserver { wrapped.subscribe(Arc::new(move |event| { let status = match event { WalEvent::WalFlushed(status) => status, - WalEvent::WalFrozen(status) => status, WalEvent::MemoryReleased(status) => status, }; if let Some(seq) = status.last_flushed_seq { oracle.advance_durable_seq(seq); } let mut guard = db_state.write(); - guard.set_next_wal_id(status.next_wal_id); + guard.set_next_wal_id(status.last_flushed_wal_id + 1); drop(guard); let _ = status_tx.send(status); })); @@ -5905,7 +5904,7 @@ mod tests { } #[tokio::test] - async fn test_wal_id_last_seen_should_exist_even_if_wal_write_fails() { + async fn test_wal_id_last_seen_should_only_reflect_flushed_wals() { let fp_registry = Arc::new(FailPointRegistry::new()); let object_store: Arc = Arc::new(InMemory::new()); let path = "/tmp/test_kv_store"; @@ -5917,6 +5916,8 @@ mod tests { .await .unwrap(), ); + // Trigger a WAL write and block until durable so WAL is written + db.put(b"foo", b"bar").await.unwrap(); fail_parallel::cfg(fp_registry.clone(), "write-wal-sst-io-error", "panic").unwrap(); @@ -5947,10 +5948,8 @@ mod tests { // Get the latest manifest let manifest = manifest_store.read_latest_manifest().await.unwrap(); - // It's possible that there exists buffered multiple wals in memory, so the next_wal_sst_id - // in manifest is greater than the next_wal_sst_id based on what's currently in the object - // store unless ALL the wals are flushed. - assert!(manifest.manifest.core.next_wal_sst_id > next_wal_sst_id); + // Assert that the manifest reflects only the flushed WAL + assert_eq!(manifest.manifest.core.next_wal_sst_id, next_wal_sst_id); } #[tokio::test] diff --git a/slatedb/src/wal_buffer.rs b/slatedb/src/wal_buffer.rs index 2fe8f62df..816339296 100644 --- a/slatedb/src/wal_buffer.rs +++ b/slatedb/src/wal_buffer.rs @@ -380,12 +380,6 @@ impl WalBufferManager { inner .immutable_wals .push_back((next_wal_id, Arc::new(current_wal))); - let status = inner.status(&self.table_store); - let listener = inner.listener.clone(); - drop(inner); - if let Some(l) = listener { - (*l)(WalEvent::WalFrozen(status)) - } Ok(()) } @@ -478,7 +472,6 @@ impl WalBufferManagerInner { let buffered_wal_entries_count = self.current_wal.len() + flushing_wal_entries_count; WalStatus { estimated_bytes: self.estimated_bytes(table_store), - next_wal_id: self.next_wal_id, last_flushed_wal_id: self.last_flushed_wal_id, last_flushed_seq: self.last_flushed_seq, last_purged_wal_id: self.last_purged_wal_id, @@ -636,9 +629,7 @@ pub(crate) struct WalObserver { pub(crate) struct WalStatus { /// The estimated in-memory bytes used by the WAL to buffer unflushed writes. pub(crate) estimated_bytes: usize, - pub(crate) next_wal_id: u64, /// The id of the last WAL file that was durably flushed - #[allow(dead_code)] pub(crate) last_flushed_wal_id: u64, /// The last sequence number that was durably flushed pub(crate) last_flushed_seq: Option, @@ -652,8 +643,6 @@ pub(crate) struct WalStatus { /// An event emitted by [`WalBufferManager`] to subscribers. #[derive(Debug, Clone)] pub(crate) enum WalEvent { - /// Emitted when the current buffer is frozen - WalFrozen(WalStatus), /// Emitted when a WAL file is durably flushed to storage. On receipt of this event, SlateDB /// notifies write tasks blocked on [`crate::config::WriteOptions::await_durable`] WalFlushed(WalStatus), From 872982018db32b023e579f3015065f70942adbb1 Mon Sep 17 00:00:00 2001 From: Jason Gustafson <12502538+hachikuji@users.noreply.github.com> Date: Thu, 9 Jul 2026 09:50:16 -0700 Subject: [PATCH 03/63] Blog post on segmentation (#1912) Blog post on segment-oriented compaction from rfc 24. --- website/public/charts/seg-compaction.html | 164 +++++++++ website/public/charts/seg-scan-p99.html | 158 ++++++++ website/public/charts/seg-throughput.html | 158 ++++++++ website/public/charts/seg-write-amp.html | 158 ++++++++ .../blog/segment-oriented-compaction.mdx | 337 ++++++++++++++++++ 5 files changed, 975 insertions(+) create mode 100644 website/public/charts/seg-compaction.html create mode 100644 website/public/charts/seg-scan-p99.html create mode 100644 website/public/charts/seg-throughput.html create mode 100644 website/public/charts/seg-write-amp.html create mode 100644 website/src/content/blog/segment-oriented-compaction.mdx diff --git a/website/public/charts/seg-compaction.html b/website/public/charts/seg-compaction.html new file mode 100644 index 000000000..ac1b2361a --- /dev/null +++ b/website/public/charts/seg-compaction.html @@ -0,0 +1,164 @@ + + + + + +SlateDB - segment-oriented compaction: comp + + + + + + +

+
+
+
+
+ + + + + + + diff --git a/website/public/charts/seg-scan-p99.html b/website/public/charts/seg-scan-p99.html new file mode 100644 index 000000000..fb32d9270 --- /dev/null +++ b/website/public/charts/seg-scan-p99.html @@ -0,0 +1,158 @@ + + + + + +SlateDB - segment-oriented compaction: p99 + + + + +
+
+
+
+
+ + + + + + + diff --git a/website/public/charts/seg-throughput.html b/website/public/charts/seg-throughput.html new file mode 100644 index 000000000..b498a5b7f --- /dev/null +++ b/website/public/charts/seg-throughput.html @@ -0,0 +1,158 @@ + + + + + +SlateDB - segment-oriented compaction: tput + + + + +
+
+
+
+
+ + + + + + + diff --git a/website/public/charts/seg-write-amp.html b/website/public/charts/seg-write-amp.html new file mode 100644 index 000000000..ba3e293dc --- /dev/null +++ b/website/public/charts/seg-write-amp.html @@ -0,0 +1,158 @@ + + + + + +SlateDB - segment-oriented compaction: wamp + + + + +
+
+
+
+
+ + + + + + + diff --git a/website/src/content/blog/segment-oriented-compaction.mdx b/website/src/content/blog/segment-oriented-compaction.mdx new file mode 100644 index 000000000..b0546893b --- /dev/null +++ b/website/src/content/blog/segment-oriented-compaction.mdx @@ -0,0 +1,337 @@ +--- +title: "Divide and Compact: Segment-Oriented Compaction in SlateDB" +pubDate: 2026-07-08 +author: Jason Gustafson +authorGithub: hachikuji +# ogImage: /img/some-custom-card.jpg # optional per-post override +--- + +import ChartEmbed from '../../components/ChartEmbed.astro'; + +LSMs typically represent all data within a single tree which is compacted over time. Controlling read/write amplification can be challenging when the data model mixes data structures with very different read/write patterns or lifetimes. Index structures, for example, tend to behave very differently from the data they point to. The single tree forces compromise to manage compaction across all structures at once. + +Segment-oriented compaction in SlateDB gives you a way to split the dataset into separate trees so that compaction strategies can be tailored to the structure of the data in each tree. It is the swiss army knife which lets you isolate data by read/write frequency, retention policy, caching strategy, or whatever other dimension matters to your application. + +This post provides an overview of segment-oriented compaction and how it can be used in your application. + +## How Compaction Works in SlateDB + +SlateDB organizes data into an LSM tree. The LSM tree is divided into two parts: L0 and the sorted runs. The L0 tables are the raw SSTs generated from memtable accumulation during ingest. Over time, the compactor takes L0 tables and rewrites them into sorted runs. We refer to the tables in these layers loosely as L0s and SRs. + +```ascii-art +╭────────────────────────────────────────────────────────────────────╮ +│ ◎ ○ ○ ░░░░░░░░░░░░░░░░░░░░ SlateDB's LSM Tree ░░░░░░░░░░░░░░░░░░░░░│ +├────────────────────────────────────────────────────────────────────┤ +│ │ +│ writes │ +│ │ │ +│ ▼ │ +│ ┌──────────┐ │ +│ │ memtable │ in memory │ +│ └────┬─────┘ │ +│ │ flush │ +│ ▼ │ +│ L0 ─ raw SSTs from each flush; newest first, key ranges overlap │ +│ ┌─────┐ ┌─────┐ ┌─────┐ ┌─────┐ │ +│ │ SST │ │ SST │ │ SST │ │ SST │ │ +│ └─────┘ └─────┘ └─────┘ └─────┘ │ +│ │ compaction rewrites L0 into sorted runs │ +│ ▼ │ +│ SORTED RUNS ─ SSTs in sorted order, no overlapping key ranges │ +│ SR0 ┌────┬────┬────┐ │ +│ │SST │SST │SST │ │ +│ └────┴────┴────┘ │ +│ SR1 ┌──────┬──────┬──────┬──────┐ │ +│ │ SST │ SST │ SST │ SST │ │ +│ └──────┴──────┴──────┴──────┘ │ +│ │ +└────────────────────────────────────────────────────────────────────┘ +``` + +

SlateDB's LSM tree: L0 holds raw SSTs flushed from the memtable (newest first, ranges may overlap); compaction rewrites them into sorted runs of non-overlapping SSTs.

+ +A compaction scheduler in SlateDB is responsible for selecting which L0s and SRs should be compacted together. Compaction strategies are typically heuristic. We don't know exactly where keys will exist within the tree, but we can build strategies which optimize for certain tree structures and read/write amplification properties. + +SlateDB's default scheduler is known as "size-tiered" compaction. It works by selecting similarly sized SRs for compaction. The number of size tiers is limited by the size of the dataset. If a dataset is bounded, size-tiered compaction provides an upper bound on write amplification. The final tier is the size of the dataset itself. However, if the dataset grows over time, then write amplification grows as well. Size-tiered compaction is similar to universal compaction in RocksDB. + +```ascii-art +╭───────────────────────────────────────────────────────────╮ +│ ◎ ○ ○ ░░░░░░░░░░░░░░ Size-Tiered Compaction ░░░░░░░░░░░░░░│ +├───────────────────────────────────────────────────────────┤ +│ │ +│ tier 0 ─ L0 SSTs; similarly-sized runs accumulate │ +│ ┌──┐ ┌──┐ ┌──┐ ┌──┐ │ +│ │██│ │██│ │██│ │██│ │ +│ └──┘ └──┘ └──┘ └──┘ │ +│ │ merge ~N similar-sized runs into one larger run │ +│ ▼ │ +│ tier 1 ─ ~N× larger │ +│ ┌────────┐ ┌────────┐ │ +│ │████████│ │████████│ │ +│ └────────┘ └────────┘ │ +│ │ merge │ +│ ▼ │ +│ tier 2 ─ ~N²× larger │ +│ ┌──────────────────┐ │ +│ │██████████████████│ │ +│ └──────────────────┘ │ +│ │ merge │ +│ ▼ │ +│ final tier ≈ size of the whole dataset │ +│ ┌────────────────────────────────────┐ │ +│ │████████████████████████████████████│ │ +│ └────────────────────────────────────┘ │ +│ │ +└───────────────────────────────────────────────────────────┘ +``` + +

Size-tiered compaction merges similarly-sized runs into a larger run at the next tier. As the dataset grows it adds tiers, and every byte is rewritten once per tier — so write amplification grows with the number of tiers.

+ +More heuristic strategies are possible with a custom compaction scheduler, but we are limited to working within the constraints of the global LSM tree. It is not always straightforward to leverage the structure of the data itself to guide our scheduling. Frequently accessed data may get mixed with infrequently accessed, long-lived data may get mixed with short-lived, often updated data may get mixed with rarely updated, etc. It is up to us to structure the keys and the compaction heuristic as well as we can in order to optimize for our data model and its access patterns. + +## Where Size-Tiered Compaction Breaks Down + +Size-tiered compaction works great for homogeneous data structures which share a common lifetime. However, complex data systems often use distinct internal structures with profoundly different access and retention characteristics. A frequently-accessed index structure might benefit from an aggressive compaction strategy, while we may be much more cautious about write amplification for its bulky data counterpart. + +Time is another dimension which forces distinct behavior on the compactor depending on the age of the data. Timeseries databases such as [Prometheus](https://prometheus.io/) and [Opendata-timeseries](https://github.com/opendata-oss/opendata) organize data into discrete windows of time (typically one hour). Write workloads are dominated by the arrival of new data points with less frequent backfills of older data. Size-tiered compaction fits poorly because it treats the whole key space as a single tree. Old windows that are effectively immutable are folded into ever-larger merges as new data arrives. Write amplification is bound by the total amount of data retained, and is dominated by the older immutable windows. + +Retention of timeseries data is also difficult with existing compaction strategies. Typically, as windows are aged out of the system, the corresponding data and index structures are removed. TTL-based expiration at the record level requires a full pass over the data that must be dropped, and all of the surviving data must be rewritten. The data has an obvious structure that the scheduler cannot easily exploit. + +Efficient compaction for timeseries databases was a primary motivation for segmented compaction. Alternative approaches include the time-window compaction strategy (TWCS) used in systems like [Cassandra](https://cassandra.apache.org/doc/4.1/cassandra/operating/compaction/twcs.html), but we will see how segmentation is much more general. It can be used in any data system which requires distinct strategies for its underlying structures. + +## Introducing Segment-Oriented Compaction + +Segment-oriented compaction was introduced in [RFC 24](https://github.com/slatedb/slatedb/blob/main/rfcs/0024-segment-oriented-compaction.md) and is first included in release [0.14.0](https://github.com/slatedb/slatedb/releases#release-v0.14.0). It gives you a direct way to control the compaction/retention behavior of your data by splitting the database into isolated LSM trees. Unlike [column families](#appendix-segments-vs-column-families), segmentation partitions the keyspace: each segment is defined by a key prefix which means that segments represent disjoint ranges of keys. + +Segments are defined using a `PrefixExtractor`. When a key is written to the database, SlateDB uses the prefix extractor to identify the segment that it belongs to. The LSM tree for each segment is created dynamically as new segment prefixes are found. + +```ascii-art +╭────────────────────────────────────────────────────────────────────╮ +│ ◎ ○ ○ ░░░░░░░░░░░░░░░░░░░░░░ Segment Routing ░░░░░░░░░░░░░░░░░░░░░░│ +├────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌────────────────────────────┐ │ +│ │ A|… B|… A|… C|… │ │ +│ └──────────────┬─────────────┘ │ +│ ▼ │ +│ ┌────────────────────────────┐ │ +│ │ PrefixExtractor │ │ +│ └──────────────┬─────────────┘ │ +│ ▼ │ +│ ┌────────────────────────────┐ │ +│ │ single memtable ( + WAL ) │ │ +│ └──────────────┬─────────────┘ │ +│ │ on flush, keys split by segment │ +│ ┌────────────────────────┼────────────────────────┐ │ +│ ▼ ▼ ▼ │ +│ segment A segment B segment C │ +│ ┌───────────┐ ┌───────────┐ ┌───────────┐ │ +│ │ L0 · SRs │ │ L0 · SRs │ │ L0 · SRs │ │ +│ └───────────┘ └───────────┘ └───────────┘ │ +│ own L0s + SRs own L0s + SRs own L0s + SRs │ +│ │ +└────────────────────────────────────────────────────────────────────┘ +``` + +

Keys share one memtable and WAL regardless of segment; the PrefixExtractor maps each key to a segment, and on flush the data fans out into a separate LSM tree (its own L0s and sorted runs) per segment.

+ +Since each segment has its own LSM tree, a compaction scheduler can target compactions to each segment using a policy which is suited to that segment. Index structures can use a separate segment so that they can be compacted more aggressively. Segmented compaction does not replace heuristic strategies, but rather complements them by allowing you to tailor the heuristic to each segment. For example, the default strategy applies size-tiered compaction within each segment. + +You can use semantic information embedded in the segment prefix to tell your scheduler how it should adjust its behavior. In a timeseries system, the prefix might embed a truncated timestamp corresponding to the start of the time window that it corresponds to. Other systems might embed a type discriminator to identify different structures. Custom compaction schedulers see all segments in the database and can choose any of them to act upon. + +The diagram above also suggests some of the tradeoffs that come from the use of segmentation. + +Each segment produces its own L0s and SRs. If we are writing to 10 active segments, then in principle, that is 10x the number of PUT requests writing to L0. The picture is somewhat clouded by SlateDB's use of multi-part upload requests, but in general, more segments means higher L0 write costs with smaller object sizes. (SlateDB's WAL, on the other hand, contains data across all segments, so there is no impact.) + +The potential for increased L0 costs depends on the steady-state write profile. Many data models tend to keep a single active segment which accumulates most of the active writes. This means only one segment is getting L0 writes at a time, so no net increase in the number of L0s being produced. Most new data points in the timeseries database discussed above would be written to the segment for the current hour. An increase in write costs into L0 or manifest bookkeeping may still be worthwhile in other use cases if it leads to better control over write amplification resulting from compaction. + +More L0s and SRs to write also implies a bigger index for SlateDB to track. This means larger manifest files. However, this depends on whether each segment is filling enough data to write sufficiently large SSTs. You don't want a bunch of dinky SSTs bloating the manifest. If your workload is able to sustain L0s and SRs at their respective size limits, then the increase in manifest overhead will be marginal because each segment is representing a disjoint portion of the keyspace. + +## Backfill + +Segmentation in the timeseries example also provides a natural way to handle backfill of older data. In a traditional LSM, a set of backfilled records would need to work through each level of the tree in order to find the right time bucket. The cost of this may show up in the effectiveness of the block cache, which would mix data across a wider range. Basically, the backfill does not poison the current bucket of data, which is more likely to be read. + +```ascii-art +╭──────────────────────────────────────────────────────────╮ +│ ◎ ○ ○ ░░░░░░░░░░░░░ Backfill · Unsegmented ░░░░░░░░░░░░░░│ +├──────────────────────────────────────────────────────────┤ +│ │ +│ backfill (old) new writes (current) │ +│ │ │ │ +│ ▼ ▼ │ +│ L0 ┌────┐ ┌────┐ ┌────┐ ┌────┐ │ +│ │ ▒▒ │ │ ██ │ │ ██ │ │ ▒▒ │ │ +│ └────┘ └────┘ └────┘ └────┘ │ +│ │ compaction rewrites old + current together │ +│ ▼ │ +│ SR ┌──────────────────────────────┐ │ +│ │ ▒▒ ████ ▒▒ ████ ▒▒ ████ ▒▒ █ │ │ +│ └──────────────────────────────┘ │ +│ │ +└──────────────────────────────────────────────────────────┘ + +╭──────────────────────────────────────────────────────────╮ +│ ◎ ○ ○ ░░░░░░░░░░░░░░ Backfill · Segmented ░░░░░░░░░░░░░░░│ +├──────────────────────────────────────────────────────────┤ +│ │ +│ backfill (old) new writes (current) │ +│ │ │ │ +│ ▼ ▼ │ +│ segment: OLD bucket segment: CURRENT │ +│ ┌────────────────┐ ┌────────────────┐ │ +│ │ ▒▒▒▒▒▒▒▒▒▒▒▒▒▒ │ │ ██████████████ │ │ +│ └────────────────┘ └────────────────┘ │ +│ compacted in isolation stays hot, untouched │ +│ │ +└──────────────────────────────────────────────────────────┘ +``` + +

Without segments, a backfill (▒) enters L0 alongside current writes (█) and is compacted together with them, churning cache across the whole key range. With segments, the backfill routes to the old bucket's own tree; the current segment is never touched, so its hot data stays cache-resident.

+ +Over time, backfills into older segments may become rarer. A system may even prohibit backfills into older segments outside of some recent window. When a segment no longer receives new data, there is no need to continue compacting it. We can schedule a final compaction and leave the segment in an immutable final state. This is a useful way to keep the compactor's working set bounded even when the total dataset itself is unbounded. + +In the timeseries data model, we might disallow backfills outside of the past day. + +```ascii-art +╭────────────────────────────────────────────────────────────────────╮ +│ ◎ ○ ○ ░░░░░░░░░░░░░░░░ Aging Out: Frozen Segments ░░░░░░░░░░░░░░░░░│ +├────────────────────────────────────────────────────────────────────┤ +│ │ +│ older ◀─────────────── time ───────────────▶ newer │ +│ │ +│ no backfills ◀ ┊ ▶ recent (writable) │ +│ ┊ writes │ +│ ┊ │ │ +│ seg 08h seg 09h seg 10h ┊ seg 11h seg 12h │ +│ ┌────────┐ ┌────────┐ ┌────────┐ ┊ ┌────────┐ ┌───────▼┐ │ +│ │████████│ │████████│ │████████│ ┊ │█ ▓ ██ ▓│ │▓ ██ ▓ █│ │ +│ └────────┘ └────────┘ └────────┘ ┊ └────────┘ └────────┘ │ +│ frozen frozen frozen ┊ active active │ +│ └───────────────┬────────────────┘ └─────────┬──────────┘ │ +│ one final run, not compacted compactor working set │ +│ │ +└────────────────────────────────────────────────────────────────────┘ +``` + +

Once a bucket ages past the writable window it receives no more data, so we run a final compaction and freeze it as a single immutable run. Only the recent segments stay in the compactor's working set — which keeps that set bounded even as the total dataset grows without bound.

+ +In addition to reducing the active compaction workload, there is a caching benefit as well for immutable segments. + +One issue with SlateDB's block cache is that compaction forces cached blocks to be reloaded. This is generally a good thing because the compacted data ought to be organized more favorably for the cache to exploit data locality. However, if there are minimal changes at the block level, then the reload is pure overhead. For an immutable segment which is no longer being actively compacted, the block references in the cache remain valid indefinitely which reduces IO overhead and churn. It depends on the read workload whether this provides a true benefit. When active segments dominate the read workload, there may be no benefit. + +## Retention + +Systems like timeseries typically do not retain data indefinitely. We need an efficient way to remove data outside of the system's retention window. Segmentation gives the compactor a new strategy to drain existing segments. + +A segment drain simply detaches a set of L0s/SRs from the current manifest. This makes them eligible for removal by SlateDB's garbage collector. + +```ascii-art +╭────────────────────────────────────────────────────────────────────╮ +│ ◎ ○ ○ ░░░░░░░░░░░░░░░ Retention: Draining a Segment ░░░░░░░░░░░░░░░│ +├────────────────────────────────────────────────────────────────────┤ +│ │ +│ manifest object storage │ +│ ┌──────────────────┐ │ +│ │ segment 10h ✂╌ ╌ ╌ ╌ ╌ ╌ ╌ ╌ ▷ ▒ 10h L0·SRs ▒ drained │ +│ │ │ ▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒ → GC │ +│ │ segment 11h ●───┼────────────▶ █ 11h L0·SRs █ live │ +│ │ │ ███████████████ │ +│ │ segment 12h ●───┼────────────▶ █ 12h L0·SRs █ live │ +│ └──────────────────┘ ███████████████ │ +│ │ +│ drain detaches the pointer; nothing references 10h, │ +│ so the garbage collector reclaims its SSTs. │ +│ │ +└────────────────────────────────────────────────────────────────────┘ +``` + +

A segment drain removes the manifest's pointer to a segment's L0s and sorted runs. No data is read or rewritten; once nothing references those SSTs, SlateDB's garbage collector reclaims them.

+ +## Query Efficiency + +Since segments are ordered by prefix, a query only touches one segment at a time. Point queries will route to exactly one segment; range queries will map to a contiguous range of segments. The active state is always limited by the size of each segment, not by the contributions from each segment that the query touches. In other words, segmentation does not lead to more SST merging. + +Query efficiency in SlateDB comes down to how well the blocks in each SST isolate the query range. If queries tend to span many segments, but within each segment, blocks provide good locality, then there is no penalty for segmentation. If, however, segments are not broad enough to give blocks sufficient data locality, then query performance may suffer. Imagine in our timeseries example if we used a window size of one minute for each bucket. A scan across one hour would need to hit 60 segments, each providing only a handful of data points. You have to find the right segment structure based on your query patterns. + +```ascii-art +╭────────────────────────────────────────────────────────────────────╮ +│ ◎ ○ ○ ░░░░░░░░░░ Segment Granularity vs. Query Locality ░░░░░░░░░░░│ +├────────────────────────────────────────────────────────────────────┤ +│ │ +│ one scan · 12:00 – 13:00 │ +│ ├───────────────────────────────────────────────┤ │ +│ │ +│ coarse — 1-hour buckets │ +│ │█████████████████████ 12h █████████████████████│ 1 segment │ +│ one contiguous read; strong block locality │ +│ │ +│ fine — 1-minute buckets │ +│ │█│█│█│█│█│█│█│█│█│█│█│█│█│█│█│█│█│█│█│█│█│█│█│█│ 60 segments │ +│ a sliver from each; poor block locality │ +│ │ +└────────────────────────────────────────────────────────────────────┘ +``` + +

The same one-hour scan over two segment granularities. With 1-hour buckets it reads a single segment as one contiguous, cache-friendly range; with 1-minute buckets the same scan fans out across 60 segments, each returning only a sliver. Match the segment width to your query patterns. (The fine row is schematic — 60 segments won't fit at scale.)

+ +## Results + +To make the impact of segmentation concrete, we ran a small load test modeling an append-heavy timeseries workload. Each sample uses a 20-byte `` tuple encoded big-endian for its key. A single writer streams samples into the current bucket — rolling forward to a new one as each fills — while readers scan recently written buckets. + +The two arms of the experiment are identical except that the segmented arm registers a `PrefixExtractor` over the 8-byte bucket prefix, giving each time bucket its own LSM tree. We ran the workload against S3 for 30 minutes. The charts below are keyed by total bytes written, so the x-axis tracks the database growing over the run. Full details — exact parameters and hardware — live alongside the [`tsdb` bencher](https://github.com/hachikuji/slatedb/blob/rfc24-segmentation-bench/slatedb-bencher/src/tsdb.rs). + +### Compaction work is bounded + +Because a sealed segment stops being compacted, the segmented arm's cumulative write amplification climbs briefly and then plateaus at roughly **3.0x**. The unsegmented arm keeps folding cold data into ever-larger merges and drifts up to **5.5x** — and would keep climbing as the dataset grows. + + + +### Compaction throughput stays steady + +The same effect shows up per window in the raw compaction work. Segmentation holds compaction to a low, steady band, whereas the unsegmented arm spikes as it rewrites progressively larger sorted runs. + + + +### Write throughput improves + +That bounded compaction work pays off in write throughput. With less bandwidth spent on compaction, the segmented arm ingests about **13% more data** over the same 30-minute window (84 GB vs. 74.5 GB). + + + +### Reads are unaffected + +The write-side win comes without a read penalty. Scan p99 latency tracks closely between the two arms, since each scan on this workload touches only recent buckets that stay cache-resident regardless of segmentation. In short, segmentation buys write-side stability essentially for free on this workload. + + + +## Conclusion + +Segmentation gives you a precise way to target the compaction strategy to the data structures in your system. A segment is defined using a key prefix and isolates its own LSM tree, which can be treated with its own heuristics and retention needs. This is a powerful way to build systems on SlateDb, but it is not a free lunch. Choosing segments effectively means ensuring enough load to fill L0s and SRs, while ensuring that segment-level blocks retain enough locality to make reads efficient. + +We expect extensions such as segment merging in the future. As the data in a system like timeseries ages, we can consolidate into coarser windows of time (e.g. hours into days) for more efficient compression. Segmentation may also provide the common foundation that we need for column families in SlateDb. + +To get started with segmented compaction today, you need SlateDb 0.14 or higher. All you need is a prefix extractor to define the segments in your system. You can then build your own compaction scheduler to tune the frequency and heurstics of each segment. A scheduler that understands the structure of your data does not need to compromise. + +## Appendix: Segments vs. Column Families + +If you've used RocksDB, segments will feel a lot like column families: both split data into independent LSM trees that can be compacted on their own schedule. Two things set them apart. + +**How they partition the keyspace.** A column family is effectively an extra dimension on the addressing — `(family, key)` — so families are parallel, independent keyspaces with no order between them, and you can't scan across them as one sorted stream. A segment instead partitions the single, ordered keyspace into contiguous prefix ranges: the data stays globally sorted, so a range scan can span a run of segments. + +**How they're managed.** Column families are a small, static set that you create and drop explicitly and write to by name. Segments are derived automatically from the key by the `PrefixExtractor`, so writes flow through one shared memtable and WAL and split into per-segment trees only at flush — never routed to a named segment. Because a segment costs nothing to declare, they can be numerous and short-lived: a new time bucket becomes a new segment with no schema change, and retention simply drains the ones that age out. From 016b676ee125f02cb14054cce0cd5a78f3524ac5 Mon Sep 17 00:00:00 2001 From: Kaivalya Apte Date: Thu, 9 Jul 2026 19:00:56 +0200 Subject: [PATCH 04/63] [1908] Wire RetryingObjectStore into Admin and Clone paths (#1909) --- slatedb/src/admin.rs | 133 +++++++++++++++++++++++++------------- slatedb/src/db/builder.rs | 6 +- 2 files changed, 92 insertions(+), 47 deletions(-) diff --git a/slatedb/src/admin.rs b/slatedb/src/admin.rs index c7160e1b2..6fbd5833b 100644 --- a/slatedb/src/admin.rs +++ b/slatedb/src/admin.rs @@ -14,6 +14,7 @@ use crate::manifest::VersionedManifest; use slatedb_common::clock::SystemClock; use crate::object_stores::{ObjectStoreType, ObjectStores}; +use crate::retrying_object_store::RetryingObjectStore; use crate::seq_tracker::FindOption; use crate::utils::IdGenerator; use bytes::Bytes; @@ -68,10 +69,7 @@ impl Admin { &self, maybe_id: Option, ) -> Result, crate::Error> { - let manifest_store = ManifestStore::new( - &self.path, - self.object_stores.store_of(ObjectStoreType::Main).clone(), - ); + let manifest_store = self.manifest_store(); let manifest = if let Some(id) = maybe_id { manifest_store .try_read_manifest(id) @@ -96,10 +94,7 @@ impl Admin { &self, range: R, ) -> Result, crate::Error> { - let manifest_store = ManifestStore::new( - &self.path, - self.object_stores.store_of(ObjectStoreType::Main).clone(), - ); + let manifest_store = self.manifest_store(); let manifest_metadata = manifest_store .list_manifests(range) .await @@ -183,10 +178,7 @@ impl Admin { /// Returns a read-only view of the current compactor state. pub async fn read_compactor_state_view(&self) -> Result { - let manifest_store = Arc::new(ManifestStore::new( - &self.path, - self.object_stores.store_of(ObjectStoreType::Main).clone(), - )); + let manifest_store = Arc::new(self.manifest_store()); let compactions_store = Arc::new(self.compactions_store()); let reader = CompactorStateReader::new(&manifest_store, &compactions_store); reader.read_view().await.map_err(crate::Error::from) @@ -249,10 +241,7 @@ impl Admin { &self, name_filter: Option<&str>, ) -> Result, crate::Error> { - let manifest_store = ManifestStore::new( - &self.path, - self.object_stores.store_of(ObjectStoreType::Main).clone(), - ); + let manifest_store = self.manifest_store(); let manifest = manifest_store .read_latest_manifest() .await @@ -495,10 +484,7 @@ impl Admin { &self, options: &CheckpointOptions, ) -> Result { - let manifest_store = Arc::new(ManifestStore::new( - &self.path, - self.object_stores.store_of(ObjectStoreType::Main).clone(), - )); + let manifest_store = Arc::new(self.manifest_store()); let mut stored_manifest = StoredManifest::load(manifest_store, self.system_clock.clone()).await?; @@ -526,10 +512,7 @@ impl Admin { id: Uuid, lifetime: Option, ) -> Result<(), crate::Error> { - let manifest_store = Arc::new(ManifestStore::new( - &self.path, - self.object_stores.store_of(ObjectStoreType::Main).clone(), - )); + let manifest_store = Arc::new(self.manifest_store()); let mut stored_manifest = StoredManifest::load(manifest_store, self.system_clock.clone()).await?; stored_manifest @@ -553,10 +536,7 @@ impl Admin { /// Deletes the checkpoint with the specified id. pub async fn delete_checkpoint(&self, id: Uuid) -> Result<(), crate::Error> { - let manifest_store = Arc::new(ManifestStore::new( - &self.path, - self.object_stores.store_of(ObjectStoreType::Main).clone(), - )); + let manifest_store = Arc::new(self.manifest_store()); let mut stored_manifest = StoredManifest::load(manifest_store, self.system_clock.clone()).await?; stored_manifest @@ -621,18 +601,26 @@ impl Admin { Ok(manifest.core().sequence_tracker.find_seq(ts, opt)) } + /// Wraps the configured object store of the given type in a + /// [`RetryingObjectStore`] so that admin operations retry transient object + /// store failures with exponential backoff. Retrying is safe here because + /// `RetryingObjectStore` verifies conditional puts via a ULID written to + /// object metadata, so an ambiguous failure after a successful write is + /// detected rather than surfaced as a spurious error. + fn retrying_store(&self, store_type: ObjectStoreType) -> Arc { + Arc::new(RetryingObjectStore::new( + self.object_stores.store_of(store_type).clone(), + self.rand.clone(), + self.system_clock.clone(), + )) + } + fn manifest_store(&self) -> ManifestStore { - ManifestStore::new( - &self.path, - self.object_stores.store_of(ObjectStoreType::Main).clone(), - ) + ManifestStore::new(&self.path, self.retrying_store(ObjectStoreType::Main)) } fn compactions_store(&self) -> CompactionsStore { - CompactionsStore::new( - &self.path, - self.object_stores.store_of(ObjectStoreType::Main).clone(), - ) + CompactionsStore::new(&self.path, self.retrying_store(ObjectStoreType::Main)) } /// Clone a database using a builder pattern. If no db already exists at the specified path, @@ -677,9 +665,9 @@ impl Admin { CloneBuilder::new( self.path.clone(), source, - self.object_stores.store_of(ObjectStoreType::Main).clone(), + self.retrying_store(ObjectStoreType::Main), ) - .with_wal_object_store(self.object_stores.store_of(ObjectStoreType::Wal).clone()) + .with_wal_object_store(self.retrying_store(ObjectStoreType::Wal)) } /// Creates a new builder for an admin client at the given path. @@ -829,7 +817,9 @@ mod tests { use crate::admin::{load_object_store_from_env, AdminBuilder}; use crate::compactions_store::{CompactionsStore, StoredCompactions}; use crate::compactor_state::{Compaction, CompactionSpec, CompactionStatus, SourceId}; - use crate::config::{CompactionWorkerOptions, CompactorOptions, GarbageCollectorOptions}; + use crate::config::{ + CheckpointOptions, CompactionWorkerOptions, CompactorOptions, GarbageCollectorOptions, + }; use crate::manifest::store::{ManifestStore, StoredManifest}; use crate::manifest::ManifestCore; use crate::test_utils::{FlakyObjectStore, StringConcatMergeOperator}; @@ -1075,19 +1065,70 @@ mod tests { } #[tokio::test] - async fn test_admin_list_manifests_list_failure_maps_to_unavailable() { + async fn test_admin_list_manifests_retries_transient_failure() { + // Admin operations wrap the object store in a RetryingObjectStore, so a + // transient list failure should be retried rather than surfaced. let inner: Arc = Arc::new(InMemory::new()); - let object_store: Arc = - Arc::new(FlakyObjectStore::new(inner, 0).with_list_failures(1, 0)); - let path = Path::from("/tmp/test_admin_list_manifests_list_failure"); - let admin = AdminBuilder::new(path, object_store).build(); + let flaky = Arc::new(FlakyObjectStore::new(inner, 0).with_list_failures(1, 0)); + let path = Path::from("/tmp/test_admin_list_manifests_retries_transient_failure"); + let admin = AdminBuilder::new(path, flaky.clone()).build(); - let err = admin + let manifests = admin .list_manifests(..) .await - .expect_err("expected list failure"); + .expect("list should succeed after retrying the transient failure"); + + assert!(manifests.is_empty()); + // 1 transient failure + 1 successful retry. + assert_eq!(flaky.list_attempts(), 2); + } + + #[tokio::test] + async fn test_admin_create_detached_checkpoint_retries_transient_put() { + // A transient put failure during checkpoint creation should be retried + // by the RetryingObjectStore rather than failing the operation. + let inner: Arc = Arc::new(InMemory::new()); + let path = Path::from("/tmp/test_admin_create_detached_checkpoint_retries_transient_put"); + let db = crate::Db::open(path.clone(), inner.clone()).await.unwrap(); + db.put(b"key", b"value").await.unwrap(); + db.close().await.unwrap(); + + // Fail the first put_opts, which the retrying store should transparently retry. + let flaky = Arc::new(FlakyObjectStore::new(inner, 1)); + let admin = AdminBuilder::new(path, flaky.clone()).build(); + + admin + .create_detached_checkpoint(&CheckpointOptions::default()) + .await + .expect("checkpoint should succeed after retrying the transient put"); + + assert!(flaky.put_attempts() >= 2); + } + + #[tokio::test] + async fn test_admin_terminal_object_store_error_maps_to_unavailable() { + // The retry layer retries transient errors forever, so it never exhausts + // and surfaces a transient failure. A terminal (non-retryable) error, + // however, must still pass through the retry wrapper and map to + // ErrorKind::Unavailable rather than being swallowed. A conditional put + // that always fails with Precondition is such a terminal error. + let inner: Arc = Arc::new(InMemory::new()); + let path = Path::from("/tmp/test_admin_terminal_object_store_error_maps_to_unavailable"); + let db = crate::Db::open(path.clone(), inner.clone()).await.unwrap(); + db.put(b"key", b"value").await.unwrap(); + db.close().await.unwrap(); + + let failing = Arc::new(FlakyObjectStore::new(inner, 0).with_put_precondition_always()); + let admin = AdminBuilder::new(path, failing.clone()).build(); + + let err = admin + .create_detached_checkpoint(&CheckpointOptions::default()) + .await + .expect_err("expected terminal precondition failure to surface"); assert_eq!(err.kind(), ErrorKind::Unavailable); + // Terminal error: attempted exactly once, no retries. + assert_eq!(failing.put_attempts(), 1); } #[tokio::test] diff --git a/slatedb/src/db/builder.rs b/slatedb/src/db/builder.rs index 10c6558a0..483796c22 100644 --- a/slatedb/src/db/builder.rs +++ b/slatedb/src/db/builder.rs @@ -858,7 +858,11 @@ impl> AdminBuilder

{ /// Builds and returns an Admin instance. pub fn build(self) -> Admin { - // No retrying object stores here, since we don't want to retry admin operations + // Store the raw object stores here. Admin wraps them in a + // `RetryingObjectStore` per-operation (see `Admin::retrying_store`) + // rather than at build time, because several admin operations delegate + // to sub-builders (compactor/GC) that add their own retry layer, and + // wrapping here would double-wrap them. Admin { path: self.path.into(), object_stores: ObjectStores::new(self.main_object_store, self.wal_object_store), From 0789090b703088a61e763ca3e2bfce5adafe19f8 Mon Sep 17 00:00:00 2001 From: Kaivalya Apte Date: Fri, 10 Jul 2026 17:32:09 +0200 Subject: [PATCH 05/63] 1707 RFC-0029 GC Safe SST Ulid Allocator (#1822) --- rfcs/0029-gc-safe-sst-ulid-timestamps.md | 394 +++++++++++++++++++++++ 1 file changed, 394 insertions(+) create mode 100644 rfcs/0029-gc-safe-sst-ulid-timestamps.md diff --git a/rfcs/0029-gc-safe-sst-ulid-timestamps.md b/rfcs/0029-gc-safe-sst-ulid-timestamps.md new file mode 100644 index 000000000..d82b7290c --- /dev/null +++ b/rfcs/0029-gc-safe-sst-ulid-timestamps.md @@ -0,0 +1,394 @@ +# GC-Safe SST ULID Timestamps + +Table of Contents: + + + +- [Summary](#summary) +- [Background](#background) +- [Motivation](#motivation) +- [Goals](#goals) +- [Non-Goals](#non-goals) +- [Design](#design) + - [Writer L0 IDs](#writer-l0-ids) + - [Compaction IDs](#compaction-ids) + - [Invariant Checks](#invariant-checks) + - [Failure Handling](#failure-handling) + - [Garbage Collection](#garbage-collection) +- [Implementation](#implementation) +- [Impact Analysis](#impact-analysis) +- [Operations](#operations) +- [Testing](#testing) +- [Rollout](#rollout) +- [Alternatives](#alternatives) +- [Open Questions](#open-questions) +- [References](#references) + + + +Status: Draft + +Authors: + +* [Kaivalya Apte](https://github.com/geeknarrator) + +## Summary + +SlateDB compacted SST garbage collection uses the timestamp embedded in SST +ULIDs as part of its deletion cutoff. Today the writer mints L0 SST IDs inside +the parallel upload workers, so mint order can differ from the order in which +L0s are published to the manifest. An uploaded but unpublished SST can then +have a ULID timestamp below the cutoff, and GC can delete it before it is +published. + +This RFC fixes the race by changing where IDs are minted, not how: + +- The writer allocates L0 physical SST IDs at dispatch, before parallel + upload, in the same sequence order that L0s are later published. +- Newly flushed L0 views use the physical SST ID as their view ID. + +This makes the fix structural. An SST that is already in an active manifest is +never deleted, because GC skips referenced SSTs. An SST that is uploaded but +not yet published always has a timestamp at or above the newest published L0, +so the `newest_l0` cutoff term protects it. + +This RFC does not try to solve clock skew. It assumes skew is bounded, and +users set `min_age` for the margin they want against GC-versus-writer skew. + +There are no manifest or SST format changes, and GC is unchanged. It stays a +pure deleter. + +## Background + +Compacted SST GC deletes SST objects that are not referenced by active +manifests or checkpoints and whose physical SST ULID timestamp is below a +calculated cutoff: + +```text +cutoff = min(now - min_age, compaction_low_watermark, newest_l0) +delete when: sst_ulid_ts < cutoff && sst is not referenced +``` + +To decide whether an SST is referenced, GC reads the latest manifest and the +manifests retained by checkpoints. It collects the physical SST IDs from every +L0 view and sorted-run view in those manifests. Any compacted SST object whose +ID is absent from that set is treated as unreferenced. Pending uploads are not +in that set until a manifest commit records them. + +The cutoff has three parts: + +- `now - min_age`: keep objects whose age is less than or equal to `min_age`. + The default `min_age` is 300 seconds. +- `compaction_low_watermark`: the minimum job ID timestamp across active + compaction jobs and the most recently finished job, read from + `.compactions`. It protects possible outputs of active compactions. +- `newest_l0`: the newest L0 physical SST timestamp in the latest manifest, + falling back to `last_compacted_l0_sst_view_id` for trees with no live L0s. + It protects L0s that are uploaded but not yet published. + +Manifest V2 also has two ULID domains: + +- `SsTableHandle.id`, the physical SST ID used by GC deletion. +- `SsTableView.id`, the view ID used by `last_compacted_l0_sst_view_id`. + +If these IDs are minted independently, GC can compare timestamps from +different domains. + +## Motivation + +The writer mints physical L0 SST IDs inside the parallel upload workers. The +manifest writer publishes uploaded immutable memtables in sequence number +order. Mint order and publish order can therefore differ. + +Publish order cannot be relaxed. `last_l0_seq` means every sequence at or +below that value is already in L0. Publishing a newer memtable while an older +one is missing would advance `last_l0_seq` past the missing range. WAL replay +skips entries at or below `last_l0_seq`, so it would not recover that range, +and with the WAL disabled there is no source to rebuild it from. + +The unsafe sequence is: + +1. Immutable memtable A has lower sequence numbers than immutable memtable B. + Both are submitted to parallel upload workers. +2. B's worker mints its SST ID before A's worker does. B's timestamp is below + A's, even though B is later by sequence number. +3. A's upload finishes and A is published. B's upload stalls, so B's SST is + uploaded but not yet in the manifest. +4. `newest_l0` is now A's timestamp, which is above B's timestamp. +5. Once the stall exceeds `min_age` and the compaction watermark is also above + B's timestamp, B's SST is unreferenced and below the cutoff. GC deletes it. +6. The manifest writer later publishes B, creating a manifest that references + a missing object. + +This is an ordering problem, not a clock problem. It happens on a single +well-behaved clock, because mint order and publish order differ. It was +reproduced by a deterministic simulation test failure in PR #1758. + +## Goals + +- Prevent GC from deleting newly flushed L0 SSTs before they are published. +- Preserve ULIDs as SST IDs. +- Avoid object-store copy or rename on the normal write path. +- Preserve parallel L0 upload throughput. +- Keep manifest and SST schemas unchanged. +- Make unsafe minting fail explicitly instead of causing silent data loss. + +## Non-Goals + +- Solve clock skew. Skew is assumed bounded, and users set `min_age` for the + margin they want against GC-versus-writer skew. +- Redesign compacted SST GC around sequence numbers or manifest IDs. +- Fix unrelated full-ULID ordering bugs, such as choosing between two + same-millisecond compaction IDs by comparing the full random suffix. + +## Design + +### Writer L0 IDs + +Move L0 physical SST ID allocation from the upload worker to +`FlushTracker::dispatch_ready_memtables`. Dispatch already happens in sequence +number order, so SST timestamp order matches the order in which the manifest +writer publishes L0s. The race above cannot happen: every published L0 was +dispatched before any still-pending L0, so `newest_l0` cannot advance past a +pending SST's timestamp. + +`UploadJob` carries the pre-allocated IDs: + +```rust +pub(crate) struct UploadJob { + pub(crate) imm_memtable: Arc, + pub(crate) segment_sst_ids: BTreeMap, +} +``` + +The uploader writes each segment SST to the pre-allocated ID instead of +minting a new ID inside the parallel upload worker. If retention removes all +entries for a segment before upload, the unused ID is discarded. + +When the manifest writer publishes a newly flushed L0, it creates an identity +view: `SsTableView.id` is the same ULID as the physical SST ID. This keeps the +timestamp used by `last_compacted_l0_sst_view_id` equal to the timestamp used +by GC deletion. + +Views created by split, union, or rescaling reference existing physical SSTs, +not newly uploaded objects. They are unchanged by this RFC. + +### Compaction IDs + +Compaction job IDs, output SST IDs, and sorted-run view IDs are minted as +today. Outputs of an active job are protected by `compaction_low_watermark`, +and clock skew within `min_age` is covered by the `now - min_age` term. The +`.compactions` invariants below reject IDs that violate the watermark rules. + +We do not check the job ID against the manifest's L0 timestamps. The +`.manifest` and `.compactions` files are updated independently, so such a +check would not hold as new L0s arrive, and it would not help anyway: GC does +not compare those timestamps, so a low job ID only lowers the cutoff and makes +GC more cautious, never less. + +### Invariant Checks + +These are `Invariant` predicates from `slatedb-txn-obj` (PR #1741), not new +fields in the manifest or `db_state.rs`. They are registered in the central +stored object construction paths, not at individual write call sites, so new +update paths don't miss them. + +For `.manifest`: + +- `l0_ulid_cutoff`: a newly added L0 physical SST ID must have a timestamp at + or above the newest L0 timestamp already in the manifest. + +For `.compactions`: + +- `compaction_job_id_cutoff`: a newly added compaction job ID must have a + timestamp at or above the maximum existing compaction job ID timestamp. +- `sorted_run_ulid_cutoff`: each output SST ID and sorted-run view ID recorded + for a compaction must have a timestamp at or above that compaction job ID + timestamp. + +The checks compare timestamp milliseconds, not full ULID ordering. Equal +milliseconds are safe because GC only deletes SSTs strictly below the cutoff. +A failure is reported as `InvalidClockTick` and the unsafe update is not +committed. The error message includes the rejected timestamp and the required +watermark. + +### Failure Handling + +With minting moved to dispatch, an invariant failure means clock skew or a new +minting path that skipped the rules, not a normal race. A minting path that +skips the rules is a bug in SlateDB, but the check only sees timestamps and +cannot tell the two causes apart. If the clocks are fine and the error keeps +happening, the user should report a bug. + +- If the error is returned in a user call path, the caller gets the error and + the `Db` stays open. +- If the error happens in a background task, such as flush or compaction, the + `Db` is marked closed with a failed state. + +Skew across a writer restart can fail the invariant: if a previous writer's +clock was ahead, a new writer's IDs fall below the committed timestamps and +are rejected. `min_age` does not help this case. The fix is to fix the clock +(run NTP) or wait until the wall clock passes the committed timestamps, then +reopen. Retrying without fixing the clock will fail again. + +### Garbage Collection + +GC does not change. It stays a pure deleter and keeps the same cutoff. All the +work in this RFC is on the minting side, so the IDs GC already reads are safe +to interpret. + +## Implementation + +- Move L0 physical SST ID allocation to + `FlushTracker::dispatch_ready_memtables`. +- Add pre-allocated segment SST IDs to `UploadJob` and update the uploader to + use them. +- Update `ManifestWriter::apply_uploaded_state` to create identity L0 views. +- Register the manifest and `.compactions` invariants in the shared + construction paths for loaded and newly created stored objects. +- Return `InvalidClockTick` on invariant failure, with the rejected timestamp + and required watermark in the error message. + +## Impact Analysis + +SlateDB features and components that this RFC interacts with: + +- [x] Error model, API errors +- [ ] Sequence numbers +- [ ] Manifest format +- [ ] Checkpoints +- [ ] Clones +- [x] Garbage collection +- [ ] Database splitting and merging +- [x] Compaction state persistence +- [ ] Compaction strategies +- [x] Distributed compaction +- [x] Compactions format +- [ ] SST format or block format +- [x] Observability (metrics/logging/tracing) + +## Operations + +### Performance & Cost + +- L0 upload and compaction output paths still write each SST once. +- ID allocation moves from the upload worker to dispatch; the work is the + same. +- The invariant checks are in-memory timestamp comparisons over the items in + an update (new L0s, job IDs, outputs). They add no I/O, and their cost is + small next to the object-store write. +- No object-store copy, rename, or extra GC CAS path is added. + +### Observability + +- Metrics: invariant failure count. +- Logging: on invariant failure, include the role, the rejected timestamp, + and the required watermark. + +### Compatibility + +- Existing SST IDs remain valid ULIDs. +- Existing manifests remain readable. +- Existing projected views with distinct view IDs remain valid. +- Invariants must be enabled only after all writers and compactors in a + deployment mint L0 IDs at dispatch, otherwise the old race can trip them. + +## Testing + +- Unit tests for identity L0 views and the manifest and `.compactions` + invariants. +- Integration tests for parallel L0 upload where upload completion order + differs from manifest publish order. +- Deterministic simulation test for the publish-order race that motivated + this RFC. +- Fault-injection tests for writer and worker clock skew, checking that + invariants fail loudly instead of losing data. + +## Rollout + +1. Move L0 SST ID allocation to dispatch and pass IDs through `UploadJob`. +2. Make newly flushed L0 views identity views. +3. Add invariants, metrics, and logs. +4. Enable strict invariant enforcement after all roles in the deployment are + upgraded. + +## Alternatives + +### Increase `min_age` alone + +- Reduces the probability that a staged SST is old enough to delete. +- Rejected as the only fix because upload stalls can exceed any practical + value. This RFC fixes the publish-order race structurally and keeps + `min_age` only as a best-effort knob. + +### Exclude external SSTs from the cutoff + +- Compute `newest_l0` from L0s owned by this database only, ignoring SSTs + inherited from a clone or union parent. This would stop a parent's + far-future L0 timestamp from raising this database's cutoff. +- Not taken. That case only happens under clock skew across databases, which + we assume is bounded and out of scope. Adding it would make the cutoff logic + more complex for a case we have not seen. + +### Calibrate writer clocks against the object store + +- Suggested in review: on each PUT, record the local time before and after, + and check the object's `last_modified` falls within that window plus an + error bound. This keeps each writer's clock close to the object store's + clock and bounds skew directly. +- Deferred. It is a reasonable way to bound skew if the bounded-skew + assumption proves too weak, but it adds a check to every write for a problem + we are treating as out of scope. + +### Monotonic allocator with a timestamp floor + +- An earlier draft of this RFC added a `MonotonicSstIdAllocator`. It computed + a floor from committed manifest and `.compactions` state, refused to mint + below the floor, waited a bounded time for a lagging clock, and returned + `InvalidClockTick` if the clock stayed behind. +- Dropped because the publish-order race does not need it, and skew within + `min_age` is already safe. It added waiting, floor plumbing across the + writer and compactor roles, and new failure modes for a problem the + invariants already catch. + +### Offset-based ULID generation + +- Suggested in review: record the wall clock when the allocator starts, then + mint timestamps as `max(last_issued_ms, max(0, floor_ms - start_ms) + now_ms)`. + A lagging clock is shifted forward past the floor instead of waiting. +- Deferred. It is a good fallback if the bounded-skew assumption proves too + weak. The trade-off is that every GC-relevant timestamp would have to be + generated this way, and shifted timestamps are written into durable state. + +### Run GC inside the `Db` + +- Suggested in review: require GC to run inside the `Db` so GC and the writer + can coordinate directly instead of relying on ID timestamps. +- Not taken because running GC as a separate process remains a supported + deployment. Worth revisiting if timestamp-based safety proves fragile. + +### Persisted GC cutoff + +- Add a monotonic `gc_sst_cutoff_ms` field to manifest and compactions state. + GC would persist the cutoff before deleting, and writers would validate new + references against the persisted value. +- Not taken because it requires a schema change and makes GC a + manifest/compactions writer. + +### Sequence or manifest IDs for SSTs + +- Replace ULID timestamp safety with sequence-number or manifest-ID safety. +- Rejected for this RFC because SSTs are written before manifest commit, + compaction outputs do not naturally belong to the input data sequence, and + clone/split/union timelines make ownership rules larger than this fix. + +## References + +- [RFC-0024: Segment-Oriented Compaction](0024-segment-oriented-compaction.md) +- [RFC-0025: Distributed Compaction](0025-distributed-compaction.md) +- [RFC-0026: Garbage Collector Boundary Files for Sequenced Metadata](0026-garbage-collector-boundary.md) +- [Issue #1707: Implement GC cutoff rule enforcement](https://github.com/slatedb/slatedb/issues/1707) +- [PR #1741: add `Invariant` predicates to slatedb-txn-obj](https://github.com/slatedb/slatedb/pull/1741) +- [PR #1747: add `l0_ulid_cutoff` invariant + L0 ULID watermark helper](https://github.com/slatedb/slatedb/pull/1747) +- [PR #1758: enforce `l0_ulid_cutoff` invariant on manifest update](https://github.com/slatedb/slatedb/pull/1758) +- [Issue #356: Use latest manifest timestamp for GC instead of `Utc::now`](https://github.com/slatedb/slatedb/issues/356) From 323ed1bcd52c81582fbee664b96757cd9c2bef22 Mon Sep 17 00:00:00 2001 From: krishna sindhur Date: Fri, 10 Jul 2026 22:00:43 +0530 Subject: [PATCH 06/63] Did small refactoring for db_state.rs triple-duplicates first/last key logic (#1916) --- slatedb/src/db_state.rs | 56 ++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 32 deletions(-) diff --git a/slatedb/src/db_state.rs b/slatedb/src/db_state.rs index 08ec6d8d5..9deefb946 100644 --- a/slatedb/src/db_state.rs +++ b/slatedb/src/db_state.rs @@ -100,16 +100,10 @@ impl SsTableView { /// Create a new view with no visible_range projection. pub(crate) fn new(id: Ulid, sst: SsTableHandle) -> Self { - let effective_range = match sst.info.first_entry.clone() { - Some(physical_first_entry) => { - let end_bound = match sst.info.last_entry.clone() { - Some(physical_last_entry) => Included(physical_last_entry), - None => Unbounded, - }; - BytesRange::new(Included(physical_first_entry), end_bound) - } - None => BytesRange::new_empty(), - }; + let effective_range = sst + .info + .physical_range() + .unwrap_or_else(BytesRange::new_empty); SsTableView { id, @@ -125,18 +119,10 @@ impl SsTableView { sst: SsTableHandle, visible_range: Option, ) -> Self { - let mut effective_range = match sst.info.first_entry.clone() { - Some(physical_first_entry) => { - let end_bound = match sst.info.last_entry.clone() { - Some(physical_last_entry) => Included(physical_last_entry), - None => Unbounded, - }; - BytesRange::new(Included(physical_first_entry), end_bound) - } - None => { - unreachable!("SST always has a first entry.") - } - }; + let mut effective_range = sst + .info + .physical_range() + .expect("SST always has a first entry."); if let Some(visible_range) = &visible_range { assert!( visible_range.is_start_bound_included_or_unbounded(), @@ -163,16 +149,10 @@ impl SsTableView { /// the range that [`Self::new_projected`] intersects a visible range /// against. fn physical_range(&self) -> BytesRange { - match self.sst.info.first_entry.clone() { - Some(physical_first_entry) => { - let end_bound = match self.sst.info.last_entry.clone() { - Some(physical_last_entry) => Included(physical_last_entry), - None => Unbounded, - }; - BytesRange::new(Included(physical_first_entry), end_bound) - } - None => unreachable!("SST always has a first entry."), - } + self.sst + .info + .physical_range() + .expect("SST always has a first entry.") } /// Like [`Self::with_visible_range`], but returns `None` instead of @@ -477,6 +457,18 @@ pub struct SsTableInfo { pub filter_format: FilterFormat, } +impl SsTableInfo { + pub(crate) fn physical_range(&self) -> Option { + self.first_entry.clone().map(|first_entry| { + let end_bound = match self.last_entry.clone() { + Some(last_entry) => Included(last_entry), + None => Unbounded, + }; + BytesRange::new(Included(first_entry), end_bound) + }) + } +} + pub(crate) trait SsTableInfoCodec: Send + Sync { fn encode(&self, manifest: &SsTableInfo) -> Bytes; From c676f114190fb61da9847dd0664d7b55c23fb92b Mon Sep 17 00:00:00 2001 From: Chris Date: Sat, 11 Jul 2026 18:02:39 -0700 Subject: [PATCH 07/63] Optimize DST binary (#1918) --- .github/workflows/dst-hourly.yaml | 2 +- Cargo.toml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dst-hourly.yaml b/.github/workflows/dst-hourly.yaml index c334a68ac..86d73ed77 100644 --- a/.github/workflows/dst-hourly.yaml +++ b/.github/workflows/dst-hourly.yaml @@ -41,7 +41,7 @@ jobs: # failure notification fires. A job-level timeout would instead cancel # the run, and GitHub does not notify on cancellations. timeout-minutes: 60 - run: cargo nextest run -p slatedb-dst --all-features --profile dst-nightly --no-capture ${{ matrix.test-filter }} + run: cargo nextest run -p slatedb-dst --all-features --cargo-profile dst --profile dst-nightly --no-capture ${{ matrix.test-filter }} env: RUSTFLAGS: "--cfg dst --cfg tokio_unstable --cfg slow" RUST_LOG: "info" diff --git a/Cargo.toml b/Cargo.toml index ce305acc0..96561815e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,12 @@ readme = "README.md" [profile.bench] lto = true +[profile.dst] +inherits = "test" +opt-level = 2 +debug-assertions = true +overflow-checks = true + [workspace.dependencies] # dependencies From 9937c2dad9e1625e85b9255ab04510425f389967 Mon Sep 17 00:00:00 2001 From: nomiero Date: Sun, 12 Jul 2026 13:09:21 -0700 Subject: [PATCH 08/63] RFC 0027 impl (4/N) - wiring object store cache for compaction and gc (#1899) --- .../src/cached_object_store/object_store.rs | 217 ++++++--- slatedb/src/cached_object_store/policy.rs | 57 +-- slatedb/src/cached_object_store/storage_fs.rs | 1 + slatedb/src/config.rs | 17 +- slatedb/src/db.rs | 417 +++++++++++++++--- slatedb/src/db/builder.rs | 68 ++- slatedb/src/garbage_collector/compacted_gc.rs | 151 ++++++- .../src/content/docs/docs/design/caching.mdx | 2 +- 8 files changed, 731 insertions(+), 199 deletions(-) diff --git a/slatedb/src/cached_object_store/object_store.rs b/slatedb/src/cached_object_store/object_store.rs index 8ef538faf..0f8f71c7f 100644 --- a/slatedb/src/cached_object_store/object_store.rs +++ b/slatedb/src/cached_object_store/object_store.rs @@ -51,7 +51,7 @@ impl CachedObjectStore { object_store: Arc, cache_storage: Arc, part_size_bytes: usize, - cache_puts: bool, + cache_put_config: CachePutConfig, stats: Arc, ) -> Result, SlateDBError> { Self::new_with_policies( @@ -61,7 +61,7 @@ impl CachedObjectStore { stats, Arc::new(DefaultGetPolicy), Arc::new(DefaultPutPolicy { - put: CachePutConfig { cache_puts }, + put: cache_put_config, }), ) } @@ -94,6 +94,23 @@ impl CachedObjectStore { })) } + /// Returns a new handle that reads through the new `object_store` on cache + /// misses while sharing everything else (all fields other than the + /// object_store in the cache are shared by ref-count clones). + /// + /// This lets a component with its own instrumented store (for example + /// the compactor) share the cache while keeping its I/O recorded under + /// its own metric labels. + pub(crate) fn clone_with_new_object_store( + &self, + object_store: Arc, + ) -> Arc { + Arc::new(Self { + object_store, + ..self.clone() + }) + } + pub(crate) async fn start_evictor(&self) { self.cache_storage.start_evictor().await; } @@ -126,7 +143,10 @@ impl CachedObjectStore { object_store, cache_storage, options.part_size_bytes, - options.cache_puts, + CachePutConfig { + cache_on_flush: options.cache_on_flush, + cache_on_compaction: options.cache_on_compaction, + }, stats, )?; cached.start_evictor().await; @@ -990,6 +1010,7 @@ mod tests { use std::time::Duration; use super::CachedObjectStore; + use crate::cached_object_store::policy::CachePutConfig; use crate::cached_object_store::stats::CachedObjectStoreStats; use crate::cached_object_store::storage::{LocalCacheStorage, PartID}; use crate::cached_object_store::storage_fs::FsCacheEntry; @@ -1026,7 +1047,14 @@ mod tests { Arc::new(DbRand::default()), 1000, )); - CachedObjectStore::new(object_store, cache_storage, 1024, false, stats).unwrap() + CachedObjectStore::new( + object_store, + cache_storage, + 1024, + CachePutConfig::default(), + stats, + ) + .unwrap() } #[tokio::test] @@ -1056,9 +1084,14 @@ mod tests { )); let part_size = 1024; - let cached_store = - CachedObjectStore::new(object_store.clone(), cache_storage, part_size, false, stats) - .unwrap(); + let cached_store = CachedObjectStore::new( + object_store.clone(), + cache_storage, + part_size, + CachePutConfig::default(), + stats, + ) + .unwrap(); let entry = cached_store.cache_storage.entry(&location, 1024); let object_size_hint = cached_store.save_get_result(&location, get_result).await?; @@ -1134,8 +1167,14 @@ mod tests { 1000, )); - let cached_store = - CachedObjectStore::new(object_store, cache_storage, part_size, false, stats).unwrap(); + let cached_store = CachedObjectStore::new( + object_store, + cache_storage, + part_size, + CachePutConfig::default(), + stats, + ) + .unwrap(); let entry = cached_store.cache_storage.entry(&location, part_size); let object_size_hint = cached_store.save_get_result(&location, get_result).await?; assert_eq!(object_size_hint, 1024 * 3); @@ -1235,8 +1274,14 @@ mod tests { 1000, )); - let cached_store = - CachedObjectStore::new(object_store, cache_storage, 1024, false, stats).unwrap(); + let cached_store = CachedObjectStore::new( + object_store, + cache_storage, + 1024, + CachePutConfig::default(), + stats, + ) + .unwrap(); struct Test { input: (Option, usize), @@ -1325,8 +1370,14 @@ mod tests { Arc::new(DbRand::default()), 1000, )); - let cached_store = - CachedObjectStore::new(object_store, cache_storage, 1024, false, stats).unwrap(); + let cached_store = CachedObjectStore::new( + object_store, + cache_storage, + 1024, + CachePutConfig::default(), + stats, + ) + .unwrap(); let aligned = cached_store.align_range(&(9..1025), 1024); assert_eq!(aligned, 0..2048); @@ -1349,8 +1400,14 @@ mod tests { Arc::new(DbRand::default()), 1000, )); - let cached_store = - CachedObjectStore::new(object_store, cache_storage, 1024, false, stats).unwrap(); + let cached_store = CachedObjectStore::new( + object_store, + cache_storage, + 1024, + CachePutConfig::default(), + stats, + ) + .unwrap(); let aligned = cached_store.align_get_range(&GetRange::Bounded(9..1025)); assert_eq!(aligned, GetRange::Bounded(0..2048)); @@ -1381,9 +1438,14 @@ mod tests { Arc::new(DbRand::default()), 1000, )); - let cached_store = - CachedObjectStore::new(object_store.clone(), cache_storage, 1024, false, stats) - .unwrap(); + let cached_store = CachedObjectStore::new( + object_store.clone(), + cache_storage, + 1024, + CachePutConfig::default(), + stats, + ) + .unwrap(); let test_path = Path::from("/data/testdata1"); let test_payload = gen_rand_bytes(1024 * 3 + 2); @@ -1467,9 +1529,14 @@ mod tests { let object_store = Arc::new(object_store::memory::InMemory::new()); - let cached_store = - CachedObjectStore::new(object_store.clone(), cache_storage, 1024, false, stats) - .unwrap(); + let cached_store = CachedObjectStore::new( + object_store.clone(), + cache_storage, + 1024, + CachePutConfig::default(), + stats, + ) + .unwrap(); // Create some test files to preload let test_paths = vec![ @@ -1519,9 +1586,14 @@ mod tests { let object_store = Arc::new(object_store::memory::InMemory::new()); - let cached_store = - CachedObjectStore::new(object_store.clone(), cache_storage, 1024, false, stats) - .unwrap(); + let cached_store = CachedObjectStore::new( + object_store.clone(), + cache_storage, + 1024, + CachePutConfig::default(), + stats, + ) + .unwrap(); // Create some test files let test_paths = vec![Path::from("file1.sst"), Path::from("file2.sst")]; @@ -1587,7 +1659,7 @@ mod tests { instrumented as Arc, cache_storage, 1024, - false, + CachePutConfig::default(), stats, ) .unwrap(); @@ -1984,7 +2056,7 @@ mod tests { object_store, Arc::clone(&cache_storage) as Arc, PART_SIZE, - false, + CachePutConfig::default(), stats, ) .unwrap(); @@ -2053,7 +2125,7 @@ mod tests { fn policy_test_store( upstream: Arc, - cache_puts: bool, + policy: CachePutConfig, ) -> Arc { let recorder = MetricsRecorderHelper::noop(); let stats = Arc::new(CachedObjectStoreStats::new(&recorder)); @@ -2066,7 +2138,7 @@ mod tests { Arc::new(DbRand::default()), 1000, )); - CachedObjectStore::new(upstream, cache_storage, 1024, cache_puts, stats).unwrap() + CachedObjectStore::new(upstream, cache_storage, 1024, policy, stats).unwrap() } fn put_opts_tagged(tag: ObjectStoreCallTag) -> object_store::PutOptions { @@ -2095,38 +2167,43 @@ mod tests { } #[rstest] - // WAL writes are never cached, even with cache_puts on. - #[case(ObjectStoreCallTag::new(TableStoreKind::Main, SstType::Wal), true, 0)] - // Compacted writes from the main store (flush) and the compactor are cached - // when cache_puts is set, and not otherwise. + // WAL writes are never cached, even with both flags enabled. + #[case( + ObjectStoreCallTag::new(TableStoreKind::Main, SstType::Wal), + CachePutConfig { cache_on_flush: true, cache_on_compaction: true }, + 0 + )] + // Flush writes (main store, compacted) cached only when cache_on_flush is set. #[case( ObjectStoreCallTag::new(TableStoreKind::Main, SstType::Compacted), - true, + CachePutConfig { cache_on_flush: true, cache_on_compaction: false }, 2 )] #[case( ObjectStoreCallTag::new(TableStoreKind::Main, SstType::Compacted), - false, + CachePutConfig { cache_on_flush: false, cache_on_compaction: true }, 0 )] + // Compaction writes (compactor store, compacted) cached only when + // cache_on_compaction is set. #[case( ObjectStoreCallTag::new(TableStoreKind::Compactor, SstType::Compacted), - true, + CachePutConfig { cache_on_flush: false, cache_on_compaction: true }, 2 )] #[case( ObjectStoreCallTag::new(TableStoreKind::Compactor, SstType::Compacted), - false, + CachePutConfig { cache_on_flush: true, cache_on_compaction: false }, 0 )] #[tokio::test] async fn test_put_caching_by_tag( #[case] tag: ObjectStoreCallTag, - #[case] cache_puts: bool, + #[case] policy: CachePutConfig, #[case] expected_parts: usize, ) { let upstream: Arc = Arc::new(object_store::memory::InMemory::new()); - let store = policy_test_store(upstream.clone(), cache_puts); + let store = policy_test_store(upstream.clone(), policy); let location = Path::from("compacted/01.sst"); let payload = gen_rand_bytes(2048); // 2 parts of 1024 bytes @@ -2145,7 +2222,13 @@ mod tests { #[tokio::test] async fn test_untagged_put_is_not_cached() { let upstream: Arc = Arc::new(object_store::memory::InMemory::new()); - let store = policy_test_store(upstream.clone(), true); + let store = policy_test_store( + upstream.clone(), + CachePutConfig { + cache_on_flush: true, + cache_on_compaction: true, + }, + ); // No tag in the options: coordination I/O (manifest, etc.) is never cached. let location = Path::from("manifest/01.manifest"); @@ -2165,7 +2248,7 @@ mod tests { #[tokio::test] async fn test_compactor_get_bypasses_cache() { let upstream: Arc = Arc::new(object_store::memory::InMemory::new()); - let store = policy_test_store(upstream.clone(), false); + let store = policy_test_store(upstream.clone(), CachePutConfig::default()); let location = Path::from("compacted/01.sst"); let payload = gen_rand_bytes(2048); @@ -2225,8 +2308,14 @@ mod tests { Arc::new(DbRand::default()), 1000, )); - let store = - CachedObjectStore::new(upstream.clone(), cache_storage, 1024, false, stats).unwrap(); + let store = CachedObjectStore::new( + upstream.clone(), + cache_storage, + 1024, + CachePutConfig::default(), + stats, + ) + .unwrap(); store.start_evictor().await; let location = Path::from("compacted/01.sst"); @@ -2302,7 +2391,7 @@ mod tests { #[tokio::test] async fn test_compactor_head_reads_without_admitting() { let upstream: Arc = Arc::new(object_store::memory::InMemory::new()); - let store = policy_test_store(upstream.clone(), false); + let store = policy_test_store(upstream.clone(), CachePutConfig::default()); let location = Path::from("compacted/01.sst"); let payload = gen_rand_bytes(512); @@ -2356,7 +2445,7 @@ mod tests { #[tokio::test] async fn test_wal_read_bypasses_cache() { let upstream: Arc = Arc::new(object_store::memory::InMemory::new()); - let store = policy_test_store(upstream.clone(), false); + let store = policy_test_store(upstream.clone(), CachePutConfig::default()); let location = Path::from("wal/00000000000000000001.sst"); let payload = gen_rand_bytes(2048); @@ -2397,7 +2486,13 @@ mod tests { #[tokio::test] async fn test_put_writes_head_and_serves_first_read_from_cache() { let upstream: Arc = Arc::new(object_store::memory::InMemory::new()); - let store = policy_test_store(upstream.clone(), true); + let store = policy_test_store( + upstream.clone(), + CachePutConfig { + cache_on_flush: true, + cache_on_compaction: false, + }, + ); // A flush write (main store, compacted) is cached and commits a head. let location = Path::from("compacted/01.sst"); @@ -2463,7 +2558,13 @@ mod tests { #[case] expected_part_sizes: Vec, ) { let upstream: Arc = Arc::new(object_store::memory::InMemory::new()); - let store = policy_test_store(upstream.clone(), true); + let store = policy_test_store( + upstream.clone(), + CachePutConfig { + cache_on_flush: false, + cache_on_compaction: true, + }, + ); // A compaction output written as a multipart upload (the path large // compacted SSTs take). The tag survives multipart init, so no fallback @@ -2507,20 +2608,24 @@ mod tests { } #[rstest] - // A compacted multipart upload is not cached when cache_puts is off. + // A compacted multipart upload is not cached when its source is disabled, + // even if the other source is enabled. #[case( ObjectStoreCallTag::new(TableStoreKind::Compactor, SstType::Compacted), - false + CachePutConfig { cache_on_flush: true, cache_on_compaction: false } + )] + // A WAL multipart upload is never cached, even with both flags on. + #[case( + ObjectStoreCallTag::new(TableStoreKind::Main, SstType::Wal), + CachePutConfig { cache_on_flush: true, cache_on_compaction: true } )] - // A WAL multipart upload is never cached, even with cache_puts on. - #[case(ObjectStoreCallTag::new(TableStoreKind::Main, SstType::Wal), true)] #[tokio::test] async fn test_multipart_upload_not_cached( #[case] tag: ObjectStoreCallTag, - #[case] cache_puts: bool, + #[case] policy: CachePutConfig, ) { let upstream: Arc = Arc::new(object_store::memory::InMemory::new()); - let store = policy_test_store(upstream.clone(), cache_puts); + let store = policy_test_store(upstream.clone(), policy); let location = Path::from("compacted/big.sst"); let mut upload = store @@ -2536,7 +2641,13 @@ mod tests { #[tokio::test] async fn test_multipart_head_is_the_commit_point() { let upstream: Arc = Arc::new(object_store::memory::InMemory::new()); - let store = policy_test_store(upstream.clone(), true); + let store = policy_test_store( + upstream.clone(), + CachePutConfig { + cache_on_flush: false, + cache_on_compaction: true, + }, + ); let location = Path::from("compacted/big.sst"); let cache_location = location.clone(); diff --git a/slatedb/src/cached_object_store/policy.rs b/slatedb/src/cached_object_store/policy.rs index e8ba6baed..d850fce12 100644 --- a/slatedb/src/cached_object_store/policy.rs +++ b/slatedb/src/cached_object_store/policy.rs @@ -58,7 +58,7 @@ pub(crate) trait PutPolicy: Send + Sync + 'static + std::fmt::Debug { fn put_action(&self, tag: Option<&ObjectStoreCallTag>) -> PutAction; } -/// The built-in put policy, configured by [`CachePutPolicy`]. +/// The built-in put policy, configured by [`CachePutConfig`]. #[derive(Debug, Clone)] pub(crate) struct DefaultPutPolicy { pub(crate) put: CachePutConfig, @@ -70,15 +70,14 @@ impl PutPolicy for DefaultPutPolicy { // Untagged writes (manifest, compaction state) are never cached. return PutAction::Skip; }; - if !self.put.cache_puts { - return PutAction::Skip; - } match tag.sst_type { SstType::Wal => PutAction::Skip, - // Only the stores that write compacted SSTs (the main store on flush - // and the compactor) are cached; other sources bypass the cache. + // Each compacted SST write source has its own config gate: the + // main store writes on flush, the compactor on compaction. Other + // sources never write compacted SSTs and skip the cache. SstType::Compacted => match tag.kind { - TableStoreKind::Main | TableStoreKind::Compactor => PutAction::Cache, + TableStoreKind::Main if self.put.cache_on_flush => PutAction::Cache, + TableStoreKind::Compactor if self.put.cache_on_compaction => PutAction::Cache, _ => PutAction::Skip, }, } @@ -126,13 +125,17 @@ pub(crate) enum PutAction { Skip, } -/// Whether compacted SST writes are cached. +/// Which compacted SST write sources are cached. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub(crate) struct CachePutConfig { - /// Cache compacted SSTs written through the cache. + /// Cache compacted SSTs written by a memtable flush. + /// + /// Default is false. + pub(crate) cache_on_flush: bool, + /// Cache compacted SSTs written by compaction. /// /// Default is false. - pub(crate) cache_puts: bool, + pub(crate) cache_on_compaction: bool, } #[cfg(test)] @@ -275,50 +278,49 @@ mod tests { } #[rstest] - // WAL writes are never cached, even with cache_puts on. + // WAL writes are never cached, even with both flags on. #[case( Some(tag(TableStoreKind::Main, SstType::Wal, None)), - CachePutConfig { cache_puts: true }, + CachePutConfig { cache_on_flush: true, cache_on_compaction: true }, PutAction::Skip )] // Untagged writes (manifest, compaction state) are never cached. #[case( None, - CachePutConfig { cache_puts: true }, + CachePutConfig { cache_on_flush: true, cache_on_compaction: true }, PutAction::Skip )] - // Compacted writes from the main store (flush) and the compactor are cached - // when cache_puts is set. + // Flush writes (main store, compacted) gated by cache_on_flush. #[case( Some(tag(TableStoreKind::Main, SstType::Compacted, None)), - CachePutConfig { cache_puts: true }, - PutAction::Cache - )] - #[case( - Some(tag(TableStoreKind::Compactor, SstType::Compacted, None)), - CachePutConfig { cache_puts: true }, + CachePutConfig { cache_on_flush: true, cache_on_compaction: false }, PutAction::Cache )] - // Nothing is cached when cache_puts is off. #[case( Some(tag(TableStoreKind::Main, SstType::Compacted, None)), - CachePutConfig { cache_puts: false }, + CachePutConfig { cache_on_flush: false, cache_on_compaction: true }, PutAction::Skip )] + // Compaction writes (compactor store, compacted) gated by cache_on_compaction. + #[case( + Some(tag(TableStoreKind::Compactor, SstType::Compacted, None)), + CachePutConfig { cache_on_flush: false, cache_on_compaction: true }, + PutAction::Cache + )] #[case( Some(tag(TableStoreKind::Compactor, SstType::Compacted, None)), - CachePutConfig { cache_puts: false }, + CachePutConfig { cache_on_flush: true, cache_on_compaction: false }, PutAction::Skip )] // Reader/GC never write compacted SSTs, but if they did the policy is Skip. #[case( Some(tag(TableStoreKind::Reader, SstType::Compacted, None)), - CachePutConfig { cache_puts: true }, + CachePutConfig { cache_on_flush: true, cache_on_compaction: true }, PutAction::Skip )] #[case( Some(tag(TableStoreKind::GC, SstType::Compacted, None)), - CachePutConfig { cache_puts: true }, + CachePutConfig { cache_on_flush: true, cache_on_compaction: true }, PutAction::Skip )] fn test_put_action( @@ -335,7 +337,8 @@ mod tests { #[test] fn test_default_put_policy_caches_nothing() { let policy = CachePutConfig::default(); - assert!(!policy.cache_puts); + assert!(!policy.cache_on_flush); + assert!(!policy.cache_on_compaction); for kind in [ TableStoreKind::Main, TableStoreKind::Compactor, diff --git a/slatedb/src/cached_object_store/storage_fs.rs b/slatedb/src/cached_object_store/storage_fs.rs index 0b26ac2d1..1a5cb602f 100644 --- a/slatedb/src/cached_object_store/storage_fs.rs +++ b/slatedb/src/cached_object_store/storage_fs.rs @@ -1149,6 +1149,7 @@ async fn delete_cache_entry( deleted_entries } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => vec![], Err(e) => { error!("FS cache failed to read_dir {path:?}: {e:?}"); vec![] diff --git a/slatedb/src/config.rs b/slatedb/src/config.rs index dbf428416..74e59541e 100644 --- a/slatedb/src/config.rs +++ b/slatedb/src/config.rs @@ -1555,9 +1555,17 @@ pub struct ObjectStoreCacheOptions { /// its default value is 4mb. pub part_size_bytes: usize, - /// Whether to cache PUT operations to disk. When enabled, data written via PUT operations - /// will be cached locally for faster subsequent reads. Default is false. - pub cache_puts: bool, + /// Whether to cache compacted SSTs produced by memtable flushes to the + /// local disk cache, for faster subsequent reads. + /// + /// Default is false. + pub cache_on_flush: bool, + + /// Whether to cache compacted SSTs produced by compaction to the local + /// disk cache, for faster subsequent reads. + /// + /// Default is false. + pub cache_on_compaction: bool, /// Whether to preload SST files into cache during database startup. When enabled, /// the database will load SST files into the cache up to the cache size limit @@ -1589,7 +1597,8 @@ impl Default for ObjectStoreCacheOptions { #[cfg(not(target_pointer_width = "32"))] max_cache_size_bytes: Some(16 * 1024 * 1024 * 1024), part_size_bytes: 4 * 1024 * 1024, - cache_puts: false, + cache_on_flush: false, + cache_on_compaction: false, preload_disk_cache_on_startup: None, scan_interval: Some(Duration::from_secs(3600)), max_open_file_handles: 1000, diff --git a/slatedb/src/db.rs b/slatedb/src/db.rs index f5e19390b..45c062cb6 100644 --- a/slatedb/src/db.rs +++ b/slatedb/src/db.rs @@ -2103,8 +2103,6 @@ impl DbWalObserver { #[cfg(test)] mod tests { use super::*; - use crate::cached_object_store::{CachedObjectStore, FsCacheStorage}; - use crate::cached_object_store_stats::CachedObjectStoreStats; use crate::config::DurabilityLevel::{Memory, Remote}; use crate::config::MetricLevel; use crate::config::{ @@ -2149,9 +2147,7 @@ mod tests { use slatedb_common::clock::MockSystemClock; use slatedb_common::metrics::{ lookup_metric, lookup_metric_with_labels, DefaultMetricsRecorder, MetricValue, - MetricsRecorderHelper, }; - use slatedb_common::DbRand; use std::collections::BTreeMap; use std::collections::Bound::Included; use std::sync::atomic::{AtomicBool, Ordering}; @@ -10927,34 +10923,62 @@ mod tests { use crate::cached_object_store::stats::{PART_ACCESS_COUNT, PART_HIT_COUNT}; use object_store::ObjectStoreExt; + /// Fixture for the object store cache tests. struct ObjectStoreCacheTest { db: Db, - store: Arc, + upstream: Arc, + cache_root: std::path::PathBuf, db_path: String, - part_size: usize, + should_compact: Option>, } - /// Builder for [`ObjectStoreCacheTest`]. Defaults: 1 KiB cache parts, a 1 KiB - /// L0 size, and cache_puts off. + /// Builder for [`ObjectStoreCacheTest`]. Defaults: 1 KiB cache parts, a + /// 1 KiB L0 size, both write sources uncached, and no compactor. struct ObjectStoreCacheTestBuilder { db_path: String, - cache_puts: bool, + object_store_cache: bool, + cache_on_flush: bool, + cache_on_compaction: bool, part_size: usize, l0_sst_size_bytes: usize, + on_demand_compactor: bool, + custom_compactor_store: bool, + metrics_recorder: Option>, } impl ObjectStoreCacheTestBuilder { fn new(db_path: &str) -> Self { Self { db_path: db_path.to_string(), - cache_puts: false, + object_store_cache: true, + cache_on_flush: false, + cache_on_compaction: false, part_size: 1024, l0_sst_size_bytes: 1024, + on_demand_compactor: false, + custom_compactor_store: false, + metrics_recorder: None, } } - fn cache_puts(mut self) -> Self { - self.cache_puts = true; + /// Leaves the object store cache unconfigured (no root folder). + fn without_object_store_cache(mut self) -> Self { + self.object_store_cache = false; + self + } + + fn metrics_recorder(mut self, recorder: Arc) -> Self { + self.metrics_recorder = Some(recorder); + self + } + + fn cache_on_flush(mut self) -> Self { + self.cache_on_flush = true; + self + } + + fn cache_on_compaction(mut self) -> Self { + self.cache_on_compaction = true; self } @@ -10968,51 +10992,89 @@ mod tests { self } + /// Adds an embedded compactor that compacts once each time + /// [`ObjectStoreCacheTest::compact_and_wait`] is called. + fn on_demand_compactor(mut self) -> Self { + self.on_demand_compactor = true; + self + } + + /// Like `on_demand_compactor`, but the compactor holds its own + /// handle to upstream, so the db builder keeps it off the cached store. + fn on_demand_compactor_with_custom_store(mut self) -> Self { + self.on_demand_compactor = true; + self.custom_compactor_store = true; + self + } + async fn build(self) -> ObjectStoreCacheTest { let Self { db_path, - cache_puts, + object_store_cache, + cache_on_flush, + cache_on_compaction, part_size, l0_sst_size_bytes, + on_demand_compactor, + custom_compactor_store, + metrics_recorder, } = self; let upstream: Arc = Arc::new(InMemory::new()); - let recorder = MetricsRecorderHelper::noop(); - let cache_stats = Arc::new(CachedObjectStoreStats::new(&recorder)); let temp_dir = tempfile::Builder::new() .prefix("objstore_cache_test_") .tempdir() .unwrap(); - let cache_storage = Arc::new(FsCacheStorage::new( - temp_dir.keep(), - None, - None, - cache_stats.clone(), - Arc::new(DefaultSystemClock::new()), - Arc::new(DbRand::default()), - 1000, - )); - let store = CachedObjectStore::new( - upstream, - cache_storage, - part_size, - cache_puts, - cache_stats, - ) - .unwrap(); - - let settings = test_db_options(0, l0_sst_size_bytes, None); - let db = Db::builder(db_path.as_str(), store.clone()) - .with_settings(settings) - .build() - .await - .unwrap(); + let cache_root = temp_dir.keep(); + + let mut opts = test_db_options(0, l0_sst_size_bytes, None); + opts.object_store_cache_options.root_folder = + object_store_cache.then(|| cache_root.clone()); + opts.object_store_cache_options.part_size_bytes = part_size; + opts.object_store_cache_options.cache_on_flush = cache_on_flush; + opts.object_store_cache_options.cache_on_compaction = cache_on_compaction; + + let mut builder = + Db::builder(db_path.as_str(), upstream.clone()).with_settings(opts); + if let Some(recorder) = metrics_recorder { + builder = builder.with_metrics_recorder(recorder); + } + let should_compact = if on_demand_compactor { + let flag = Arc::new(AtomicBool::new(false)); + let flag_clone = flag.clone(); + let scheduler = Arc::new(OnDemandCompactionSchedulerSupplier::new(Arc::new( + move |_state| flag_clone.swap(false, Ordering::SeqCst), + ))); + // A different Arc over the same storage; open gates make + // GatedObjectStore a pass-through. + let compactor_store: Arc = if custom_compactor_store { + Arc::new(GatedObjectStore::new(upstream.clone())) + } else { + upstream.clone() + }; + // One subcompaction writes one output SST, keeping exact + // part counts deterministic. + let mut compactor_options = fast_compactor_options(); + if let Some(worker) = compactor_options.worker.as_mut() { + worker.max_subcompactions = 1; + } + builder = builder.with_compactor_builder( + CompactorBuilder::new(db_path.as_str(), compactor_store) + .with_scheduler_supplier(scheduler) + .with_options(compactor_options), + ); + Some(flag) + } else { + None + }; + let db = builder.build().await.unwrap(); ObjectStoreCacheTest { db, - store, + upstream, + cache_root, db_path, - part_size, + should_compact, } } } @@ -11031,30 +11093,37 @@ mod tests { object_store::path::Path::from(format!("{}/{}", self.db_path, suffix)) } - async fn cached_part_count(&self, path: &object_store::path::Path) -> usize { - self.store - .cache_storage - .entry(path, self.part_size) - .cached_parts() - .await - .unwrap() - .len() + /// Number of cached part files for an object. + fn cached_part_count(&self, path: &object_store::path::Path) -> usize { + let dir = self.cache_root.join(path.to_string()); + let Ok(entries) = std::fs::read_dir(dir) else { + return 0; + }; + entries + .filter(|e| { + e.as_ref() + .unwrap() + .file_name() + .to_string_lossy() + .starts_with("_part") + }) + .count() } - async fn assert_cached(&self, path: &object_store::path::Path, expected_parts: usize) { + fn assert_cached(&self, path: &object_store::path::Path, expected_parts: usize) { assert_eq!( - self.cached_part_count(path).await, + self.cached_part_count(path), expected_parts, "expected {path} to be cached as {expected_parts} part(s)" ); } /// Asserts each of `suffixes` (relative to the db root) is uncached. - async fn assert_uncached(&self, suffixes: &[&str]) { + fn assert_uncached(&self, suffixes: &[&str]) { for suffix in suffixes { let path = self.sub_path(suffix); assert_eq!( - self.cached_part_count(&path).await, + self.cached_part_count(&path), 0, "expected {suffix} to be uncached" ); @@ -11064,7 +11133,7 @@ mod tests { /// Lists the compacted SSTs currently in the object store. async fn compacted_locations(&self) -> Vec { let prefix = self.sub_path("compacted"); - self.store + self.upstream .list(Some(&prefix)) .map(|meta| meta.unwrap().location) .collect() @@ -11073,7 +11142,48 @@ mod tests { /// The size of an object as stored upstream, in bytes. async fn object_size(&self, path: &object_store::path::Path) -> u64 { - self.store.head(path).await.unwrap().size + self.upstream.head(path).await.unwrap().size + } + + /// The upstream path of a compacted SST id. + fn compacted_sst_path(&self, id: &SsTableId) -> object_store::path::Path { + self.sub_path(&format!("compacted/{}.sst", id.unwrap_compacted_id())) + } + + fn l0_ids(&self) -> Vec { + self.db.manifest().l0().iter().map(|v| v.sst.id).collect() + } + + /// Triggers one on-demand compaction and waits for a sorted run to + /// land in the manifest. Requires `on_demand_compactor`. + async fn compact_and_wait(&self) { + self.should_compact + .as_ref() + .expect("fixture built without on_demand_compactor") + .store(true, Ordering::SeqCst); + tokio::time::timeout(Duration::from_secs(30), async { + loop { + if !self.db.manifest().compacted().is_empty() { + return; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("compaction did not land within timeout"); + } + + /// The SSTs the compaction wrote: sorted run members that were not + /// among the flushed L0s. + fn compaction_output_ids(&self, l0_ids: &[SsTableId]) -> Vec { + self.db + .manifest() + .compacted() + .iter() + .flat_map(|sr| sr.sst_views.iter()) + .map(|v| v.sst.id) + .filter(|id| !l0_ids.contains(id)) + .collect() } async fn close(self) { @@ -11115,7 +11225,7 @@ mod tests { .await .unwrap(); - // First (cold) get. cache_puts is off, so the SST is not cached on the + // First (cold) get. cache_on_flush is off, so the SST is not cached on the // write. The whole SST is a single cache part, read as three sub-ranges // (index, filter and block). The first is a cold read that fetches and // caches the part (a miss) and the next two are served from the cache @@ -11208,12 +11318,12 @@ mod tests { } /// A flushed L0 SST is a compacted SST written by the main store, so - /// cache_puts admits it. The manifest (untagged) and the WAL (skipped by - /// policy) are never cached. + /// cache_on_flush admits it. The manifest (untagged) and the WAL + /// (skipped by policy) are never cached. #[tokio::test] async fn test_object_store_cache_caches_flushed_sst_only() { let fixture = ObjectStoreCacheTest::builder("/tmp/test_object_store_cache_flush_only") - .cache_puts() + .cache_on_flush() .build() .await; @@ -11229,20 +11339,18 @@ mod tests { .await .unwrap(); - fixture - .assert_uncached(&[ - "manifest/00000000000000000001.manifest", - "manifest/00000000000000000002.manifest", - "wal/00000000000000000001.sst", - "wal/00000000000000000002.sst", - ]) - .await; + fixture.assert_uncached(&[ + "manifest/00000000000000000001.manifest", + "manifest/00000000000000000002.manifest", + "wal/00000000000000000001.sst", + "wal/00000000000000000002.sst", + ]); // The single explicit memtable flush produces one L0 SST, cached as one // part (the key/value is well under the 1 KiB part size). let compacted = fixture.compacted_locations().await; assert_eq!(compacted.len(), 1, "expected exactly one flushed SST"); - fixture.assert_cached(&compacted[0], 1).await; + fixture.assert_cached(&compacted[0], 1); fixture.close().await; } @@ -11254,7 +11362,7 @@ mod tests { const MIB: usize = 1024 * 1024; let fixture = ObjectStoreCacheTest::builder("/tmp/test_object_store_cache_large_flush") - .cache_puts() + .cache_on_flush() .part_size(MIB) // Large enough that the whole write flushes as a single L0 SST. .l0_sst_size_bytes(64 * MIB) @@ -11288,8 +11396,177 @@ mod tests { expected_parts > 10, "expected a large multipart SST, got {expected_parts} part(s)" ); - fixture.assert_cached(&compacted[0], expected_parts).await; + fixture.assert_cached(&compacted[0], expected_parts); fixture.close().await; } + + /// cache_on_compaction admits the embedded compactor's output; with + /// cache_on_flush off, the flushed L0 inputs stay uncached. + #[tokio::test] + async fn test_object_store_cache_caches_compaction_output() { + let t = ObjectStoreCacheTest::builder("/tmp/test_object_store_cache_compaction_output") + .cache_on_compaction() + .on_demand_compactor() + .build() + .await; + + for i in 0..2u32 { + let key = format!("key{:04}", i); + t.db().put(key.as_bytes(), &[b'v'; 64]).await.unwrap(); + t.db() + .flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .unwrap(); + } + let l0_ids = t.l0_ids(); + assert_eq!(l0_ids.len(), 2); + + t.compact_and_wait().await; + + for id in &l0_ids { + t.assert_cached(&t.compacted_sst_path(id), 0); + } + + let output_ids = t.compaction_output_ids(&l0_ids); + assert!(!output_ids.is_empty(), "expected compaction output SSTs"); + for id in &output_ids { + let path = t.compacted_sst_path(id); + assert!( + t.cached_part_count(&path) > 0, + "expected compaction output {path} to be cached" + ); + } + t.close().await; + } + + /// Compaction output above the multipart threshold is cached in full. + #[tokio::test] + async fn test_object_store_cache_caches_large_multipart_compaction_output() { + const MIB: usize = 1024 * 1024; + + let t = ObjectStoreCacheTest::builder( + "/tmp/test_object_store_cache_large_compaction_output", + ) + .cache_on_compaction() + .on_demand_compactor() + .part_size(MIB) + // Large enough that each write batch flushes as a single L0 SST. + .l0_sst_size_bytes(64 * MIB) + .build() + .await; + + // Two ~10 MiB L0s; the ~20 MiB output crosses the multipart threshold. + for sst in 0..2u32 { + for i in 0..10u32 { + let key = format!("k{:04}", sst * 10 + i); + t.db() + .put(key.as_bytes(), &vec![i as u8; MIB]) + .await + .unwrap(); + } + t.db() + .flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .unwrap(); + } + let l0_ids = t.l0_ids(); + assert_eq!(l0_ids.len(), 2); + + t.compact_and_wait().await; + + let output_ids = t.compaction_output_ids(&l0_ids); + assert_eq!(output_ids.len(), 1, "expected one output SST"); + let path = t.compacted_sst_path(&output_ids[0]); + let expected_parts = (t.object_size(&path).await as usize).div_ceil(MIB); + assert_eq!( + expected_parts, 21, + "update this count if an SST encoding change shifts the size" + ); + t.assert_cached(&path, expected_parts); + t.close().await; + } + + /// A compactor builder with its own object store stays cacheless: + /// output is not admitted even with cache_on_compaction on. + #[tokio::test] + async fn test_object_store_cache_skips_compaction_output_from_custom_store() { + let t = ObjectStoreCacheTest::builder( + "/tmp/test_object_store_cache_custom_compactor_store", + ) + .cache_on_compaction() + .on_demand_compactor_with_custom_store() + .build() + .await; + + for i in 0..2u32 { + let key = format!("key{:04}", i); + t.db().put(key.as_bytes(), &[b'v'; 64]).await.unwrap(); + t.db() + .flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .unwrap(); + } + let l0_ids = t.l0_ids(); + assert_eq!(l0_ids.len(), 2); + + t.compact_and_wait().await; + + let output_ids = t.compaction_output_ids(&l0_ids); + assert!(!output_ids.is_empty(), "expected compaction output SSTs"); + for id in &output_ids { + let path = t.compacted_sst_path(id); + assert_eq!( + t.cached_part_count(&path), + 0, + "expected compaction output {path} to stay uncached" + ); + } + t.close().await; + } + + /// An embedded compactor on the DB's own store records its object + /// store I/O under the compactor component, with and without the + /// object store cache. + #[tokio::test] + async fn test_embedded_compactor_io_recorded_under_compactor_component() { + for object_store_cache in [true, false] { + let recorder = Arc::new(DefaultMetricsRecorder::new()); + let mut builder = + ObjectStoreCacheTest::builder("/tmp/test_compactor_component_metrics") + .cache_on_compaction() + .on_demand_compactor() + .metrics_recorder(recorder.clone()); + if !object_store_cache { + builder = builder.without_object_store_cache(); + } + let t = builder.build().await; + + for i in 0..2u32 { + let key = format!("key{:04}", i); + t.db().put(key.as_bytes(), &[b'v'; 64]).await.unwrap(); + t.db() + .flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .unwrap(); + } + t.compact_and_wait().await; + + let gets = + lookup_object_store_op_request_count(&recorder, "compactor", "main", "get"); + let puts = + lookup_object_store_op_request_count(&recorder, "compactor", "main", "put"); + assert!(gets > 0, "no compactor gets [cache={object_store_cache}]"); + assert!(puts > 0, "no compactor puts [cache={object_store_cache}]"); + t.close().await; + } + } } } diff --git a/slatedb/src/db/builder.rs b/slatedb/src/db/builder.rs index 483796c22..03fe0b80e 100644 --- a/slatedb/src/db/builder.rs +++ b/slatedb/src/db/builder.rs @@ -637,16 +637,43 @@ impl> DbBuilder

{ write_rx, &tokio_handle, )?; - // The compactor and GC each get their own cacheless store (so background - // reads do not pollute the foreground cache), tagged with their kind. + + // Wraps a background component's (compactor, GC) raw main store in + // the component's own retry and instrumentation layer, so its I/O is + // recorded under its own metric labels. Returns (main, uncached). + // + // main: when the component runs against the DB's own store (the auto + // from settings path, or a caller-supplied builder holding a clone of + // the DB's store) and object store caching is configured, the DB's + // cache is shared on top of that layer, so cache fills and evictions + // stay coherent with the DB's. A different caller-supplied store is + // used as given (so a custom compaction reader takes effect instead + // of being silently ignored) and stays cacheless. // - // The compactor reads/writes through whichever object store its builder - // holds: the DB's own store on the auto-from-settings path, or the one - // the caller supplied on their own `CompactorBuilder` (so a custom - // compaction read path — e.g. one that bypasses a prefetch wrapper used - // by the foreground read path — actually takes effect instead of being - // silently ignored). Either way it is wrapped in its own Compactor-tagged - // retry/instrumentation layer below. + // uncached: the same wrapped store without the cache, for I/O that + // must bypass it. + let background_component_stores = + |raw_store: Arc, component: ObjectStoreComponent| { + let retrying = instrumented_retrying_object_store( + raw_store.clone(), + &recorder, + component, + ObjectStoreType::Main, + rand.clone(), + system_clock.clone(), + ); + let main: Arc = match &cached_object_store { + Some(cached) if Arc::ptr_eq(&raw_store, &self.main_object_store) => { + cached.clone_with_new_object_store(retrying.clone()) + } + _ => retrying.clone(), + }; + (main, retrying) + }; + + // The compactor reads/writes through the object store held by its + // builder: the DB's own store on the auto from settings path, or the + // store the caller passed to their own `CompactorBuilder`. let compactor_builder = self.compactor_builder.or_else(|| { self.settings.compactor_options.as_ref().map(|opts| { CompactorBuilder::new(path.clone(), self.main_object_store.clone()) @@ -669,15 +696,9 @@ impl> DbBuilder

{ } builder = builder.with_fp_registry(self.fp_registry.clone()); - // Wrap whatever object store the builder holds in the compactor's - // own cacheless, Compactor-tagged retry/instrumentation layer. - let compactor_main_object_store = instrumented_retrying_object_store( + let (compactor_main_object_store, _) = background_component_stores( builder.main_object_store.clone(), - &recorder, ObjectStoreComponent::Compactor, - ObjectStoreType::Main, - rand.clone(), - system_clock.clone(), ); let compactor_table_store = Arc::new(TableStore::new_with_fp_registry( ObjectStores::new( @@ -713,12 +734,14 @@ impl> DbBuilder

{ } } + // Same store selection as the compactor above. Sharing the DB's cache + // also means an SST deleted by the GC has its cache entries evicted. let gc_builder = self.gc_builder.or_else(|| { self.settings .garbage_collector_options .filter(|opts| !opts.is_empty()) .map(|opts| { - GarbageCollectorBuilder::new(path.clone(), retrying_main_object_store.clone()) + GarbageCollectorBuilder::new(path.clone(), self.main_object_store.clone()) .with_options(opts) }) }); @@ -728,11 +751,12 @@ impl> DbBuilder

{ .options .metric_level .or(Some(self.settings.metric_level)); + let (gc_main_object_store, gc_object_store) = background_component_stores( + gc_builder.main_object_store.clone(), + ObjectStoreComponent::Gc, + ); let gc_table_store = Arc::new(TableStore::new_with_fp_registry( - ObjectStores::new( - retrying_main_object_store.clone(), - retrying_wal_object_store.clone(), - ), + ObjectStores::new(gc_main_object_store, retrying_wal_object_store.clone()), sst_format.clone(), path_resolver.clone(), self.fp_registry.clone(), @@ -747,7 +771,7 @@ impl> DbBuilder

{ gc_table_store, manifest_store.clone(), compactions_store.clone(), - retrying_main_object_store.clone(), + gc_object_store, ); // Garbage collector only uses tickers, so pass in a dummy rx channel let (_, rx) = async_channel::unbounded(); diff --git a/slatedb/src/garbage_collector/compacted_gc.rs b/slatedb/src/garbage_collector/compacted_gc.rs index 16172fff6..6cd3597ba 100644 --- a/slatedb/src/garbage_collector/compacted_gc.rs +++ b/slatedb/src/garbage_collector/compacted_gc.rs @@ -111,6 +111,32 @@ impl CompactedGcTask { None => DateTime::::UNIX_EPOCH, } } + + /// Deletes the given compacted SSTs from the table store. + /// + /// In case of dryrun, the actual deletion doesn't happen. + async fn maybe_delete_compacted_ssts(&self, sst_ids: Vec) { + if self.compacted_options.dry_run { + if !sst_ids.is_empty() { + log::info!("dry run: skipping SST deletion [count={}]", sst_ids.len()); + } + for id in sst_ids { + log::debug!("dry run: would delete SST but skipped [id={:?}]", id); + } + return; + } + + futures::stream::iter(sst_ids) + .for_each_concurrent(GC_DELETE_CONCURRENCY, |id| async move { + log::info!("deleting SST [id={:?}]", id); + if let Err(e) = self.table_store.delete_sst(&id).await { + error!("error deleting SST [id={:?}, error={}]", id, e); + } else { + self.stats.gc_compacted_count.increment(1); + } + }) + .await; + } } /// Collect every SST id referenced by `manifests`, across the unsegmented @@ -230,28 +256,7 @@ impl GcTask for CompactedGcTask { .map(|sst| sst.id) .collect::>(); - if self.compacted_options.dry_run { - if !sst_ids_to_delete.is_empty() { - log::info!( - "dry run: skipping SST deletion [count={}]", - sst_ids_to_delete.len() - ); - } - for id in sst_ids_to_delete { - log::debug!("dry run: would delete SST but skipped [id={:?}]", id); - } - return Ok(()); - } - futures::stream::iter(sst_ids_to_delete) - .for_each_concurrent(GC_DELETE_CONCURRENCY, |id| async move { - log::info!("deleting SST [id={:?}]", id); - if let Err(e) = self.table_store.delete_sst(&id).await { - error!("error deleting SST [id={:?}, error={}]", id, e); - } else { - self.stats.gc_compacted_count.increment(1); - } - }) - .await; + self.maybe_delete_compacted_ssts(sst_ids_to_delete).await; Ok(()) } @@ -264,6 +269,9 @@ impl GcTask for CompactedGcTask { #[cfg(test)] mod tests { use super::*; + use crate::cached_object_store::policy::CachePutConfig; + use crate::cached_object_store::stats::CachedObjectStoreStats; + use crate::cached_object_store::{CachedObjectStore, FsCacheStorage}; use crate::compactions_store::{CompactionsStore, StoredCompactions}; use crate::compactor_state::{Compaction, CompactionSpec, SourceId}; use crate::db_state::{SortedRun, SsTableHandle, SsTableId, SsTableInfo, SsTableView}; @@ -276,6 +284,7 @@ mod tests { use bytes::Bytes; use object_store::{memory::InMemory, path::Path}; use slatedb_common::clock::DefaultSystemClock; + use slatedb_common::DbRand; use std::collections::{BTreeMap, VecDeque}; use std::time::Duration; @@ -832,4 +841,102 @@ mod tests { let manifest = manifest_with(LsmTreeState::default(), vec![]); assert_eq!(newest_l0_dt(&manifest), DateTime::::UNIX_EPOCH); } + + #[tokio::test] + async fn test_compacted_gc_evicts_deleted_sst_from_object_store_cache() { + let recorder = slatedb_common::metrics::MetricsRecorderHelper::noop(); + let main_store = Arc::new(InMemory::new()); + let cache_stats = Arc::new(CachedObjectStoreStats::new(&recorder)); + let temp_dir = tempfile::Builder::new() + .prefix("gc_cache_evict_test_") + .tempdir() + .unwrap(); + let part_size = 1024; + let cache_storage = Arc::new(FsCacheStorage::new( + temp_dir.keep(), + None, + None, + cache_stats.clone(), + Arc::new(DefaultSystemClock::new()), + Arc::new(DbRand::default()), + 1000, + )); + let cached_store = CachedObjectStore::new( + main_store.clone(), + cache_storage, + part_size, + CachePutConfig { + cache_on_flush: true, + cache_on_compaction: false, + }, + cache_stats, + ) + .unwrap(); + + let format = SsTableFormat::default(); + // The GC store deletes through the cache; the Main store caches on write. + let gc_table_store = Arc::new(TableStore::new( + ObjectStores::new(cached_store.clone(), None), + format.clone(), + Path::from("/root"), + None, + TableStoreKind::GC, + )); + let main_table_store = Arc::new(TableStore::new( + ObjectStores::new(cached_store.clone(), None), + format.clone(), + Path::from("/root"), + None, + TableStoreKind::Main, + )); + + // Written through the Main store so cache_on_flush admits it. + let id_to_delete = SsTableId::Compacted(ulid::Ulid::from_parts(1_000, 0)); + let sst = build_test_sst(&format, 1).await; + main_table_store + .write_sst(&id_to_delete, &sst, false) + .await + .unwrap(); + + let location = gc_table_store + .list_compacted_ssts(..) + .await + .unwrap() + .into_iter() + .find(|m| m.id == id_to_delete) + .expect("sst to delete should be listed") + .metadata + .location; + let entry = cached_store.cache_storage.entry(&location, part_size); + assert!( + !entry.cached_parts().await.unwrap().is_empty(), + "sst should be cached before delete" + ); + + // Call GC deletion directly, no need to test the decision here. + let manifest_store = Arc::new(ManifestStore::new(&Path::from("/root"), main_store.clone())); + let compactions_store = Arc::new(CompactionsStore::new( + &Path::from("/root"), + main_store.clone(), + )); + let task = CompactedGcTask::new( + manifest_store, + compactions_store, + gc_table_store.clone(), + Arc::new(GcStats::new(&recorder)), + GarbageCollectorDirectoryOptions { + interval: None, + min_age: Duration::from_secs(5), + dry_run: false, + }, + None, + ); + task.maybe_delete_compacted_ssts(vec![id_to_delete]).await; + + let entry = cached_store.cache_storage.entry(&location, part_size); + assert!( + entry.cached_parts().await.unwrap().is_empty(), + "sst should be evicted after delete" + ); + } } diff --git a/website/src/content/docs/docs/design/caching.mdx b/website/src/content/docs/docs/design/caching.mdx index 74000cc5a..e375be750 100644 --- a/website/src/content/docs/docs/design/caching.mdx +++ b/website/src/content/docs/docs/design/caching.mdx @@ -33,7 +33,7 @@ The defaults reflect the usual access patterns. Point reads default to `cache_bl The disk cache stays disabled unless `object_store_cache_options.root_folder` is set. If you want it warm before serving traffic, you can preload it on startup with [`PreloadLevel::L0Sst`](https://docs.rs/slatedb/latest/slatedb/config/enum.PreloadLevel.html#variant.L0Sst) or [`PreloadLevel::AllSst`](https://docs.rs/slatedb/latest/slatedb/config/enum.PreloadLevel.html#variant.AllSst). SlateDB loads recent SSTs, or all SSTs, into the local cache until the cache size limit is reached. -By default, writes go straight to the upstream object store and do not populate the object-store cache. Setting [`cache_puts`](https://docs.rs/slatedb/latest/slatedb/config/struct.ObjectStoreCacheOptions.html#structfield.cache_puts) to `true` also stores `PUT` payloads locally, which can help if readers are likely to touch freshly written SSTs soon afterward. +By default, writes go straight to the upstream object store and do not populate the object-store cache. Each SST write source has its own admission flag: [`cache_on_flush`](https://docs.rs/slatedb/latest/slatedb/config/struct.ObjectStoreCacheOptions.html#structfield.cache_on_flush) stores SSTs written by memtable flushes locally, and [`cache_on_compaction`](https://docs.rs/slatedb/latest/slatedb/config/struct.ObjectStoreCacheOptions.html#structfield.cache_on_compaction) does the same for compaction output. Enabling them can help if readers are likely to touch freshly written SSTs soon afterward. WAL and manifest writes are never cached. ## Sharing Between Instances From d304e71d3031e57d518a161c6232c81b8daf8876 Mon Sep 17 00:00:00 2001 From: nomiero Date: Sun, 12 Jul 2026 16:29:35 -0700 Subject: [PATCH 09/63] extract gc deletion into maybe_delete methods (#1919) --- .../src/garbage_collector/compactions_gc.rs | 71 +++++++++------- slatedb/src/garbage_collector/manifest_gc.rs | 65 ++++++++------- slatedb/src/garbage_collector/wal_gc.rs | 80 ++++++++++--------- 3 files changed, 119 insertions(+), 97 deletions(-) diff --git a/slatedb/src/garbage_collector/compactions_gc.rs b/slatedb/src/garbage_collector/compactions_gc.rs index 25b3273bd..6cc0ab3e8 100644 --- a/slatedb/src/garbage_collector/compactions_gc.rs +++ b/slatedb/src/garbage_collector/compactions_gc.rs @@ -64,6 +64,41 @@ impl CompactionsGcTask { fn compactions_min_age(&self) -> chrono::Duration { chrono::Duration::from_std(self.compactions_options.min_age).expect("invalid duration") } + + /// Deletes the given compactions files from the compactions store. + /// + /// In case of dryrun, the actual deletion doesn't happen. + async fn maybe_delete_compactions(&self, compactions_ids: Vec) { + if self.compactions_options.dry_run { + if !compactions_ids.is_empty() { + log::info!( + "dry run: skipping compactions deletion [count={}]", + compactions_ids.len() + ); + } + for id in compactions_ids { + log::debug!( + "dry run: would delete compactions but skipped [id={:?}]", + id + ); + } + return; + } + + futures::stream::iter(compactions_ids) + .for_each_concurrent(GC_DELETE_CONCURRENCY, |id| async move { + if let Err(e) = self + .compactions_store + .delete_compactions_unchecked(id) + .await + { + error!("error deleting compactions [id={:?}, error={}]", id, e); + } else { + self.stats.gc_compactions_count.increment(1); + } + }) + .await; + } } impl GcTask for CompactionsGcTask { @@ -96,36 +131,12 @@ impl GcTask for CompactionsGcTask { } let compactions_to_delete = retain_allowed_by_gc_filter(&self.gc_filter, compactions_to_delete).await; - if self.compactions_options.dry_run { - if !compactions_to_delete.is_empty() { - log::info!( - "dry run: skipping compactions deletion [count={}]", - compactions_to_delete.len() - ); - } - for compactions_metadata in compactions_to_delete { - log::debug!( - "dry run: would delete compactions but skipped [id={:?}]", - compactions_metadata.id - ); - } - return Ok(()); - } - futures::stream::iter(compactions_to_delete) - .for_each_concurrent(GC_DELETE_CONCURRENCY, |compactions_metadata| async move { - if let Err(e) = self - .compactions_store - .delete_compactions_unchecked(compactions_metadata.id) - .await - { - error!( - "error deleting compactions [id={:?}, error={}]", - compactions_metadata.id, e - ); - } else { - self.stats.gc_compactions_count.increment(1); - } - }) + let compactions_ids_to_delete = compactions_to_delete + .into_iter() + .map(|compactions_metadata| compactions_metadata.id) + .collect::>(); + + self.maybe_delete_compactions(compactions_ids_to_delete) .await; Ok(()) diff --git a/slatedb/src/garbage_collector/manifest_gc.rs b/slatedb/src/garbage_collector/manifest_gc.rs index 18fcb754b..f3d3af8d6 100644 --- a/slatedb/src/garbage_collector/manifest_gc.rs +++ b/slatedb/src/garbage_collector/manifest_gc.rs @@ -44,6 +44,34 @@ impl ManifestGcTask { fn manifest_min_age(&self) -> chrono::Duration { chrono::Duration::from_std(self.manifest_options.min_age).expect("invalid duration") } + + /// Deletes the given manifests from the manifest store. + /// + /// In case of dryrun, the actual deletion doesn't happen. + async fn maybe_delete_manifests(&self, manifest_ids: Vec) { + if self.manifest_options.dry_run { + if !manifest_ids.is_empty() { + log::info!( + "dry run: skipping manifest deletion [count={}]", + manifest_ids.len() + ); + } + for id in manifest_ids { + log::debug!("dry run: would delete manifest but skipped [id={:?}]", id); + } + return; + } + + futures::stream::iter(manifest_ids) + .for_each_concurrent(GC_DELETE_CONCURRENCY, |id| async move { + if let Err(e) = self.manifest_store.delete_manifest_unchecked(id).await { + error!("error deleting manifest [id={:?}, error={}]", id, e); + } else { + self.stats.gc_manifest_count.increment(1); + } + }) + .await; + } } impl GcTask for ManifestGcTask { @@ -92,37 +120,12 @@ impl GcTask for ManifestGcTask { } let manifests_to_delete = retain_allowed_by_gc_filter(&self.gc_filter, manifests_to_delete).await; - if self.manifest_options.dry_run { - if !manifests_to_delete.is_empty() { - log::info!( - "dry run: skipping manifest deletion [count={}]", - manifests_to_delete.len() - ); - } - for manifest_metadata in manifests_to_delete { - log::debug!( - "dry run: would delete manifest but skipped [id={:?}]", - manifest_metadata.id - ); - } - return Ok(()); - } - futures::stream::iter(manifests_to_delete) - .for_each_concurrent(GC_DELETE_CONCURRENCY, |manifest_metadata| async move { - if let Err(e) = self - .manifest_store - .delete_manifest_unchecked(manifest_metadata.id) - .await - { - error!( - "error deleting manifest [id={:?}, error={}]", - manifest_metadata.id, e - ); - } else { - self.stats.gc_manifest_count.increment(1); - } - }) - .await; + let manifest_ids_to_delete = manifests_to_delete + .into_iter() + .map(|manifest_metadata| manifest_metadata.id) + .collect::>(); + + self.maybe_delete_manifests(manifest_ids_to_delete).await; Ok(()) } diff --git a/slatedb/src/garbage_collector/wal_gc.rs b/slatedb/src/garbage_collector/wal_gc.rs index 9e9df11fe..f4b06c2f5 100644 --- a/slatedb/src/garbage_collector/wal_gc.rs +++ b/slatedb/src/garbage_collector/wal_gc.rs @@ -88,6 +88,49 @@ impl WalGcTask { fn wal_sst_min_age(&self) -> chrono::Duration { chrono::Duration::from_std(self.wal_options.min_age).expect("invalid duration") } + + /// Deletes the given WAL SSTs from the table store. + /// + /// In case of dryrun, the actual deletion doesn't happen. + async fn maybe_delete_wal_ssts(&self, sst_ids: Vec) { + if self.wal_options.dry_run { + if !sst_ids.is_empty() { + log::info!( + "dry run: skipping {} deletion [count={}]", + self.resource(), + sst_ids.len() + ); + if matches!(self.mode, WalGcMode::Fence) { + log::info!( + "WAL fence GC is dry-run by default. This is a conservative setting. \ + Set wal_fence_options.dry_run=false and use a conservative min_age to enable. \ + Silence this log with wal_fence_options=None. See #352 for details." + ); + } + } + for id in sst_ids { + log::debug!( + "dry run: would delete {} but skipped [id={:?}]", + self.resource(), + id + ); + } + return; + } + + futures::stream::iter(sst_ids) + .for_each_concurrent(GC_DELETE_CONCURRENCY, |id| async move { + if let Err(e) = self.table_store.delete_sst(&id).await { + error!("error deleting WAL SST [id={:?}, error={}]", id, e); + } else { + match self.mode { + WalGcMode::Regular => self.stats.gc_wal_count.increment(1), + WalGcMode::Fence => self.stats.gc_wal_fence_count.increment(1), + } + } + }) + .await; + } } impl GcTask for WalGcTask { @@ -131,42 +174,7 @@ impl GcTask for WalGcTask { .map(|wal_sst| wal_sst.id) .collect::>(); - if self.wal_options.dry_run { - if !sst_ids_to_delete.is_empty() { - log::info!( - "dry run: skipping {} deletion [count={}]", - self.resource(), - sst_ids_to_delete.len() - ); - if matches!(self.mode, WalGcMode::Fence) { - log::info!( - "WAL fence GC is dry-run by default. This is a conservative setting. \ - Set wal_fence_options.dry_run=false and use a conservative min_age to enable. \ - Silence this log with wal_fence_options=None. See #352 for details." - ); - } - } - for id in sst_ids_to_delete { - log::debug!( - "dry run: would delete {} but skipped [id={:?}]", - self.resource(), - id - ); - } - return Ok(()); - } - futures::stream::iter(sst_ids_to_delete) - .for_each_concurrent(GC_DELETE_CONCURRENCY, |id| async move { - if let Err(e) = self.table_store.delete_sst(&id).await { - error!("error deleting WAL SST [id={:?}, error={}]", id, e); - } else { - match self.mode { - WalGcMode::Regular => self.stats.gc_wal_count.increment(1), - WalGcMode::Fence => self.stats.gc_wal_fence_count.increment(1), - } - } - }) - .await; + self.maybe_delete_wal_ssts(sst_ids_to_delete).await; Ok(()) } From 62047c3a82cb6eee11c210fb5a828c5f091be89c Mon Sep 17 00:00:00 2001 From: Chris Date: Sun, 12 Jul 2026 22:29:04 -0700 Subject: [PATCH 10/63] Add a DST for DB rescaling (#1920) --- .github/workflows/dst-hourly.yaml | 2 + slatedb-dst/src/actors/mod.rs | 1 + slatedb-dst/src/actors/workload.rs | 8 +- slatedb-dst/src/harness.rs | 17 +- slatedb-dst/src/lib.rs | 2 + slatedb-dst/src/rescaling.rs | 433 +++++++++++++++++++++++++++++ slatedb-dst/tests/rescaling.rs | 52 ++++ 7 files changed, 513 insertions(+), 2 deletions(-) create mode 100644 slatedb-dst/src/rescaling.rs create mode 100644 slatedb-dst/tests/rescaling.rs diff --git a/.github/workflows/dst-hourly.yaml b/.github/workflows/dst-hourly.yaml index 86d73ed77..944e7af8b 100644 --- a/.github/workflows/dst-hourly.yaml +++ b/.github/workflows/dst-hourly.yaml @@ -20,6 +20,8 @@ jobs: test-filter: test_dst_bank_with_toxics - name: determinism test-filter: test_dst_is_deterministic + - name: rescaling + test-filter: test_dst_rescaling_preserves_data - name: segments test-filter: test_dst_segments_is_deterministic # Hard backstop for the whole job. The "Run DST Tests" step below sets a diff --git a/slatedb-dst/src/actors/mod.rs b/slatedb-dst/src/actors/mod.rs index 92eaa99e3..25023ab76 100644 --- a/slatedb-dst/src/actors/mod.rs +++ b/slatedb-dst/src/actors/mod.rs @@ -29,6 +29,7 @@ pub use self::fencer::{DbFencerActor, DbFencerActorOptions, SuppressFenced}; pub use self::flusher::FlusherActor; pub use self::shutdown::ShutdownActor; pub use self::suppress_errors::SuppressErrorActor; +pub(crate) use self::workload::decode_workload_value; pub use self::workload::{WorkloadActor, WorkloadActorOptions, WorkloadMergeOperator}; /// Emit one progress log line every N completed steps for the looping actors. diff --git a/slatedb-dst/src/actors/workload.rs b/slatedb-dst/src/actors/workload.rs index 547e845d2..a4221d59d 100644 --- a/slatedb-dst/src/actors/workload.rs +++ b/slatedb-dst/src/actors/workload.rs @@ -113,6 +113,12 @@ impl WorkloadActor { key_prefix: None, }) } + + /// Overrides the version assigned to the next generated workload value. + pub fn with_next_value_version(mut self, next_value_version: u64) -> Self { + self.next_value_version = AtomicU64::new(next_value_version); + self + } } #[derive(Clone, Copy, Debug)] @@ -434,7 +440,7 @@ fn observe_absent(key: &Bytes, observed: &mut BTreeMap) { } } -fn decode_workload_value(value: &[u8]) -> u64 { +pub(crate) fn decode_workload_value(value: &[u8]) -> u64 { let version_bytes: [u8; WORKLOAD_VALUE_VERSION_SIZE] = value[..WORKLOAD_VALUE_VERSION_SIZE] .try_into() .expect("workload value version slice has fixed size"); diff --git a/slatedb-dst/src/harness.rs b/slatedb-dst/src/harness.rs index cbcae3f23..5416d1af7 100644 --- a/slatedb-dst/src/harness.rs +++ b/slatedb-dst/src/harness.rs @@ -598,7 +598,22 @@ impl Harness { .rng_seed(RngSeed::from_bytes(&runtime_seed.to_le_bytes())) .build_local(Default::default()) .expect("failed to build dst harness runtime"); - runtime.block_on(async move { self.run_inner().await }) + runtime.block_on(self.run_async()) + } + + /// Runs the harness to completion on the current Tokio runtime. + /// + /// This is useful when multiple harnesses must share one deterministic + /// scheduler. The caller is responsible for providing and seeding the + /// runtime. + /// + /// ## Returns + /// - `Ok(())`: All actors completed successfully, or an actor requested + /// shutdown and all remaining actor exits were neutral. + /// - `Err(Error)`: Database startup failed, no actors were configured, or + /// an actor returned or joined with an error. + pub async fn run_async(self) -> Result<(), Error> { + self.run_inner().await } async fn run_inner(self) -> Result<(), Error> { diff --git a/slatedb-dst/src/lib.rs b/slatedb-dst/src/lib.rs index a68e4ef6e..cc4e47135 100644 --- a/slatedb-dst/src/lib.rs +++ b/slatedb-dst/src/lib.rs @@ -13,6 +13,7 @@ mod deterministic_local_filesystem; pub mod failing_object_store; mod harness; mod prefix_extractor; +mod rescaling; mod scenarios; pub mod utils; @@ -23,4 +24,5 @@ pub use self::failing_object_store::{ }; pub use self::harness::*; pub use self::prefix_extractor::FirstDelimiterPrefixExtractor; +pub use self::rescaling::RescalingScenario; pub use self::scenarios::DeterministicScenario; diff --git a/slatedb-dst/src/rescaling.rs b/slatedb-dst/src/rescaling.rs new file mode 100644 index 000000000..8bc874b95 --- /dev/null +++ b/slatedb-dst/src/rescaling.rs @@ -0,0 +1,433 @@ +//! Data-preservation checks for RFC-0004 split and merge scenarios. +//! +//! [`RescalingScenario`] keeps [`Harness`] focused on one physical database at +//! a time. Rescaling happens between harness runs: the scenario stops the +//! source database, uses SlateDB's administrative clone operations to project +//! or union manifests, and starts new harnesses for the resulting databases. +//! +//! Each run performs these phases: +//! - run four prefix-scoped workload actors against a root database +//! - scan the quiesced root and project it at `workload-3/` into adjacent left +//! and right databases +//! - verify that both projections exactly match their ranges in the root +//! - run the left and right databases concurrently, assigning actors 1-2 to +//! the left database and actors 3-4 to the right +//! - scan the quiesced children, union them into a merged database, and verify +//! that the merged rows exactly equal both child snapshots +//! - run all four workload actors against the merged database to verify that it +//! remains usable after the union +//! +//! Every workload operation remains inside one actor prefix, so point +//! operations, write batches, and scans stay within one child. The child +//! harnesses run concurrently on one seeded current-thread runtime and share +//! one underlying [`DeterministicLocalFilesystem`]. This exercises interleaved +//! access to the same object store while keeping operation ordering +//! reproducible from the scenario seed. +//! +//! Garbage collection remains enabled, but detach GC is disabled because both +//! projected children reference the root checkpoint. WALs are disabled because +//! manifest union does not combine live WAL state. Projection and union run +//! only after their source harnesses have stopped. +//! +//! The exact snapshot comparisons at the split and merge barriers are the +//! primary assertions: every root row must appear in exactly one child, and +//! every child row must appear in the merged database. + +use std::ops::Bound; +use std::sync::Arc; +use std::time::Duration; + +use bytes::Bytes; +use log::{error, info}; +use object_store::path::Path; +use object_store::ObjectStore; +use rand::RngCore; +use slatedb::admin::{AdminBuilder, CloneSourceSpec}; +use slatedb::config::DurabilityLevel; +use slatedb::{Db, DbRand, DbReader}; +use slatedb_common::clock::{MockSystemClock, SystemClock}; +use tempfile::TempDir; +use tokio::runtime::RngSeed; +use tracing::instrument; + +use crate::actors::{ + decode_workload_value, FlusherActor, ShutdownActor, WorkloadActor, WorkloadActorOptions, + WorkloadMergeOperator, +}; +use crate::utils::{build_settings, build_toxic, dst_seeds}; +use crate::{DeterministicLocalFilesystem, Harness}; + +type ScenarioError = Box; +type ScenarioResult = Result; +type Rows = Vec<(Bytes, Bytes)>; + +const ROOT_ACTORS: &[&str] = &["workload-1", "workload-2", "workload-3", "workload-4"]; +const LEFT_ACTORS: &[&str] = &["workload-1", "workload-2"]; +const RIGHT_ACTORS: &[&str] = &["workload-3", "workload-4"]; +const SPLIT_KEY: &[u8] = b"workload-3/"; + +/// Configuration for the RFC-0004 split/merge data-preservation scenario. +/// +/// A run exercises one root database, projects it into two children, runs the +/// children concurrently, unions their quiesced states, and then runs the +/// merged database. Exact snapshots verify that projection and union preserve +/// every row. +pub struct RescalingScenario { + /// Logical name used for harness labels and database paths. + pub name: &'static str, + /// Per-harness mock-clock duration, in milliseconds. + pub shutdown_at_ms: i64, +} + +impl RescalingScenario { + /// Runs the rescaling scenario across the configured DST seed budget. + /// + /// Each seed worker runs all phases on one seeded current-thread runtime. + /// The disjoint child databases run as concurrent tasks on that shared + /// deterministic scheduler, so the default seed count matches the number + /// of available cores. + pub fn run(self) -> ScenarioResult<()> { + let Self { + name, + shutdown_at_ms, + } = self; + let num_cores = std::thread::available_parallelism() + .map(|p| p.get()) + .unwrap_or(1); + let seeds = dst_seeds(num_cores)?; + + let handles = seeds + .into_iter() + .enumerate() + .map(|(core, seed)| { + info!("dst {name} seed [core={core}, seed={seed}]"); + ( + core, + seed, + std::thread::spawn(move || run_seed(name, seed, shutdown_at_ms)), + ) + }) + .collect::>(); + + for (core, seed, handle) in handles { + match handle.join() { + Ok(result) => result?, + Err(payload) => { + error!("dst {name} panicked [core={core}, seed={seed}]"); + std::panic::resume_unwind(payload); + } + } + } + + Ok(()) + } +} + +#[instrument(level = "debug", skip_all, fields(scenario = name, seed = seed))] +fn run_seed(name: &'static str, seed: u64, shutdown_at_ms: i64) -> ScenarioResult<()> { + let runtime = tokio::runtime::Builder::new_current_thread() + .rng_seed(RngSeed::from_bytes(&seed.to_le_bytes())) + .build_local(Default::default()) + .expect("failed to build rescaling scenario runtime"); + runtime.block_on(run_seed_async(name, seed, shutdown_at_ms)) +} + +async fn run_seed_async(name: &'static str, seed: u64, shutdown_at_ms: i64) -> ScenarioResult<()> { + let tempdir = TempDir::new()?; + let object_store: Arc = Arc::new( + DeterministicLocalFilesystem::new_with_prefix(tempdir.path())?, + ); + let seed_rng = DbRand::new(seed); + let next_seed = || seed_rng.rng().next_u64(); + let root_path = Path::from(format!("{name}/root")); + let left_path = Path::from(format!("{name}/split/left")); + let right_path = Path::from(format!("{name}/split/right")); + let merged_path = Path::from(format!("{name}/merged")); + + let root_end_ms = run_harness_phase( + format!("{name}-root"), + root_path.clone(), + object_store.clone(), + next_seed(), + 0, + shutdown_at_ms, + 1, + ROOT_ACTORS, + ) + .await?; + let root_rows = snapshot_rows( + root_path.clone(), + object_store.clone(), + next_seed(), + root_end_ms, + ) + .await?; + let child_next_value_version = next_workload_value_version(&root_rows); + + let split_key = Bytes::from_static(SPLIT_KEY); + let left_range = (Bound::Unbounded, Bound::Excluded(split_key.clone())); + let right_range = (Bound::Included(split_key), Bound::Unbounded); + create_clone( + left_path.clone(), + vec![CloneSourceSpec::new(root_path.clone()).with_projection_range(left_range.clone())], + object_store.clone(), + next_seed(), + root_end_ms, + ) + .await?; + create_clone( + right_path.clone(), + vec![CloneSourceSpec::new(root_path).with_projection_range(right_range.clone())], + object_store.clone(), + next_seed(), + root_end_ms, + ) + .await?; + + let left_after_split = snapshot_rows( + left_path.clone(), + object_store.clone(), + next_seed(), + root_end_ms, + ) + .await?; + let right_after_split = snapshot_rows( + right_path.clone(), + object_store.clone(), + next_seed(), + root_end_ms, + ) + .await?; + let (expected_left, expected_right): (Rows, Rows) = root_rows + .into_iter() + .partition(|(key, _)| key.as_ref() < SPLIT_KEY); + assert_eq!( + left_after_split, expected_left, + "projection mismatch [scenario={name}, seed={seed}, phase=split, partition=left]" + ); + assert_eq!( + right_after_split, expected_right, + "projection mismatch [scenario={name}, seed={seed}, phase=split, partition=right]" + ); + + let (left_end_ms, right_end_ms) = tokio::try_join!( + run_harness_phase( + format!("{name}-left"), + left_path.clone(), + object_store.clone(), + next_seed(), + root_end_ms, + shutdown_at_ms, + child_next_value_version, + LEFT_ACTORS, + ), + run_harness_phase( + format!("{name}-right"), + right_path.clone(), + object_store.clone(), + next_seed(), + root_end_ms, + shutdown_at_ms, + child_next_value_version, + RIGHT_ACTORS, + ), + )?; + let children_end_ms = left_end_ms.max(right_end_ms); + + let left_rows = snapshot_rows( + left_path.clone(), + object_store.clone(), + next_seed(), + children_end_ms, + ) + .await?; + let right_rows = snapshot_rows( + right_path.clone(), + object_store.clone(), + next_seed(), + children_end_ms, + ) + .await?; + assert!( + left_rows.iter().all(|(key, _)| key.as_ref() < SPLIT_KEY), + "child contains key outside its range [scenario={name}, seed={seed}, phase=children, partition=left]" + ); + assert!( + right_rows.iter().all(|(key, _)| key.as_ref() >= SPLIT_KEY), + "child contains key outside its range [scenario={name}, seed={seed}, phase=children, partition=right]" + ); + + create_clone( + merged_path.clone(), + vec![ + CloneSourceSpec::new(left_path).with_projection_range(left_range), + CloneSourceSpec::new(right_path).with_projection_range(right_range), + ], + object_store.clone(), + next_seed(), + children_end_ms, + ) + .await?; + + let merged_rows = snapshot_rows( + merged_path.clone(), + object_store.clone(), + next_seed(), + children_end_ms, + ) + .await?; + let expected_merged = left_rows.into_iter().chain(right_rows).collect::(); + let merged_next_value_version = next_workload_value_version(&expected_merged); + assert_eq!( + merged_rows, expected_merged, + "union mismatch [scenario={name}, seed={seed}, phase=merge]" + ); + + run_harness_phase( + format!("{name}-merged"), + merged_path, + object_store, + next_seed(), + children_end_ms, + shutdown_at_ms, + merged_next_value_version, + ROOT_ACTORS, + ) + .await?; + + Ok(()) +} + +async fn run_harness_phase( + name: String, + path: Path, + object_store: Arc, + seed: u64, + start_at_ms: i64, + shutdown_at_ms: i64, + next_value_version: u64, + actor_names: &'static [&'static str], +) -> ScenarioResult { + let system_clock = Arc::new(MockSystemClock::with_time(start_at_ms)); + let shutdown_at_ms = start_at_ms + .checked_add(shutdown_at_ms) + .expect("rescaling phase shutdown timestamp must not overflow"); + let workload_options = WorkloadActorOptions { + read_durability: DurabilityLevel::Remote, + ..WorkloadActorOptions::default() + }; + let harness_name = name.clone(); + let mut harness = Harness::new(name, seed, move |ctx| async move { + let failures = ctx.failure_controller(); + for index in 0..10 { + failures.add_toxic(build_toxic(ctx.rand(), ctx.path().as_ref(), index)); + } + + let db_seed = ctx.rand().rng().next_u64(); + let mut settings = build_settings(ctx.rand()).await; + settings.l0_sst_size_bytes = 1024; + settings.l0_max_ssts = 4; + settings.max_unflushed_bytes = 64 * 1024; + settings.manifest_poll_interval = Duration::from_millis(10); + settings + .garbage_collector_options + .as_mut() + .expect("rescaling scenario requires garbage collection") + .detach_options = None; + // Manifest union rejects sources with live WAL data. + settings.wal_enabled = false; + + let db = Db::builder(ctx.path().clone(), ctx.main_object_store()) + .with_system_clock(ctx.system_clock()) + .with_fp_registry(ctx.fp_registry()) + .with_seed(db_seed) + .with_settings(settings) + .with_merge_operator( + ctx.merge_operator() + .expect("rescaling workload requires a merge operator"), + ) + .build() + .await?; + Ok(Arc::new(db)) + }) + .with_path(path) + .with_main_object_store(object_store) + .with_system_clock(system_clock.clone()) + .with_merge_operator(Arc::new(WorkloadMergeOperator)); + + for actor_name in actor_names { + let actor = WorkloadActor::new(workload_options.clone())? + .with_next_value_version(next_value_version); + harness = harness.actor(*actor_name, actor); + } + harness = harness + .actor("flusher", FlusherActor::new(1_u64..=5_u64)?) + .actor("shutdown", ShutdownActor::new(shutdown_at_ms)?); + + info!("starting rescaling harness phase [name={harness_name}]"); + harness.run_async().await?; + Ok(system_clock.now().timestamp_millis()) +} + +fn next_workload_value_version(rows: &Rows) -> u64 { + rows.iter() + .map(|(_, value)| decode_workload_value(value.as_ref())) + .max() + .unwrap_or(0) + .checked_add(1) + .expect("workload value version must not overflow") +} + +async fn create_clone( + clone_path: Path, + sources: Vec, Bound)>>, + object_store: Arc, + seed: u64, + start_at_ms: i64, +) -> Result<(), slatedb::Error> { + let system_clock: Arc = Arc::new(MockSystemClock::with_time(start_at_ms)); + let admin = AdminBuilder::new(clone_path, object_store) + .with_system_clock(system_clock.clone()) + .with_seed(seed) + .build(); + let mut sources = sources.into_iter(); + let first = sources + .next() + .expect("rescaling clone requires at least one source"); + let mut builder = admin + .create_clone_builder_from_source(first) + .with_system_clock(system_clock) + .with_seed(seed); + for source in sources { + builder = builder.with_source(source); + } + builder.build().await +} + +async fn snapshot_rows( + path: Path, + object_store: Arc, + seed: u64, + start_at_ms: i64, +) -> ScenarioResult { + let system_clock: Arc = Arc::new(MockSystemClock::with_time(start_at_ms)); + let reader = DbReader::builder(path, object_store) + .with_system_clock(system_clock) + .with_seed(seed) + .with_merge_operator(Arc::new(WorkloadMergeOperator)) + .build() + .await?; + let rows_result = async { + let mut iter = reader.scan(..).await?; + let mut rows = Vec::new(); + while let Some(kv) = iter.next().await? { + rows.push((kv.key, kv.value)); + } + Ok::<_, slatedb::Error>(rows) + } + .await; + let close_result = reader.close().await; + let rows = rows_result?; + close_result?; + Ok(rows) +} diff --git a/slatedb-dst/tests/rescaling.rs b/slatedb-dst/tests/rescaling.rs new file mode 100644 index 000000000..6c2c7abd6 --- /dev/null +++ b/slatedb-dst/tests/rescaling.rs @@ -0,0 +1,52 @@ +//! Verifies that RFC-0004 projection and union preserve database contents under +//! DST workloads. +//! +//! Each simulation run: +//! - opens a root database on a deterministic filesystem-backed object store +//! - runs four prefix-scoped workload actors with randomized SlateDB settings, +//! object-store faults, flushing, compaction, and garbage collection +//! - quiesces the root and records its complete ordered key/value state +//! - projects the root at `workload-3/` into adjacent left and right databases +//! - checks that each projection exactly matches its range in the root snapshot +//! - runs the children concurrently in separate [`slatedb_dst::Harness`] +//! instances on one seeded runtime, assigning actors 1-2 to the left child +//! and actors 3-4 to the right +//! - quiesces both children and records their post-workload state +//! - unions the children and checks that the merged database exactly matches +//! the two child snapshots +//! - runs all four workload actors against the merged database to confirm that +//! it remains readable and writable after the union +//! +//! Workload actor names are also key prefixes. The split boundary therefore +//! keeps every point operation, write batch, and prefix scan inside one child. +//! The child harnesses share the underlying object store and seeded runtime but +//! have independent clocks and fault controllers. The single-threaded runtime +//! makes their interleaved object-store operations reproducible from the seed. +//! +//! Garbage collection remains enabled during harness runs. Detach GC is +//! disabled because both children retain checkpoints in the root manifest. +//! WALs are disabled because manifest union rejects sources with live WAL data. +//! Projection and union happen only after their source harnesses have stopped. +//! +//! A snapshot mismatch means projection dropped or misplaced a root row, or +//! union failed to preserve the complete logical state of both children. +#![cfg(dst)] + +use rstest::rstest; +use slatedb_dst::RescalingScenario; + +type TestError = Box; +type TestResult = Result; + +#[rstest] +#[cfg_attr(not(slow), case::regular(200))] +// Four physical harness clocks (root, left, right, and merged) make this a +// 4.8M ms aggregate mock-clock budget per seed. +#[cfg_attr(slow, case::slow(1_200_000))] +fn test_dst_rescaling_preserves_data(#[case] shutdown_at_ms: i64) -> TestResult<()> { + RescalingScenario { + name: "rescaling", + shutdown_at_ms, + } + .run() +} From 44295b069ed85e4cbdee9cc6032d6eb3e19b004a Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 13 Jul 2026 10:17:27 -0700 Subject: [PATCH 11/63] Allow users to disable boundary files in GC (#1917) --- Cargo.lock | 1 + bindings/go/uniffi/slatedb.go | 6 ++ bindings/go/uniffi/slatedb_test.go | 13 ++-- .../io/slatedb/uniffi/SlateDbAdminTest.java | 3 +- bindings/node/tests/admin.test.mjs | 1 + bindings/python/tests/test_admin.py | 1 + bindings/uniffi/src/config.rs | 30 ++++++++ slatedb-cli/src/args.rs | 15 +++- slatedb-cli/src/main.rs | 19 ++++- slatedb-dst/src/utils.rs | 1 + slatedb-txn-obj/Cargo.toml | 1 + slatedb-txn-obj/src/object_store.rs | 40 ++++++++++ slatedb/src/config.rs | 30 ++++++++ slatedb/src/db.rs | 1 + slatedb/src/fence.rs | 1 + slatedb/src/garbage_collector.rs | 14 ++++ .../src/garbage_collector/compactions_gc.rs | 74 ++++++++++++++++-- slatedb/src/garbage_collector/manifest_gc.rs | 75 +++++++++++++++++-- .../src/content/docs/docs/design/files.mdx | 6 ++ website/src/content/docs/docs/design/gc.mdx | 16 +++- .../src/content/docs/docs/operations/cli.mdx | 5 ++ .../docs/docs/operations/configuration.mdx | 11 +++ .../standalone-garbage-collector.mdx | 6 ++ 23 files changed, 346 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ce786a76a..3ee67f46f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3358,6 +3358,7 @@ dependencies = [ "object_store", "parking_lot", "slatedb-common", + "tempfile", "thiserror 1.0.69", "tokio", ] diff --git a/bindings/go/uniffi/slatedb.go b/bindings/go/uniffi/slatedb.go index b1151eec0..3c2fdece8 100644 --- a/bindings/go/uniffi/slatedb.go +++ b/bindings/go/uniffi/slatedb.go @@ -9457,6 +9457,9 @@ type GarbageCollectorOptions struct { CompactionsOptions *GarbageCollectorDirectoryOptions // Options for detaching clone references. `None` disables detach garbage collection. DetachOptions *GarbageCollectorScheduleOptions + // Whether GC should delete eligible manifest/compactions metadata without advancing boundary + // files. + DisableBoundaryFiles bool } func (r *GarbageCollectorOptions) Destroy() { @@ -9466,6 +9469,7 @@ func (r *GarbageCollectorOptions) Destroy() { FfiDestroyerOptionalGarbageCollectorDirectoryOptions{}.Destroy(r.CompactedOptions) FfiDestroyerOptionalGarbageCollectorDirectoryOptions{}.Destroy(r.CompactionsOptions) FfiDestroyerOptionalGarbageCollectorScheduleOptions{}.Destroy(r.DetachOptions) + FfiDestroyerBool{}.Destroy(r.DisableBoundaryFiles) } type FfiConverterGarbageCollectorOptions struct{} @@ -9484,6 +9488,7 @@ func (c FfiConverterGarbageCollectorOptions) Read(reader io.Reader) GarbageColle FfiConverterOptionalGarbageCollectorDirectoryOptionsINSTANCE.Read(reader), FfiConverterOptionalGarbageCollectorDirectoryOptionsINSTANCE.Read(reader), FfiConverterOptionalGarbageCollectorScheduleOptionsINSTANCE.Read(reader), + FfiConverterBoolINSTANCE.Read(reader), } } @@ -9502,6 +9507,7 @@ func (c FfiConverterGarbageCollectorOptions) Write(writer io.Writer, value Garba FfiConverterOptionalGarbageCollectorDirectoryOptionsINSTANCE.Write(writer, value.CompactedOptions) FfiConverterOptionalGarbageCollectorDirectoryOptionsINSTANCE.Write(writer, value.CompactionsOptions) FfiConverterOptionalGarbageCollectorScheduleOptionsINSTANCE.Write(writer, value.DetachOptions) + FfiConverterBoolINSTANCE.Write(writer, value.DisableBoundaryFiles) } type FfiDestroyerGarbageCollectorOptions struct{} diff --git a/bindings/go/uniffi/slatedb_test.go b/bindings/go/uniffi/slatedb_test.go index 4dbe5cf6a..6ec3c87c1 100644 --- a/bindings/go/uniffi/slatedb_test.go +++ b/bindings/go/uniffi/slatedb_test.go @@ -1925,12 +1925,13 @@ func TestAdminRunGcOnce(t *testing.T) { DryRun: true, } options := &slatedb.GarbageCollectorOptions{ - ManifestOptions: nil, - WalOptions: directoryOptions, - WalFenceOptions: directoryOptions, - CompactedOptions: nil, - CompactionsOptions: nil, - DetachOptions: &slatedb.GarbageCollectorScheduleOptions{IntervalMs: nil}, + ManifestOptions: nil, + WalOptions: directoryOptions, + WalFenceOptions: directoryOptions, + CompactedOptions: nil, + CompactionsOptions: nil, + DetachOptions: &slatedb.GarbageCollectorScheduleOptions{IntervalMs: nil}, + DisableBoundaryFiles: true, } if err := admin.RunGcOnce(options); err != nil { diff --git a/bindings/java/slatedb-uniffi/src/test/java/io/slatedb/uniffi/SlateDbAdminTest.java b/bindings/java/slatedb-uniffi/src/test/java/io/slatedb/uniffi/SlateDbAdminTest.java index 895c09eb9..d285f7531 100644 --- a/bindings/java/slatedb-uniffi/src/test/java/io/slatedb/uniffi/SlateDbAdminTest.java +++ b/bindings/java/slatedb-uniffi/src/test/java/io/slatedb/uniffi/SlateDbAdminTest.java @@ -183,7 +183,8 @@ void adminRunGcOnceAcceptsDefaultAndCustomOptions() throws Exception { directoryOptions, null, null, - scheduleOptions); + scheduleOptions, + true); TestSupport.await(admin.runGcOnce(options)); } diff --git a/bindings/node/tests/admin.test.mjs b/bindings/node/tests/admin.test.mjs index ad6ea417b..08bf06a27 100644 --- a/bindings/node/tests/admin.test.mjs +++ b/bindings/node/tests/admin.test.mjs @@ -162,6 +162,7 @@ test("admin run_gc_once accepts default and custom options", async (t) => { compacted_options: undefined, compactions_options: undefined, detach_options: { interval_ms: undefined }, + disable_boundary_files: true, }); }); diff --git a/bindings/python/tests/test_admin.py b/bindings/python/tests/test_admin.py index 0e3853f34..f65b1c628 100644 --- a/bindings/python/tests/test_admin.py +++ b/bindings/python/tests/test_admin.py @@ -144,6 +144,7 @@ async def test_admin_run_gc_once_accepts_default_and_custom_options() -> None: compacted_options=None, compactions_options=None, detach_options=schedule_options, + disable_boundary_files=True, ) await admin.run_gc_once(options) diff --git a/bindings/uniffi/src/config.rs b/bindings/uniffi/src/config.rs index ec6bccc81..97a287f8c 100644 --- a/bindings/uniffi/src/config.rs +++ b/bindings/uniffi/src/config.rs @@ -424,6 +424,10 @@ pub struct GarbageCollectorOptions { /// Options for detaching clone references. `None` disables detach garbage collection. #[uniffi(default = None)] pub detach_options: Option, + /// Whether GC should delete eligible manifest/compactions metadata without advancing boundary + /// files. + #[uniffi(default = false)] + pub disable_boundary_files: bool, } impl Default for GarbageCollectorOptions { @@ -436,6 +440,7 @@ impl Default for GarbageCollectorOptions { compacted_options: core.compacted_options.map(Into::into), compactions_options: core.compactions_options.map(Into::into), detach_options: core.detach_options.map(Into::into), + disable_boundary_files: !core.boundary_files_enabled, } } } @@ -468,10 +473,35 @@ impl From for slatedb::config::GarbageCollectorOptions compactions_options: value.compactions_options.map(Into::into), detach_options: value.detach_options.map(Into::into), metric_level: None, + boundary_files_enabled: !value.disable_boundary_files, } } } +#[cfg(test)] +mod tests { + use super::GarbageCollectorOptions; + + #[test] + fn boundary_files_are_enabled_by_default() { + let gc: slatedb::config::GarbageCollectorOptions = + GarbageCollectorOptions::default().into(); + + assert!(gc.boundary_files_enabled); + } + + #[test] + fn boundary_files_can_be_disabled() { + let gc: slatedb::config::GarbageCollectorOptions = GarbageCollectorOptions { + disable_boundary_files: true, + ..GarbageCollectorOptions::default() + } + .into(); + + assert!(!gc.boundary_files_enabled); + } +} + /// Specify options to provide when creating a checkpoint. #[derive(Clone, Debug, PartialEq, Eq, uniffi::Record, Default)] pub struct CheckpointOptions { diff --git a/slatedb-cli/src/args.rs b/slatedb-cli/src/args.rs index 99b46586b..c22208cc3 100644 --- a/slatedb-cli/src/args.rs +++ b/slatedb-cli/src/args.rs @@ -195,6 +195,10 @@ pub(crate) enum CliCommands { #[arg(short, long)] #[clap(value_parser = humantime::parse_duration)] min_age: Duration, + + /// Delete eligible metadata without advancing boundary files. + #[arg(long, default_value_t = false)] + disable_boundary_files: bool, }, /// Runs the compactor coordinator until interrupted (Ctrl-C). @@ -311,6 +315,10 @@ pub(crate) enum CliCommands { /// the period is how often to attempt a GC #[arg(long, value_parser = parse_gc_schedule)] compactions: Option, + + /// Delete eligible metadata without advancing boundary files. + #[arg(long, default_value_t = false)] + disable_boundary_files: bool, }, } @@ -454,9 +462,14 @@ mod tests { .unwrap(); match args.command { - CliCommands::RunGarbageCollection { resource, min_age } => { + CliCommands::RunGarbageCollection { + resource, + min_age, + disable_boundary_files, + } => { assert!(matches!(resource, GcResource::WalFence)); assert_eq!(min_age, Duration::from_secs(60)); + assert!(!disable_boundary_files); } command => panic!("unexpected command: {command:?}"), } diff --git a/slatedb-cli/src/main.rs b/slatedb-cli/src/main.rs index dc18d4369..fd12de8bd 100644 --- a/slatedb-cli/src/main.rs +++ b/slatedb-cli/src/main.rs @@ -67,9 +67,11 @@ async fn main() -> Result<(), Box> { } CliCommands::DeleteCheckpoint { id } => exec_delete_checkpoint(&admin, id).await?, CliCommands::ListCheckpoints { name } => exec_list_checkpoints(&admin, name).await?, - CliCommands::RunGarbageCollection { resource, min_age } => { - exec_gc_once(&admin, resource, min_age).await? - } + CliCommands::RunGarbageCollection { + resource, + min_age, + disable_boundary_files, + } => exec_gc_once(&admin, resource, min_age, !disable_boundary_files).await?, CliCommands::RunCompactor { no_embedded_worker } => { admin .run_compactor_with_options( @@ -112,6 +114,7 @@ async fn main() -> Result<(), Box> { wal_fence, compacted, compactions, + disable_boundary_files, } => { schedule_gc( &admin, @@ -120,6 +123,7 @@ async fn main() -> Result<(), Box> { wal_fence, compacted, compactions, + !disable_boundary_files, cancellation_token.clone(), ) .await? @@ -290,6 +294,7 @@ async fn exec_gc_once( admin: &Admin, resource: GcResource, min_age: Duration, + boundary_files_enabled: bool, ) -> Result<(), Box> { fn create_gc_dir_opts(min_age: Duration) -> Option { Some(GarbageCollectorDirectoryOptions { @@ -307,6 +312,7 @@ async fn exec_gc_once( compactions_options: None, detach_options: None, metric_level: None, + boundary_files_enabled, }, GcResource::Wal => GarbageCollectorOptions { manifest_options: None, @@ -316,6 +322,7 @@ async fn exec_gc_once( compactions_options: None, detach_options: None, metric_level: None, + boundary_files_enabled, }, GcResource::WalFence => GarbageCollectorOptions { manifest_options: None, @@ -325,6 +332,7 @@ async fn exec_gc_once( compactions_options: None, detach_options: None, metric_level: None, + boundary_files_enabled, }, GcResource::Compacted => GarbageCollectorOptions { manifest_options: None, @@ -334,6 +342,7 @@ async fn exec_gc_once( compactions_options: None, detach_options: None, metric_level: None, + boundary_files_enabled, }, GcResource::Compactions => GarbageCollectorOptions { manifest_options: None, @@ -343,12 +352,14 @@ async fn exec_gc_once( compactions_options: create_gc_dir_opts(min_age), detach_options: None, metric_level: None, + boundary_files_enabled, }, }; admin.run_gc_once(gc_opts).await?; Ok(()) } +#[allow(clippy::too_many_arguments)] async fn schedule_gc( admin: &Admin, manifest_schedule: Option, @@ -356,6 +367,7 @@ async fn schedule_gc( wal_fence_schedule: Option, compacted_schedule: Option, compactions_schedule: Option, + boundary_files_enabled: bool, cancellation_token: CancellationToken, ) -> Result<(), Box> { fn create_gc_dir_opts(schedule: GcSchedule) -> Option { @@ -373,6 +385,7 @@ async fn schedule_gc( compactions_options: compactions_schedule.and_then(create_gc_dir_opts), detach_options: None, metric_level: None, + boundary_files_enabled, }; admin diff --git a/slatedb-dst/src/utils.rs b/slatedb-dst/src/utils.rs index 4be8fcbbc..127e195cd 100644 --- a/slatedb-dst/src/utils.rs +++ b/slatedb-dst/src/utils.rs @@ -157,6 +157,7 @@ pub fn build_settings_gc(rng: &mut impl Rng) -> GarbageCollectorOptions { interval: Some(rng.random_range(Duration::from_millis(1)..Duration::from_secs(600))), }), metric_level: None, + boundary_files_enabled: true, } } diff --git a/slatedb-txn-obj/Cargo.toml b/slatedb-txn-obj/Cargo.toml index 2e41faa76..f15ec8b5b 100644 --- a/slatedb-txn-obj/Cargo.toml +++ b/slatedb-txn-obj/Cargo.toml @@ -22,4 +22,5 @@ thiserror = { workspace = true } test-util = [] [dev-dependencies] +tempfile = { workspace = true } tokio = { workspace = true, features = ["macros", "rt", "time"] } diff --git a/slatedb-txn-obj/src/object_store.rs b/slatedb-txn-obj/src/object_store.rs index 84bdd2f96..14a191d7c 100644 --- a/slatedb-txn-obj/src/object_store.rs +++ b/slatedb-txn-obj/src/object_store.rs @@ -437,6 +437,7 @@ mod tests { use chrono::Utc; use futures::stream::{self, BoxStream}; use futures::StreamExt; + use object_store::local::LocalFileSystem; use object_store::memory::InMemory; use object_store::path::Path; use object_store::{ @@ -683,6 +684,45 @@ mod tests { boundary.check(MonotonicId::new(1)).await.unwrap(); } + #[tokio::test] + async fn test_boundary_check_supports_local_filesystem_conditional_get() { + let tempdir = tempfile::tempdir().unwrap(); + let object_store: Arc = Arc::new( + LocalFileSystem::new_with_prefix(tempdir.path()).expect("create local object store"), + ); + let root = Path::from("root"); + let boundary_path = root.clone().join("gc").join("manifest.boundary"); + object_store + .put(&boundary_path, PutPayload::from("2")) + .await + .unwrap(); + let boundary = ObjectStoreBoundaryObject::new(&root, object_store.clone(), "manifest"); + + // The first check reads and caches the boundary and its filesystem ETag. + boundary.check(MonotonicId::new(3)).await.unwrap(); + assert!(boundary + .cache + .lock() + .as_ref() + .and_then(|(_, version)| version.e_tag.as_ref()) + .is_some()); + + // The second check sends GET If-None-Match. LocalFileSystem returns NotModified, + // which the boundary implementation must handle by reusing its cache. + boundary.check(MonotonicId::new(3)).await.unwrap(); + + // Replace the file directly rather than calling BoundaryObject::advance, since + // LocalFileSystem does not support PutMode::Update. The stale ETag should cause + // the next check to read and enforce the new boundary. + object_store + .put(&boundary_path, PutPayload::from("4")) + .await + .unwrap(); + let err = boundary.check(MonotonicId::new(3)).await.unwrap_err(); + assert!(matches!(err, TransactionalObjectError::ObjectVersionExists)); + boundary.check(MonotonicId::new(5)).await.unwrap(); + } + #[tokio::test] async fn test_boundary_advance_creates_boundary_and_rejects_at_or_below_it() { let object_store = Arc::new(InMemory::new()); diff --git a/slatedb/src/config.rs b/slatedb/src/config.rs index 74e59541e..a3989769c 100644 --- a/slatedb/src/config.rs +++ b/slatedb/src/config.rs @@ -210,6 +210,10 @@ use crate::error::SlateDBError; use crate::garbage_collector::{DEFAULT_INTERVAL, DEFAULT_MIN_AGE}; +fn default_boundary_files_enabled() -> bool { + true +} + /// Enum representing different levels of cache preloading on startup #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] pub enum PreloadLevel { @@ -1431,6 +1435,13 @@ pub struct GarbageCollectorOptions { /// a garbage collector is owned by a [`Settings`] configured DB, unset means /// inherit [`Settings::metric_level`]. pub metric_level: Option, + + /// Whether manifest and compactions boundary files are advanced before deletion. + /// + /// When disabled, garbage collection still deletes eligible metadata but does not update the + /// durable boundary. Every garbage collector for the database must use the same policy. + #[serde(default = "default_boundary_files_enabled")] + pub boundary_files_enabled: bool, } impl GarbageCollectorOptions { @@ -1531,6 +1542,7 @@ impl Default for GarbageCollectorOptions { compactions_options: Some(GarbageCollectorDirectoryOptions::default()), detach_options: Some(GarbageCollectorScheduleOptions::default()), metric_level: None, + boundary_files_enabled: true, } } } @@ -1703,6 +1715,24 @@ mod tests { assert_eq!(MetricLevel::default(), options.metric_level); } + #[test] + fn test_gc_boundary_files_are_enabled_by_default_when_omitted() { + fn without_boundary_setting(value: T) -> serde_json::Value { + let mut value = serde_json::to_value(value).unwrap(); + value + .as_object_mut() + .unwrap() + .remove("boundary_files_enabled"); + value + } + + let gc: GarbageCollectorOptions = + serde_json::from_value(without_boundary_setting(GarbageCollectorOptions::default())) + .unwrap(); + + assert!(gc.boundary_files_enabled); + } + #[test] fn test_db_options_load_from_json_file() { figment::Jail::expect_with(|jail| { diff --git a/slatedb/src/db.rs b/slatedb/src/db.rs index 45c062cb6..91800dafc 100644 --- a/slatedb/src/db.rs +++ b/slatedb/src/db.rs @@ -7790,6 +7790,7 @@ mod tests { }), detach_options: None, metric_level: None, + boundary_files_enabled: true, }; let gc = GarbageCollectorBuilder::new(path.clone(), object_store.clone()) diff --git a/slatedb/src/fence.rs b/slatedb/src/fence.rs index e29b03891..1ee2f58d1 100644 --- a/slatedb/src/fence.rs +++ b/slatedb/src/fence.rs @@ -309,6 +309,7 @@ mod tests { compactions_options: None, detach_options: None, metric_level: None, + boundary_files_enabled: true, }; let gc = GarbageCollector::new( self.manifest_store.clone(), diff --git a/slatedb/src/garbage_collector.rs b/slatedb/src/garbage_collector.rs index 70ccc88d1..e3910040f 100644 --- a/slatedb/src/garbage_collector.rs +++ b/slatedb/src/garbage_collector.rs @@ -278,6 +278,7 @@ impl GarbageCollector { stats.clone(), compactions_options, gc_filter.clone(), + options.boundary_files_enabled, ) }); let manifest_gc_task = options.manifest_options.map(|manifest_options| { @@ -286,6 +287,7 @@ impl GarbageCollector { stats.clone(), manifest_options, gc_filter.clone(), + options.boundary_files_enabled, ) }); let detach_gc_task = options.detach_options.map(|detach_options| { @@ -1167,6 +1169,7 @@ mod tests { compactions_options: None, detach_options: None, metric_level: None, + boundary_files_enabled: true, }; let gc = GarbageCollector::new( manifest_store.clone(), @@ -1233,6 +1236,7 @@ mod tests { compactions_options: None, detach_options: None, metric_level: None, + boundary_files_enabled: true, }; let recorder = Arc::new(DefaultMetricsRecorder::new()); let helper = MetricsRecorderHelper::new(recorder.clone(), Default::default()); @@ -1298,6 +1302,7 @@ mod tests { compactions_options: None, detach_options: None, metric_level: None, + boundary_files_enabled: true, }; let gc = GarbageCollector::new( manifest_store.clone(), @@ -1376,6 +1381,7 @@ mod tests { compactions_options: None, detach_options: None, metric_level: None, + boundary_files_enabled: true, }; let gc = GarbageCollector::new( manifest_store.clone(), @@ -1830,6 +1836,7 @@ mod tests { }), detach_options: None, metric_level: None, + boundary_files_enabled: true, }; let gc = GarbageCollector::new( @@ -1905,6 +1912,7 @@ mod tests { }), detach_options: None, metric_level: None, + boundary_files_enabled: true, }; let mut gc = GarbageCollector::new( @@ -1975,6 +1983,7 @@ mod tests { }), detach_options: None, metric_level: None, + boundary_files_enabled: true, }; let gc = GarbageCollector::new( @@ -2024,6 +2033,7 @@ mod tests { }), detach_options: None, metric_level: None, + boundary_files_enabled: true, }; let mut gc = GarbageCollector::new( @@ -2077,6 +2087,7 @@ mod tests { }), detach_options: None, metric_level: None, + boundary_files_enabled: true, }; let gc = GarbageCollector::new( @@ -2407,6 +2418,7 @@ mod tests { compactions_options: Some(options), detach_options: None, metric_level: None, + boundary_files_enabled: true, }; let recorder = MetricsRecorderHelper::noop(); let gc = GarbageCollector::new( @@ -2505,6 +2517,7 @@ mod tests { compactions_options: None, detach_options: None, metric_level: None, + boundary_files_enabled: true, }; let recorder = Arc::new(DefaultMetricsRecorder::new()); let helper = MetricsRecorderHelper::new(recorder.clone(), Default::default()); @@ -2638,6 +2651,7 @@ mod tests { compactions_options: Some(dry_run_options), detach_options: None, metric_level: None, + boundary_files_enabled: true, }; let recorder = MetricsRecorderHelper::noop(); let gc = GarbageCollector::new( diff --git a/slatedb/src/garbage_collector/compactions_gc.rs b/slatedb/src/garbage_collector/compactions_gc.rs index 6cc0ab3e8..f4634dd2c 100644 --- a/slatedb/src/garbage_collector/compactions_gc.rs +++ b/slatedb/src/garbage_collector/compactions_gc.rs @@ -36,12 +36,14 @@ pub(crate) struct CompactionsGcTask { stats: Arc, compactions_options: GarbageCollectorDirectoryOptions, gc_filter: Option>, + boundary_files_enabled: bool, } impl std::fmt::Debug for CompactionsGcTask { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("CompactionsGcTask") .field("compactions_options", &self.compactions_options) + .field("boundary_files_enabled", &self.boundary_files_enabled) .finish() } } @@ -52,12 +54,14 @@ impl CompactionsGcTask { stats: Arc, compactions_options: GarbageCollectorDirectoryOptions, gc_filter: Option>, + boundary_files_enabled: bool, ) -> Self { Self { compactions_store, stats, compactions_options, gc_filter, + boundary_files_enabled, } } @@ -122,12 +126,14 @@ impl GcTask for CompactionsGcTask { // Advance the boundary to the latest compactions file selected by the GC model. The // optional GC filter only gates the final deletion pass. - if let Some(boundary) = compactions_to_delete - .iter() - .map(|compactions_metadata| compactions_metadata.id) - .max() - { - self.compactions_store.advance_boundary(boundary).await?; + if self.boundary_files_enabled { + if let Some(boundary) = compactions_to_delete + .iter() + .map(|compactions_metadata| compactions_metadata.id) + .max() + { + self.compactions_store.advance_boundary(boundary).await?; + } } let compactions_to_delete = retain_allowed_by_gc_filter(&self.gc_filter, compactions_to_delete).await; @@ -197,6 +203,7 @@ mod tests { dry_run: false, }, None, + true, ); task.collect(Utc::now() + TimeDelta::hours(1)) .await @@ -221,6 +228,60 @@ mod tests { ); } + #[tokio::test] + async fn test_collect_without_boundary_advancement_deletes_and_preserves_boundary() { + let object_store = Arc::new(InMemory::new()); + let compactions_store = Arc::new(CompactionsStore::new( + &Path::from("/root"), + object_store.clone(), + )); + let mut stored_compactions = StoredCompactions::create(compactions_store.clone(), 0) + .await + .unwrap(); + stored_compactions + .update(stored_compactions.prepare_dirty().unwrap()) + .await + .unwrap(); + compactions_store.advance_boundary(1).await.unwrap(); + stored_compactions + .update(stored_compactions.prepare_dirty().unwrap()) + .await + .unwrap(); + + let recorder = MetricsRecorderHelper::noop(); + let task = CompactionsGcTask::new( + compactions_store.clone(), + Arc::new(GcStats::new(&recorder)), + GarbageCollectorDirectoryOptions { + min_age: Duration::from_secs(1), + interval: None, + dry_run: false, + }, + None, + false, + ); + task.collect(Utc::now() + TimeDelta::hours(1)) + .await + .unwrap(); + + let raw_boundary = object_store + .get(&Path::from("/root/gc/compactions.boundary")) + .await + .unwrap() + .bytes() + .await + .unwrap(); + assert_eq!("1", std::str::from_utf8(&raw_boundary).unwrap()); + let compactions = compactions_store.list_compactions(..).await.unwrap(); + assert_eq!( + vec![3], + compactions + .iter() + .map(|compactions| compactions.id) + .collect::>() + ); + } + #[tokio::test] async fn test_collect_advances_boundary_before_filtering_compactions_files() { let object_store = Arc::new(InMemory::new()); @@ -250,6 +311,7 @@ mod tests { dry_run: false, }, Some(Arc::new(DenyAllGcFilter) as Arc), + true, ); task.collect(Utc::now() + TimeDelta::hours(1)) .await diff --git a/slatedb/src/garbage_collector/manifest_gc.rs b/slatedb/src/garbage_collector/manifest_gc.rs index f3d3af8d6..709ec3c80 100644 --- a/slatedb/src/garbage_collector/manifest_gc.rs +++ b/slatedb/src/garbage_collector/manifest_gc.rs @@ -16,12 +16,14 @@ pub(crate) struct ManifestGcTask { stats: Arc, manifest_options: GarbageCollectorDirectoryOptions, gc_filter: Option>, + boundary_files_enabled: bool, } impl std::fmt::Debug for ManifestGcTask { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("ManifestGcTask") .field("manifest_options", &self.manifest_options) + .field("boundary_files_enabled", &self.boundary_files_enabled) .finish() } } @@ -32,12 +34,14 @@ impl ManifestGcTask { stats: Arc, manifest_options: GarbageCollectorDirectoryOptions, gc_filter: Option>, + boundary_files_enabled: bool, ) -> Self { ManifestGcTask { manifest_store, stats, manifest_options, gc_filter, + boundary_files_enabled, } } @@ -111,12 +115,14 @@ impl GcTask for ManifestGcTask { // Advance the boundary to the latest manifest selected by the GC model. The optional GC // filter only gates the final deletion pass. - if let Some(boundary) = manifests_to_delete - .iter() - .map(|manifest_metadata| manifest_metadata.id) - .max() - { - self.manifest_store.advance_boundary(boundary).await?; + if self.boundary_files_enabled { + if let Some(boundary) = manifests_to_delete + .iter() + .map(|manifest_metadata| manifest_metadata.id) + .max() + { + self.manifest_store.advance_boundary(boundary).await?; + } } let manifests_to_delete = retain_allowed_by_gc_filter(&self.gc_filter, manifests_to_delete).await; @@ -192,6 +198,7 @@ mod tests { dry_run: false, }, None, + true, ); task.collect(Utc::now() + TimeDelta::hours(1)) .await @@ -216,6 +223,61 @@ mod tests { ); } + #[tokio::test] + async fn test_collect_without_boundary_advancement_deletes_without_creating_boundary() { + let object_store = Arc::new(InMemory::new()); + let manifest_store = Arc::new(ManifestStore::new( + &Path::from("/root"), + object_store.clone(), + )); + let mut stored_manifest = StoredManifest::create_new_db( + manifest_store.clone(), + ManifestCore::new(), + Arc::new(DefaultSystemClock::new()), + ) + .await + .unwrap(); + stored_manifest + .update(stored_manifest.prepare_dirty().unwrap()) + .await + .unwrap(); + stored_manifest + .update(stored_manifest.prepare_dirty().unwrap()) + .await + .unwrap(); + + let recorder = MetricsRecorderHelper::noop(); + let task = ManifestGcTask::new( + manifest_store.clone(), + Arc::new(GcStats::new(&recorder)), + GarbageCollectorDirectoryOptions { + min_age: Duration::from_secs(1), + interval: None, + dry_run: false, + }, + None, + false, + ); + task.collect(Utc::now() + TimeDelta::hours(1)) + .await + .unwrap(); + + assert!(matches!( + object_store + .get(&Path::from("/root/gc/manifest.boundary")) + .await, + Err(object_store::Error::NotFound { .. }) + )); + let manifests = manifest_store.list_manifests(..).await.unwrap(); + assert_eq!( + vec![3], + manifests + .iter() + .map(|manifest| manifest.id) + .collect::>() + ); + } + #[tokio::test] async fn test_collect_advances_boundary_before_filtering_manifest_files() { let object_store = Arc::new(InMemory::new()); @@ -249,6 +311,7 @@ mod tests { dry_run: false, }, Some(Arc::new(DenyAllGcFilter) as Arc), + true, ); task.collect(Utc::now() + TimeDelta::hours(1)) .await diff --git a/website/src/content/docs/docs/design/files.mdx b/website/src/content/docs/docs/design/files.mdx index 68fa01532..347db2a23 100644 --- a/website/src/content/docs/docs/design/files.mdx +++ b/website/src/content/docs/docs/design/files.mdx @@ -70,3 +70,9 @@ The `gc` directory contains boundary files used for garbage collection coordinat Each boundary file stores a single unsigned 64-bit integer representing an inclusive high-watermark. A boundary value `B` means that object IDs `<= B` are eligible for deletion. Before the garbage collector deletes old sequenced metadata files, it advances the namespace boundary. After a writer creates a sequenced metadata file, it checks the boundary before returning success. If the created ID is at or behind the boundary, the write is treated as failed. Boundary files use conditional updates (ETag-based) to ensure monotonic advancement and prevent concurrent GC processes from interfering with each other. They provide a persistent marker that allows garbage collectors to safely delete objects below the boundary without risking deletion of still-referenced data. + +Garbage collectors advance boundary files by default. This can be disabled through +`GarbageCollectorOptions`; eligible metadata is still deleted, but the durable boundary is not +advanced. Metadata writers always check any existing boundary. All garbage collectors for a +database must use a compatible policy, and the metadata `min_age` settings must be long enough to +outlive stale processes when boundary advancement is disabled. diff --git a/website/src/content/docs/docs/design/gc.mdx b/website/src/content/docs/docs/design/gc.mdx index 363eb3f39..ba51c2b04 100644 --- a/website/src/content/docs/docs/design/gc.mdx +++ b/website/src/content/docs/docs/design/gc.mdx @@ -9,6 +9,20 @@ The garbage collector has a configurable minimum age and interval for each file Each garbage collection directory type supports a `dry_run` option. When enabled, the collector logs files that would be deleted without actually deleting them. This is useful for testing or verifying garbage collection behavior before enabling actual deletion. +## Boundary files + +Before deleting old manifest or compactions metadata, SlateDB normally advances a durable boundary +file. Metadata writers check that boundary after creating a new version, which prevents a stale +writer from successfully publishing metadata that GC has already passed. + +Boundary advancement can be disabled with +`GarbageCollectorOptions::boundary_files_enabled`. This supports object stores without conditional +overwrite (`If-Match`) while allowing GC to continue deleting eligible metadata. Metadata readers +and writers still check any existing boundary; on a new database, no boundary file is created while +all garbage collectors use this mode. Disabling advancement removes the stale-writer fail-safe, so +every garbage collector must use a compatible setting and the manifest and compactions `min_age` +values must exceed the maximum lifetime of a stale process. + ## Filtering Deletion Candidates SlateDB supports custom filtering of garbage collection candidates through the `GcFilter` trait. This allows users to intercept files before deletion and approve or reject them based on custom logic. @@ -63,4 +77,4 @@ By default, garbage collection is enabled for all managed directories (manifest, WAL fence garbage collection runs in dry-run mode by default. This means it logs files that would be deleted without actually deleting them. This conservative default prevents accidental data loss while still providing visibility into what would be cleaned up. -To enable actual deletion for WAL fence GC, set `dry_run: false` with a high `min_age` to safely clean up old fences. Alternatively, to silence the dry-run logging entirely, set `wal_fence_options: None`. \ No newline at end of file +To enable actual deletion for WAL fence GC, set `dry_run: false` with a high `min_age` to safely clean up old fences. Alternatively, to silence the dry-run logging entirely, set `wal_fence_options: None`. diff --git a/website/src/content/docs/docs/operations/cli.mdx b/website/src/content/docs/docs/operations/cli.mdx index 5687abcc5..663b8c982 100644 --- a/website/src/content/docs/docs/operations/cli.mdx +++ b/website/src/content/docs/docs/operations/cli.mdx @@ -76,6 +76,11 @@ slatedb --env-file .env --path schedule-garbage-collection \ The scheduled process runs until interrupted (Ctrl-C), then shuts down gracefully. +Pass `--disable-boundary-files` to `run-garbage-collection` or +`schedule-garbage-collection` to delete eligible manifest and compactions metadata without +advancing boundary files. Use the same setting for every garbage collector operating on the +database. + :::note The garbage collector expects the database to already exist (a manifest must be present). If you're creating a new database, open it once before starting the garbage collector. diff --git a/website/src/content/docs/docs/operations/configuration.mdx b/website/src/content/docs/docs/operations/configuration.mdx index 75b268037..e7ce9c2d6 100644 --- a/website/src/content/docs/docs/operations/configuration.mdx +++ b/website/src/content/docs/docs/operations/configuration.mdx @@ -67,6 +67,17 @@ default is `Info`; set it to `Debug` to enable debug-level metrics when using a metrics recorder. See [Metric levels](/docs/operations/metrics/#metric-levels) for more information. +## Garbage-collection boundary files + +`GarbageCollectorOptions::boundary_files_enabled` controls whether manifest and compactions garbage +collection advances durable boundary files before deleting eligible metadata. Boundary advancement +is enabled by default. Set it to `false` for object stores that do not support conditional overwrite +(`If-Match`), and use the same setting for every garbage collector operating on the database. + +Metadata readers and writers always honor existing boundary files. Without boundary advancement, +configure manifest and compactions garbage collection `min_age` values longer than any stale writer +or compactor can remain alive. + ## Reconfiguring Object Stores Reconfiguring an object store only changes how SlateDB reaches storage. It does not migrate data between stores for you. The new `ObjectStore` must still have the existing database contents for the database path. diff --git a/website/src/content/docs/docs/tutorials/standalone-garbage-collector.mdx b/website/src/content/docs/docs/tutorials/standalone-garbage-collector.mdx index 52f0af9fa..74245cf8b 100644 --- a/website/src/content/docs/docs/tutorials/standalone-garbage-collector.mdx +++ b/website/src/content/docs/docs/tutorials/standalone-garbage-collector.mdx @@ -52,6 +52,12 @@ You can also embed the garbage collector in your own process using `GarbageColle With `GarbageCollectorOptions::default()`, garbage collection runs every 60 seconds and uses a 5 minute minimum age for managed directories. WAL fence deletion stays in dry-run mode by default. +To delete manifest and compactions metadata without advancing boundary files, set +`GarbageCollectorOptions::boundary_files_enabled` to `false` on every garbage collector for the +database. Metadata readers and writers continue to honor any existing boundary. Increase the +manifest and compactions `min_age` values beyond the longest time a stale process could remain +alive before using this mode. + :::note The standalone garbage collector expects the database to already exist (a manifest must be present). From 6b4857b65b1c8acfdded16df682e4c0a88c98304 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Mon, 13 Jul 2026 14:54:08 -0700 Subject: [PATCH 12/63] adds schema evolution docs wrt `PrefixExtractor` (#1921) --- bindings/uniffi/src/builder.rs | 5 ++- bindings/uniffi/src/filter_policy.rs | 2 +- slatedb/src/db/builder.rs | 6 ++- slatedb/src/prefix_extractor.rs | 2 +- .../docs/docs/design/segmented-compaction.mdx | 39 +++++++++++++++++-- 5 files changed, 47 insertions(+), 7 deletions(-) diff --git a/bindings/uniffi/src/builder.rs b/bindings/uniffi/src/builder.rs index c45c6fea5..aa04414d4 100644 --- a/bindings/uniffi/src/builder.rs +++ b/bindings/uniffi/src/builder.rs @@ -124,7 +124,10 @@ impl DbBuilder { /// Sets the segment extractor (RFC-0024). When configured, every write is /// routed through the extractor and the database tracks per-segment LSM /// state. The extractor must be configured at database creation time and - /// cannot be changed thereafter. + /// remain configured thereafter. Its name must remain stable; its + /// implementation may evolve only if it preserves routing for all existing + /// key schemas and keeps segment prefixes across schema versions an + /// antichain (no prefix may be a proper prefix of another). pub fn with_segment_extractor(&self, extractor: Arc) -> Result<(), Error> { self.update_builder(|builder| { builder.with_segment_extractor(adapt_prefix_extractor(extractor)) diff --git a/bindings/uniffi/src/filter_policy.rs b/bindings/uniffi/src/filter_policy.rs index 4b375dd4c..b7be18eb3 100644 --- a/bindings/uniffi/src/filter_policy.rs +++ b/bindings/uniffi/src/filter_policy.rs @@ -39,7 +39,7 @@ impl From<&slatedb::PrefixTarget> for PrefixTarget { } /// Application-provided prefix extractor used to configure prefix-based -/// bloom filters. +/// bloom filters and segmented compaction. #[uniffi::export(with_foreign)] pub trait PrefixExtractor: Send + Sync { /// Stable identifier for this extractor's configuration. Included in the diff --git a/slatedb/src/db/builder.rs b/slatedb/src/db/builder.rs index 03fe0b80e..6fc621628 100644 --- a/slatedb/src/db/builder.rs +++ b/slatedb/src/db/builder.rs @@ -220,7 +220,11 @@ impl> DbBuilder

{ /// Set the segment extractor (RFC-0024). When configured, every /// write is routed through the extractor and the database tracks /// per-segment LSM state. The extractor must be configured at - /// database creation time and cannot be changed thereafter. + /// database creation time and remain configured thereafter. Its name + /// must remain stable; its implementation may evolve only if it preserves + /// routing for all existing key schemas and keeps segment prefixes across + /// schema versions an antichain (no prefix may be a proper prefix of + /// another). pub fn with_segment_extractor( mut self, extractor: Arc, diff --git a/slatedb/src/prefix_extractor.rs b/slatedb/src/prefix_extractor.rs index 968435c80..18c095f57 100644 --- a/slatedb/src/prefix_extractor.rs +++ b/slatedb/src/prefix_extractor.rs @@ -1,7 +1,7 @@ use bytes::Bytes; /// Extractor for a prefix from a byte string, used to build and probe -/// prefix-based bloom filters. +/// prefix-based bloom filters and segmented compaction. /// /// This trait is specific to `BloomFilterPolicy` — it is not part of the core /// `FilterPolicy`/`FilterBuilder`/`Filter` traits. Custom filter policies that diff --git a/website/src/content/docs/docs/design/segmented-compaction.mdx b/website/src/content/docs/docs/design/segmented-compaction.mdx index 361a6183f..eb347e6a7 100644 --- a/website/src/content/docs/docs/design/segmented-compaction.mdx +++ b/website/src/content/docs/docs/design/segmented-compaction.mdx @@ -158,14 +158,47 @@ every key to the segment named by its first three bytes. :::caution -The extractor is fixed for the life of the database. Its `name()` is persisted -in the manifest; opening with a different (or newly-added/removed) extractor -fails. +Whether segmentation is enabled is fixed for the life of the database. The +extractor's `name()` is persisted in the manifest; opening with a different +name, or adding or removing an extractor, fails. This is primarily as a soft +check against accidental misuse. ::: List the segments that currently exist (in the manifest or in memtables) with [`DbStatus::list_segments()`](https://docs.rs/slatedb/latest/slatedb/struct.DbStatus.html#method.list_segments). +## Evolving the key schema + +An extractor's name is fixed, but its implementation may evolve +when the change is backward-compatible. For example, a schema byte can +reserve disjoint key ranges for successive segmentation policies: + +```text +# all records below use a schema marker as the first part of the key prefix + +# initially use weekly partitions +[01][week]... -> [01][week] # v1: weekly segments +# later migrate to daily partitions +[02][day]... -> [02][day] # v2: daily segments +``` + +The new implementation must keep extracting the original weekly prefixes for +all v1 keys while adding daily prefixes only for v2 keys. In general: + +- Keep the same stable name and preserve the behavior of previous PrefixExtractor + implementations. +- Ensure segment prefixes from all versions form an antichain: no + prefix may be a prefix of another (a schema discriminator at the + start of the key is a simple way to reserve disjoint ranges). + +SlateDB checks the name and, on open, asks the configured extractor to +recognize every persisted segment prefix. It also rejects new writes that +would introduce a nested prefix. These are guardrails rather than complete +compatibility verification: SlateDB does not persist the implementation's +version or revalidate every existing key. Maintaining backward-compatible +routing under a stable name is therefore the application's responsibility. + + ## How it compacts Each segment is compacted on its own schedule. The default size-tiered From 593c62e3f7acfb81372e48a627529c91dc521ada Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 13 Jul 2026 15:35:24 -0700 Subject: [PATCH 13/63] Regenerate Go bindings (#1922) --- bindings/go/uniffi/slatedb.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/bindings/go/uniffi/slatedb.go b/bindings/go/uniffi/slatedb.go index 3c2fdece8..27aa2afd5 100644 --- a/bindings/go/uniffi/slatedb.go +++ b/bindings/go/uniffi/slatedb.go @@ -682,7 +682,7 @@ func uniffiCheckChecksums() { checksum := rustCall(func(_uniffiStatus *C.RustCallStatus) C.uint16_t { return C.uniffi_slatedb_uniffi_checksum_method_dbbuilder_with_segment_extractor() }) - if checksum != 14261 { + if checksum != 12566 { // If this happens try cleaning and rebuilding your project panic("slatedb: uniffi_slatedb_uniffi_checksum_method_dbbuilder_with_segment_extractor: UniFFI API checksum mismatch") } @@ -4225,7 +4225,10 @@ type DbBuilderInterface interface { // Sets the segment extractor (RFC-0024). When configured, every write is // routed through the extractor and the database tracks per-segment LSM // state. The extractor must be configured at database creation time and - // cannot be changed thereafter. + // remain configured thereafter. Its name must remain stable; its + // implementation may evolve only if it preserves routing for all existing + // key schemas and keeps segment prefixes across schema versions an + // antichain (no prefix may be a proper prefix of another). WithSegmentExtractor(extractor PrefixExtractor) error // Applies a [`crate::Settings`] object to the builder. WithSettings(settings *Settings) error @@ -4361,7 +4364,10 @@ func (_self *DbBuilder) WithSeed(seed uint64) error { // Sets the segment extractor (RFC-0024). When configured, every write is // routed through the extractor and the database tracks per-segment LSM // state. The extractor must be configured at database creation time and -// cannot be changed thereafter. +// remain configured thereafter. Its name must remain stable; its +// implementation may evolve only if it preserves routing for all existing +// key schemas and keeps segment prefixes across schema versions an +// antichain (no prefix may be a proper prefix of another). func (_self *DbBuilder) WithSegmentExtractor(extractor PrefixExtractor) error { _pointer := _self.ffiObject.incrementPointer("*DbBuilder") defer _self.ffiObject.decrementPointer() @@ -7730,7 +7736,7 @@ func (_ FfiDestroyerObjectStore) Destroy(value *ObjectStore) { } // Application-provided prefix extractor used to configure prefix-based -// bloom filters. +// bloom filters and segmented compaction. type PrefixExtractor interface { // Stable identifier for this extractor's configuration. Included in the // bloom filter policy name so filters built with different extractors @@ -7742,7 +7748,7 @@ type PrefixExtractor interface { } // Application-provided prefix extractor used to configure prefix-based -// bloom filters. +// bloom filters and segmented compaction. type PrefixExtractorImpl struct { ffiObject FfiObject } From 4ff907a48d835084600d189ea3ac47bc60be52ad Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 14 Jul 2026 10:31:51 -0700 Subject: [PATCH 14/63] Add `DbReaderMode` and latest manifest polling (#1915) --- bindings/go/uniffi/doc.go | 6 +- bindings/go/uniffi/slatedb.go | 126 +++- bindings/go/uniffi/slatedb.h | 22 +- bindings/go/uniffi/slatedb_test.go | 10 +- .../io/slatedb/uniffi/SlateDbReaderTest.java | 6 +- bindings/node/tests/reader.test.mjs | 5 +- bindings/python/tests/test_reader.py | 7 +- bindings/uniffi/src/builder.rs | 12 +- bindings/uniffi/src/config.rs | 26 + bindings/uniffi/src/lib.rs | 4 +- slatedb-cli/README.md | 2 +- slatedb-cli/src/args.rs | 7 +- slatedb-cli/src/scan.rs | 7 +- slatedb/src/admin.rs | 6 +- slatedb/src/clone.rs | 2 +- slatedb/src/config.rs | 18 +- slatedb/src/db/builder.rs | 23 +- slatedb/src/db_reader.rs | 682 +++++++++++------- slatedb/src/lib.rs | 2 +- .../content/docs/docs/design/checkpoints.mdx | 8 +- 20 files changed, 649 insertions(+), 332 deletions(-) diff --git a/bindings/go/uniffi/doc.go b/bindings/go/uniffi/doc.go index 284c11c6c..a3a5daad7 100644 --- a/bindings/go/uniffi/doc.go +++ b/bindings/go/uniffi/doc.go @@ -87,9 +87,9 @@ // insertion, and scan fetch parallelism. // // For long-lived read-only access, open a [DbReader] with -// [NewDbReaderBuilder]. A reader can be pinned to an existing checkpoint with -// [DbReaderBuilder.WithCheckpointId], configured with [ReaderOptions], and -// given a [MergeOperator] for merge-aware reads. +// [NewDbReaderBuilder]. A reader's state selection can be configured with +// [DbReaderBuilder.WithReaderMode] and [ReaderMode]. It can also be configured +// with [ReaderOptions] and given a [MergeOperator] for merge-aware reads. // // [Db.Snapshot] creates a consistent read-only [DbSnapshot] from a writable // database handle. diff --git a/bindings/go/uniffi/slatedb.go b/bindings/go/uniffi/slatedb.go index 27aa2afd5..150e08a8c 100644 --- a/bindings/go/uniffi/slatedb.go +++ b/bindings/go/uniffi/slatedb.go @@ -723,15 +723,6 @@ func uniffiCheckChecksums() { panic("slatedb: uniffi_slatedb_uniffi_checksum_method_dbreaderbuilder_build: UniFFI API checksum mismatch") } } - { - checksum := rustCall(func(_uniffiStatus *C.RustCallStatus) C.uint16_t { - return C.uniffi_slatedb_uniffi_checksum_method_dbreaderbuilder_with_checkpoint_id() - }) - if checksum != 41016 { - // If this happens try cleaning and rebuilding your project - panic("slatedb: uniffi_slatedb_uniffi_checksum_method_dbreaderbuilder_with_checkpoint_id: UniFFI API checksum mismatch") - } - } { checksum := rustCall(func(_uniffiStatus *C.RustCallStatus) C.uint16_t { return C.uniffi_slatedb_uniffi_checksum_method_dbreaderbuilder_with_filter_policies() @@ -768,6 +759,15 @@ func uniffiCheckChecksums() { panic("slatedb: uniffi_slatedb_uniffi_checksum_method_dbreaderbuilder_with_options: UniFFI API checksum mismatch") } } + { + checksum := rustCall(func(_uniffiStatus *C.RustCallStatus) C.uint16_t { + return C.uniffi_slatedb_uniffi_checksum_method_dbreaderbuilder_with_reader_mode() + }) + if checksum != 45455 { + // If this happens try cleaning and rebuilding your project + panic("slatedb: uniffi_slatedb_uniffi_checksum_method_dbreaderbuilder_with_reader_mode: UniFFI API checksum mismatch") + } + } { checksum := rustCall(func(_uniffiStatus *C.RustCallStatus) C.uint16_t { return C.uniffi_slatedb_uniffi_checksum_method_dbreaderbuilder_with_segment_extractor() @@ -5206,8 +5206,6 @@ func (_ FfiDestroyerDbReader) Destroy(value *DbReader) { type DbReaderBuilderInterface interface { // Opens the reader and consumes this builder. Build() (*DbReader, error) - // Pins the reader to an existing checkpoint UUID string. - WithCheckpointId(checkpointId string) error // Sets the filter policies used when decoding SST filter blocks. // // Must match (or be a superset of) the writer's policies so SST filter @@ -5220,6 +5218,8 @@ type DbReaderBuilderInterface interface { WithMetricsRecorder(metricsRecorder MetricsRecorder) error // Applies custom reader options. WithOptions(options ReaderOptions) error + // Sets how the reader chooses and refreshes database state. + WithReaderMode(mode ReaderMode) error // Sets the segment extractor (RFC-0024). A reader opening a segmented // database must configure an extractor matching the one the database // was created with. @@ -5276,18 +5276,6 @@ func (_self *DbReaderBuilder) Build() (*DbReader, error) { return res, err } -// Pins the reader to an existing checkpoint UUID string. -func (_self *DbReaderBuilder) WithCheckpointId(checkpointId string) error { - _pointer := _self.ffiObject.incrementPointer("*DbReaderBuilder") - defer _self.ffiObject.decrementPointer() - _, _uniffiErr := rustCallWithError[*Error](FfiConverterError{}, func(_uniffiStatus *C.RustCallStatus) bool { - C.uniffi_slatedb_uniffi_fn_method_dbreaderbuilder_with_checkpoint_id( - _pointer, FfiConverterStringINSTANCE.Lower(checkpointId), _uniffiStatus) - return false - }) - return _uniffiErr.AsError() -} - // Sets the filter policies used when decoding SST filter blocks. // // Must match (or be a superset of) the writer's policies so SST filter @@ -5340,6 +5328,18 @@ func (_self *DbReaderBuilder) WithOptions(options ReaderOptions) error { return _uniffiErr.AsError() } +// Sets how the reader chooses and refreshes database state. +func (_self *DbReaderBuilder) WithReaderMode(mode ReaderMode) error { + _pointer := _self.ffiObject.incrementPointer("*DbReaderBuilder") + defer _self.ffiObject.decrementPointer() + _, _uniffiErr := rustCallWithError[*Error](FfiConverterError{}, func(_uniffiStatus *C.RustCallStatus) bool { + C.uniffi_slatedb_uniffi_fn_method_dbreaderbuilder_with_reader_mode( + _pointer, FfiConverterReaderModeINSTANCE.Lower(mode), _uniffiStatus) + return false + }) + return _uniffiErr.AsError() +} + // Sets the segment extractor (RFC-0024). A reader opening a segmented // database must configure an extractor matching the one the database // was created with. @@ -12256,6 +12256,86 @@ func (_ FfiDestroyerPrefixTarget) Destroy(value PrefixTarget) { value.Destroy() } +// Determines how a [`crate::DbReader`] chooses and refreshes database state. +type ReaderMode interface { + Destroy() +} + +// Create and maintain checkpoints while following the latest database state. +type ReaderModeManagedCheckpoint struct { +} + +func (e ReaderModeManagedCheckpoint) Destroy() { +} + +// Remain pinned to the database state referenced by the supplied checkpoint UUID string. +type ReaderModeCheckpoint struct { + Field0 string +} + +func (e ReaderModeCheckpoint) Destroy() { + FfiDestroyerString{}.Destroy(e.Field0) +} + +// Follow the latest manifest without creating or maintaining a checkpoint. +type ReaderModeFollowLatest struct { +} + +func (e ReaderModeFollowLatest) Destroy() { +} + +type FfiConverterReaderMode struct{} + +var FfiConverterReaderModeINSTANCE = FfiConverterReaderMode{} + +func (c FfiConverterReaderMode) Lift(rb RustBufferI) ReaderMode { + return LiftFromRustBuffer[ReaderMode](c, rb) +} + +func (c FfiConverterReaderMode) Lower(value ReaderMode) C.RustBuffer { + return LowerIntoRustBuffer[ReaderMode](c, value) +} + +func (c FfiConverterReaderMode) LowerExternal(value ReaderMode) ExternalCRustBuffer { + return RustBufferFromC(LowerIntoRustBuffer[ReaderMode](c, value)) +} +func (FfiConverterReaderMode) Read(reader io.Reader) ReaderMode { + id := readInt32(reader) + switch id { + case 1: + return ReaderModeManagedCheckpoint{} + case 2: + return ReaderModeCheckpoint{ + FfiConverterStringINSTANCE.Read(reader), + } + case 3: + return ReaderModeFollowLatest{} + default: + panic(fmt.Sprintf("invalid enum value %v in FfiConverterReaderMode.Read()", id)) + } +} + +func (FfiConverterReaderMode) Write(writer io.Writer, value ReaderMode) { + switch variant_value := value.(type) { + case ReaderModeManagedCheckpoint: + writeInt32(writer, 1) + case ReaderModeCheckpoint: + writeInt32(writer, 2) + FfiConverterStringINSTANCE.Write(writer, variant_value.Field0) + case ReaderModeFollowLatest: + writeInt32(writer, 3) + default: + _ = variant_value + panic(fmt.Sprintf("invalid enum value `%v` in FfiConverterReaderMode.Write", value)) + } +} + +type FfiDestroyerReaderMode struct{} + +func (_ FfiDestroyerReaderMode) Destroy(value ReaderMode) { + value.Destroy() +} + // Kind of row entry stored in WAL iteration results. type RowEntryKind uint diff --git a/bindings/go/uniffi/slatedb.h b/bindings/go/uniffi/slatedb.h index 3e71c1dcd..27f6de14c 100644 --- a/bindings/go/uniffi/slatedb.h +++ b/bindings/go/uniffi/slatedb.h @@ -864,11 +864,6 @@ uint64_t uniffi_slatedb_uniffi_fn_constructor_dbreaderbuilder_new(RustBuffer pat uint64_t uniffi_slatedb_uniffi_fn_method_dbreaderbuilder_build(uint64_t ptr ); #endif -#ifndef UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_FN_METHOD_DBREADERBUILDER_WITH_CHECKPOINT_ID -#define UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_FN_METHOD_DBREADERBUILDER_WITH_CHECKPOINT_ID -void uniffi_slatedb_uniffi_fn_method_dbreaderbuilder_with_checkpoint_id(uint64_t ptr, RustBuffer checkpoint_id, RustCallStatus *out_status -); -#endif #ifndef UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_FN_METHOD_DBREADERBUILDER_WITH_FILTER_POLICIES #define UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_FN_METHOD_DBREADERBUILDER_WITH_FILTER_POLICIES void uniffi_slatedb_uniffi_fn_method_dbreaderbuilder_with_filter_policies(uint64_t ptr, RustBuffer policies, RustCallStatus *out_status @@ -889,6 +884,11 @@ void uniffi_slatedb_uniffi_fn_method_dbreaderbuilder_with_metrics_recorder(uint6 void uniffi_slatedb_uniffi_fn_method_dbreaderbuilder_with_options(uint64_t ptr, RustBuffer options, RustCallStatus *out_status ); #endif +#ifndef UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_FN_METHOD_DBREADERBUILDER_WITH_READER_MODE +#define UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_FN_METHOD_DBREADERBUILDER_WITH_READER_MODE +void uniffi_slatedb_uniffi_fn_method_dbreaderbuilder_with_reader_mode(uint64_t ptr, RustBuffer mode, RustCallStatus *out_status +); +#endif #ifndef UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_FN_METHOD_DBREADERBUILDER_WITH_SEGMENT_EXTRACTOR #define UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_FN_METHOD_DBREADERBUILDER_WITH_SEGMENT_EXTRACTOR void uniffi_slatedb_uniffi_fn_method_dbreaderbuilder_with_segment_extractor(uint64_t ptr, uint64_t extractor, RustCallStatus *out_status @@ -2239,12 +2239,6 @@ uint16_t uniffi_slatedb_uniffi_checksum_method_dbbuilder_with_wal_object_store(v #define UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_CHECKSUM_METHOD_DBREADERBUILDER_BUILD uint16_t uniffi_slatedb_uniffi_checksum_method_dbreaderbuilder_build(void -); -#endif -#ifndef UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_CHECKSUM_METHOD_DBREADERBUILDER_WITH_CHECKPOINT_ID -#define UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_CHECKSUM_METHOD_DBREADERBUILDER_WITH_CHECKPOINT_ID -uint16_t uniffi_slatedb_uniffi_checksum_method_dbreaderbuilder_with_checkpoint_id(void - ); #endif #ifndef UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_CHECKSUM_METHOD_DBREADERBUILDER_WITH_FILTER_POLICIES @@ -2269,6 +2263,12 @@ uint16_t uniffi_slatedb_uniffi_checksum_method_dbreaderbuilder_with_metrics_reco #define UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_CHECKSUM_METHOD_DBREADERBUILDER_WITH_OPTIONS uint16_t uniffi_slatedb_uniffi_checksum_method_dbreaderbuilder_with_options(void +); +#endif +#ifndef UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_CHECKSUM_METHOD_DBREADERBUILDER_WITH_READER_MODE +#define UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_CHECKSUM_METHOD_DBREADERBUILDER_WITH_READER_MODE +uint16_t uniffi_slatedb_uniffi_checksum_method_dbreaderbuilder_with_reader_mode(void + ); #endif #ifndef UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_CHECKSUM_METHOD_DBREADERBUILDER_WITH_SEGMENT_EXTRACTOR diff --git a/bindings/go/uniffi/slatedb_test.go b/bindings/go/uniffi/slatedb_test.go index 6ec3c87c1..384ec5164 100644 --- a/bindings/go/uniffi/slatedb_test.go +++ b/bindings/go/uniffi/slatedb_test.go @@ -1531,9 +1531,9 @@ func TestDbReaderBuilderValidationAndErrors(t *testing.T) { builder := slatedb.NewDbReaderBuilder(testDBPath, store) defer builder.Destroy() - err := builder.WithCheckpointId("not-a-uuid") + err := builder.WithReaderMode(slatedb.ReaderModeCheckpoint{Field0: "not-a-uuid"}) if !errors.Is(err, slatedb.ErrErrorInvalid) { - t.Fatalf("DbReaderBuilder.WithCheckpointId(invalid): got %v, want invalid error", err) + t.Fatalf("DbReaderBuilder.WithReaderMode(invalid checkpoint): got %v, want invalid error", err) } }) @@ -1550,8 +1550,10 @@ func TestDbReaderBuilderValidationAndErrors(t *testing.T) { builder := slatedb.NewDbReaderBuilder(testDBPath, store) defer builder.Destroy() - if err := builder.WithCheckpointId("ffffffff-ffff-ffff-ffff-ffffffffffff"); err != nil { - t.Fatalf("DbReaderBuilder.WithCheckpointId(valid): %v", err) + if err := builder.WithReaderMode(slatedb.ReaderModeCheckpoint{ + Field0: "ffffffff-ffff-ffff-ffff-ffffffffffff", + }); err != nil { + t.Fatalf("DbReaderBuilder.WithReaderMode(checkpoint): %v", err) } _, err := builder.Build() diff --git a/bindings/java/slatedb-uniffi/src/test/java/io/slatedb/uniffi/SlateDbReaderTest.java b/bindings/java/slatedb-uniffi/src/test/java/io/slatedb/uniffi/SlateDbReaderTest.java index 066b7ff90..dfc385373 100644 --- a/bindings/java/slatedb-uniffi/src/test/java/io/slatedb/uniffi/SlateDbReaderTest.java +++ b/bindings/java/slatedb-uniffi/src/test/java/io/slatedb/uniffi/SlateDbReaderTest.java @@ -266,7 +266,9 @@ void readerRejectsInvalidCheckpointId() throws Exception { TestSupport.await(dbHandle.db().put(TestSupport.bytes("seed"), TestSupport.bytes("value"))); TestSupport.await(dbHandle.db().flushWithOptions(new FlushOptions(FlushType.MEM_TABLE))); - TestSupport.expectFailure(Error.Invalid.class, () -> builder.withCheckpointId("not-a-uuid")); + TestSupport.expectFailure( + Error.Invalid.class, + () -> builder.withReaderMode(new ReaderMode.Checkpoint("not-a-uuid"))); } } @@ -278,7 +280,7 @@ void readerMissingCheckpointIdFailsBuild() throws Exception { TestSupport.await(dbHandle.db().put(TestSupport.bytes("seed"), TestSupport.bytes("value"))); TestSupport.await(dbHandle.db().flushWithOptions(new FlushOptions(FlushType.MEM_TABLE))); - builder.withCheckpointId("ffffffff-ffff-ffff-ffff-ffffffffffff"); + builder.withReaderMode(new ReaderMode.Checkpoint("ffffffff-ffff-ffff-ffff-ffffffffffff")); TestSupport.awaitFailure(Error.Data.class, builder.build()); } } diff --git a/bindings/node/tests/reader.test.mjs b/bindings/node/tests/reader.test.mjs index a06b71934..ca0192841 100644 --- a/bindings/node/tests/reader.test.mjs +++ b/bindings/node/tests/reader.test.mjs @@ -8,6 +8,7 @@ import { DbReaderBuilder, ErrorData, FlushType, + ReaderMode, SsTableId, WriteBatch, } from "../index.js"; @@ -320,13 +321,13 @@ test("reader builder validation and errors", async (t) => { const invalidBuilder = cleanup.track(new DbReaderBuilder(TEST_DB_PATH, store), { shutdown: false }); const invalidCheckpointError = await expectInvalid( - () => invalidBuilder.with_checkpoint_id("not-a-uuid"), + () => invalidBuilder.with_reader_mode(ReaderMode.Checkpoint("not-a-uuid")), ); assert.match(invalidCheckpointError.message, /^invalid checkpoint_id UUID:/); const missingCheckpointId = "ffffffff-ffff-ffff-ffff-ffffffffffff"; const missingBuilder = cleanup.track(new DbReaderBuilder(TEST_DB_PATH, store), { shutdown: false }); - missingBuilder.with_checkpoint_id(missingCheckpointId); + missingBuilder.with_reader_mode(ReaderMode.Checkpoint(missingCheckpointId)); const missingCheckpointError = await expectError( () => missingBuilder.build(), ErrorData, diff --git a/bindings/python/tests/test_reader.py b/bindings/python/tests/test_reader.py index d8bf466a4..7142882f2 100644 --- a/bindings/python/tests/test_reader.py +++ b/bindings/python/tests/test_reader.py @@ -22,6 +22,7 @@ FlushOptions, FlushType, KeyRange, + ReaderMode, ) @@ -267,11 +268,13 @@ async def test_reader_builder_validation_and_errors() -> None: invalid_builder = DbReaderBuilder(TEST_DB_PATH, store) with pytest.raises(Error.Invalid) as exc: - invalid_builder.with_checkpoint_id("not-a-uuid") + invalid_builder.with_reader_mode(ReaderMode.CHECKPOINT("not-a-uuid")) assert exc.value.message.startswith("invalid checkpoint_id UUID:") missing_builder = DbReaderBuilder(TEST_DB_PATH, store) - missing_builder.with_checkpoint_id("ffffffff-ffff-ffff-ffff-ffffffffffff") + missing_builder.with_reader_mode( + ReaderMode.CHECKPOINT("ffffffff-ffff-ffff-ffff-ffffffffffff") + ) with pytest.raises(Error.Data) as exc: await missing_builder.build() assert "checkpoint missing" in exc.value.message diff --git a/bindings/uniffi/src/builder.rs b/bindings/uniffi/src/builder.rs index aa04414d4..51acfc349 100644 --- a/bindings/uniffi/src/builder.rs +++ b/bindings/uniffi/src/builder.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use crate::admin::Admin; -use crate::config::{ReaderOptions, SstBlockSize}; +use crate::config::{ReaderMode, ReaderOptions, SstBlockSize}; use crate::db::Db; use crate::db_cache::DbCache; use crate::db_reader::DbReader; @@ -17,7 +17,6 @@ use crate::settings::Settings; use crate::types::{CloneSourceSpec, KeyRange}; use crate::MetricsRecorder; use parking_lot::Mutex; -use uuid::Uuid; /// Builder for opening a writable [`crate::Db`]. /// @@ -184,11 +183,10 @@ impl DbReaderBuilder { }) } - /// Pins the reader to an existing checkpoint UUID string. - pub fn with_checkpoint_id(&self, checkpoint_id: String) -> Result<(), Error> { - let checkpoint_id = Uuid::parse_str(&checkpoint_id) - .map_err(|source| SlateDbError::InvalidCheckpointId { source })?; - self.update_builder(|builder| builder.with_checkpoint_id(checkpoint_id)) + /// Sets how the reader chooses and refreshes database state. + pub fn with_reader_mode(&self, mode: ReaderMode) -> Result<(), Error> { + let mode = mode.try_into()?; + self.update_builder(|builder| builder.with_reader_mode(mode)) .map_err(Into::into) } diff --git a/bindings/uniffi/src/config.rs b/bindings/uniffi/src/config.rs index 97a287f8c..86e7f219d 100644 --- a/bindings/uniffi/src/config.rs +++ b/bindings/uniffi/src/config.rs @@ -157,6 +157,32 @@ impl From for slatedb::config::ReadOptions { } } +/// Determines how a [`crate::DbReader`] chooses and refreshes database state. +#[derive(Clone, Debug, Default, uniffi::Enum)] +pub enum ReaderMode { + /// Create and maintain checkpoints while following the latest database state. + #[default] + ManagedCheckpoint, + /// Remain pinned to the database state referenced by the supplied checkpoint UUID string. + Checkpoint(String), + /// Follow the latest manifest without creating or maintaining a checkpoint. + FollowLatest, +} + +impl TryFrom for slatedb::DbReaderMode { + type Error = Error; + + fn try_from(value: ReaderMode) -> Result { + Ok(match value { + ReaderMode::ManagedCheckpoint => Self::ManagedCheckpoint, + ReaderMode::Checkpoint(checkpoint_id) => { + Self::Checkpoint(try_checkpoint_id_from_str(&checkpoint_id)?) + } + ReaderMode::FollowLatest => Self::FollowLatest, + }) + } +} + /// Options for opening a [`crate::DbReader`]. #[derive(Clone, Debug, uniffi::Record)] pub struct ReaderOptions { diff --git a/bindings/uniffi/src/lib.rs b/bindings/uniffi/src/lib.rs index 3aeaa9e82..310e4bc94 100644 --- a/bindings/uniffi/src/lib.rs +++ b/bindings/uniffi/src/lib.rs @@ -25,8 +25,8 @@ pub use builder::{AdminBuilder, CloneBuilder, DbBuilder, DbReaderBuilder}; pub use config::{ DurabilityLevel, FlushOptions, FlushType, GarbageCollectorDirectoryOptions, GarbageCollectorOptions, GarbageCollectorScheduleOptions, IsolationLevel, IterationOrder, - MergeOptions, PutOptions, ReadOptions, ReaderOptions, ScanOptions, SstBlockSize, Ttl, - WriteOptions, + MergeOptions, PutOptions, ReadOptions, ReaderMode, ReaderOptions, ScanOptions, SstBlockSize, + Ttl, WriteOptions, }; pub use db::Db; pub use db_reader::DbReader; diff --git a/slatedb-cli/README.md b/slatedb-cli/README.md index 158589a7f..5c9a85606 100644 --- a/slatedb-cli/README.md +++ b/slatedb-cli/README.md @@ -119,7 +119,7 @@ Options: - `--value `: How values are rendered. `none` prints keys only. Default `auto`. - `--max-keys `: Stop after emitting `N` entries. - `--count`: Print only ` entries, bytes` instead of the entries themselves. When combined with `--max-keys`, only the entries up to the cap are counted. -- `--checkpoint `: Scan an existing checkpoint for a point-in-time scan that needs only read access to the store. Without it, the reader writes a transient checkpoint, so it needs write access to the store. +- `--checkpoint `: Scan an existing checkpoint for a point-in-time view protected from garbage collection. Without it, the reader follows the latest manifest without writing a checkpoint, so concurrent garbage collection may delete objects referenced by the scan. Encoding rules: - `--key`/`--value` control both how the bound arguments are parsed and how keys/values are rendered. In `hex`, bound arguments are hex digits with an optional `0x` prefix. In `utf8`, they are taken literally. In `auto`, a bound is hex-decoded when it starts with `0x`/`0X` and taken literally otherwise. diff --git a/slatedb-cli/src/args.rs b/slatedb-cli/src/args.rs index c22208cc3..838a7fd35 100644 --- a/slatedb-cli/src/args.rs +++ b/slatedb-cli/src/args.rs @@ -177,9 +177,10 @@ pub(crate) enum CliCommands { #[arg(long)] count: bool, - /// Scan an existing checkpoint by its UUID, rather than the database's current - /// state. This needs only read access to the store; without it the reader writes - /// a transient checkpoint and so needs write access. + /// Scan an existing checkpoint by its UUID, rather than following the latest + /// manifest. This provides a point-in-time view protected from garbage collection. + /// Without it, the scan remains read-only but concurrent garbage collection may + /// delete objects referenced by the scan. #[arg(long)] #[clap(value_parser = uuid::Uuid::parse_str)] checkpoint: Option, diff --git a/slatedb-cli/src/scan.rs b/slatedb-cli/src/scan.rs index 503defdf1..1ee6ad6e7 100644 --- a/slatedb-cli/src/scan.rs +++ b/slatedb-cli/src/scan.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use object_store::path::Path; use object_store::ObjectStore; use slatedb::config::ScanOptions; -use slatedb::DbReader; +use slatedb::{DbReader, DbReaderMode}; use uuid::Uuid; type KeyRange = (Bound>, Bound>); @@ -58,9 +58,10 @@ pub(crate) async fn exec_scan( .transpose()?; let range = build_range(from.as_deref(), to.as_deref(), key_mode)?; - let mut builder = DbReader::builder(path, object_store); + let mut builder = + DbReader::builder(path, object_store).with_reader_mode(DbReaderMode::FollowLatest); if let Some(checkpoint_id) = checkpoint { - builder = builder.with_checkpoint_id(checkpoint_id); + builder = builder.with_reader_mode(DbReaderMode::Checkpoint(checkpoint_id)); } let reader = builder.build().await?; diff --git a/slatedb/src/admin.rs b/slatedb/src/admin.rs index 6fbd5833b..5fb4c3d2a 100644 --- a/slatedb/src/admin.rs +++ b/slatedb/src/admin.rs @@ -451,9 +451,9 @@ impl Admin { /// If you have a [`crate::Db`] instance open, you can use the [`crate::Db::create_checkpoint`] /// method instead. That method will flush the memtables and WALs before creating the checkpoint. /// - /// If you're using a [`crate::DbReader`], you might wish to have the reader manage the checkpoint - /// for you by calling [`crate::DbReader::open`] with no `checkpoint_id` set. The reader will - /// create a checkpoint for you and periodically refresh it. + /// If you're using a [`crate::DbReader`], you might wish to use + /// [`crate::DbReaderMode::ManagedCheckpoint`]. The reader will create a checkpoint for you and + /// periodically refresh it. /// /// # Examples /// diff --git a/slatedb/src/clone.rs b/slatedb/src/clone.rs index 96d9ac72f..b14c5e5da 100644 --- a/slatedb/src/clone.rs +++ b/slatedb/src/clone.rs @@ -722,7 +722,7 @@ mod tests { // A reader pinned to the checkpoint must resolve the external SSTs // referenced by the checkpoint's manifest. let reader = DbReader::builder(clone_path.clone(), object_store.clone()) - .with_checkpoint_id(checkpoint_id) + .with_reader_mode(crate::DbReaderMode::Checkpoint(checkpoint_id)) .build() .await .unwrap(); diff --git a/slatedb/src/config.rs b/slatedb/src/config.rs index a3989769c..62e3108bf 100644 --- a/slatedb/src/config.rs +++ b/slatedb/src/config.rs @@ -1007,14 +1007,14 @@ impl Default for Settings { pub struct DbReaderOptions { /// How frequently to poll for new manifest files and WAL data. Refreshing the manifest /// file allows readers to detect newly compacted data. The reader will also look for - /// new writes to the WAL at this poll interval. If the reader is using an explicit checkpoint, - /// then the manifest and WAL will not be polled. + /// new writes to the WAL at this poll interval. Readers using + /// [`crate::DbReaderMode::Checkpoint`] do not poll the manifest or WAL. pub manifest_poll_interval: Duration, - /// For readers that do not provide an explicit checkpoint, the client will - /// maintain its own checkpoint against the latest database state. The checkpoint's - /// expire time will be set to the current time plus this value. This lifetime - /// must always be greater than manifest_poll_interval x 2. + /// For readers using [`crate::DbReaderMode::ManagedCheckpoint`], the client maintains a + /// checkpoint against the latest database state. The checkpoint's expire time is set to the + /// current time plus this value. This lifetime must always be greater than + /// `manifest_poll_interval * 2`. This option is ignored by other reader modes. pub checkpoint_lifetime: Duration, /// The max size of a single in-memory table used to buffer WAL entries @@ -1031,10 +1031,10 @@ pub struct DbReaderOptions { /// don't need to see the most recent uncommitted writes and want to minimize the /// cost of opening many readers. /// - /// WAL replay is also skipped when the reader is opened from a checkpoint. + /// WAL replay is also skipped in [`crate::DbReaderMode::Checkpoint`] mode. /// - /// When combined with manifest polling (no explicit checkpoint), the reader will - /// still see newly compacted data as manifests are updated. + /// When combined with a reader mode that polls manifests, the reader will still see newly + /// compacted data as manifests are updated. /// /// Defaults to false. pub skip_wal_replay: bool, diff --git a/slatedb/src/db/builder.rs b/slatedb/src/db/builder.rs index 6fc621628..f618e6742 100644 --- a/slatedb/src/db/builder.rs +++ b/slatedb/src/db/builder.rs @@ -139,7 +139,7 @@ use crate::db::Db; use crate::db::DbInner; use crate::db_cache::SplitCache; use crate::db_cache::{DbCache, DbCacheWrapper, UnownedDbCache}; -use crate::db_reader::DbReader; +use crate::db_reader::{DbReader, DbReaderMode}; use crate::db_status::{ClosedResultWriter, DbStatusManager}; use crate::dispatcher::MessageHandlerExecutor; use crate::error::SlateDBError; @@ -1587,7 +1587,7 @@ pub struct DbReaderBuilder> { object_store: Arc, wal_object_store: Option>, db_cache: Option>, - checkpoint_id: Option, + mode: DbReaderMode, merge_operator: Option, block_transformer: Option>, filter_policies: Vec>, @@ -1606,7 +1606,7 @@ impl> DbReaderBuilder

{ object_store, wal_object_store: None, db_cache: default_db_cache(), - checkpoint_id: None, + mode: DbReaderMode::default(), merge_operator: None, block_transformer: None, filter_policies: default_filter_policies(), @@ -1618,10 +1618,9 @@ impl> DbReaderBuilder

{ } } - /// Sets the checkpoint ID to use for the reader. - /// If not set, the reader will create and manage its own checkpoint. - pub fn with_checkpoint_id(mut self, checkpoint_id: uuid::Uuid) -> Self { - self.checkpoint_id = Some(checkpoint_id); + /// Sets how the reader chooses and refreshes database state. + pub fn with_reader_mode(mut self, mode: DbReaderMode) -> Self { + self.mode = mode; self } @@ -1792,8 +1791,8 @@ impl> DbReaderBuilder

{ // read from: the pinned checkpoint's manifest when a checkpoint id is // given (compaction may have pruned re-localized external SSTs from // the latest manifest), and the latest manifest otherwise. - let external_ssts = match (&latest_manifest, self.checkpoint_id) { - (Some(latest_stored_manifest), Some(checkpoint_id)) => { + let external_ssts = match (&latest_manifest, self.mode) { + (Some(latest_stored_manifest), DbReaderMode::Checkpoint(checkpoint_id)) => { let checkpoint = latest_stored_manifest .db_state() .find_checkpoint(checkpoint_id) @@ -1803,9 +1802,7 @@ impl> DbReaderBuilder

{ .await? .external_ssts() } - (Some(latest_stored_manifest), None) => { - latest_stored_manifest.manifest().external_ssts() - } + (Some(latest_stored_manifest), _) => latest_stored_manifest.manifest().external_ssts(), (None, _) => HashMap::new(), }; @@ -1835,7 +1832,7 @@ impl> DbReaderBuilder

{ let reader = DbReader::open_internal( manifest_store, table_store, - self.checkpoint_id, + self.mode, self.merge_operator, self.segment_extractor, self.options, diff --git a/slatedb/src/db_reader.rs b/slatedb/src/db_reader.rs index 83b16b719..f588e5a21 100644 --- a/slatedb/src/db_reader.rs +++ b/slatedb/src/db_reader.rs @@ -43,6 +43,30 @@ use uuid::Uuid; pub(crate) const DB_READER_TASK_NAME: &str = "manifest_poller"; +/// Determines how a [`DbReader`] chooses and refreshes the database state it reads. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub enum DbReaderMode { + /// Create and maintain checkpoints while following the latest database state. + /// + /// The reader will automatically create a checkpoint and refresh it periodically to ensure + /// that the reader can continue to read the latest database state without being affected by + /// garbage collection. + #[default] + ManagedCheckpoint, + + /// Remain pinned to the database state referenced by the supplied checkpoint. + Checkpoint(Uuid), + + /// Follow the latest manifest without creating a checkpoint. + /// + /// This mode performs no object-store writes and provides no protection from garbage + /// collection. Reads using an older manifest may fail if referenced objects are deleted. + /// This mode is useful for read-only access to a database that is not being actively written + /// to, for mirrored databases where manifest changes might not be allowed, or for readers + /// that are willing to handle missing objects gracefully. + FollowLatest, +} + /// Read-only interface for accessing a database from either /// the latest persistent state or from an arbitrary checkpoint. pub struct DbReader { @@ -54,9 +78,9 @@ struct DbReaderInner { manifest_store: Arc, table_store: Arc, options: DbReaderOptions, - state: RwLock>, + mode: DbReaderMode, + state: RwLock>, system_clock: Arc, - user_checkpoint_id: Option, oracle: Arc, reader: Reader, status_manager: DbStatusManager, @@ -75,8 +99,9 @@ enum DbReaderMessage { } #[derive(Clone)] -struct CheckpointState { - checkpoint: Checkpoint, +struct ReaderState { + manifest_id: u64, + checkpoint: Option, manifest: Manifest, imm_memtable: VecDeque>, last_wal_id: u64, @@ -85,7 +110,7 @@ struct CheckpointState { static EMPTY_TABLE: LazyLock> = LazyLock::new(|| Arc::new(KVTable::new())); -impl DbStateReader for CheckpointState { +impl DbStateReader for ReaderState { fn memtable(&self) -> Arc { Arc::clone(&EMPTY_TABLE) } @@ -99,9 +124,9 @@ impl DbStateReader for CheckpointState { } } -impl From<&CheckpointState> for VersionedManifest { - fn from(state: &CheckpointState) -> Self { - Self::from_manifest(state.checkpoint.manifest_id, state.manifest.clone()) +impl From<&ReaderState> for VersionedManifest { + fn from(state: &ReaderState) -> Self { + Self::from_manifest(state.manifest_id, state.manifest.clone()) } } @@ -110,7 +135,7 @@ impl DbReaderInner { manifest_store: Arc, table_store: Arc, options: DbReaderOptions, - checkpoint_id: Option, + mode: DbReaderMode, merge_operator: Option, segment_extractor: Option>, system_clock: Arc, @@ -119,18 +144,27 @@ impl DbReaderInner { mut manifest: StoredManifest, ) -> Result { let checkpoint = - Self::get_or_create_checkpoint(&mut manifest, checkpoint_id, &options, rand.clone()) - .await?; - - let replay_new_wals = checkpoint_id.is_none() && !options.skip_wal_replay; + Self::get_or_create_checkpoint(&mut manifest, mode, &options, rand.clone()).await?; + let (manifest_id, initial_manifest) = if let Some(checkpoint) = checkpoint.as_ref() { + ( + checkpoint.manifest_id, + manifest_store.read_manifest(checkpoint.manifest_id).await?, + ) + } else { + (manifest.id(), manifest.manifest().clone()) + }; + let replay_new_wals = + !matches!(mode, DbReaderMode::Checkpoint(_)) && !options.skip_wal_replay; let initial_state = Arc::new( - Self::build_initial_checkpoint_state( - Arc::clone(&manifest_store), + Self::build_reader_state( + checkpoint, + manifest_id, + initial_manifest, + VecDeque::new(), + replay_new_wals, Arc::clone(&table_store), &options, segment_extractor.as_ref(), - checkpoint, - replay_new_wals, ) .await?, ); @@ -170,9 +204,9 @@ impl DbReaderInner { manifest_store, table_store, options, + mode, state, system_clock, - user_checkpoint_id: checkpoint_id, oracle, reader, status_manager, @@ -185,25 +219,32 @@ impl DbReaderInner { async fn get_or_create_checkpoint( manifest: &mut StoredManifest, - checkpoint_id: Option, + mode: DbReaderMode, options: &DbReaderOptions, rand: Arc, - ) -> Result { - let checkpoint = if let Some(checkpoint_id) = checkpoint_id { - manifest - .db_state() - .find_checkpoint(checkpoint_id) - .ok_or(SlateDBError::CheckpointMissing(checkpoint_id))? - .clone() - } else { - let options = CheckpointOptions { - lifetime: Some(options.checkpoint_lifetime), - ..CheckpointOptions::default() - }; - let checkpoint_id = rand.rng().gen_uuid(); - manifest.write_checkpoint(checkpoint_id, &options).await? - }; - Ok(checkpoint) + ) -> Result, SlateDBError> { + match mode { + DbReaderMode::Checkpoint(checkpoint_id) => Ok(Some( + manifest + .db_state() + .find_checkpoint(checkpoint_id) + .ok_or(SlateDBError::CheckpointMissing(checkpoint_id))? + .clone(), + )), + DbReaderMode::ManagedCheckpoint => { + let checkpoint_options = CheckpointOptions { + lifetime: Some(options.checkpoint_lifetime), + ..CheckpointOptions::default() + }; + let checkpoint_id = rand.rng().gen_uuid(); + Ok(Some( + manifest + .write_checkpoint(checkpoint_id, &checkpoint_options) + .await?, + )) + } + DbReaderMode::FollowLatest => Ok(None), + } } async fn get_with_options + Send>( @@ -268,7 +309,13 @@ impl DbReaderInner { &self, stored_manifest: &mut StoredManifest, ) -> Result { - let current_checkpoint_id = self.state.read().checkpoint.id; + let current_checkpoint_id = self + .state + .read() + .checkpoint + .as_ref() + .expect("managed reader must have a checkpoint") + .id; let options = CheckpointOptions { lifetime: Some(self.options.checkpoint_lifetime), ..CheckpointOptions::default() @@ -280,18 +327,21 @@ impl DbReaderInner { } async fn reestablish_checkpoint(&self, checkpoint: Checkpoint) -> Result<(), SlateDBError> { - let new_checkpoint_state = self.rebuild_checkpoint_state(checkpoint).await?; - let durable_seq = new_checkpoint_state.last_remote_persisted_seq; - let versioned_manifest = VersionedManifest::from(&new_checkpoint_state); + let new_state = self.rebuild_checkpoint_state(checkpoint).await?; + self.install_state(new_state); + Ok(()) + } + + fn install_state(&self, new_state: ReaderState) { + let durable_seq = new_state.last_remote_persisted_seq; + let versioned_manifest = VersionedManifest::from(&new_state); + let touched_segments = collect_touched_segments(&new_state); self.oracle.advance_durable_seq(durable_seq); let mut write_guard = self.state.write(); - *write_guard = Arc::new(new_checkpoint_state); + *write_guard = Arc::new(new_state); drop(write_guard); - self.status_manager.report_manifest_and_memtable_segments( - versioned_manifest, - collect_touched_segments(self.state.read().as_ref()), - ); - Ok(()) + self.status_manager + .report_manifest_and_memtable_segments(versioned_manifest, touched_segments); } async fn maybe_replay_new_wals(&self) -> Result<(), SlateDBError> { @@ -304,13 +354,13 @@ impl DbReaderInner { .last_seen_wal_id(last_replayed_wal_id) .await?; if last_seen_wal_id > last_replayed_wal_id { - let current_checkpoint = Arc::clone(&self.state.read()); - let mut imm_memtable = current_checkpoint.imm_memtable().clone(); + let current_state = Arc::clone(&self.state.read()); + let mut imm_memtable = current_state.imm_memtable().clone(); let (last_wal_id, last_committed_seq) = Self::replay_wal_into( Arc::clone(&self.table_store), &self.options, - current_checkpoint.core(), + current_state.core(), &mut imm_memtable, true, self.segment_extractor.as_ref(), @@ -319,9 +369,10 @@ impl DbReaderInner { self.oracle.advance_durable_seq(last_committed_seq); let mut write_guard = self.state.write(); - *write_guard = Arc::new(CheckpointState { - checkpoint: current_checkpoint.checkpoint.clone(), - manifest: current_checkpoint.manifest.clone(), + *write_guard = Arc::new(ReaderState { + manifest_id: current_state.manifest_id, + checkpoint: current_state.checkpoint.clone(), + manifest: current_state.manifest.clone(), imm_memtable, last_wal_id, last_remote_persisted_seq: last_committed_seq, @@ -333,37 +384,23 @@ impl DbReaderInner { Ok(()) } - async fn build_initial_checkpoint_state( - manifest_store: Arc, - table_store: Arc, - options: &DbReaderOptions, - segment_extractor: Option<&Arc>, - checkpoint: Checkpoint, - replay_new_wals: bool, - ) -> Result { - let manifest = manifest_store.read_manifest(checkpoint.manifest_id).await?; - let imm_memtable = VecDeque::new(); - Self::build_checkpoint_state( - checkpoint, - manifest, - imm_memtable, - replay_new_wals, - Arc::clone(&table_store), - options, - segment_extractor, - ) - .await - } - async fn rebuild_checkpoint_state( &self, new_checkpoint: Checkpoint, - ) -> Result { + ) -> Result { + let manifest_id = new_checkpoint.manifest_id; + let manifest = self.manifest_store.read_manifest(manifest_id).await?; + self.rebuild_state(Some(new_checkpoint), manifest_id, manifest) + .await + } + + async fn rebuild_state( + &self, + checkpoint: Option, + manifest_id: u64, + manifest: Manifest, + ) -> Result { let prior = self.state.read().clone(); - let manifest = self - .manifest_store - .read_manifest(new_checkpoint.manifest_id) - .await?; let mut imm_memtable = VecDeque::new(); for table in prior.imm_memtable.iter() { @@ -390,8 +427,9 @@ impl DbReaderInner { } } - Self::build_checkpoint_state( - new_checkpoint, + Self::build_reader_state( + checkpoint, + manifest_id, manifest, imm_memtable, !self.options.skip_wal_replay, @@ -402,15 +440,16 @@ impl DbReaderInner { .await } - async fn build_checkpoint_state( - checkpoint: Checkpoint, + async fn build_reader_state( + checkpoint: Option, + manifest_id: u64, manifest: Manifest, mut imm_memtable: VecDeque>, replay_new_wals: bool, table_store: Arc, options: &DbReaderOptions, segment_extractor: Option<&Arc>, - ) -> Result { + ) -> Result { let (last_wal_id, last_committed_seq) = Self::replay_wal_into( Arc::clone(&table_store), options, @@ -421,7 +460,8 @@ impl DbReaderInner { ) .await?; - Ok(CheckpointState { + Ok(ReaderState { + manifest_id, checkpoint, manifest, imm_memtable, @@ -430,11 +470,38 @@ impl DbReaderInner { }) } + async fn refresh_latest_manifest(&self) -> Result<(), SlateDBError> { + let latest_manifest = self.manifest_store.read_latest_manifest().await?; + self.apply_latest_manifest(latest_manifest).await + } + + async fn apply_latest_manifest( + &self, + latest_manifest: VersionedManifest, + ) -> Result<(), SlateDBError> { + let manifest_id = latest_manifest.id; + if manifest_id <= self.state.read().manifest_id { + return self.maybe_replay_new_wals().await; + } + + let new_state = self + .rebuild_state(None, manifest_id, latest_manifest.manifest) + .await?; + self.install_state(new_state); + info!("refreshed reader to latest manifest [manifest_id={manifest_id}]"); + Ok(()) + } + async fn maybe_refresh_checkpoint( &self, stored_manifest: &mut StoredManifest, ) -> Result<(), SlateDBError> { - let checkpoint = self.state.read().checkpoint.clone(); + let checkpoint = self + .state + .read() + .checkpoint + .clone() + .expect("managed reader must have a checkpoint"); let half_lifetime = self .options .checkpoint_lifetime @@ -450,12 +517,11 @@ impl DbReaderInner { .await { Ok(refreshed_checkpoint) => refreshed_checkpoint, - Err(SlateDBError::CheckpointMissing(id)) if self.user_checkpoint_id.is_none() => { + Err(SlateDBError::CheckpointMissing(id)) => { // Our self-established checkpoint lapsed (e.g. a stalled poll tick // outlived the lease during an object-store outage) and the writer's // GC reaped it. Re-establish a fresh checkpoint against the latest - // manifest instead of failing the reader permanently. A user-supplied - // checkpoint must still fail loud: the caller's pinned view is gone. + // manifest instead of failing the reader permanently. warn!("reader checkpoint missing, re-establishing [checkpoint_id={id}]"); let checkpoint = self.replace_checkpoint(stored_manifest).await?; self.reestablish_checkpoint(checkpoint).await?; @@ -470,11 +536,16 @@ impl DbReaderInner { let mut write_guard = self.state.write(); let current_state = write_guard.as_ref(); // Defensively, only update checkpoint if the id and expiry still match. - if current_state.checkpoint.id == checkpoint.id - && current_state.checkpoint.expire_time == checkpoint.expire_time + if current_state + .checkpoint + .as_ref() + .is_some_and(|current_checkpoint| { + current_checkpoint.id == checkpoint.id + && current_checkpoint.expire_time == checkpoint.expire_time + }) { let mut updated_state = current_state.clone(); - updated_state.checkpoint = refreshed_checkpoint.clone(); + updated_state.checkpoint = Some(refreshed_checkpoint.clone()); *write_guard = Arc::new(updated_state); } } @@ -649,24 +720,37 @@ impl MessageHandler for ManifestPoller { async fn handle(&mut self, message: DbReaderMessage) -> Result<(), SlateDBError> { assert!(matches!(message, DbReaderMessage::PollManifest)); - let mut manifest = StoredManifest::load( - Arc::clone(&self.inner.manifest_store), - self.inner.system_clock.clone(), - ) - .await?; + match self.inner.mode { + DbReaderMode::ManagedCheckpoint => { + let mut manifest = StoredManifest::load( + Arc::clone(&self.inner.manifest_store), + self.inner.system_clock.clone(), + ) + .await?; - let latest_manifest = manifest.manifest(); - if self - .inner - .should_reestablish_checkpoint(&latest_manifest.core) - { - let checkpoint = self.inner.replace_checkpoint(&mut manifest).await?; - self.inner.reestablish_checkpoint(checkpoint).await?; - } else { - self.inner.maybe_replay_new_wals().await?; - } + let latest_manifest = manifest.manifest(); + if self + .inner + .should_reestablish_checkpoint(&latest_manifest.core) + { + let checkpoint = self.inner.replace_checkpoint(&mut manifest).await?; + self.inner.reestablish_checkpoint(checkpoint).await?; + } else { + self.inner.maybe_replay_new_wals().await?; + } - self.inner.maybe_refresh_checkpoint(&mut manifest).await + self.inner.maybe_refresh_checkpoint(&mut manifest).await + } + DbReaderMode::FollowLatest => { + let result = self.inner.refresh_latest_manifest().await; + if let Err(error) = result { + warn!("failed to refresh reader to latest manifest [error={error:?}]"); + } + Ok(()) + } + // No polling is needed for a pinned checkpoint, so we just return Ok(()). + DbReaderMode::Checkpoint(_) => Ok(()), + } } async fn cleanup( @@ -674,25 +758,36 @@ impl MessageHandler for ManifestPoller { _messages: BoxStream<'async_trait, DbReaderMessage>, _result: Result<(), SlateDBError>, ) -> Result<(), SlateDBError> { + if self.inner.mode != DbReaderMode::ManagedCheckpoint { + return Ok(()); + } let mut manifest = StoredManifest::load( Arc::clone(&self.inner.manifest_store), self.inner.system_clock.clone(), ) .await?; - let checkpoint_id = self.inner.state.read().checkpoint.id; - if Some(checkpoint_id) != self.inner.user_checkpoint_id { - info!( - "deleting reader established checkpoint for shutdown [checkpoint_id={}]", - checkpoint_id - ); - manifest.delete_checkpoint(checkpoint_id).await?; - } + let checkpoint_id = self + .inner + .state + .read() + .checkpoint + .as_ref() + .expect("managed reader must have a checkpoint") + .id; + info!( + "deleting reader established checkpoint for shutdown [checkpoint_id={}]", + checkpoint_id + ); + manifest.delete_checkpoint(checkpoint_id).await?; Ok(()) } } impl DbReader { - fn validate_options(options: &DbReaderOptions) -> Result<(), SlateDBError> { + fn validate_options(mode: DbReaderMode, options: &DbReaderOptions) -> Result<(), SlateDBError> { + if mode != DbReaderMode::ManagedCheckpoint { + return Ok(()); + } if options.checkpoint_lifetime.as_millis() < 1000 { return Err(SlateDBError::InvalidCheckpointLifetime( options.checkpoint_lifetime, @@ -732,23 +827,21 @@ impl DbReader { } /// Creates a database reader that can read the contents of a database (but cannot write any - /// data). The caller can provide an optional checkpoint. If the checkpoint is provided, the - /// reader will read using the specified checkpoint and will not periodically refresh the - /// checkpoint. Otherwise, the reader creates a new checkpoint pointing to the current manifest - /// and refreshes it periodically as specified in the options. It also removes the previous - /// checkpoint once any ongoing reads have completed. + /// data). [`DbReaderMode`] controls whether the reader manages a checkpoint, remains pinned to + /// a supplied checkpoint, or follows the latest manifest without garbage-collection + /// protection. pub async fn open>( path: P, object_store: Arc, - checkpoint_id: Option, + mode: DbReaderMode, options: DbReaderOptions, ) -> Result { // Use the builder API internally - let mut builder = Self::builder(path, object_store).with_options(options); - if let Some(id) = checkpoint_id { - builder = builder.with_checkpoint_id(id); - } - builder.build().await + Self::builder(path, object_store) + .with_options(options) + .with_reader_mode(mode) + .build() + .await } /// Creates a new builder for a database reader at the given path. @@ -792,7 +885,7 @@ impl DbReader { pub(crate) async fn open_internal( manifest_store: Arc, table_store: Arc, - checkpoint_id: Option, + mode: DbReaderMode, merge_operator: Option, segment_extractor: Option>, options: DbReaderOptions, @@ -800,7 +893,7 @@ impl DbReader { rand: Arc, recorder: slatedb_common::metrics::MetricsRecorderHelper, ) -> Result { - Self::validate_options(&options)?; + Self::validate_options(mode, &options)?; let manifest = StoredManifest::load(Arc::clone(&manifest_store), system_clock.clone()).await?; @@ -817,7 +910,7 @@ impl DbReader { manifest_store, table_store, options, - checkpoint_id, + mode, merge_operator, segment_extractor, system_clock.clone(), @@ -832,10 +925,9 @@ impl DbReader { system_clock.clone(), ); - // If no checkpoint was provided, then we have established a new checkpoint - // from the latest state, and we need to refresh it according to the params - // of `DbReaderOptions`. - if checkpoint_id.is_none() { + // Pinned checkpoints never advance. Managed checkpoints and unprotected readers both + // poll for newer database state according to `DbReaderOptions`. + if !matches!(mode, DbReaderMode::Checkpoint(_)) { inner.spawn_manifest_poller(&task_executor)?; } @@ -866,7 +958,7 @@ impl DbReader { /// ## Examples /// /// ``` - /// use slatedb::{Db, DbReader, config::DbReaderOptions, Error}; + /// use slatedb::{Db, DbReader, DbReaderMode, config::DbReaderOptions, Error}; /// use slatedb::object_store::{ObjectStore, memory::InMemory}; /// use std::sync::Arc; /// @@ -880,7 +972,7 @@ impl DbReader { /// let reader = DbReader::open( /// "test_db", /// Arc::clone(&object_store), - /// None, + /// DbReaderMode::ManagedCheckpoint, /// DbReaderOptions::default(), /// ).await?; /// assert_eq!(reader.get(b"key").await?, Some("value".into())); @@ -914,7 +1006,7 @@ impl DbReader { /// ## Examples /// /// ``` - /// use slatedb::{Db, DbReader, config::DbReaderOptions, config::ReadOptions, Error}; + /// use slatedb::{Db, DbReader, DbReaderMode, config::DbReaderOptions, config::ReadOptions, Error}; /// use slatedb::object_store::{ObjectStore, memory::InMemory}; /// use std::sync::Arc; /// @@ -928,7 +1020,7 @@ impl DbReader { /// let reader = DbReader::open( /// "test_db", /// Arc::clone(&object_store), - /// None, + /// DbReaderMode::ManagedCheckpoint, /// DbReaderOptions::default(), /// ).await?; /// assert_eq!(db.get_with_options(b"key", &ReadOptions::default()).await?, Some("value".into())); @@ -985,7 +1077,7 @@ impl DbReader { /// ## Examples /// /// ``` - /// use slatedb::{Db, DbReader, config::DbReaderOptions, Error}; + /// use slatedb::{Db, DbReader, DbReaderMode, config::DbReaderOptions, Error}; /// use slatedb::object_store::{ObjectStore, memory::InMemory}; /// use std::sync::Arc; /// @@ -1000,7 +1092,7 @@ impl DbReader { /// let reader = DbReader::open( /// "test_db", /// Arc::clone(&object_store), - /// None, + /// DbReaderMode::ManagedCheckpoint, /// DbReaderOptions::default(), /// ).await?; /// let mut iter = reader.scan("a".."b").await?; @@ -1036,7 +1128,7 @@ impl DbReader { /// ## Examples /// /// ``` - /// use slatedb::{Db, DbReader, config::DbReaderOptions, config::ScanOptions, config::DurabilityLevel, Error}; + /// use slatedb::{Db, DbReader, DbReaderMode, config::DbReaderOptions, config::ScanOptions, config::DurabilityLevel, Error}; /// use slatedb::object_store::{ObjectStore, memory::InMemory}; /// use std::sync::Arc; /// @@ -1051,7 +1143,7 @@ impl DbReader { /// let reader = DbReader::open( /// "test_db", /// Arc::clone(&object_store), - /// None, + /// DbReaderMode::ManagedCheckpoint, /// DbReaderOptions::default(), /// ).await?; /// let mut iter = reader.scan_with_options("a".."b", &ScanOptions { @@ -1146,7 +1238,7 @@ impl DbReader { /// ## Examples /// /// ``` - /// use slatedb::{Db, DbReader, config::DbReaderOptions, Error}; + /// use slatedb::{Db, DbReader, DbReaderMode, config::DbReaderOptions, Error}; /// use slatedb::object_store::{ObjectStore, memory::InMemory}; /// use std::sync::Arc; /// @@ -1155,7 +1247,12 @@ impl DbReader { /// let object_store: Arc = Arc::new(InMemory::new()); /// let db = Db::open("test_db", object_store.clone()).await?; /// let options = DbReaderOptions::default(); - /// let reader = DbReader::open("test_db", object_store.clone(), None, options).await?; + /// let reader = DbReader::open( + /// "test_db", + /// object_store.clone(), + /// DbReaderMode::ManagedCheckpoint, + /// options, + /// ).await?; /// reader.close().await?; /// Ok(()) /// } @@ -1288,16 +1385,17 @@ fn has_not_found_object_store_error(err: &(dyn std::error::Error + 'static)) -> #[cfg(test)] mod tests { - use super::CheckpointState; + use super::{DbReaderMessage, ManifestPoller, ReaderState}; use crate::clock::MonotonicClock; use crate::config::{ CheckpointOptions, CheckpointScope, FlushOptions, FlushType, MergeOptions, PutOptions, Settings, WriteOptions, }; - use crate::db_reader::{DbReader, DbReaderInner, DbReaderOptions}; + use crate::db_reader::{DbReader, DbReaderInner, DbReaderMode, DbReaderOptions}; use crate::db_state::SsTableId; use crate::db_stats::DbStats; use crate::db_status::DbStatusManager; + use crate::dispatcher::MessageHandler; use crate::format::sst::SsTableFormat; use crate::iter::IterationOrder; use crate::manifest::store::{ManifestStore, StoredManifest}; @@ -1317,7 +1415,7 @@ mod tests { use fail_parallel::FailPointRegistry; use object_store::memory::InMemory; use object_store::path::Path; - use object_store::ObjectStore; + use object_store::{ObjectStore, ObjectStoreExt}; use rstest::rstest; use slatedb_common::clock::{DefaultSystemClock, SystemClock}; use slatedb_common::DbRand; @@ -1343,7 +1441,7 @@ mod tests { let reader = DbReader::open( path.clone(), Arc::clone(&object_store), - None, + DbReaderMode::ManagedCheckpoint, DbReaderOptions::default(), ) .await @@ -1365,9 +1463,14 @@ mod tests { db.put(b"test_key", b"test_value").await.unwrap(); db.flush().await.unwrap(); - let reader = DbReader::open(path, object_store, None, DbReaderOptions::default()) - .await - .unwrap(); + let reader = DbReader::open( + path, + object_store, + DbReaderMode::ManagedCheckpoint, + DbReaderOptions::default(), + ) + .await + .unwrap(); let manifest = reader.manifest(); let expected: VersionedManifest = @@ -1397,7 +1500,7 @@ mod tests { let reader = DbReader::open_internal( test_provider.manifest_store(), test_provider.table_store(), - Some(checkpoint_result.id), + DbReaderMode::Checkpoint(checkpoint_result.id), None, None, DbReaderOptions::default(), @@ -1435,7 +1538,7 @@ mod tests { let reader = DbReader::open( path.clone(), Arc::clone(&object_store), - Some(checkpoint_result.id), + DbReaderMode::Checkpoint(checkpoint_result.id), DbReaderOptions::default(), ) .await @@ -1581,7 +1684,7 @@ mod tests { // when let reader = DbReader::builder(path, object_store) .with_segment_extractor(Arc::new(test_utils::FixedThreeBytePrefixExtractor)) - .with_checkpoint_id(checkpoint.id) + .with_reader_mode(DbReaderMode::Checkpoint(checkpoint.id)) .build() .await .unwrap(); @@ -1661,6 +1764,165 @@ mod tests { .await; } + #[tokio::test(start_paused = true)] + async fn follow_latest_should_refresh_without_object_store_writes() { + let object_store: Arc = Arc::new(InMemory::new()); + let path = Path::from("/tmp/test_follow_latest_reader"); + let test_provider = TestProvider::new(path.clone(), Arc::clone(&object_store)); + let db = test_provider.new_db(Settings::default()).await.unwrap(); + + let key = b"key"; + db.put(key, b"initial").await.unwrap(); + db.flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .unwrap(); + + let recording_store = Arc::new(test_utils::RecordingObjectStore::new(Arc::clone( + &object_store, + ))); + let reader_store: Arc = recording_store.clone(); + let reader = DbReader::open( + path, + reader_store, + DbReaderMode::FollowLatest, + DbReaderOptions { + manifest_poll_interval: Duration::from_millis(100), + // FollowLatest does not create a checkpoint, so checkpoint validation is + // intentionally inapplicable to this mode. + checkpoint_lifetime: Duration::ZERO, + ..DbReaderOptions::default() + }, + ) + .await + .unwrap(); + + assert_eq!( + reader.get(key).await.unwrap(), + Some(Bytes::from_static(b"initial")) + ); + let initial_manifest = reader.manifest(); + let initial_manifest_id = initial_manifest.id(); + + db.put(key, b"updated").await.unwrap(); + db.flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .unwrap(); + let latest_manifest = test_provider + .manifest_store() + .read_latest_manifest() + .await + .unwrap(); + assert!(latest_manifest.id > initial_manifest_id); + assert!(latest_manifest.manifest.core.checkpoints.is_empty()); + + let mut poller = ManifestPoller { + inner: Arc::clone(&reader.inner), + }; + poller.handle(DbReaderMessage::PollManifest).await.unwrap(); + + assert!(reader.manifest().id() >= latest_manifest.id); + assert_eq!( + reader.get(key).await.unwrap(), + Some(Bytes::from_static(b"updated")) + ); + + let refreshed_manifest_id = reader.manifest().id(); + reader + .inner + .apply_latest_manifest(initial_manifest) + .await + .unwrap(); + assert_eq!(reader.manifest().id(), refreshed_manifest_id); + assert!(recording_store.write_kinds().is_empty()); + + reader.close().await.unwrap(); + assert!(recording_store.write_kinds().is_empty()); + } + + #[tokio::test] + async fn follow_latest_refresh_failure_should_keep_last_good_state() { + let object_store: Arc = Arc::new(InMemory::new()); + let path = Path::from("/tmp/test_follow_latest_refresh_failure"); + let test_provider = TestProvider::new(path, Arc::clone(&object_store)); + let db = test_provider.new_db(Settings::default()).await.unwrap(); + + db.put(b"key", b"value").await.unwrap(); + db.flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .unwrap(); + db.close().await.unwrap(); + + let reader = DbReader::open_internal( + test_provider.manifest_store(), + test_provider.table_store(), + DbReaderMode::FollowLatest, + None, + None, + DbReaderOptions { + manifest_poll_interval: Duration::from_secs(60 * 60), + ..DbReaderOptions::default() + }, + test_provider.system_clock.clone(), + test_provider.rand.clone(), + slatedb_common::metrics::MetricsRecorderHelper::noop(), + ) + .await + .unwrap(); + let manifest_id = reader.manifest().id(); + + let manifest_store = test_provider.manifest_store(); + let mut saved_manifests = Vec::new(); + for manifest in manifest_store.list_manifests(..).await.unwrap() { + let location = manifest.metadata.location; + let bytes = object_store + .get(&location) + .await + .unwrap() + .bytes() + .await + .unwrap(); + object_store.delete(&location).await.unwrap(); + saved_manifests.push((location, bytes)); + } + + let mut poller = ManifestPoller { + inner: Arc::clone(&reader.inner), + }; + poller.handle(DbReaderMessage::PollManifest).await.unwrap(); + + assert_eq!(reader.manifest().id(), manifest_id); + assert_eq!( + reader.get(b"key").await.unwrap(), + Some(Bytes::from_static(b"value")) + ); + + for (location, bytes) in saved_manifests { + object_store.put(&location, bytes.into()).await.unwrap(); + } + let db = test_provider.new_db(Settings::default()).await.unwrap(); + db.put(b"key", b"updated").await.unwrap(); + db.flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .unwrap(); + db.close().await.unwrap(); + + poller.handle(DbReaderMessage::PollManifest).await.unwrap(); + assert!(reader.manifest().id() > manifest_id); + assert_eq!( + reader.get(b"key").await.unwrap(), + Some(Bytes::from_static(b"updated")) + ); + reader.close().await.unwrap(); + } + #[tokio::test(start_paused = true)] async fn should_reestablish_reader_checkpoint() { let object_store: Arc = Arc::new(InMemory::new()); @@ -1800,7 +2062,7 @@ mod tests { checkpoint_lifetime: Duration::from_millis(1000), ..DbReaderOptions::default() }, - None, + DbReaderMode::ManagedCheckpoint, None, None, clock.clone(), @@ -1895,7 +2157,7 @@ mod tests { checkpoint_lifetime: Duration::from_millis(1000), ..DbReaderOptions::default() }, - None, + DbReaderMode::ManagedCheckpoint, None, None, clock.clone(), @@ -1905,7 +2167,7 @@ mod tests { ) .await .unwrap(); - let reader_checkpoint_id = inner.state.read().checkpoint.id; + let reader_checkpoint_id = inner.state.read().checkpoint.as_ref().unwrap().id; // Simulate the writer's GC reaping the expired checkpoint. let mut stored_manifest = StoredManifest::load(Arc::clone(&manifest_store), clock.clone()) @@ -1927,7 +2189,7 @@ mod tests { .unwrap(); // The reader should have replaced the reaped checkpoint with a new one. - let new_checkpoint_id = inner.state.read().checkpoint.id; + let new_checkpoint_id = inner.state.read().checkpoint.as_ref().unwrap().id; assert_ne!(reader_checkpoint_id, new_checkpoint_id); let latest_manifest = manifest_store.read_latest_manifest().await.unwrap(); let checkpoints = &latest_manifest.manifest.core.checkpoints; @@ -1935,79 +2197,6 @@ mod tests { assert_eq!(new_checkpoint_id, checkpoints[0].id); } - // A missing user-supplied checkpoint must still fail loud rather than be - // silently replaced (RFC-0004): the caller's pinned view is gone. - #[tokio::test] - async fn should_fail_refresh_when_user_checkpoint_missing() { - let object_store: Arc = Arc::new(InMemory::new()); - let path = Path::from(format!( - "/tmp/test_db_reader_user_checkpoint_missing_{}", - Uuid::new_v4() - )); - let clock = Arc::new(MockSystemClock::new()); - let mut test_provider = TestProvider::new(path, Arc::clone(&object_store)); - test_provider.system_clock = clock.clone(); - - let manifest_store = test_provider.manifest_store(); - let table_store = test_provider.table_store(); - - let mut stored_manifest = StoredManifest::create_new_db( - Arc::clone(&manifest_store), - ManifestCore::new(), - clock.clone(), - ) - .await - .unwrap(); - let user_checkpoint_id = Uuid::new_v4(); - stored_manifest - .write_checkpoint( - user_checkpoint_id, - &CheckpointOptions { - lifetime: Some(Duration::from_millis(1000)), - ..CheckpointOptions::default() - }, - ) - .await - .unwrap(); - let recorder = slatedb_common::metrics::MetricsRecorderHelper::noop(); - - let inner = DbReaderInner::new( - Arc::clone(&manifest_store), - table_store, - DbReaderOptions { - manifest_poll_interval: Duration::from_millis(100), - checkpoint_lifetime: Duration::from_millis(1000), - ..DbReaderOptions::default() - }, - Some(user_checkpoint_id), - None, - None, - clock.clone(), - test_provider.rand.clone(), - recorder, - stored_manifest, - ) - .await - .unwrap(); - - let mut stored_manifest = StoredManifest::load(Arc::clone(&manifest_store), clock.clone()) - .await - .unwrap(); - stored_manifest - .delete_checkpoint(user_checkpoint_id) - .await - .unwrap(); - - clock.advance(Duration::from_millis(501)).await; - let mut stored_manifest = StoredManifest::load(Arc::clone(&manifest_store), clock.clone()) - .await - .unwrap(); - let result = inner.maybe_refresh_checkpoint(&mut stored_manifest).await; - assert!( - matches!(result, Err(SlateDBError::CheckpointMissing(id)) if id == user_checkpoint_id) - ); - } - #[tokio::test(start_paused = true)] async fn should_replay_new_wals() { let object_store: Arc = Arc::new(InMemory::new()); @@ -2577,10 +2766,11 @@ mod tests { checkpoint: Option, merge_operator: Option, ) -> Result { + let mode = checkpoint.map_or(DbReaderMode::ManagedCheckpoint, DbReaderMode::Checkpoint); DbReader::open_internal( self.manifest_store(), self.table_store(), - checkpoint, + mode, merge_operator, None, options, @@ -2724,8 +2914,12 @@ mod tests { // Seed the prior checkpoint state with IMMs. let input_tables: Vec<_> = case.tables.iter().map(InputMemtable::build).collect(); - let prior_state = CheckpointState { - checkpoint: test_checkpoint(stored_manifest.id(), test_provider.system_clock.clone()), + let prior_state = ReaderState { + manifest_id: stored_manifest.id(), + checkpoint: Some(test_checkpoint( + stored_manifest.id(), + test_provider.system_clock.clone(), + )), manifest: stored_manifest.manifest().clone(), imm_memtable: input_tables.iter().cloned().collect(), last_wal_id: 0, @@ -2769,9 +2963,9 @@ mod tests { skip_wal_replay: true, ..DbReaderOptions::default() }, + mode: DbReaderMode::ManagedCheckpoint, state: parking_lot::RwLock::new(Arc::new(prior_state)), system_clock: test_provider.system_clock.clone(), - user_checkpoint_id: None, oracle, reader, status_manager: DbStatusManager::new(0), @@ -2824,8 +3018,9 @@ mod tests { let manifest_store = test_provider.manifest_store(); let table_store = test_provider.table_store(); - let prior_state = CheckpointState { - checkpoint: test_checkpoint(1, test_provider.system_clock.clone()), + let prior_state = ReaderState { + manifest_id: 1, + checkpoint: Some(test_checkpoint(1, test_provider.system_clock.clone())), manifest: Manifest::initial(current_core.clone()), imm_memtable: VecDeque::from([immutable_memtable( 1, @@ -2851,9 +3046,9 @@ mod tests { manifest_store, table_store, options: DbReaderOptions::default(), + mode: DbReaderMode::ManagedCheckpoint, state: parking_lot::RwLock::new(Arc::new(prior_state)), system_clock: test_provider.system_clock.clone(), - user_checkpoint_id: None, oracle, reader, status_manager: DbStatusManager::new(0), @@ -3014,9 +3209,14 @@ mod tests { reader_opts.object_store_cache_options.root_folder = Some(cache_path.clone()); reader_opts.object_store_cache_options.part_size_bytes = 1024; - let reader = DbReader::open(path.clone(), Arc::clone(&object_store), None, reader_opts) - .await - .unwrap(); + let reader = DbReader::open( + path.clone(), + Arc::clone(&object_store), + DbReaderMode::ManagedCheckpoint, + reader_opts, + ) + .await + .unwrap(); // Read data to populate the cache let val = reader.get(b"key1").await.unwrap(); diff --git a/slatedb/src/lib.rs b/slatedb/src/lib.rs index 89a0027cb..bb37dc97d 100644 --- a/slatedb/src/lib.rs +++ b/slatedb/src/lib.rs @@ -50,7 +50,7 @@ pub use db::{Db, DbBuilder, DbReaderBuilder, DbStatus, SegmentPrefix, WriteHandl pub use db_cache::stats as db_cache_stats; pub use db_cache_manager::CacheTarget; pub use db_iter::{DbIterator, DbRecencyIterator}; -pub use db_reader::DbReader; +pub use db_reader::{DbReader, DbReaderMode}; pub use db_snapshot::DbSnapshot; pub use db_transaction::DbTransaction; pub use error::{CloseReason, Error, ErrorKind}; diff --git a/website/src/content/docs/docs/design/checkpoints.mdx b/website/src/content/docs/docs/design/checkpoints.mdx index 0158d093a..e2798661d 100644 --- a/website/src/content/docs/docs/design/checkpoints.mdx +++ b/website/src/content/docs/docs/design/checkpoints.mdx @@ -31,7 +31,13 @@ A new checkpoint can also be created from an existing checkpoint through [`Check ## Readers and Background Tasks -[`DbReader`](https://docs.rs/slatedb/latest/slatedb/struct.DbReader.html) uses checkpoints to separate reader lifetime from writer lifetime. If you open a reader without a checkpoint ID, the reader creates its own checkpoint, refreshes it, and replaces it as the manifest advances. If you open a reader with an explicit checkpoint ID, the reader stays on that fixed view and does not follow newer manifests or newer WAL data. [`DbReaderOptions::checkpoint_lifetime`](https://docs.rs/slatedb/latest/slatedb/config/struct.DbReaderOptions.html#structfield.checkpoint_lifetime) controls how long a reader-managed checkpoint lives before it must be refreshed. +[`DbReader`](https://docs.rs/slatedb/latest/slatedb/struct.DbReader.html) uses checkpoints to separate reader lifetime from writer lifetime. You choose how the reader tracks database state by passing a [`DbReaderMode`](https://docs.rs/slatedb/latest/slatedb/enum.DbReaderMode.html): + +- `ManagedCheckpoint` (default): The reader creates and maintains checkpoints while following the latest database state. +- `Checkpoint(id)`: The reader remains pinned to the database state referenced by the supplied checkpoint. It does not follow newer manifests or newer WAL data. +- `FollowLatest`: The reader follows the latest manifest without creating a checkpoint. This mode performs no object-store writes and provides no protection from garbage collection. Reads may fail if referenced objects are deleted. This mode is useful for read-only access to databases not being actively written to, for mirrored databases where manifest changes might not be allowed, or for readers willing to handle missing objects gracefully. + +[`DbReaderOptions::checkpoint_lifetime`](https://docs.rs/slatedb/latest/slatedb/config/struct.DbReaderOptions.html#structfield.checkpoint_lifetime) controls how long a `ManagedCheckpoint` reader's checkpoint lives before it must be refreshed. SlateDB also uses short-lived internal checkpoints during some background transitions. For example, the compactor writes a temporary checkpoint before publishing a manifest that drops obsolete SST references. That keeps recently replaced SSTs readable until GC can safely remove them. From 8f4eada73e860dc4a099f307b130999c6c2487bc Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 14 Jul 2026 11:40:53 -0700 Subject: [PATCH 15/63] Clarify impact of disabled boundary files (#1925) --- bindings/go/uniffi/slatedb.go | 7 ++++++- bindings/uniffi/src/config.rs | 7 ++++++- slatedb/src/config.rs | 9 +++++++-- website/src/content/docs/docs/design/gc.mdx | 9 ++++++--- .../docs/docs/tutorials/standalone-garbage-collector.mdx | 9 ++++++--- 5 files changed, 31 insertions(+), 10 deletions(-) diff --git a/bindings/go/uniffi/slatedb.go b/bindings/go/uniffi/slatedb.go index 150e08a8c..b4ba2ec42 100644 --- a/bindings/go/uniffi/slatedb.go +++ b/bindings/go/uniffi/slatedb.go @@ -9464,7 +9464,12 @@ type GarbageCollectorOptions struct { // Options for detaching clone references. `None` disables detach garbage collection. DetachOptions *GarbageCollectorScheduleOptions // Whether GC should delete eligible manifest/compactions metadata without advancing boundary - // files. + // files. This supports object stores without conditional overwrites (`If-Match`), but allows a + // SlateDB client or compactor to begin updating a manifest or compactions file, stop making + // progress (for example, because its process or host is suspended), then resume after GC's + // `min_age`. It can then recreate a deleted metadata ID and incorrectly report its stale update + // as successful. Set `min_age` longer than the maximum lifetime of a stale process, and use the + // same setting for every GC operating on the database. DisableBoundaryFiles bool } diff --git a/bindings/uniffi/src/config.rs b/bindings/uniffi/src/config.rs index 86e7f219d..1a57fb73e 100644 --- a/bindings/uniffi/src/config.rs +++ b/bindings/uniffi/src/config.rs @@ -451,7 +451,12 @@ pub struct GarbageCollectorOptions { #[uniffi(default = None)] pub detach_options: Option, /// Whether GC should delete eligible manifest/compactions metadata without advancing boundary - /// files. + /// files. This supports object stores without conditional overwrites (`If-Match`), but allows a + /// SlateDB client or compactor to begin updating a manifest or compactions file, stop making + /// progress (for example, because its process or host is suspended), then resume after GC's + /// `min_age`. It can then recreate a deleted metadata ID and incorrectly report its stale update + /// as successful. Set `min_age` longer than the maximum lifetime of a stale process, and use the + /// same setting for every GC operating on the database. #[uniffi(default = false)] pub disable_boundary_files: bool, } diff --git a/slatedb/src/config.rs b/slatedb/src/config.rs index 62e3108bf..f4be2da18 100644 --- a/slatedb/src/config.rs +++ b/slatedb/src/config.rs @@ -1438,8 +1438,13 @@ pub struct GarbageCollectorOptions { /// Whether manifest and compactions boundary files are advanced before deletion. /// - /// When disabled, garbage collection still deletes eligible metadata but does not update the - /// durable boundary. Every garbage collector for the database must use the same policy. + /// Disable this only for object stores that do not support conditional overwrites (`If-Match`). + /// Without boundary advancement, a SlateDB client or compactor can begin updating a manifest or + /// compactions file, stop making progress (for example, because its process or host is + /// suspended), then resume after the garbage collector's `min_age`. It can then recreate a + /// deleted metadata ID and incorrectly report its stale update as successful. Set `min_age` + /// longer than the maximum lifetime of a stale process, and use the same setting for every + /// garbage collector operating on the database. #[serde(default = "default_boundary_files_enabled")] pub boundary_files_enabled: bool, } diff --git a/website/src/content/docs/docs/design/gc.mdx b/website/src/content/docs/docs/design/gc.mdx index ba51c2b04..82d6aa1f3 100644 --- a/website/src/content/docs/docs/design/gc.mdx +++ b/website/src/content/docs/docs/design/gc.mdx @@ -19,9 +19,12 @@ Boundary advancement can be disabled with `GarbageCollectorOptions::boundary_files_enabled`. This supports object stores without conditional overwrite (`If-Match`) while allowing GC to continue deleting eligible metadata. Metadata readers and writers still check any existing boundary; on a new database, no boundary file is created while -all garbage collectors use this mode. Disabling advancement removes the stale-writer fail-safe, so -every garbage collector must use a compatible setting and the manifest and compactions `min_age` -values must exceed the maximum lifetime of a stale process. +all garbage collectors use this mode. Without a boundary, a SlateDB client or compactor can begin +updating a manifest or compactions file, stop making progress (for example, because its process or +host is suspended), then resume after `min_age`. GC may have deleted the ID it intended to create, +so create-if-absent can reuse that ID and report a stale update as successful even though newer +metadata has superseded it. Every garbage collector must use a compatible setting, and the manifest +and compactions `min_age` values must exceed the maximum lifetime of a stale process. ## Filtering Deletion Candidates diff --git a/website/src/content/docs/docs/tutorials/standalone-garbage-collector.mdx b/website/src/content/docs/docs/tutorials/standalone-garbage-collector.mdx index 74245cf8b..94aaacdca 100644 --- a/website/src/content/docs/docs/tutorials/standalone-garbage-collector.mdx +++ b/website/src/content/docs/docs/tutorials/standalone-garbage-collector.mdx @@ -54,9 +54,12 @@ With `GarbageCollectorOptions::default()`, garbage collection runs every 60 seco To delete manifest and compactions metadata without advancing boundary files, set `GarbageCollectorOptions::boundary_files_enabled` to `false` on every garbage collector for the -database. Metadata readers and writers continue to honor any existing boundary. Increase the -manifest and compactions `min_age` values beyond the longest time a stale process could remain -alive before using this mode. +database. Metadata readers and writers continue to honor any existing boundary. Without boundary +advancement, a SlateDB client or compactor can begin updating a manifest or compactions file, stop +making progress (for example, because its process or host is suspended), then resume after +`min_age`. It can then recreate a deleted metadata ID and incorrectly report its stale update as +successful. Set the manifest and compactions `min_age` values longer than the maximum lifetime of a +stale process before using this mode. :::note From e7d6b4f262d1cc976e57f93b1dd8687298aacb09 Mon Sep 17 00:00:00 2001 From: Rohan Date: Tue, 14 Jul 2026 16:03:22 -0400 Subject: [PATCH 16/63] [k/N] wal refactor: reorganize WalBufferManager into separate handles that share inner (#1886) --- slatedb/src/batch_write.rs | 20 +- slatedb/src/db.rs | 70 +++-- slatedb/src/db/builder.rs | 4 +- slatedb/src/wal_buffer.rs | 519 ++++++++++++++++++++----------------- 4 files changed, 334 insertions(+), 279 deletions(-) diff --git a/slatedb/src/batch_write.rs b/slatedb/src/batch_write.rs index 7560768c6..82e75833a 100644 --- a/slatedb/src/batch_write.rs +++ b/slatedb/src/batch_write.rs @@ -109,11 +109,11 @@ impl std::fmt::Debug for BatchWriterMessage { pub(crate) struct WriteBatchEventHandler { db_inner: Arc, is_first_write: bool, - wal_buffer: Arc, + wal_buffer: WalBufferManager, } impl WriteBatchEventHandler { - pub(crate) fn new(db_inner: Arc, wal_buffer: Arc) -> Self { + pub(crate) fn new(db_inner: Arc, wal_buffer: WalBufferManager) -> Self { Self { db_inner, is_first_write: true, @@ -134,7 +134,7 @@ impl MessageHandler for WriteBatchEventHandler { }) => { let result = self .db_inner - .write_batch(batch, &options, txn.as_ref(), self.wal_buffer.as_ref()) + .write_batch(batch, &options, txn.as_ref(), &self.wal_buffer) .await; // if this is the first write and the WAL is disabled, make sure users are flushing // their memtables in a timely manner. @@ -158,7 +158,7 @@ impl MessageHandler for WriteBatchEventHandler { } = flush_msg; let result = self .db_inner - .flush_batch_writer(freeze_memtable, self.wal_buffer.as_ref()); + .flush_batch_writer(freeze_memtable, &self.wal_buffer); let _ = done.send(result); Ok(()) } @@ -558,14 +558,14 @@ mod tests { ) .await .unwrap(); - let wal_buffer = Arc::new(WalBufferManager::new( + let wal_buffer = WalBufferManager::new( db.inner.status_manager.clone(), &db.inner.recorder, 0, db.inner.table_store.clone(), 1024, None, - )); + ); let mut handler = WriteBatchEventHandler::new(db.inner.clone(), wal_buffer); assert!(handler.is_first_write); @@ -587,14 +587,14 @@ mod tests { let db = Db::open("/tmp/test_user_defined_seqnum", object_store) .await .unwrap(); - let wal_buffer = Arc::new(WalBufferManager::new( + let wal_buffer = WalBufferManager::new( db.inner.status_manager.clone(), &db.inner.recorder, 0, db.inner.table_store.clone(), 1024, None, - )); + ); let mut handler = WriteBatchEventHandler::new(db.inner.clone(), wal_buffer); @@ -630,14 +630,14 @@ mod tests { ) .await .unwrap(); - let wal_buffer = Arc::new(WalBufferManager::new( + let wal_buffer = WalBufferManager::new( db.inner.status_manager.clone(), &db.inner.recorder, 0, db.inner.table_store.clone(), 1024, None, - )); + ); let mut handler = WriteBatchEventHandler::new(db.inner.clone(), wal_buffer); diff --git a/slatedb/src/db.rs b/slatedb/src/db.rs index 91800dafc..a4db740fa 100644 --- a/slatedb/src/db.rs +++ b/slatedb/src/db.rs @@ -37,7 +37,7 @@ use crate::dispatcher::MessageHandlerExecutor; use crate::garbage_collector::GC_TASK_NAME; use crate::transaction_manager::IsolationLevel; use crate::CloseReason; -use log::{info, trace, warn}; +use log::{debug, info, trace, warn}; use parking_lot::RwLock; use std::time::Duration; @@ -69,7 +69,7 @@ use crate::sst_iter::SstIteratorOptions; use crate::tablestore::TableStore; use crate::transaction_manager::TransactionManager; use crate::types::KeyValue; -use crate::utils::{format_bytes_si, SafeSender}; +use crate::utils::{format_bytes_si, SafeSender, WatchableOnceCellReader}; use crate::wal_buffer::{WalEvent, WalObserver, WalStatus, WAL_BUFFER_TASK_NAME}; use crate::wal_replay::{WalReplayIterator, WalReplayOptions}; use crate::{DbCacheManagerOps, DbMetadataOps, DbReadOps, DbWriteOps}; @@ -174,7 +174,12 @@ impl DbInner { let txn_manager = Arc::new(TransactionManager::new(oracle.clone(), rand.clone())); let snapshot_manager = Arc::new(SnapshotManager::new(oracle.clone(), rand.clone())); - let wal_observer = DbWalObserver::new(wal_observer, oracle.clone(), state.clone()); + let wal_observer = DbWalObserver::new( + wal_observer, + oracle.clone(), + state.clone(), + status_manager.result_reader(), + ); let db_inner = Self { state, @@ -386,9 +391,11 @@ impl DbInner { }; tokio::select! { + biased; + + result = await_closed => result?, result = await_memtable_uploaded => result?, result = await_flush_wal => result?, - result = await_closed => result?, _ = timeout_fut => { warn!("backpressure timeout: waited 30s, no memtable/WAL flushed yet"); } @@ -2055,26 +2062,38 @@ impl WriteHandle { #[derive(Clone)] pub(crate) struct DbWalObserver { status_rx: tokio::sync::watch::Receiver, + closed_reader: WatchableOnceCellReader>, wrapped: WalObserver, } impl DbWalObserver { - fn new(wrapped: WalObserver, oracle: Arc, db_state: Arc>) -> Self { + fn new( + wrapped: WalObserver, + oracle: Arc, + db_state: Arc>, + closed_reader: WatchableOnceCellReader>, + ) -> Self { let (status_tx, status_rx) = tokio::sync::watch::channel(wrapped.status()); - wrapped.subscribe(Arc::new(move |event| { - let status = match event { - WalEvent::WalFlushed(status) => status, - WalEvent::MemoryReleased(status) => status, - }; - if let Some(seq) = status.last_flushed_seq { - oracle.advance_durable_seq(seq); - } - let mut guard = db_state.write(); - guard.set_next_wal_id(status.last_flushed_wal_id + 1); - drop(guard); - let _ = status_tx.send(status); - })); - Self { status_rx, wrapped } + wrapped + .subscribe(Arc::new(move |event| { + let status = match event { + WalEvent::WalFlushed(status) => status, + WalEvent::MemoryReleased(status) => status, + }; + if let Some(seq) = status.last_flushed_seq { + oracle.advance_durable_seq(seq); + } + let mut guard = db_state.write(); + guard.set_next_wal_id(status.last_flushed_wal_id + 1); + drop(guard); + let _ = status_tx.send(status); + })) + .expect("failed to subscribe to wal"); + Self { + status_rx, + closed_reader, + wrapped, + } } pub(crate) fn status(&self) -> WalStatus { @@ -2086,11 +2105,14 @@ impl DbWalObserver { predicate: impl FnMut(&WalStatus) -> bool, ) -> Result<(), SlateDBError> { let mut status_rx = self.status_rx.clone(); - status_rx - .wait_for(predicate) - .await - .map_err(|_| SlateDBError::Closed)?; - Ok(()) + let result = status_rx.wait_for(predicate).await.map(|_| ()); + match result { + Ok(_) => Ok(()), + Err(_) => { + debug!("wal listener tx dropped - wait on db close"); + self.closed_reader.clone().await_value().await + } + } } /// Waits until the wal a given wal id is released by the wal writer diff --git a/slatedb/src/db/builder.rs b/slatedb/src/db/builder.rs index f618e6742..31e54c56e 100644 --- a/slatedb/src/db/builder.rs +++ b/slatedb/src/db/builder.rs @@ -592,14 +592,14 @@ impl> DbBuilder

{ ); let recent_flushed_wal_id = replay_range.end - 1; - let wal_buffer = Arc::new(WalBufferManager::new( + let mut wal_buffer = WalBufferManager::new( status_manager.clone(), &recorder, recent_flushed_wal_id, table_store.clone(), self.settings.l0_sst_size_bytes, self.settings.flush_interval, - )); + ); // Setup communication channels wired to the shared closed state. let reader = status_manager.result_reader(); diff --git a/slatedb/src/wal_buffer.rs b/slatedb/src/wal_buffer.rs index 816339296..25df51833 100644 --- a/slatedb/src/wal_buffer.rs +++ b/slatedb/src/wal_buffer.rs @@ -1,4 +1,5 @@ use std::collections::VecDeque; +use std::fmt::{Debug, Formatter}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::Duration; @@ -11,6 +12,7 @@ use crate::tablestore::TableStore; use crate::types::RowEntry; use crate::utils::SafeSender; use crate::utils::{format_bytes_si, WatchableOnceCell, WatchableOnceCellReader}; +use crate::wal_buffer_stats::WalBufferStats; use async_trait::async_trait; use futures::{stream::BoxStream, StreamExt}; use log::{error, trace}; @@ -47,8 +49,7 @@ pub(crate) type WalStatusListener = Arc>, - status_manager: crate::db_status::DbStatusManager, - stats: stats::WalBufferStats, + stats: Arc, table_store: Arc, max_wal_bytes_size: usize, max_flush_interval: Option, @@ -56,6 +57,12 @@ pub(crate) struct WalBufferManager { /// sent. Compared against `flush_epoch` in the inner struct to avoid sending /// redundant flush requests for the same WAL. last_flush_requested_epoch: AtomicU64, + /// The channel to send the flush work to the background worker. + flush_tx: SafeSender, + /// The channel that the flush task waits on to receive work. Will be consumed by init + flush_rx: Option>, + /// task executor for the background worker. + task_executor: Option>, } struct WalBufferManagerInner { @@ -63,10 +70,6 @@ struct WalBufferManagerInner { /// When the current WAL is ready to be flushed, it'll be moved to the `immutable_wals`. /// The flusher will try flush all the immutable wals to remote storage. immutable_wals: VecDeque<(u64, Arc)>, - /// The channel to send the flush work to the background worker. - flush_tx: Option>, - /// task executor for the background worker. - task_executor: Option>, /// Whenever a WAL is applied to Memtable and successfully flushed to remote storage, /// the immutable wal can be recycled in memory. last_applied_seq: Option, @@ -82,8 +85,6 @@ struct WalBufferManagerInner { last_flushed_seq: Option, /// The last wal id that was deallocated from the buffer last_purged_wal_id: u64, - /// A listener to which the wal sends events. Currently the only event is a wal file flush. - listener: Option, } /// Stores entries to the write-ahead log (WAL) in memory. @@ -132,47 +133,45 @@ impl WalBufferManager { last_purged_wal_id: last_flushed_wal_id, next_wal_id: last_flushed_wal_id + 1, last_flushed_seq: None, - flush_tx: None, - task_executor: None, - listener: None, }; + let (flush_tx, flush_rx) = SafeSender::unbounded_channel(status_manager.result_reader()); Self { inner: Arc::new(parking_lot::RwLock::new(inner)), - status_manager, - stats: stats::WalBufferStats::new(recorder), + stats: Arc::new(WalBufferStats::new(recorder)), table_store, max_wal_bytes_size, max_flush_interval, last_flush_requested_epoch: AtomicU64::new(0), + flush_tx, + flush_rx: Some(flush_rx), + task_executor: None, } } // todo: consider consolidating with new pub(crate) async fn init( - self: &Arc, + &mut self, task_executor: Arc, ) -> Result<(), SlateDBError> { - let (flush_tx, flush_rx) = - SafeSender::unbounded_channel(self.status_manager.result_reader()); - { - let mut inner = self.inner.write(); - inner.flush_tx = Some(flush_tx); - } + let Some(flush_rx) = self.flush_rx.take() else { + error!("WalBufferManager#init called multiple times"); + return Err(SlateDBError::InvalidDBState); + }; + assert!(self.task_executor.is_none()); let wal_flush_handler = WalFlushHandler { max_flush_interval: self.max_flush_interval, - wal_buffer_manager: self.clone(), + inner: self.inner.clone(), + table_store: self.table_store.clone(), + stats: self.stats.clone(), + listener: None, }; - let result = task_executor.add_handler( WAL_BUFFER_TASK_NAME.to_string(), Box::new(wal_flush_handler), flush_rx, &Handle::current(), ); - { - let mut inner = self.inner.write(); - inner.task_executor = Some(task_executor); - } + self.task_executor = Some(task_executor); result } @@ -181,31 +180,17 @@ impl WalBufferManager { inner.last_flushed_wal_id } - fn subscribe(&self, listener: WalStatusListener) { - // TODO: consider extending to multiple listeners - let mut inner = self.inner.write(); - assert!(inner.listener.is_none()); - inner.listener = Some(listener); - } - pub(crate) fn status(&self) -> WalStatus { - let inner = self.inner.read(); - inner.status(&self.table_store) + self.inner.read().status(&self.table_store) } /// Append row entries to the current WAL. Returns a watcher for durability notification. - /// TODO: validate the seq number is always increasing. pub(crate) fn append( &self, entries: &[RowEntry], ) -> Result>, SlateDBError> { // TODO: check if the wal buffer is in a fatal error state. - - let mut inner = self.inner.write(); - for entry in entries { - inner.current_wal.append(entry.clone()); - } - Ok(inner.current_wal.durable_watcher()) + self.inner.write().append(entries) } /// Check if we need to flush the wal with considering max_wal_size. the checking over `max_wal_size` @@ -215,23 +200,12 @@ impl WalBufferManager { pub(crate) fn maybe_trigger_flush( &self, ) -> Result>, SlateDBError> { - // check the size of the current wal let (durable_watcher, need_flush, flush_epoch) = { let inner = self.inner.read(); - let current_wal_size = self - .table_store - .estimate_encoded_size_wal(inner.current_wal.len(), inner.current_wal.size()); - trace!( - "checking flush trigger [current_wal_size={}, max_wal_bytes_size={}]", - format_bytes_si(current_wal_size as u64), - format_bytes_si(self.max_wal_bytes_size as u64), - ); - let need_flush = current_wal_size >= self.max_wal_bytes_size; - ( - inner.current_wal.durable_watcher(), - need_flush, - inner.flush_epoch, - ) + // checks the size of the current wal + let (need_flush, flush_epoch) = + inner.needs_flush(&self.table_store, self.max_wal_bytes_size); + (inner.current_wal.durable_watcher(), need_flush, flush_epoch) }; if need_flush { // Only send a flush request if one hasn't already been sent for this epoch. @@ -254,9 +228,11 @@ impl WalBufferManager { Ok(durable_watcher) } - pub(crate) fn observer(self: &Arc) -> WalObserver { + pub(crate) fn observer(&self) -> WalObserver { WalObserver { - wal_buffer: self.clone(), + inner: self.inner.clone(), + table_store: self.table_store.clone(), + flush_tx: self.flush_tx.clone(), } } @@ -266,13 +242,7 @@ impl WalBufferManager { result_tx: Option>>, ) -> Result<(), SlateDBError> { self.stats.flush_requests.increment(1); - let flush_tx = self - .inner - .read() - .flush_tx - .clone() - .expect("flush_tx not initialized, please call init first."); - flush_tx.send(WalFlushWork { result_tx }) + self.flush_tx.send(WalFlushWork::Flush { result_tx }) } pub(crate) fn flush( @@ -283,132 +253,125 @@ impl WalBufferManager { Ok(result_rx) } - /// Returns the list of immutable WALs that need to be flushed. - /// Used by the handler to determine which WALs to write to storage. - fn flushing_wals(&self) -> Vec<(u64, Arc)> { - let inner = self.inner.read(); - let mut flushing_wals = Vec::new(); - for (wal_id, wal) in inner.immutable_wals.iter() { - if *wal_id > inner.last_flushed_wal_id { - flushing_wals.push((*wal_id, wal.clone())); - } + /// Track the last applied sequence number. It's called when some WAL entries are applied to the memtable. + /// This information of the last applied seq is used to determine if the immutable wals can be recycled. + /// + /// It's the caller's duty to ensure the seq is monotonically increasing. + pub(crate) fn track_last_applied_seq(&self, seq: u64) { + { + let mut inner = self.inner.write(); + inner.last_applied_seq = Some(seq); + inner.maybe_release_immutable_wals(); } - flushing_wals } - #[instrument(level = "trace", skip_all, err(level = tracing::Level::DEBUG))] - async fn do_flush(&self) -> Result<(), SlateDBError> { - self.freeze_current_wal()?; - let flushing_wals = self.flushing_wals(); + #[allow(dead_code)] + pub(crate) async fn close(&self) -> Result<(), SlateDBError> { + let task_executor = self + .task_executor + .as_ref() + .expect("task executor should be initialized"); + task_executor.shutdown_task(WAL_BUFFER_TASK_NAME).await + } +} - if flushing_wals.is_empty() { - return Ok(()); +impl WalBufferManagerInner { + fn append( + &mut self, + entries: &[RowEntry], + ) -> Result>, SlateDBError> { + // TODO: validate the seq number is always increasing. + for entry in entries { + self.current_wal.append(entry.clone()); } + Ok(self.current_wal.durable_watcher()) + } - for (wal_id, wal) in flushing_wals.iter() { - let result = self.do_flush_one_wal(*wal_id, wal.clone()).await; - if let Err(e) = &result { - // a WAL buffer can be retried to flush multiple times, but WatchableOnceCell is only set once. - // we do NOT call `wal.notify_durable` as soon as encountered any error here, but notify - // the error when we're sure enters fatal state in `do_cleanup`. - error!("failed to flush WAL [wal_id={}]", wal_id); - return Err(e.clone()); - } - - // increment the last flushed wal id, and last flushed seq - let (status, listener) = { - let mut inner = self.inner.write(); - inner.last_flushed_wal_id = *wal_id; - if let Some(seq) = wal.last_seq() { - if let Some(last_flushed_seq) = inner.last_flushed_seq { - assert!(seq >= last_flushed_seq); - } - inner.last_flushed_seq = Some(seq); - } - let status = inner.status(&self.table_store); - let listener = inner.listener.clone(); - (status, listener) - }; + fn needs_flush(&self, table_store: &TableStore, max_wal_bytes_size: usize) -> (bool, u64) { + // check the size of the current wal + let current_wal_size = + table_store.estimate_encoded_size_wal(self.current_wal.len(), self.current_wal.size()); + trace!( + "checking flush trigger [current_wal_size={}, max_wal_bytes_size={}]", + format_bytes_si(current_wal_size as u64), + format_bytes_si(max_wal_bytes_size as u64), + ); + let need_flush = current_wal_size >= max_wal_bytes_size; + (need_flush, self.flush_epoch) + } - // TODO: we probably want to release immutable wals first for the backpressure check - // notify durable only when the flush is successful. - if let Some(l) = listener { - (*l)(WalEvent::WalFlushed(status)) + /// Returns the list of immutable WALs that need to be flushed. + /// Used by the handler to determine which WALs to write to storage. + fn flushing_wals(&self) -> Vec<(u64, Arc)> { + let mut flushing_wals = Vec::new(); + for (wal_id, wal) in self.immutable_wals.iter() { + if *wal_id > self.last_flushed_wal_id { + flushing_wals.push((*wal_id, wal.clone())); } - wal.notify_durable(result.clone()); - } - - self.maybe_release_immutable_wals(); - let status = self.status(); - let listener = self.inner.read().listener.clone(); - if let Some(l) = listener { - (*l)(WalEvent::MemoryReleased(status)) } - - Ok(()) + flushing_wals } - async fn do_flush_one_wal(&self, wal_id: u64, wal: Arc) -> Result<(), SlateDBError> { - self.stats.flushes.increment(1); + /// Returns the total size of all unflushed WALs in bytes. + fn estimated_bytes(&self, table_store: &TableStore) -> usize { + let current_wal_size = + table_store.estimate_encoded_size_wal(self.current_wal.len(), self.current_wal.size()); + let imm_wal_size = self + .immutable_wals + .iter() + .map(|(_, wal)| table_store.estimate_encoded_size_wal(wal.len(), wal.size())) + .sum::(); + current_wal_size + imm_wal_size + } - let mut sst_builder = self.table_store.wal_table_builder(); - let mut iter = wal.iter(); - while let Some(entry) = iter.next() { - sst_builder.add(entry).await?; + fn status(&self, table_store: &TableStore) -> WalStatus { + let flushing_wal_entries_count = self + .immutable_wals + .iter() + .map(|(_, wal)| wal.len()) + .sum::(); + let buffered_wal_entries_count = self.current_wal.len() + flushing_wal_entries_count; + WalStatus { + estimated_bytes: self.estimated_bytes(table_store), + last_flushed_wal_id: self.last_flushed_wal_id, + last_purged_wal_id: self.last_purged_wal_id, + last_flushed_seq: self.last_flushed_seq, + buffered_wal_entries_count, } - - let encoded_sst = sst_builder.build().await?; - let written_bytes = encoded_sst.remaining_len() as u64; - self.table_store - .write_sst(&SsTableId::Wal(wal_id), &encoded_sst, false) - .await?; - self.stats.flush_bytes.increment(written_bytes); - Ok(()) } - fn freeze_current_wal(&self) -> Result<(), SlateDBError> { - let is_empty = self.inner.read().current_wal.is_empty(); - if is_empty { - return Ok(()); + fn freeze_current_wal(&mut self) { + if self.current_wal.is_empty() { + return; } - - let mut inner = self.inner.write(); - let next_wal_id = inner.next_wal_id; - inner.next_wal_id += 1; - let current_wal = std::mem::replace(&mut inner.current_wal, WalBuffer::new()); - inner.flush_epoch += 1; - inner - .immutable_wals + let next_wal_id = self.next_wal_id; + self.next_wal_id += 1; + let current_wal = std::mem::replace(&mut self.current_wal, WalBuffer::new()); + self.flush_epoch += 1; + self.immutable_wals .push_back((next_wal_id, Arc::new(current_wal))); - Ok(()) } - /// Track the last applied sequence number. It's called when some WAL entries are applied to the memtable. - /// This information of the last applied seq is used to determine if the immutable wals can be recycled. - /// - /// It's the caller's duty to ensure the seq is monotonically increasing. - pub(crate) fn track_last_applied_seq(&self, seq: u64) { - { - let mut inner = self.inner.write(); - inner.last_applied_seq = Some(seq); + fn record_flushed_wal(&mut self, wal_id: u64, wal: &Arc) { + self.last_flushed_wal_id = wal_id; + if let Some(seq) = wal.last_seq() { + if let Some(last_flushed_seq) = self.last_flushed_seq { + assert!(seq >= last_flushed_seq); + } + self.last_flushed_seq = Some(seq); } - self.maybe_release_immutable_wals(); - // don't notify here - notifications should only be issued from the flush task } - /// Recycle the immutable WALs that are flushed to the remote storage. - fn maybe_release_immutable_wals(&self) { - let mut inner = self.inner.write(); - - let last_applied_seq = match inner.last_applied_seq { + fn maybe_release_immutable_wals(&mut self) -> usize { + let last_applied_seq = match self.last_applied_seq { Some(seq) => seq, - None => return, + None => return 0, }; - let last_flushed_seq = inner.last_flushed_seq; + let last_flushed_seq = self.last_flushed_seq; let mut releaseable_count = 0; - for (_, wal) in inner.immutable_wals.iter() { + for (_, wal) in self.immutable_wals.iter() { if wal .last_seq() // TODO: check me (make sure seq starts at 1) @@ -426,57 +389,16 @@ impl WalBufferManager { "draining immutable wals [releaseable_count={}]", releaseable_count ); - let last_purged = inner + let last_purged = self .immutable_wals .drain(..releaseable_count) .map(|(id, _wal)| id) .max(); if let Some(last_purged) = last_purged { - inner.last_purged_wal_id = last_purged; + self.last_purged_wal_id = last_purged; } } - } - - #[allow(dead_code)] - pub(crate) async fn close(&self) -> Result<(), SlateDBError> { - let task_executor = { - let inner = self.inner.read(); - inner - .task_executor - .clone() - .expect("task executor should be initialized") - }; - task_executor.shutdown_task(WAL_BUFFER_TASK_NAME).await - } -} - -impl WalBufferManagerInner { - /// Returns the total size of all unflushed WALs in bytes. - fn estimated_bytes(&self, table_store: &TableStore) -> usize { - let current_wal_size = - table_store.estimate_encoded_size_wal(self.current_wal.len(), self.current_wal.size()); - let imm_wal_size = self - .immutable_wals - .iter() - .map(|(_, wal)| table_store.estimate_encoded_size_wal(wal.len(), wal.size())) - .sum::(); - current_wal_size + imm_wal_size - } - - fn status(&self, table_store: &TableStore) -> WalStatus { - let flushing_wal_entries_count = self - .immutable_wals - .iter() - .map(|(_, wal)| wal.len()) - .sum::(); - let buffered_wal_entries_count = self.current_wal.len() + flushing_wal_entries_count; - WalStatus { - estimated_bytes: self.estimated_bytes(table_store), - last_flushed_wal_id: self.last_flushed_wal_id, - last_flushed_seq: self.last_flushed_seq, - last_purged_wal_id: self.last_purged_wal_id, - buffered_wal_entries_count, - } + releaseable_count } } @@ -558,14 +480,102 @@ impl WalBufferIterator { } } -#[derive(Debug)] -struct WalFlushWork { - result_tx: Option>>, +enum WalFlushWork { + Flush { + result_tx: Option>>, + }, + Subscribe { + listener: WalStatusListener, + }, +} + +impl Debug for WalFlushWork { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + WalFlushWork::Flush { .. } => f.write_str("Flush"), + WalFlushWork::Subscribe { .. } => f.write_str("Subscribe"), + } + } } struct WalFlushHandler { max_flush_interval: Option, - wal_buffer_manager: Arc, + inner: Arc>, + table_store: Arc, + stats: Arc, + listener: Option, +} + +impl WalFlushHandler { + #[instrument(level = "trace", skip_all, err(level = tracing::Level::DEBUG))] + async fn do_flush(&self) -> Result<(), SlateDBError> { + let flushing_wals = { + let mut inner = self.inner.write(); + inner.freeze_current_wal(); + inner.flushing_wals() + }; + + for (wal_id, wal) in flushing_wals.iter() { + let result = self.do_flush_one_wal(*wal_id, wal.clone()).await; + if let Err(e) = &result { + // a WAL buffer can be retried to flush multiple times, but WatchableOnceCell is only set once. + // we do NOT call `wal.notify_durable` as soon as encountered any error here, but notify + // the error when we're sure enters fatal state in `do_cleanup`. + error!("failed to flush WAL [wal_id={}]", wal_id); + return Err(e.clone()); + } + + // increment the last flushed wal id, and last flushed seq + let status = { + let mut inner = self.inner.write(); + inner.record_flushed_wal(*wal_id, wal); + inner.status(&self.table_store) + }; + + self.notify_listener(WalEvent::WalFlushed(status)); + wal.notify_durable(result.clone()); + } + + self.maybe_release_immutable_wals(); + + Ok(()) + } + + fn maybe_release_immutable_wals(&self) { + let (status, released_wals) = { + let mut inner = self.inner.write(); + let released_wals = inner.maybe_release_immutable_wals(); + let status = inner.status(&self.table_store); + (status, released_wals) + }; + if released_wals > 0 { + self.notify_listener(WalEvent::MemoryReleased(status)); + } + } + + async fn do_flush_one_wal(&self, wal_id: u64, wal: Arc) -> Result<(), SlateDBError> { + self.stats.flushes.increment(1); + + let mut sst_builder = self.table_store.wal_table_builder(); + let mut iter = wal.iter(); + while let Some(entry) = iter.next() { + sst_builder.add(entry).await?; + } + + let encoded_sst = sst_builder.build().await?; + let written_bytes = encoded_sst.remaining_len() as u64; + self.table_store + .write_sst(&SsTableId::Wal(wal_id), &encoded_sst, false) + .await?; + self.stats.flush_bytes.increment(written_bytes); + Ok(()) + } + + fn notify_listener(&self, event: WalEvent) { + if let Some(l) = self.listener.as_ref() { + (*l)(event); + } + } } #[async_trait] @@ -574,20 +584,29 @@ impl MessageHandler for WalFlushHandler { if let Some(max_flush_interval) = self.max_flush_interval { return vec![MessageTickerDef::new( max_flush_interval, - Box::new(|| WalFlushWork { result_tx: None }), + Box::new(|| WalFlushWork::Flush { result_tx: None }), )]; } vec![] } async fn handle(&mut self, message: WalFlushWork) -> Result<(), SlateDBError> { - let WalFlushWork { result_tx } = message; - if let Some(result_tx) = result_tx { - let result = self.wal_buffer_manager.do_flush().await; - let _ = result_tx.send(result.clone()); - result - } else { - self.wal_buffer_manager.do_flush().await + match message { + WalFlushWork::Flush { result_tx } => { + if let Some(result_tx) = result_tx { + let result = self.do_flush().await; + let _ = result_tx.send(result.clone()); + result + } else { + self.do_flush().await + } + } + WalFlushWork::Subscribe { listener } => { + // TODO: support multiple listeners. For now, the db listener is the only one + assert!(self.listener.is_none()); + self.listener = Some(listener); + Ok(()) + } } } @@ -599,18 +618,25 @@ impl MessageHandler for WalFlushHandler { let error = result.err().unwrap_or(SlateDBError::Closed); // drain remaining messages - while let Some(WalFlushWork { result_tx }) = messages.next().await { - if let Some(result_tx) = result_tx { - let _ = result_tx.send(Err(error.clone())); + while let Some(msg) = messages.next().await { + match msg { + WalFlushWork::Flush { result_tx } => { + if let Some(result_tx) = result_tx { + let _ = result_tx.send(Err(error.clone())); + } + } + WalFlushWork::Subscribe { listener: _ } => {} } } // notify all the flushing wals to be finished with fatal error or shutdown // error. we need ensure all the wal tables finally get notified. freeze current // WAL to notify writers in the subsequent flushing_wals loop. - self.wal_buffer_manager.freeze_current_wal()?; - - let flushing_wals = self.wal_buffer_manager.flushing_wals(); + let flushing_wals = { + let mut inner = self.inner.write(); + inner.freeze_current_wal(); + inner.flushing_wals() + }; for (_, wal) in flushing_wals.iter() { wal.notify_durable(Err(error.clone())); } @@ -621,7 +647,9 @@ impl MessageHandler for WalFlushHandler { /// Interface for getting information about the current state of the Wal #[derive(Clone)] pub(crate) struct WalObserver { - wal_buffer: Arc, + inner: Arc>, + table_store: Arc, + flush_tx: SafeSender, } /// Describes the current status of the WAL @@ -653,11 +681,13 @@ pub(crate) enum WalEvent { impl WalObserver { /// Gets information about the Wal buffer's current state pub(crate) fn status(&self) -> WalStatus { - self.wal_buffer.status() + self.inner.read().status(self.table_store.as_ref()) } - pub(crate) fn subscribe(&self, listener: WalStatusListener) { - self.wal_buffer.subscribe(listener); + pub(crate) fn subscribe(&self, listener: WalStatusListener) -> Result<(), SlateDBError> { + self.flush_tx + .send(WalFlushWork::Subscribe { listener }) + .map_err(|_err| SlateDBError::Closed) } } @@ -889,7 +919,7 @@ mod tests { } async fn setup_wal_buffer() -> ( - Arc, + WalBufferManager, Arc, Arc, Arc, @@ -900,7 +930,7 @@ mod tests { async fn setup_wal_buffer_with_flush_interval( flush_interval: Duration, ) -> ( - Arc, + WalBufferManager, Arc, Arc, Arc, @@ -912,7 +942,7 @@ mod tests { flush_interval: Duration, listener: WalStatusListener, ) -> ( - Arc, + WalBufferManager, Arc, Arc, Arc, @@ -931,21 +961,24 @@ mod tests { let oracle = Arc::new(DbOracle::new(0, 0, 0, status_manager.clone())); let recorder = Arc::new(DefaultMetricsRecorder::new()); let helper = MetricsRecorderHelper::new(recorder.clone(), MetricLevel::default()); - let wal_buffer = Arc::new(WalBufferManager::new( + let mut wal_buffer = WalBufferManager::new( status_manager.clone(), &helper, 0, // recent_flushed_wal_id table_store.clone(), 1000, // max_wal_bytes_size Some(flush_interval), // max_flush_interval - )); - wal_buffer.subscribe(Arc::new(move |status| { - (*listener)(status.clone()); - let WalEvent::WalFlushed(status) = status else { - return; - }; - oracle.advance_durable_seq(status.last_flushed_seq.unwrap_or(0)) - })); + ); + let observer = wal_buffer.observer(); + observer + .subscribe(Arc::new(move |status| { + (*listener)(status.clone()); + let WalEvent::WalFlushed(status) = status else { + return; + }; + oracle.advance_durable_seq(status.last_flushed_seq.unwrap_or(0)) + })) + .unwrap(); let task_executor = Arc::new(MessageHandlerExecutor::new( Arc::new(status_manager), system_clock.clone(), From cc69461d902560bb5f4407a506f32cd154ede79d Mon Sep 17 00:00:00 2001 From: Almog Gavra Date: Tue, 14 Jul 2026 16:52:44 -0700 Subject: [PATCH 17/63] fix accounting for max_concurrent_compactions (#1926) --- slatedb/src/compactor_state.rs | 21 ++++++++++++- slatedb/src/size_tiered_compaction.rs | 43 +++++++++++++++++++++++++-- 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/slatedb/src/compactor_state.rs b/slatedb/src/compactor_state.rs index 5004d2b56..a95d1aaf3 100644 --- a/slatedb/src/compactor_state.rs +++ b/slatedb/src/compactor_state.rs @@ -274,6 +274,14 @@ impl CompactionStatus { ) } + /// Returns whether this compaction still consumes scheduler capacity. + pub(crate) fn counts_against_max_concurrent(self) -> bool { + matches!( + self, + CompactionStatus::Submitted | CompactionStatus::Scheduled | CompactionStatus::Running + ) + } + fn finished(self) -> bool { matches!(self, CompactionStatus::Completed | CompactionStatus::Failed) } @@ -1365,14 +1373,25 @@ mod tests { } #[test] - fn test_compaction_status_active_and_finished() { + fn test_compaction_status_classifications() { assert!(CompactionStatus::Submitted.active()); + assert!(CompactionStatus::Scheduled.active()); assert!(CompactionStatus::Running.active()); + assert!(CompactionStatus::Compacted.active()); assert!(!CompactionStatus::Completed.active()); assert!(!CompactionStatus::Failed.active()); + assert!(CompactionStatus::Submitted.counts_against_max_concurrent()); + assert!(CompactionStatus::Scheduled.counts_against_max_concurrent()); + assert!(CompactionStatus::Running.counts_against_max_concurrent()); + assert!(!CompactionStatus::Compacted.counts_against_max_concurrent()); + assert!(!CompactionStatus::Completed.counts_against_max_concurrent()); + assert!(!CompactionStatus::Failed.counts_against_max_concurrent()); + assert!(!CompactionStatus::Submitted.finished()); + assert!(!CompactionStatus::Scheduled.finished()); assert!(!CompactionStatus::Running.finished()); + assert!(!CompactionStatus::Compacted.finished()); assert!(CompactionStatus::Completed.finished()); assert!(CompactionStatus::Failed.finished()); } diff --git a/slatedb/src/size_tiered_compaction.rs b/slatedb/src/size_tiered_compaction.rs index 0a722a9a1..616ad5a2b 100644 --- a/slatedb/src/size_tiered_compaction.rs +++ b/slatedb/src/size_tiered_compaction.rs @@ -185,6 +185,14 @@ impl CompactionScheduler for SizeTieredCompactionScheduler { .flat_map(|c| c.core().recent_compactions()) .filter(|c| c.active()) .collect::>(); + // A Compacted job has finished using a worker slot, even though it + // remains active until its output is committed to the manifest. Keep + // it in `active_compactions` for conflict checks and destination-id + // reservation, but do not count it against execution capacity. + let compaction_slots_in_use = active_compactions + .iter() + .filter(|c| c.status().counts_against_max_concurrent()) + .count(); let mut next_fresh_sr_id = next_global_sr_id(db_state, &active_compactions); // Precompute per-tree (sources, conflict checker, backpressure) once @@ -235,7 +243,7 @@ impl CompactionScheduler for SizeTieredCompactionScheduler { loop { let mut picked_any = false; for tree in &mut trees { - if active_compactions.len() + compactions.len() >= self.max_concurrent_compactions { + if compaction_slots_in_use + compactions.len() >= self.max_concurrent_compactions { break; } if let Some(compaction) = self.pick_next_compaction(tree, &mut next_fresh_sr_id) { @@ -501,7 +509,7 @@ mod tests { use crate::compactor::{CompactionScheduler, CompactionSchedulerSupplier}; use crate::compactor_state::{ - Compaction, CompactionSpec, Compactions, CompactorState, SourceId, + Compaction, CompactionSpec, CompactionStatus, Compactions, CompactorState, SourceId, }; use crate::config::{CompactorOptions, SizeTieredCompactionSchedulerOptions}; use crate::db_state::{SortedRun, SsTableHandle, SsTableId, SsTableInfo, SsTableView}; @@ -694,6 +702,37 @@ mod tests { assert_eq!(requests.len(), 0); } + #[test] + fn test_compacted_job_does_not_consume_compaction_slot() { + // A finished worker job in one tree is still waiting for its manifest + // commit, while a disjoint tree has eligible work. With one worker + // slot, the scheduler should immediately refill that slot. + let scheduler = + SizeTieredCompactionScheduler::new(SizeTieredCompactionSchedulerOptions::default(), 1); + let root_l0: Vec = (0..4).map(|_| create_sst_view(1)).collect(); + let segment_l0: Vec = (0..4).map(|_| create_sst_view(1)).collect(); + let mut core = create_db_state(root_l0.iter().cloned().collect(), Vec::new()); + core.segments = vec![segment_with( + b"finished/", + segment_l0.iter().cloned().collect(), + Vec::new(), + )]; + let mut state = create_compactor_state(core); + + let completed_worker_job = Compaction::new( + ulid::Ulid::new(), + create_segment_l0_compaction(b"finished/", &segment_l0, 0), + ) + .with_status(CompactionStatus::Compacted); + state.insert_compaction_for_test(completed_worker_job); + + let requests = scheduler.propose(&(&state).into()); + + assert_eq!(requests.len(), 1); + assert!(requests[0].segment().is_empty()); + assert_eq!(requests[0].destination(), Some(1)); + } + #[test] fn test_should_not_compact_srs_if_fewer_than_min_threshold() { // given: From 844f7246c0ab140becbffdb0f8ff820280398e10 Mon Sep 17 00:00:00 2001 From: Rohan Date: Wed, 15 Jul 2026 00:31:08 -0400 Subject: [PATCH 18/63] k/N wal refactor: drop last seq tracking from the wal (#1927) --- slatedb/src/batch_write.rs | 4 --- slatedb/src/wal_buffer.rs | 52 ++++++++------------------------------ 2 files changed, 11 insertions(+), 45 deletions(-) diff --git a/slatedb/src/batch_write.rs b/slatedb/src/batch_write.rs index 82e75833a..bb92715f1 100644 --- a/slatedb/src/batch_write.rs +++ b/slatedb/src/batch_write.rs @@ -269,10 +269,6 @@ impl DbInner { // after merge operators and overwrites are collapsed self.db_stats.memtable_write_bytes.increment(entries_size); - // update the last_applied_seq to wal buffer. if a chunk of WAL entries are applied to the memtable - // and flushed to the remote storage, WAL buffer manager will recycle these WAL entries. - wal_buffer.track_last_applied_seq(commit_seq); - // insert a fail point to make it easier to test the case where the last_committed_seq is not updated. // this is useful for testing the case where the reader is not able to see the writes. fail_point!( diff --git a/slatedb/src/wal_buffer.rs b/slatedb/src/wal_buffer.rs index 25df51833..ea4ba6e2b 100644 --- a/slatedb/src/wal_buffer.rs +++ b/slatedb/src/wal_buffer.rs @@ -70,9 +70,6 @@ struct WalBufferManagerInner { /// When the current WAL is ready to be flushed, it'll be moved to the `immutable_wals`. /// The flusher will try flush all the immutable wals to remote storage. immutable_wals: VecDeque<(u64, Arc)>, - /// Whenever a WAL is applied to Memtable and successfully flushed to remote storage, - /// the immutable wal can be recycled in memory. - last_applied_seq: Option, /// The next wal id that will be generated next_wal_id: u64, /// Monotonically increasing epoch incremented each time the current WAL is @@ -127,7 +124,6 @@ impl WalBufferManager { let inner = WalBufferManagerInner { current_wal, immutable_wals, - last_applied_seq: None, flush_epoch: 1, last_flushed_wal_id, last_purged_wal_id: last_flushed_wal_id, @@ -253,18 +249,6 @@ impl WalBufferManager { Ok(result_rx) } - /// Track the last applied sequence number. It's called when some WAL entries are applied to the memtable. - /// This information of the last applied seq is used to determine if the immutable wals can be recycled. - /// - /// It's the caller's duty to ensure the seq is monotonically increasing. - pub(crate) fn track_last_applied_seq(&self, seq: u64) { - { - let mut inner = self.inner.write(); - inner.last_applied_seq = Some(seq); - inner.maybe_release_immutable_wals(); - } - } - #[allow(dead_code)] pub(crate) async fn close(&self) -> Result<(), SlateDBError> { let task_executor = self @@ -363,19 +347,13 @@ impl WalBufferManagerInner { } fn maybe_release_immutable_wals(&mut self) -> usize { - let last_applied_seq = match self.last_applied_seq { - Some(seq) => seq, - None => return 0, - }; - let last_flushed_seq = self.last_flushed_seq; let mut releaseable_count = 0; - for (_, wal) in self.immutable_wals.iter() { + for (_id, wal) in self.immutable_wals.iter() { if wal .last_seq() - // TODO: check me (make sure seq starts at 1) - .map(|seq| seq <= last_applied_seq && seq <= last_flushed_seq.unwrap_or(0)) + .map(|seq| seq <= last_flushed_seq.unwrap_or(0)) .unwrap_or(false) { releaseable_count += 1; @@ -1061,34 +1039,27 @@ mod tests { wal_buffer.flush().unwrap().await.unwrap().unwrap(); } assert_eq!(wal_buffer.last_flushed_wal_id(), 100); - assert_eq!(wal_buffer.inner.read().immutable_wals.len(), 100); - - wal_buffer.track_last_applied_seq(50); - assert_eq!(wal_buffer.inner.read().immutable_wals.len(), 50); + assert_eq!(wal_buffer.inner.read().immutable_wals.len(), 0); } #[tokio::test] async fn test_immutable_wal_reclaim_with_flush_check() { - let (wal_buffer, _, _, _) = setup_wal_buffer().await; + let (wal_buffer, _, _, _) = setup_wal_buffer_with_flush_interval(Duration::MAX).await; // Append entries to create multiple WALs for i in 0..100 { let seq = i + 1; let entry = make_entry(&format!("key{}", i), &format!("value{}", i), seq, None); wal_buffer.append(&[entry]).unwrap(); - wal_buffer.flush().unwrap().await.unwrap().unwrap(); + wal_buffer.inner.write().freeze_current_wal(); } - wal_buffer.track_last_applied_seq(50); - assert_eq!(wal_buffer.inner.read().immutable_wals.len(), 50); - assert_eq!(wal_buffer.last_flushed_wal_id(), 100); + // simulate flushing just some of the wals + wal_buffer.inner.write().last_flushed_seq = Some(50); - // set flush seq to 80, and track last applied seq to 90, it should release 20 wals - { - let mut inner = wal_buffer.inner.write(); - inner.last_flushed_seq = Some(80); - } - wal_buffer.track_last_applied_seq(90); - assert_eq!(wal_buffer.inner.read().immutable_wals.len(), 20); + let released = wal_buffer.inner.write().maybe_release_immutable_wals(); + + assert_eq!(released, 50); + assert_eq!(wal_buffer.inner.read().immutable_wals.len(), 50); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -1175,7 +1146,6 @@ mod tests { // given: let (listener, events) = recording_listener(); let (wal_buffer, _, _, _) = setup_wal_buffer_with_args(Duration::MAX, listener).await; - wal_buffer.track_last_applied_seq(10); // when: wal_buffer From 7e5108d48157103621dbf3d1cf1a647ae561f7a0 Mon Sep 17 00:00:00 2001 From: Rohan Date: Wed, 15 Jul 2026 01:11:55 -0400 Subject: [PATCH 19/63] switch fence fault injection over to fail channel from fail-parallel (#1928) --- Cargo.lock | 18 ++++---- Cargo.toml | 2 +- slatedb/src/fence.rs | 107 +++++++++++-------------------------------- 3 files changed, 38 insertions(+), 89 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3ee67f46f..609dcfc66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -888,7 +888,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -925,9 +925,9 @@ dependencies = [ [[package]] name = "fail-parallel" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f4a147ba57fcd64323c54c8f69fd4e045456a99574163010ccf5eea3168aaa" +checksum = "c29b33a0187823f1fa88b36980227dc96c7504ede2288e7d2a77d9d6d88b260c" dependencies = [ "log", "once_cell", @@ -1749,7 +1749,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2107,7 +2107,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2879,7 +2879,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2936,7 +2936,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3535,7 +3535,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4384,7 +4384,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 96561815e..c0bb1d446 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,7 @@ crossbeam-channel = "0.5.15" crossbeam-skiplist = "0.1.3" dotenvy = "0.15.7" duration-str = { version = "0.11.2", default-features = false } -fail-parallel = "0.5.2" +fail-parallel = "0.6.0" figment = "0.10.19" flate2 = "1.1.2" flatbuffers = "25.2.10" diff --git a/slatedb/src/fence.rs b/slatedb/src/fence.rs index 1ee2f58d1..cca8aa8fc 100644 --- a/slatedb/src/fence.rs +++ b/slatedb/src/fence.rs @@ -2,13 +2,10 @@ use crate::error::SlateDBError; use crate::manifest::store::{FenceableManifest, StoredManifest}; use crate::tablestore::TableStore; use crate::Settings; -#[cfg(test)] -use fail_parallel::fail_point; -use fail_parallel::FailPointRegistry; +use fail_parallel::{fail_point_send, FailPointTx}; use slatedb_common::SystemClock; -use std::collections::HashSet; use std::ops::Range; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use std::time::Duration; pub(crate) struct WriterFencer { @@ -16,7 +13,7 @@ pub(crate) struct WriterFencer { manifest_update_timeout: Duration, system_clock: Arc, #[cfg_attr(not(test), allow(dead_code))] - fp_ctl: Arc, + fp_tx: FailPointTx, } pub(crate) struct WriterFenceResult { @@ -24,79 +21,33 @@ pub(crate) struct WriterFenceResult { pub(crate) replay_range: Range, } -#[cfg_attr(not(test), allow(dead_code))] -struct FailPointCtl { - fp_registry: Arc, - event_tx: tokio::sync::mpsc::UnboundedSender, - event_toggles: Mutex>, -} - -impl FailPointCtl { - fn new( - fp_registry: Arc, - event_tx: tokio::sync::mpsc::UnboundedSender, - ) -> Self { - Self { - fp_registry, - event_tx, - event_toggles: Mutex::new(HashSet::new()), - } - } - - #[cfg(test)] - fn enable_fp(&self, event: impl ToString) { - self.event_toggles.lock().unwrap().insert(event.to_string()); - } -} - impl WriterFencer { pub(crate) fn new( table_store: Arc, settings: &Settings, system_clock: Arc, ) -> Self { - let (event_tx, _) = tokio::sync::mpsc::unbounded_channel(); - Self::new_with_fp_ctl( - table_store, - settings, - system_clock, - Arc::new(FailPointCtl::new( - Arc::new(FailPointRegistry::new()), - event_tx, - )), - ) + Self::new_with_fp_handle(table_store, settings, system_clock, FailPointTx::dummy()) } - fn new_with_fp_ctl( + fn new_with_fp_handle( table_store: Arc, settings: &Settings, system_clock: Arc, - fp_ctl: Arc, + fp_tx: FailPointTx, ) -> Self { Self { table_store, manifest_update_timeout: settings.manifest_update_timeout, system_clock, - fp_ctl, + fp_tx, } } - #[cfg(test)] - fn fp_notify(&self, event: impl ToString) { - let event = event.to_string(); - let _ = self.fp_ctl.event_tx.send(event.clone()); - let event_toggle = HashSet::clone(&*self.fp_ctl.event_toggles.lock().unwrap()); - fail_point!( - Arc::clone(&self.fp_ctl.fp_registry), - "fence_event", - event_toggle.contains(&event), - |_| {} - ); + fn fail_point_send(&self, _name: impl ToString) { + fail_point_send!(self.fp_tx, _name, |_| {}); } - #[cfg(not(test))] - fn fp_notify(&self, _event: impl ToString) {} - /// Fences all writers with an older epoch than the provided `stored_manifest` by (1) writing /// a new `FenceableManifest` with a bumped epoch, and (2) writing an empty WAL file that acts /// as a barrier. Any parallel old writers will fail with `SlateDBError::Fenced` when trying @@ -110,7 +61,7 @@ impl WriterFencer { .table_store .next_wal_sst_id(stored_manifest.manifest().core.replay_after_wal_id) .await?; - self.fp_notify("LoadEmptyWalId"); + self.fail_point_send("LoadEmptyWalId"); let mut manifest = FenceableManifest::init_writer( stored_manifest, @@ -118,7 +69,7 @@ impl WriterFencer { self.system_clock.clone(), ) .await?; - self.fp_notify("FenceManifest"); + self.fail_point_send("FenceManifest"); let mut manifest_dirty = manifest.prepare_dirty()?; // verify that the empty_wal_id we computed is still valid. Its possible that between @@ -133,7 +84,7 @@ impl WriterFencer { .await?; manifest.refresh().await?; manifest_dirty = manifest.prepare_dirty()?; - self.fp_notify("ReloadEmptyWalId"); + self.fail_point_send("ReloadEmptyWalId"); // at this point we still hold the epoch, so it should not be possible for the barrier // to have advanced past the computed empty_wal_id assert!(empty_wal_id > manifest_dirty.value.core.replay_after_wal_id); @@ -147,13 +98,13 @@ impl WriterFencer { Err(SlateDBError::Fenced) => false, Err(err) => return Err(err), }; - self.fp_notify(format!("{}:{}", "WriteWalFence", attempt)); + self.fail_point_send(format!("{}:{}", "WriteWalFence", attempt)); // Refresh validates that we own the latest epoch still. manifest.refresh().await?; let dirty_manifest = manifest.prepare_dirty()?; let replay_after_wal_id = dirty_manifest.value.core.replay_after_wal_id; - self.fp_notify(format!("{}:{}", "RefreshManifest", attempt)); + self.fail_point_send(format!("{}:{}", "RefreshManifest", attempt)); if wrote_fence { // this writer is the only writer that could have written replay_after_wal_id, @@ -180,7 +131,7 @@ mod tests { FlushOptions, FlushType, GarbageCollectorDirectoryOptions, GarbageCollectorOptions, }; use crate::error::SlateDBError; - use crate::fence::{FailPointCtl, WriterFencer}; + use crate::fence::WriterFencer; use crate::format::sst::SsTableFormat; use crate::garbage_collector::GarbageCollector; use crate::manifest::store::{ManifestStore, StoredManifest}; @@ -190,6 +141,7 @@ mod tests { use crate::tablestore::{TableStore, TableStoreKind}; use crate::{CloseReason, Db, ErrorKind, Settings}; use bytes::Bytes; + use fail_parallel::fail_point_channel; use fail_parallel::FailPointRegistry; use object_store::memory::InMemory; use object_store::path::Path; @@ -207,7 +159,6 @@ mod tests { manifest_store: Arc, table_store: Arc, fp_registry: Arc, - fp_ctl: Arc, event_rx: tokio::sync::mpsc::UnboundedReceiver, fencer: Option, stored_manifest: Option, @@ -236,13 +187,12 @@ mod tests { .await .unwrap(); let fp_registry = Arc::new(FailPointRegistry::new()); - let (event_tx, event_rx) = tokio::sync::mpsc::unbounded_channel(); - let fp_ctl = Arc::new(FailPointCtl::new(fp_registry.clone(), event_tx)); - let fencer = WriterFencer::new_with_fp_ctl( + let (fp_tx, event_rx) = fail_point_channel(fp_registry.clone()); + let fencer = WriterFencer::new_with_fp_handle( table_store.clone(), &settings, system_clock.clone(), - fp_ctl.clone(), + fp_tx, ); Self { object_store, @@ -250,7 +200,6 @@ mod tests { manifest_store, table_store, fp_registry, - fp_ctl, event_rx, fencer: Some(fencer), stored_manifest: Some(stored_manifest), @@ -470,9 +419,8 @@ mod tests { // initialize a fencer. configure it to pause at LoadEmptyWalId (so the // fenced writer can race ahead) and at the case's event (so a new // writer can claim the epoch out from under the fencer). - h.fp_ctl.enable_fp("LoadEmptyWalId"); - h.fp_ctl.enable_fp(case.pause_event); - fail_parallel::cfg(h.fp_registry.clone(), "fence_event", "pause").unwrap(); + fail_parallel::cfg(h.fp_registry.clone(), "LoadEmptyWalId", "pause").unwrap(); + fail_parallel::cfg(h.fp_registry.clone(), case.pause_event, "pause").unwrap(); let fencer = h.fencer.take().unwrap(); let stored_manifest = h.stored_manifest.take().unwrap(); @@ -482,7 +430,7 @@ mod tests { // after LoadEmptyWalId pause, have the fenced db write some wals and flush and gc. // This advances replay_after_wal_id past the fencer's stale empty_wal_id, so the - // recompute branch (and ReloadEmptyWalId fp_notify) fires when the fencer resumes. + // recompute branch (and ReloadEmptyWalId fail_point_send) fires when the fencer resumes. h.put(&db, 1, false).await; h.put(&db, 2, false).await; db.flush_with_options(FlushOptions { @@ -503,7 +451,8 @@ mod tests { // resume the fencer. re-issuing "pause" wakes the current pause and // keeps the action set to "pause" so the next toggled event also // pauses. - fail_parallel::cfg(h.fp_registry.clone(), "fence_event", "pause").unwrap(); + fail_parallel::cfg(h.fp_registry.clone(), "LoadEmptyWalId", "pause").unwrap(); + fail_parallel::cfg(h.fp_registry.clone(), case.pause_event, "pause").unwrap(); // wait for the case's pause event. fp_notify sends an event for every // failpoint regardless of whether it pauses, so drain intermediate @@ -540,7 +489,8 @@ mod tests { } // resume the fencer - fail_parallel::cfg(h.fp_registry.clone(), "fence_event", "off").unwrap(); + fail_parallel::cfg(h.fp_registry.clone(), "LoadEmptyWalId", "off").unwrap(); + fail_parallel::cfg(h.fp_registry.clone(), case.pause_event, "off").unwrap(); // validate that its fenced — the fencer's manifest.refresh sees the new db's // bumped epoch and returns Fenced. @@ -571,8 +521,7 @@ mod tests { h.put(&db, 0, false).await; // configure the fencer to pause - h.fp_ctl.enable_fp(case.event); - fail_parallel::cfg(h.fp_registry.clone(), "fence_event", "pause").unwrap(); + fail_parallel::cfg(h.fp_registry.clone(), case.event, "pause").unwrap(); // spawn WriterFencer on another task let fencer = h.fencer.take().unwrap(); @@ -612,7 +561,7 @@ mod tests { } // unpause WriterFencer - fail_parallel::cfg(h.fp_registry.clone(), "fence_event", "off").unwrap(); + fail_parallel::cfg(h.fp_registry.clone(), case.event, "off").unwrap(); // verify it returns successfully let result = jh.await.unwrap().unwrap(); // The fencer's stale empty_wal_id was retried above the fenced writer's possibly From 6a28896b30dc98f2371ccc59d7c9e1a1871473f0 Mon Sep 17 00:00:00 2001 From: Kaivalya Apte Date: Thu, 16 Jul 2026 01:23:10 +0200 Subject: [PATCH 20/63] [1923] Add configurable object-store retry bound (#1924) --- bindings/uniffi/src/config.rs | 1 + slatedb-cli/src/main.rs | 6 ++ slatedb-dst/src/utils.rs | 2 + slatedb/src/admin.rs | 3 + slatedb/src/compactions_store.rs | 1 + slatedb/src/config.rs | 30 ++++++ slatedb/src/db.rs | 2 + slatedb/src/db/builder.rs | 20 +++- slatedb/src/db_transaction.rs | 1 + slatedb/src/fence.rs | 1 + slatedb/src/garbage_collector.rs | 12 +++ slatedb/src/instrumented_object_store.rs | 1 + slatedb/src/manifest/store.rs | 1 + slatedb/src/retrying_object_store.rs | 113 ++++++++++++++++------- slatedb/src/tablestore.rs | 1 + 15 files changed, 159 insertions(+), 36 deletions(-) diff --git a/bindings/uniffi/src/config.rs b/bindings/uniffi/src/config.rs index 1a57fb73e..024a9d3ec 100644 --- a/bindings/uniffi/src/config.rs +++ b/bindings/uniffi/src/config.rs @@ -505,6 +505,7 @@ impl From for slatedb::config::GarbageCollectorOptions detach_options: value.detach_options.map(Into::into), metric_level: None, boundary_files_enabled: !value.disable_boundary_files, + object_store_max_retries: None, } } } diff --git a/slatedb-cli/src/main.rs b/slatedb-cli/src/main.rs index fd12de8bd..08d5fd44e 100644 --- a/slatedb-cli/src/main.rs +++ b/slatedb-cli/src/main.rs @@ -313,6 +313,7 @@ async fn exec_gc_once( detach_options: None, metric_level: None, boundary_files_enabled, + object_store_max_retries: None, }, GcResource::Wal => GarbageCollectorOptions { manifest_options: None, @@ -323,6 +324,7 @@ async fn exec_gc_once( detach_options: None, metric_level: None, boundary_files_enabled, + object_store_max_retries: None, }, GcResource::WalFence => GarbageCollectorOptions { manifest_options: None, @@ -333,6 +335,7 @@ async fn exec_gc_once( detach_options: None, metric_level: None, boundary_files_enabled, + object_store_max_retries: None, }, GcResource::Compacted => GarbageCollectorOptions { manifest_options: None, @@ -343,6 +346,7 @@ async fn exec_gc_once( detach_options: None, metric_level: None, boundary_files_enabled, + object_store_max_retries: None, }, GcResource::Compactions => GarbageCollectorOptions { manifest_options: None, @@ -353,6 +357,7 @@ async fn exec_gc_once( detach_options: None, metric_level: None, boundary_files_enabled, + object_store_max_retries: None, }, }; admin.run_gc_once(gc_opts).await?; @@ -386,6 +391,7 @@ async fn schedule_gc( detach_options: None, metric_level: None, boundary_files_enabled, + object_store_max_retries: None, }; admin diff --git a/slatedb-dst/src/utils.rs b/slatedb-dst/src/utils.rs index 127e195cd..690c554b8 100644 --- a/slatedb-dst/src/utils.rs +++ b/slatedb-dst/src/utils.rs @@ -126,6 +126,7 @@ pub fn build_settings_compactor(rng: &mut impl Rng) -> CompactorOptions { commit_compacted_interval: rng .random_range(Duration::from_millis(1)..Duration::from_secs(5)), worker_heartbeat_timeout, + object_store_max_retries: None, } } @@ -158,6 +159,7 @@ pub fn build_settings_gc(rng: &mut impl Rng) -> GarbageCollectorOptions { }), metric_level: None, boundary_files_enabled: true, + object_store_max_retries: None, } } diff --git a/slatedb/src/admin.rs b/slatedb/src/admin.rs index 5fb4c3d2a..7555ba789 100644 --- a/slatedb/src/admin.rs +++ b/slatedb/src/admin.rs @@ -50,6 +50,8 @@ pub struct Admin { pub(crate) system_clock: Arc, /// The random number generator to use for randomness. pub(crate) rand: Arc, + /// The retry policy applied to admin object-store operations. + pub(crate) object_store_max_retries: Option, #[cfg(feature = "compaction_filters")] pub(crate) compaction_filter_supplier: Option>, @@ -612,6 +614,7 @@ impl Admin { self.object_stores.store_of(store_type).clone(), self.rand.clone(), self.system_clock.clone(), + self.object_store_max_retries, )) } diff --git a/slatedb/src/compactions_store.rs b/slatedb/src/compactions_store.rs index 71fc8520b..e264b244a 100644 --- a/slatedb/src/compactions_store.rs +++ b/slatedb/src/compactions_store.rs @@ -501,6 +501,7 @@ mod tests { flaky.clone(), Arc::new(DbRand::default()), Arc::new(DefaultSystemClock::new()), + None, )); let store = Arc::new(CompactionsStore::new(&Path::from(ROOT), retrying.clone())); diff --git a/slatedb/src/config.rs b/slatedb/src/config.rs index f4be2da18..01b72e724 100644 --- a/slatedb/src/config.rs +++ b/slatedb/src/config.rs @@ -761,6 +761,16 @@ pub struct Settings { /// Default: no TTL (insertions will remain until deleted) pub default_ttl: Option, + /// Maximum number of wrapper-level retries for a single object-store + /// operation, on top of the `object_store` client's own HTTP retries. + /// Applies to both foreground (user API) and background-task operations, + /// since both share the same retrying object store. + /// + /// * `None` (default): retry transient errors indefinitely (historical behavior). + /// * `Some(n)`: give up after `n` retries and return the underlying error. + #[serde(default)] + pub object_store_max_retries: Option, + /// The block format for SST files. This is only available in tests /// to verify backward compatibility between V1 and V2 formats. #[cfg(test)] @@ -997,6 +1007,7 @@ impl Default for Settings { garbage_collector_options: Some(GarbageCollectorOptions::default()), metric_level: MetricLevel::default(), default_ttl: None, + object_store_max_retries: None, #[cfg(test)] block_format: None, } @@ -1042,6 +1053,11 @@ pub struct DbReaderOptions { /// Optional metrics reporting level for standalone readers. Defaults to /// [`MetricLevel::default`] when unset. pub metric_level: Option, + + /// Controls wrapper-level retries for this reader's object-store operations. + /// Defaults to unbounded retries. + #[serde(default)] + pub object_store_max_retries: Option, } impl Default for DbReaderOptions { @@ -1053,6 +1069,7 @@ impl Default for DbReaderOptions { object_store_cache_options: ObjectStoreCacheOptions::default(), skip_wal_replay: false, metric_level: None, + object_store_max_retries: None, } } } @@ -1145,6 +1162,11 @@ pub struct CompactorOptions { #[serde(deserialize_with = "deserialize_duration")] #[serde(serialize_with = "serialize_duration")] pub worker_heartbeat_timeout: Duration, + + /// Controls wrapper-level retries for this compactor's object-store + /// operations. Defaults to unbounded retries. + #[serde(default)] + pub object_store_max_retries: Option, } /// Default options for the compactor. Currently, only a @@ -1162,6 +1184,7 @@ impl Default for CompactorOptions { metric_level: None, commit_compacted_interval: Duration::from_secs(1), worker_heartbeat_timeout: Duration::from_secs(30), + object_store_max_retries: None, } } } @@ -1181,6 +1204,7 @@ impl std::fmt::Debug for CompactorOptions { .field("metric_level", &self.metric_level) .field("commit_compacted_interval", &self.commit_compacted_interval) .field("worker_heartbeat_timeout", &self.worker_heartbeat_timeout) + .field("object_store_max_retries", &self.object_store_max_retries) .finish() } } @@ -1447,6 +1471,11 @@ pub struct GarbageCollectorOptions { /// garbage collector operating on the database. #[serde(default = "default_boundary_files_enabled")] pub boundary_files_enabled: bool, + + /// Controls wrapper-level retries for this garbage collector's object-store + /// operations. Defaults to unbounded retries. + #[serde(default)] + pub object_store_max_retries: Option, } impl GarbageCollectorOptions { @@ -1548,6 +1577,7 @@ impl Default for GarbageCollectorOptions { detach_options: Some(GarbageCollectorScheduleOptions::default()), metric_level: None, boundary_files_enabled: true, + object_store_max_retries: None, } } } diff --git a/slatedb/src/db.rs b/slatedb/src/db.rs index a4db740fa..a7676304c 100644 --- a/slatedb/src/db.rs +++ b/slatedb/src/db.rs @@ -7001,6 +7001,7 @@ mod tests { garbage_collector_options: None, metric_level: MetricLevel::default(), default_ttl: ttl, + object_store_max_retries: None, block_format: None, } } @@ -7813,6 +7814,7 @@ mod tests { detach_options: None, metric_level: None, boundary_files_enabled: true, + object_store_max_retries: None, }; let gc = GarbageCollectorBuilder::new(path.clone(), object_store.clone()) diff --git a/slatedb/src/db/builder.rs b/slatedb/src/db/builder.rs index 31e54c56e..8f8cee183 100644 --- a/slatedb/src/db/builder.rs +++ b/slatedb/src/db/builder.rs @@ -433,6 +433,7 @@ impl> DbBuilder

{ let metrics_recorder = self.metrics_recorder.clone(); let recorder = MetricsRecorderHelper::new(self.metrics_recorder, self.settings.metric_level); + let max_retries = self.settings.object_store_max_retries; let retrying_main_object_store = instrumented_retrying_object_store( self.main_object_store.clone(), &recorder, @@ -440,6 +441,7 @@ impl> DbBuilder

{ ObjectStoreType::Main, rand.clone(), system_clock.clone(), + max_retries, ); let retrying_wal_object_store: Option> = self.wal_object_store.map(|s| { @@ -450,6 +452,7 @@ impl> DbBuilder

{ ObjectStoreType::Wal, rand.clone(), system_clock.clone(), + max_retries, ) }); @@ -665,6 +668,7 @@ impl> DbBuilder

{ ObjectStoreType::Main, rand.clone(), system_clock.clone(), + max_retries, ); let main: Arc = match &cached_object_store { Some(cached) if Arc::ptr_eq(&raw_store, &self.main_object_store) => { @@ -823,6 +827,7 @@ pub struct AdminBuilder> { wal_object_store: Option>, system_clock: Arc, rand: Arc, + object_store_max_retries: Option, #[cfg(feature = "compaction_filters")] compaction_filter_supplier: Option>, merge_operator: Option, @@ -837,6 +842,7 @@ impl> AdminBuilder

{ wal_object_store: None, system_clock: Arc::new(DefaultSystemClock::new()), rand: Arc::new(DbRand::default()), + object_store_max_retries: None, #[cfg(feature = "compaction_filters")] compaction_filter_supplier: None, merge_operator: None, @@ -896,6 +902,7 @@ impl> AdminBuilder

{ object_stores: ObjectStores::new(self.main_object_store, self.wal_object_store), system_clock: self.system_clock, rand: self.rand, + object_store_max_retries: self.object_store_max_retries, #[cfg(feature = "compaction_filters")] compaction_filter_supplier: self.compaction_filter_supplier, merge_operator: self.merge_operator, @@ -1022,6 +1029,7 @@ impl> GarbageCollectorBuilder

{ ObjectStoreType::Main, self.rand.clone(), self.system_clock.clone(), + self.options.object_store_max_retries, ); let retrying_wal_object_store = self.wal_object_store.map(|s| { instrumented_retrying_object_store( @@ -1031,6 +1039,7 @@ impl> GarbageCollectorBuilder

{ ObjectStoreType::Wal, self.rand.clone(), self.system_clock.clone(), + self.options.object_store_max_retries, ) }); let manifest_store = Arc::new(ManifestStore::new( @@ -1253,6 +1262,7 @@ impl> CompactorBuilder

{ ObjectStoreType::Main, self.rand.clone(), self.system_clock.clone(), + self.options.object_store_max_retries, ); let manifest_store = Arc::new(ManifestStore::new( &path, @@ -1747,6 +1757,7 @@ impl> DbReaderBuilder

{ ObjectStoreType::Main, self.rand.clone(), self.system_clock.clone(), + self.options.object_store_max_retries, ); let retrying_wal_object_store: Option> = @@ -1758,6 +1769,7 @@ impl> DbReaderBuilder

{ ObjectStoreType::Wal, self.rand.clone(), self.system_clock.clone(), + self.options.object_store_max_retries, ) }); @@ -2044,6 +2056,7 @@ fn instrumented_retrying_object_store( store_type: ObjectStoreType, rand: Arc, system_clock: Arc, + max_retries: Option, ) -> Arc { let instrumented: Arc = Arc::new(InstrumentedObjectStore::new( object_store, @@ -2051,7 +2064,12 @@ fn instrumented_retrying_object_store( component, store_type, )); - Arc::new(RetryingObjectStore::new(instrumented, rand, system_clock)) + Arc::new(RetryingObjectStore::new( + instrumented, + rand, + system_clock, + max_retries, + )) } #[allow(unreachable_code)] diff --git a/slatedb/src/db_transaction.rs b/slatedb/src/db_transaction.rs index 063c8400e..ff5d49492 100644 --- a/slatedb/src/db_transaction.rs +++ b/slatedb/src/db_transaction.rs @@ -2099,6 +2099,7 @@ mod tests { garbage_collector_options: None, metric_level: MetricLevel::default(), default_ttl: None, + object_store_max_retries: None, block_format: None, } } diff --git a/slatedb/src/fence.rs b/slatedb/src/fence.rs index cca8aa8fc..b1cc40147 100644 --- a/slatedb/src/fence.rs +++ b/slatedb/src/fence.rs @@ -259,6 +259,7 @@ mod tests { detach_options: None, metric_level: None, boundary_files_enabled: true, + object_store_max_retries: None, }; let gc = GarbageCollector::new( self.manifest_store.clone(), diff --git a/slatedb/src/garbage_collector.rs b/slatedb/src/garbage_collector.rs index e3910040f..bfc0ea9dd 100644 --- a/slatedb/src/garbage_collector.rs +++ b/slatedb/src/garbage_collector.rs @@ -1170,6 +1170,7 @@ mod tests { detach_options: None, metric_level: None, boundary_files_enabled: true, + object_store_max_retries: None, }; let gc = GarbageCollector::new( manifest_store.clone(), @@ -1237,6 +1238,7 @@ mod tests { detach_options: None, metric_level: None, boundary_files_enabled: true, + object_store_max_retries: None, }; let recorder = Arc::new(DefaultMetricsRecorder::new()); let helper = MetricsRecorderHelper::new(recorder.clone(), Default::default()); @@ -1303,6 +1305,7 @@ mod tests { detach_options: None, metric_level: None, boundary_files_enabled: true, + object_store_max_retries: None, }; let gc = GarbageCollector::new( manifest_store.clone(), @@ -1382,6 +1385,7 @@ mod tests { detach_options: None, metric_level: None, boundary_files_enabled: true, + object_store_max_retries: None, }; let gc = GarbageCollector::new( manifest_store.clone(), @@ -1837,6 +1841,7 @@ mod tests { detach_options: None, metric_level: None, boundary_files_enabled: true, + object_store_max_retries: None, }; let gc = GarbageCollector::new( @@ -1913,6 +1918,7 @@ mod tests { detach_options: None, metric_level: None, boundary_files_enabled: true, + object_store_max_retries: None, }; let mut gc = GarbageCollector::new( @@ -1984,6 +1990,7 @@ mod tests { detach_options: None, metric_level: None, boundary_files_enabled: true, + object_store_max_retries: None, }; let gc = GarbageCollector::new( @@ -2034,6 +2041,7 @@ mod tests { detach_options: None, metric_level: None, boundary_files_enabled: true, + object_store_max_retries: None, }; let mut gc = GarbageCollector::new( @@ -2088,6 +2096,7 @@ mod tests { detach_options: None, metric_level: None, boundary_files_enabled: true, + object_store_max_retries: None, }; let gc = GarbageCollector::new( @@ -2419,6 +2428,7 @@ mod tests { detach_options: None, metric_level: None, boundary_files_enabled: true, + object_store_max_retries: None, }; let recorder = MetricsRecorderHelper::noop(); let gc = GarbageCollector::new( @@ -2518,6 +2528,7 @@ mod tests { detach_options: None, metric_level: None, boundary_files_enabled: true, + object_store_max_retries: None, }; let recorder = Arc::new(DefaultMetricsRecorder::new()); let helper = MetricsRecorderHelper::new(recorder.clone(), Default::default()); @@ -2652,6 +2663,7 @@ mod tests { detach_options: None, metric_level: None, boundary_files_enabled: true, + object_store_max_retries: None, }; let recorder = MetricsRecorderHelper::noop(); let gc = GarbageCollector::new( diff --git a/slatedb/src/instrumented_object_store.rs b/slatedb/src/instrumented_object_store.rs index 5cc336455..1528a1191 100644 --- a/slatedb/src/instrumented_object_store.rs +++ b/slatedb/src/instrumented_object_store.rs @@ -651,6 +651,7 @@ mod tests { instrumented, Arc::new(DbRand::default()), Arc::new(DefaultSystemClock::default()), + None, ); // when: diff --git a/slatedb/src/manifest/store.rs b/slatedb/src/manifest/store.rs index 0ecd709ec..829a97d2a 100644 --- a/slatedb/src/manifest/store.rs +++ b/slatedb/src/manifest/store.rs @@ -939,6 +939,7 @@ mod tests { flaky.clone(), Arc::new(DbRand::default()), Arc::new(DefaultSystemClock::new()), + None, )); let ms = Arc::new(ManifestStore::new(&Path::from(ROOT), retrying.clone())); diff --git a/slatedb/src/retrying_object_store.rs b/slatedb/src/retrying_object_store.rs index 164ef18df..b054a04a8 100644 --- a/slatedb/src/retrying_object_store.rs +++ b/slatedb/src/retrying_object_store.rs @@ -46,12 +46,20 @@ impl Sleeper for SystemClockSleeper { } /// A thin wrapper around an `ObjectStore` that retries transient errors with -/// exponential backoff forever using the configured [`SystemClock`] for sleeps. +/// exponential backoff using the configured [`SystemClock`] for sleeps. +/// +/// Retries are unbounded by default; a bound can be configured via +/// `max_retries`, in which case an operation that keeps failing eventually +/// returns its underlying error instead of retrying forever. This applies to +/// both foreground and background object-store operations, since both go +/// through this wrapper. #[derive(Debug, Clone)] pub(crate) struct RetryingObjectStore { inner: Arc, rand: Arc, clock: Arc, + /// Maximum wrapper-level retries per operation. `None` = unbounded. + max_retries: Option, } impl RetryingObjectStore { @@ -59,16 +67,25 @@ impl RetryingObjectStore { inner: Arc, rand: Arc, clock: Arc, + max_retries: Option, ) -> Self { - Self { inner, rand, clock } + Self { + inner, + rand, + clock, + max_retries, + } } #[inline] - fn retry_builder() -> ExponentialBuilder { - ExponentialBuilder::default() - .without_max_times() + fn retry_builder(&self) -> ExponentialBuilder { + let builder = ExponentialBuilder::default() .with_min_delay(Duration::from_millis(100)) - .with_max_delay(Duration::from_secs(1)) + .with_max_delay(Duration::from_secs(1)); + match self.max_retries { + Some(max_retries) => builder.with_max_times(max_retries as usize), + None => builder.without_max_times(), + } } #[inline] @@ -116,7 +133,7 @@ impl RetryingObjectStore { ..Default::default() }; let result = (|| async { self.inner.get_opts(location, get_opts.clone()).await }) - .retry(Self::retry_builder()) + .retry(self.retry_builder()) .sleep(self.sleeper()) .notify(Self::notify) .when(Self::should_retry) @@ -283,7 +300,7 @@ impl ObjectStore for RetryingObjectStore { extensions, }) }) - .retry(Self::retry_builder()) + .retry(self.retry_builder()) .sleep(self.sleeper()) .notify(Self::notify) .when(Self::should_retry) @@ -321,7 +338,7 @@ impl ObjectStore for RetryingObjectStore { .put_opts(location, payload.clone(), opts_with_id.clone()) .await }) - .retry(Self::retry_builder()) + .retry(self.retry_builder()) .sleep(self.sleeper()) .notify(Self::notify) .when(Self::should_retry) @@ -339,7 +356,7 @@ impl ObjectStore for RetryingObjectStore { .put_opts(location, payload.clone(), opts.clone()) .await }) - .retry(Self::retry_builder()) + .retry(self.retry_builder()) .sleep(self.sleeper()) .notify(Self::notify) .when(Self::should_retry) @@ -379,7 +396,7 @@ impl ObjectStore for RetryingObjectStore { .put_multipart_opts(location, opts_with_id.clone()) .await }) - .retry(Self::retry_builder()) + .retry(self.retry_builder()) .sleep(self.sleeper()) .notify(Self::notify) .when(Self::should_retry) @@ -393,7 +410,7 @@ impl ObjectStore for RetryingObjectStore { | object_store::Error::NotImplemented { .. }, ) => { (|| async { self.inner.put_multipart_opts(location, opts.clone()).await }) - .retry(Self::retry_builder()) + .retry(self.retry_builder()) .sleep(self.sleeper()) .notify(Self::notify) .when(Self::should_retry) @@ -416,7 +433,7 @@ impl ObjectStore for RetryingObjectStore { ) -> BoxStream<'static, object_store::Result> { let inner = Arc::clone(&self.inner); let sleeper = self.sleeper(); - let retry_builder = Self::retry_builder(); + let retry_builder = self.retry_builder(); locations .then(move |loc| { let inner = Arc::clone(&inner); @@ -438,6 +455,7 @@ impl ObjectStore for RetryingObjectStore { fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { let inner = Arc::clone(&self.inner); let sleeper = self.sleeper(); + let retry_builder = self.retry_builder(); let prefix_owned = prefix.cloned(); // list() is a little more complex than the other functions because: @@ -456,7 +474,7 @@ impl ObjectStore for RetryingObjectStore { // Any error in the stream will return an error for try_collect stream.try_collect::>().await }) - .retry(Self::retry_builder()) + .retry(retry_builder) .sleep(sleeper) .notify(Self::notify) .when(Self::should_retry) @@ -483,6 +501,7 @@ impl ObjectStore for RetryingObjectStore { ) -> BoxStream<'static, object_store::Result> { let inner = Arc::clone(&self.inner); let sleeper = self.sleeper(); + let retry_builder = self.retry_builder(); let prefix_owned = prefix.cloned(); let offset_owned = offset.clone(); @@ -492,7 +511,7 @@ impl ObjectStore for RetryingObjectStore { let stream = inner.list_with_offset(prefix_owned.as_ref(), &offset_owned); stream.try_collect::>().await }) - .retry(Self::retry_builder()) + .retry(retry_builder) .sleep(sleeper) .notify(Self::notify) .when(Self::should_retry) @@ -512,7 +531,7 @@ impl ObjectStore for RetryingObjectStore { async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { (|| async { self.inner.list_with_delimiter(prefix).await }) - .retry(Self::retry_builder()) + .retry(self.retry_builder()) .sleep(self.sleeper()) .notify(Self::notify) .when(Self::should_retry) @@ -526,7 +545,7 @@ impl ObjectStore for RetryingObjectStore { options: CopyOptions, ) -> object_store::Result<()> { (|| async { self.inner.copy_opts(from, to, options.clone()).await }) - .retry(Self::retry_builder()) + .retry(self.retry_builder()) .sleep(self.sleeper()) .notify(Self::notify) .when(Self::should_retry) @@ -540,7 +559,7 @@ impl ObjectStore for RetryingObjectStore { options: RenameOptions, ) -> object_store::Result<()> { (|| async { self.inner.rename_opts(from, to, options.clone()).await }) - .retry(Self::retry_builder()) + .retry(self.retry_builder()) .sleep(self.sleeper()) .notify(Self::notify) .when(Self::should_retry) @@ -575,7 +594,7 @@ mod tests { async fn test_put_opts_retries_transient_until_success() { let inner: Arc = Arc::new(InMemory::new()); let flaky = Arc::new(FlakyObjectStore::new(inner, 1)); - let retrying = RetryingObjectStore::new(flaky.clone(), test_rand(), test_clock()); + let retrying = RetryingObjectStore::new(flaky.clone(), test_rand(), test_clock(), None); let path = Path::from("/data/obj"); retrying @@ -598,7 +617,7 @@ mod tests { async fn test_put_opts_preserves_extensions() { let inner: Arc = Arc::new(InMemory::new()); let marking: Arc = Arc::new(ExtensionObjectStore::new(inner)); - let retrying = RetryingObjectStore::new(marking, test_rand(), test_clock()); + let retrying = RetryingObjectStore::new(marking, test_rand(), test_clock(), None); let path = Path::from("/data/extension-put"); let result = retrying @@ -625,7 +644,7 @@ mod tests { .await .unwrap(); let marking: Arc = Arc::new(ExtensionObjectStore::new(inner)); - let retrying = RetryingObjectStore::new(marking, test_rand(), test_clock()); + let retrying = RetryingObjectStore::new(marking, test_rand(), test_clock(), None); let result = retrying .get_opts( @@ -647,7 +666,7 @@ mod tests { let inner: Arc = Arc::new(InMemory::new()); let flaky = Arc::new(FlakyObjectStore::new(inner, 1)); let clock = Arc::new(MockSystemClock::new()); - let retrying = RetryingObjectStore::new(flaky.clone(), test_rand(), clock.clone()); + let retrying = RetryingObjectStore::new(flaky.clone(), test_rand(), clock.clone(), None); let path = Path::from("/data/obj"); let handle = tokio::spawn({ @@ -690,7 +709,7 @@ mod tests { async fn test_put_opts_does_not_retry_on_already_exists() { let inner: Arc = Arc::new(InMemory::new()); let flaky = Arc::new(FlakyObjectStore::new(inner, 0)); - let retrying = RetryingObjectStore::new(flaky.clone(), test_rand(), test_clock()); + let retrying = RetryingObjectStore::new(flaky.clone(), test_rand(), test_clock(), None); let path = Path::from("/data/obj"); retrying @@ -732,7 +751,7 @@ mod tests { .unwrap(); let flaky = Arc::new(FlakyObjectStore::new(inner, 0).with_head_failures(1)); - let retrying = RetryingObjectStore::new(flaky.clone(), test_rand(), test_clock()); + let retrying = RetryingObjectStore::new(flaky.clone(), test_rand(), test_clock(), None); let meta = retrying.head(&path).await.expect("head should succeed"); assert_eq!(meta.size, 4); @@ -743,7 +762,7 @@ mod tests { async fn test_put_opts_does_not_retry_on_precondition() { let inner: Arc = Arc::new(InMemory::new()); let failing = Arc::new(FlakyObjectStore::new(inner, 0).with_put_precondition_always()); - let retrying = RetryingObjectStore::new(failing.clone(), test_rand(), test_clock()); + let retrying = RetryingObjectStore::new(failing.clone(), test_rand(), test_clock(), None); let path = Path::from("/p"); let err = retrying @@ -765,7 +784,7 @@ mod tests { #[tokio::test] async fn test_get_opts_does_not_retry_on_not_modified() { let inner: Arc = Arc::new(InMemory::new()); - let retrying = RetryingObjectStore::new(inner.clone(), test_rand(), test_clock()); + let retrying = RetryingObjectStore::new(inner.clone(), test_rand(), test_clock(), None); let path = Path::from("/data/obj"); retrying @@ -810,7 +829,7 @@ mod tests { } let flaky = Arc::new(FlakyObjectStore::new(inner, 0).with_list_failures(1, 1)); - let retrying = RetryingObjectStore::new(flaky.clone(), test_rand(), test_clock()); + let retrying = RetryingObjectStore::new(flaky.clone(), test_rand(), test_clock(), None); let listed: Vec<_> = retrying .list(None) @@ -845,7 +864,7 @@ mod tests { } let flaky = Arc::new(FlakyObjectStore::new(inner, 0).with_list_with_offset_failures(1, 1)); - let retrying = RetryingObjectStore::new(flaky.clone(), test_rand(), test_clock()); + let retrying = RetryingObjectStore::new(flaky.clone(), test_rand(), test_clock(), None); let offset = Path::from("/items/a"); let listed: Vec<_> = retrying @@ -870,7 +889,7 @@ mod tests { let flaky = Arc::new( FlakyObjectStore::new(inner, 0).with_put_succeeds_but_returns_already_exists(), ); - let retrying = RetryingObjectStore::new(flaky.clone(), test_rand(), test_clock()); + let retrying = RetryingObjectStore::new(flaky.clone(), test_rand(), test_clock(), None); let path = Path::from("/data/obj"); // Must use PutMode::Create to trigger ULID verification @@ -905,7 +924,7 @@ mod tests { .unwrap(); // Now try to write via RetryingObjectStore - should fail because ULID won't match - let retrying = RetryingObjectStore::new(inner.clone(), test_rand(), test_clock()); + let retrying = RetryingObjectStore::new(inner.clone(), test_rand(), test_clock(), None); let err = retrying .put_opts( &path, @@ -940,7 +959,7 @@ mod tests { .unwrap(); let flaky = Arc::new(FlakyObjectStore::new(inner, 0).with_get_range_failures(2)); - let retrying = RetryingObjectStore::new(flaky.clone(), test_rand(), test_clock()); + let retrying = RetryingObjectStore::new(flaky.clone(), test_rand(), test_clock(), None); let result = retrying .get_range(&path, 0..5) @@ -965,7 +984,7 @@ mod tests { // get_ranges calls get_range internally, so flaky get_range failures will trigger retries let flaky = Arc::new(FlakyObjectStore::new(inner, 0).with_get_range_failures(2)); - let retrying = RetryingObjectStore::new(flaky.clone(), test_rand(), test_clock()); + let retrying = RetryingObjectStore::new(flaky.clone(), test_rand(), test_clock(), None); let ranges = vec![0..5, 6..11]; let result = retrying @@ -983,7 +1002,7 @@ mod tests { use std::borrow::Cow; let inner: Arc = Arc::new(InMemory::new()); - let retrying = RetryingObjectStore::new(inner.clone(), test_rand(), test_clock()); + let retrying = RetryingObjectStore::new(inner.clone(), test_rand(), test_clock(), None); let path = Path::from("/data/obj"); let mut user_attrs = Attributes::new(); @@ -1038,7 +1057,7 @@ mod tests { #[tokio::test] async fn test_get_opts_range_read_size_check_passes() { let inner: Arc = Arc::new(InMemory::new()); - let retrying = RetryingObjectStore::new(inner.clone(), test_rand(), test_clock()); + let retrying = RetryingObjectStore::new(inner.clone(), test_rand(), test_clock(), None); let path = Path::from("/data/obj"); inner @@ -1077,7 +1096,7 @@ mod tests { .unwrap(); let flaky = Arc::new(FlakyObjectStore::new(inner, 0).with_truncate_get_range_bytes(1, 1)); - let retrying = RetryingObjectStore::new(flaky.clone(), test_rand(), test_clock()); + let retrying = RetryingObjectStore::new(flaky.clone(), test_rand(), test_clock(), None); // First attempt returns truncated body (1 byte vs 5 expected), // triggering the size check error. The retry succeeds normally. @@ -1097,4 +1116,28 @@ mod tests { // 1 truncated attempt + 1 successful retry assert_eq!(flaky.get_range_attempts(), 2); } + + #[tokio::test] + async fn test_bounded_max_retries_gives_up_instead_of_retrying_forever() { + // Store fails more times (5) than the configured retry bound (2). + let inner: Arc = Arc::new(InMemory::new()); + let flaky = Arc::new(FlakyObjectStore::new(inner, 5)); + let retrying = RetryingObjectStore::new(flaky.clone(), test_rand(), test_clock(), Some(2)); + + let path = Path::from("/data/obj"); + let err = retrying + .put_opts( + &path, + PutPayload::from_bytes(Bytes::from_static(b"hello")), + PutOptions::default(), + ) + .await + .expect_err("bounded retries should exhaust and surface the underlying error"); + + // The underlying transient error is returned rather than being retried + // forever, so callers/background tasks can fail fast. + assert!(matches!(err, object_store::Error::Generic { .. })); + // 1 initial attempt + 2 retries = 3 total attempts. + assert_eq!(flaky.put_attempts(), 3); + } } diff --git a/slatedb/src/tablestore.rs b/slatedb/src/tablestore.rs index 42f4b7b68..98ae8ab5f 100644 --- a/slatedb/src/tablestore.rs +++ b/slatedb/src/tablestore.rs @@ -2396,6 +2396,7 @@ mod tests { flaky.clone(), Arc::new(DbRand::default()), Arc::new(DefaultSystemClock::new()), + None, )); let format = SsTableFormat { From c8e62bca081d3775d10563e40e3d35a5cc4bc021 Mon Sep 17 00:00:00 2001 From: Ryan Dielhenn Date: Wed, 15 Jul 2026 20:02:29 -0700 Subject: [PATCH 21/63] Eliminate unnecessary object store requests made on each `CommitCompacted` tick and increase worker default heartbeat_interval (#1931) --- slatedb/src/compactor.rs | 103 +++++++++++++++++++++++++++++++++++++-- slatedb/src/config.rs | 2 +- 2 files changed, 100 insertions(+), 5 deletions(-) diff --git a/slatedb/src/compactor.rs b/slatedb/src/compactor.rs index 39eaf95b1..c322496d8 100644 --- a/slatedb/src/compactor.rs +++ b/slatedb/src/compactor.rs @@ -550,9 +550,15 @@ impl MessageHandler for CompactorEventHandler { CompactorMessage::LogStats => self.handle_log_ticker(), CompactorMessage::PollManifest => self.handle_ticker().await?, CompactorMessage::CommitCompacted => { - self.state_writer.load_compactions().await?; - self.update_distributed_compaction_metrics(); - self.commit_compacted_entries().await?; + // A remote worker can only produce a new Compacted result for a + // job the coordinator already tracks as active. When there are no + // active jobs, the regular manifest poll is sufficient to discover + // new submissions. We can avoid an otherwise idle object-store refresh. + if self.state().active_compactions().next().is_some() { + self.state_writer.load_compactions().await?; + self.update_distributed_compaction_metrics(); + self.commit_compacted_entries().await?; + } } } Ok(()) @@ -867,6 +873,7 @@ impl CompactorEventHandler { return Ok(()); } + let mut manifest_changed = false; for compaction in compacted { let id = compaction.id(); match self.validate_compaction(&compaction) { @@ -884,6 +891,7 @@ impl CompactorEventHandler { .collect(), }; self.state_mut().finish_compaction(id, output_sr); + manifest_changed = true; self.stats .last_compaction_ts .set(self.system_clock.now().timestamp()); @@ -902,7 +910,13 @@ impl CompactorEventHandler { } self.log_compaction_state(); - self.state_writer.write_state_safely().await?; + if manifest_changed { + self.state_writer.write_state_safely().await?; + } else { + // Validation failures only change `.compactions`. Avoid creating a + // checkpoint and writing an unchanged manifest. + self.state_writer.write_compactions_safely().await?; + } Ok(()) } @@ -5020,6 +5034,71 @@ mod tests { ); } + #[tokio::test] + async fn test_commit_compacted_ticker_skips_remote_refresh_when_idle() { + let mut fixture = CompactorEventHandlerTestFixture::new().await; + assert!(fixture + .handler + .state() + .active_compactions() + .next() + .is_none()); + + // Simulate an external submission arriving after the coordinator's last + // regular poll. An idle fast-commit tick must not read it from storage. + let remote_id = Ulid::new(); + let mut external = StoredCompactions::try_load(fixture.compactions_store.clone()) + .await + .unwrap() + .unwrap(); + let mut dirty = external.prepare_dirty().unwrap(); + dirty.value.insert(Compaction::new( + remote_id, + CompactionSpec::new(Vec::new(), 0), + )); + external.update(dirty).await.unwrap(); + + fixture + .handler + .handle(CompactorMessage::CommitCompacted) + .await + .unwrap(); + assert!( + !fixture + .handler + .state() + .compactions() + .value + .contains(&remote_id), + "idle fast-commit tick should not refresh .compactions" + ); + + // Once the coordinator has active work, the same fast tick must resume + // refreshing so it can observe worker transitions promptly. + let local_id = Ulid::new(); + fixture + .handler + .state_mut() + .insert_compaction_for_test(Compaction::new( + local_id, + CompactionSpec::new(Vec::new(), 0), + )); + fixture + .handler + .handle(CompactorMessage::CommitCompacted) + .await + .unwrap(); + assert!( + fixture + .handler + .state() + .compactions() + .value + .contains(&remote_id), + "active fast-commit tick should refresh .compactions" + ); + } + #[tokio::test] async fn test_handle_ticker_starts_preexisting_submitted_compaction() { let compactor_options = Arc::new(compactor_options()); @@ -6155,6 +6234,12 @@ mod tests { .handler .state_mut() .insert_compaction_for_test(compaction); + let manifest_id_before = fixture + .manifest_store + .read_latest_manifest() + .await + .unwrap() + .id; // when: fixture @@ -6177,6 +6262,16 @@ mod tests { .status(), CompactionStatus::Failed, ); + let manifest_id_after = fixture + .manifest_store + .read_latest_manifest() + .await + .unwrap() + .id; + assert_eq!( + manifest_id_after, manifest_id_before, + "validation-only failures must not checkpoint or rewrite the manifest" + ); } /// A Running compaction whose heartbeat is older than the timeout must be diff --git a/slatedb/src/config.rs b/slatedb/src/config.rs index 01b72e724..55cd2331a 100644 --- a/slatedb/src/config.rs +++ b/slatedb/src/config.rs @@ -1289,7 +1289,7 @@ impl Default for CompactionWorkerOptions { Self { max_concurrent_compactions: 4, compactions_poll_interval: Duration::from_secs(5), - heartbeat_interval: Duration::from_secs(5), + heartbeat_interval: Duration::from_secs(10), max_sst_size: 256 * 1024 * 1024, max_fetch_tasks: 4, bytes_to_fetch: 2 * 1024 * 1024, From 5ddac5b9ff95583d1b79c4132a13cd37ff3a27dc Mon Sep 17 00:00:00 2001 From: Rui Fan <1996fanrui@gmail.com> Date: Thu, 16 Jul 2026 20:25:01 +0200 Subject: [PATCH 22/63] [1941] Fix stale nightly benchmark description in slatedb-bencher README (#1942) --- slatedb-bencher/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slatedb-bencher/README.md b/slatedb-bencher/README.md index 9e7fa5678..7aca293c3 100644 --- a/slatedb-bencher/README.md +++ b/slatedb-bencher/README.md @@ -78,7 +78,7 @@ The script also has a `SLATEDB_BENCH_CLEAN` environment variable which can be se ### `nightly.yaml` -`benchmark-db.sh` is also used in `.github/workflows/nightly.yaml` to benchmark the nightly build and generate plots for the [SlateDB website](https://slatedb.io/performance/benchmarks/main/). The tests are run using [WarpBuild](https://warpbuild.com) and the results are uploaded using [github-action-benchmark](https://github.com/benchmark-action/github-action-benchmark). The job will also fail if the results are not within 200% of the previous results. +`benchmark-db.sh` is also used in `.github/workflows/nightly.yaml` to benchmark the nightly build. The tests are run using [WarpBuild](https://warpbuild.com), and each run appends to mermaid `xyChart` files that are posted to the workflow's GitHub Actions job summary. ## `compaction` Subcommand From 3f8e3017264fff13c69821eb3a6346dffe353f0e Mon Sep 17 00:00:00 2001 From: Kaivalya Apte Date: Thu, 16 Jul 2026 20:33:04 +0200 Subject: [PATCH 23/63] [1923] Expose object_store_max_retries on UniFFI (#1933) --- bindings/go/uniffi/slatedb.go | 16 ++++++ .../io/slatedb/uniffi/SlateDbAdminTest.java | 3 +- .../java/io/slatedb/uniffi/TestSupport.java | 2 +- bindings/uniffi/src/config.rs | 56 ++++++++++++++++++- 4 files changed, 73 insertions(+), 4 deletions(-) diff --git a/bindings/go/uniffi/slatedb.go b/bindings/go/uniffi/slatedb.go index b4ba2ec42..0683f3f83 100644 --- a/bindings/go/uniffi/slatedb.go +++ b/bindings/go/uniffi/slatedb.go @@ -9471,6 +9471,11 @@ type GarbageCollectorOptions struct { // as successful. Set `min_age` longer than the maximum lifetime of a stale process, and use the // same setting for every GC operating on the database. DisableBoundaryFiles bool + // Maximum number of wrapper-level retries for a single object-store + // operation, on top of the `object_store` client's own HTTP retries. + // `None` (default) retries transient errors indefinitely; `Some(n)` gives + // up after `n` retries and surfaces the underlying error. + ObjectStoreMaxRetries *uint32 } func (r *GarbageCollectorOptions) Destroy() { @@ -9481,6 +9486,7 @@ func (r *GarbageCollectorOptions) Destroy() { FfiDestroyerOptionalGarbageCollectorDirectoryOptions{}.Destroy(r.CompactionsOptions) FfiDestroyerOptionalGarbageCollectorScheduleOptions{}.Destroy(r.DetachOptions) FfiDestroyerBool{}.Destroy(r.DisableBoundaryFiles) + FfiDestroyerOptionalUint32{}.Destroy(r.ObjectStoreMaxRetries) } type FfiConverterGarbageCollectorOptions struct{} @@ -9500,6 +9506,7 @@ func (c FfiConverterGarbageCollectorOptions) Read(reader io.Reader) GarbageColle FfiConverterOptionalGarbageCollectorDirectoryOptionsINSTANCE.Read(reader), FfiConverterOptionalGarbageCollectorScheduleOptionsINSTANCE.Read(reader), FfiConverterBoolINSTANCE.Read(reader), + FfiConverterOptionalUint32INSTANCE.Read(reader), } } @@ -9519,6 +9526,7 @@ func (c FfiConverterGarbageCollectorOptions) Write(writer io.Writer, value Garba FfiConverterOptionalGarbageCollectorDirectoryOptionsINSTANCE.Write(writer, value.CompactionsOptions) FfiConverterOptionalGarbageCollectorScheduleOptionsINSTANCE.Write(writer, value.DetachOptions) FfiConverterBoolINSTANCE.Write(writer, value.DisableBoundaryFiles) + FfiConverterOptionalUint32INSTANCE.Write(writer, value.ObjectStoreMaxRetries) } type FfiDestroyerGarbageCollectorOptions struct{} @@ -10244,6 +10252,11 @@ type ReaderOptions struct { MaxMemtableBytes uint64 // Whether WAL replay should be skipped entirely. SkipWalReplay bool + // Maximum number of wrapper-level retries for a single object-store + // operation, on top of the `object_store` client's own HTTP retries. + // `None` (default) retries transient errors indefinitely; `Some(n)` gives + // up after `n` retries and surfaces the underlying error. + ObjectStoreMaxRetries *uint32 } func (r *ReaderOptions) Destroy() { @@ -10251,6 +10264,7 @@ func (r *ReaderOptions) Destroy() { FfiDestroyerUint64{}.Destroy(r.CheckpointLifetimeMs) FfiDestroyerUint64{}.Destroy(r.MaxMemtableBytes) FfiDestroyerBool{}.Destroy(r.SkipWalReplay) + FfiDestroyerOptionalUint32{}.Destroy(r.ObjectStoreMaxRetries) } type FfiConverterReaderOptions struct{} @@ -10267,6 +10281,7 @@ func (c FfiConverterReaderOptions) Read(reader io.Reader) ReaderOptions { FfiConverterUint64INSTANCE.Read(reader), FfiConverterUint64INSTANCE.Read(reader), FfiConverterBoolINSTANCE.Read(reader), + FfiConverterOptionalUint32INSTANCE.Read(reader), } } @@ -10283,6 +10298,7 @@ func (c FfiConverterReaderOptions) Write(writer io.Writer, value ReaderOptions) FfiConverterUint64INSTANCE.Write(writer, value.CheckpointLifetimeMs) FfiConverterUint64INSTANCE.Write(writer, value.MaxMemtableBytes) FfiConverterBoolINSTANCE.Write(writer, value.SkipWalReplay) + FfiConverterOptionalUint32INSTANCE.Write(writer, value.ObjectStoreMaxRetries) } type FfiDestroyerReaderOptions struct{} diff --git a/bindings/java/slatedb-uniffi/src/test/java/io/slatedb/uniffi/SlateDbAdminTest.java b/bindings/java/slatedb-uniffi/src/test/java/io/slatedb/uniffi/SlateDbAdminTest.java index d285f7531..ccac9f4df 100644 --- a/bindings/java/slatedb-uniffi/src/test/java/io/slatedb/uniffi/SlateDbAdminTest.java +++ b/bindings/java/slatedb-uniffi/src/test/java/io/slatedb/uniffi/SlateDbAdminTest.java @@ -184,7 +184,8 @@ void adminRunGcOnceAcceptsDefaultAndCustomOptions() throws Exception { null, null, scheduleOptions, - true); + true, + null); TestSupport.await(admin.runGcOnce(options)); } diff --git a/bindings/java/slatedb-uniffi/src/test/java/io/slatedb/uniffi/TestSupport.java b/bindings/java/slatedb-uniffi/src/test/java/io/slatedb/uniffi/TestSupport.java index ff1512541..9ffe5af99 100644 --- a/bindings/java/slatedb-uniffi/src/test/java/io/slatedb/uniffi/TestSupport.java +++ b/bindings/java/slatedb-uniffi/src/test/java/io/slatedb/uniffi/TestSupport.java @@ -350,7 +350,7 @@ static ScanOptions scanOptions(long readAheadBytes, boolean cacheBlocks, long ma } static ReaderOptions readerOptions(boolean skipWalReplay) { - return new ReaderOptions(100L, 1000L, 64L * 1024 * 1024, skipWalReplay); + return new ReaderOptions(100L, 1000L, 64L * 1024 * 1024, skipWalReplay, null); } private static Throwable unwrap(Throwable thrown) { diff --git a/bindings/uniffi/src/config.rs b/bindings/uniffi/src/config.rs index 024a9d3ec..ecebff4e5 100644 --- a/bindings/uniffi/src/config.rs +++ b/bindings/uniffi/src/config.rs @@ -194,6 +194,12 @@ pub struct ReaderOptions { pub max_memtable_bytes: u64, /// Whether WAL replay should be skipped entirely. pub skip_wal_replay: bool, + /// Maximum number of wrapper-level retries for a single object-store + /// operation, on top of the `object_store` client's own HTTP retries. + /// `None` (default) retries transient errors indefinitely; `Some(n)` gives + /// up after `n` retries and surfaces the underlying error. + #[uniffi(default = None)] + pub object_store_max_retries: Option, } impl Default for ReaderOptions { @@ -203,6 +209,7 @@ impl Default for ReaderOptions { checkpoint_lifetime_ms: 600_000, max_memtable_bytes: 64 * 1024 * 1024, skip_wal_replay: false, + object_store_max_retries: None, } } } @@ -214,6 +221,7 @@ impl From for slatedb::config::DbReaderOptions { checkpoint_lifetime: Duration::from_millis(value.checkpoint_lifetime_ms), max_memtable_bytes: value.max_memtable_bytes, skip_wal_replay: value.skip_wal_replay, + object_store_max_retries: value.object_store_max_retries, ..Default::default() } } @@ -459,6 +467,12 @@ pub struct GarbageCollectorOptions { /// same setting for every GC operating on the database. #[uniffi(default = false)] pub disable_boundary_files: bool, + /// Maximum number of wrapper-level retries for a single object-store + /// operation, on top of the `object_store` client's own HTTP retries. + /// `None` (default) retries transient errors indefinitely; `Some(n)` gives + /// up after `n` retries and surfaces the underlying error. + #[uniffi(default = None)] + pub object_store_max_retries: Option, } impl Default for GarbageCollectorOptions { @@ -472,6 +486,7 @@ impl Default for GarbageCollectorOptions { compactions_options: core.compactions_options.map(Into::into), detach_options: core.detach_options.map(Into::into), disable_boundary_files: !core.boundary_files_enabled, + object_store_max_retries: core.object_store_max_retries, } } } @@ -505,14 +520,14 @@ impl From for slatedb::config::GarbageCollectorOptions detach_options: value.detach_options.map(Into::into), metric_level: None, boundary_files_enabled: !value.disable_boundary_files, - object_store_max_retries: None, + object_store_max_retries: value.object_store_max_retries, } } } #[cfg(test)] mod tests { - use super::GarbageCollectorOptions; + use super::{GarbageCollectorOptions, ReaderOptions}; #[test] fn boundary_files_are_enabled_by_default() { @@ -532,6 +547,43 @@ mod tests { assert!(!gc.boundary_files_enabled); } + + #[test] + fn gc_object_store_max_retries_defaults_to_unbounded() { + let gc: slatedb::config::GarbageCollectorOptions = + GarbageCollectorOptions::default().into(); + + assert_eq!(gc.object_store_max_retries, None); + } + + #[test] + fn gc_object_store_max_retries_threads_through() { + let gc: slatedb::config::GarbageCollectorOptions = GarbageCollectorOptions { + object_store_max_retries: Some(3), + ..GarbageCollectorOptions::default() + } + .into(); + + assert_eq!(gc.object_store_max_retries, Some(3)); + } + + #[test] + fn reader_object_store_max_retries_defaults_to_unbounded() { + let reader: slatedb::config::DbReaderOptions = ReaderOptions::default().into(); + + assert_eq!(reader.object_store_max_retries, None); + } + + #[test] + fn reader_object_store_max_retries_threads_through() { + let reader: slatedb::config::DbReaderOptions = ReaderOptions { + object_store_max_retries: Some(5), + ..ReaderOptions::default() + } + .into(); + + assert_eq!(reader.object_store_max_retries, Some(5)); + } } /// Specify options to provide when creating a checkpoint. From 5f7ac90f6ca71de72f57798b6a80b9a839188c49 Mon Sep 17 00:00:00 2001 From: Chris Date: Thu, 16 Jul 2026 12:11:28 -0700 Subject: [PATCH 24/63] Fix flaky local filesystem boundary test (#1940) --- slatedb-txn-obj/src/object_store.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/slatedb-txn-obj/src/object_store.rs b/slatedb-txn-obj/src/object_store.rs index 14a191d7c..d0abda30e 100644 --- a/slatedb-txn-obj/src/object_store.rs +++ b/slatedb-txn-obj/src/object_store.rs @@ -712,15 +712,15 @@ mod tests { boundary.check(MonotonicId::new(3)).await.unwrap(); // Replace the file directly rather than calling BoundaryObject::advance, since - // LocalFileSystem does not support PutMode::Update. The stale ETag should cause - // the next check to read and enforce the new boundary. + // LocalFileSystem does not support PutMode::Update. Use a differently sized value + // so filesystems with coarse mtime resolution still produce a different ETag. object_store - .put(&boundary_path, PutPayload::from("4")) + .put(&boundary_path, PutPayload::from("40")) .await .unwrap(); let err = boundary.check(MonotonicId::new(3)).await.unwrap_err(); assert!(matches!(err, TransactionalObjectError::ObjectVersionExists)); - boundary.check(MonotonicId::new(5)).await.unwrap(); + boundary.check(MonotonicId::new(41)).await.unwrap(); } #[tokio::test] From 5beefe1f0ca425ec8c04b5bf091c6ceff097f60f Mon Sep 17 00:00:00 2001 From: nomiero Date: Thu, 16 Jul 2026 16:09:35 -0700 Subject: [PATCH 25/63] remove fsync from cached object store (#1945) --- slatedb/src/cached_object_store/storage_fs.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/slatedb/src/cached_object_store/storage_fs.rs b/slatedb/src/cached_object_store/storage_fs.rs index 1a5cb602f..8a1500e53 100644 --- a/slatedb/src/cached_object_store/storage_fs.rs +++ b/slatedb/src/cached_object_store/storage_fs.rs @@ -270,7 +270,13 @@ impl FsCacheEntry { .open(tmp_path) .map_err(wrap_io_err)?; file.write_all(&buf).map_err(wrap_io_err)?; - file.sync_all().map_err(wrap_io_err)?; + + // Note: There is no fsync before the rename. The cache holds copies + // of durable upstream bytes, so part durability is not required, only + // correctness. The tmp file plus atomic rename means a reader never + // observes a partially written part. If a crash leaves a renamed but + // not yet flushed part that reads back corrupt, block validation + // fails on read, the retried GET will override the cached entry. std::fs::rename(tmp_path, path).map_err(wrap_io_err) }) .await? From 4df9dcf747ab597c82004e8a15c0b447fc7361e5 Mon Sep 17 00:00:00 2001 From: Rui Fan <1996fanrui@gmail.com> Date: Fri, 17 Jul 2026 01:14:53 +0200 Subject: [PATCH 26/63] [1932] Expose manifest structural counts as DbStats gauges (#1934) --- slatedb/src/db.rs | 40 +++++++++++--- slatedb/src/db_stats.rs | 12 +++++ .../src/memtable_flusher/manifest_writer.rs | 53 ++++++++++++++----- .../content/docs/docs/operations/metrics.mdx | 4 ++ 4 files changed, 89 insertions(+), 20 deletions(-) diff --git a/slatedb/src/db.rs b/slatedb/src/db.rs index a7676304c..f159e54f8 100644 --- a/slatedb/src/db.rs +++ b/slatedb/src/db.rs @@ -9622,16 +9622,19 @@ mod tests { } #[tokio::test] - async fn test_should_record_l0_sst_count() { + async fn test_should_record_manifest_structural_counts() { // given: let object_store: Arc = Arc::new(InMemory::new()); let metrics_recorder = Arc::new(DefaultMetricsRecorder::new()); - let db = Db::builder("/tmp/test_should_record_l0_sst_count", object_store) - .with_settings(test_db_options(0, 1024, None)) - .with_metrics_recorder(metrics_recorder.clone()) - .build() - .await - .unwrap(); + let db = Db::builder( + "/tmp/test_should_record_manifest_structural_counts", + object_store, + ) + .with_settings(test_db_options(0, 1024, None)) + .with_metrics_recorder(metrics_recorder.clone()) + .build() + .await + .unwrap(); // when: write data and flush memtable to L0 db.put(b"k1", b"v1").await.unwrap(); @@ -9660,6 +9663,29 @@ mod tests { segment_max, l0_count, "expected segment_max_l0_sst_count == l0_sst_count for an unsegmented DB" ); + + // The single flushed L0 SST shows up as one SST view, with no sorted + // runs and no external DBs. These gauges are set at the same call site. + assert_eq!( + lookup_metric(&metrics_recorder, crate::db_stats::SST_VIEW_COUNT), + Some(1), + "expected sst_view_count == 1 for a single flushed L0 SST" + ); + assert_eq!( + lookup_metric(&metrics_recorder, crate::db_stats::SST_COUNT), + Some(1), + "expected sst_count == 1 (one distinct physical SST) for a single flushed L0 SST" + ); + assert_eq!( + lookup_metric(&metrics_recorder, crate::db_stats::SORTED_RUN_COUNT), + Some(0), + "expected sorted_run_count == 0 before any compaction" + ); + assert_eq!( + lookup_metric(&metrics_recorder, crate::db_stats::EXTERNAL_DB_COUNT), + Some(0), + "expected external_db_count == 0 for a standalone DB" + ); db.close().await.unwrap(); } diff --git a/slatedb/src/db_stats.rs b/slatedb/src/db_stats.rs index 8c4475634..89a40c7c6 100644 --- a/slatedb/src/db_stats.rs +++ b/slatedb/src/db_stats.rs @@ -26,6 +26,10 @@ pub const IMMUTABLE_MEMTABLE_FLUSHES: &str = db_stat_name!("immutable_memtable_f pub const TOTAL_MEM_SIZE_BYTES: &str = db_stat_name!("total_mem_size_bytes"); pub const L0_SST_COUNT: &str = db_stat_name!("l0_sst_count"); pub const SEGMENT_MAX_L0_SST_COUNT: &str = db_stat_name!("segment_max_l0_sst_count"); +pub const SORTED_RUN_COUNT: &str = db_stat_name!("sorted_run_count"); +pub const SST_VIEW_COUNT: &str = db_stat_name!("sst_view_count"); +pub const SST_COUNT: &str = db_stat_name!("sst_count"); +pub const EXTERNAL_DB_COUNT: &str = db_stat_name!("external_db_count"); pub const L0_FLUSH_BYTES: &str = db_stat_name!("l0_flush_bytes"); pub const SST_FILTER_FALSE_POSITIVE_COUNT: &str = db_stat_name!("sst_filter_false_positive_count"); pub const SST_FILTER_POSITIVE_COUNT: &str = db_stat_name!("sst_filter_positive_count"); @@ -63,6 +67,10 @@ pub(crate) struct DbStatsInner { pub(crate) total_mem_size_bytes: Arc, pub(crate) l0_sst_count: Arc, pub(crate) segment_max_l0_sst_count: Arc, + pub(crate) sorted_run_count: Arc, + pub(crate) sst_view_count: Arc, + pub(crate) sst_count: Arc, + pub(crate) external_db_count: Arc, pub(crate) l0_flush_bytes: Arc, pub(crate) merge_operator_read_operands: Arc, pub(crate) merge_operator_flush_operands: Arc, @@ -137,6 +145,10 @@ impl DbStats { total_mem_size_bytes: recorder.gauge(TOTAL_MEM_SIZE_BYTES).register(), l0_sst_count: recorder.gauge(L0_SST_COUNT).register(), segment_max_l0_sst_count: recorder.gauge(SEGMENT_MAX_L0_SST_COUNT).register(), + sorted_run_count: recorder.gauge(SORTED_RUN_COUNT).register(), + sst_view_count: recorder.gauge(SST_VIEW_COUNT).register(), + sst_count: recorder.gauge(SST_COUNT).register(), + external_db_count: recorder.gauge(EXTERNAL_DB_COUNT).register(), l0_flush_bytes: recorder.counter(L0_FLUSH_BYTES).register(), merge_operator_read_operands: recorder .counter(MERGE_OPERATOR_OPERANDS) diff --git a/slatedb/src/memtable_flusher/manifest_writer.rs b/slatedb/src/memtable_flusher/manifest_writer.rs index 6758adb2b..09e81414a 100644 --- a/slatedb/src/memtable_flusher/manifest_writer.rs +++ b/slatedb/src/memtable_flusher/manifest_writer.rs @@ -19,7 +19,7 @@ use super::uploader::UploadedMemtable; use crate::checkpoint::CheckpointCreateResult; use crate::config::CheckpointOptions; use crate::db::DbInner; -use crate::db_state::{collect_touched_segments, DbState, SsTableView}; +use crate::db_state::{collect_touched_segments, COWDbState, DbState, SsTableId, SsTableView}; use crate::dispatcher::MessageHandler; use crate::error::SlateDBError; use crate::manifest::store::FenceableManifest; @@ -33,7 +33,7 @@ use futures::stream::BoxStream; use futures::StreamExt; use parking_lot::RwLockWriteGuard; use std::cmp; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashSet}; use std::sync::Arc; use std::time::Duration; use tokio::runtime::Handle; @@ -663,17 +663,7 @@ impl ManifestWriterHandler { let mut wguard_state = self.db.state.write(); wguard_state.merge_remote_manifest(remote_dirty); let cow = wguard_state.state(); - // L0 SST counters span every tree (root + each named segment per - // RFC-0024). `l0_sst_count` reports the total; `segment_max_*` - // reports the largest single tree, which is the right quantity - // for backpressure since `l0_max_ssts` is enforced per-tree. - let (total, max) = cow - .core() - .trees() - .map(|t| t.l0.len()) - .fold((0usize, 0usize), |(sum, max), n| (sum + n, max.max(n))); - self.db.db_stats.l0_sst_count.set(total as i64); - self.db.db_stats.segment_max_l0_sst_count.set(max as i64); + self.update_stats_for_manifest(&cow); cow.manifest.clone() }; self.db @@ -681,6 +671,43 @@ impl ManifestWriterHandler { .report_manifest(dirty_manifest.into()); } + fn update_stats_for_manifest(&self, cow: &COWDbState) { + let mut l0_ssts = 0usize; + let mut segment_max_l0_ssts = 0usize; + let mut sorted_runs = 0usize; + let mut sst_views = 0usize; + let mut distinct_ssts: HashSet = HashSet::new(); + for tree in cow.core().trees() { + l0_ssts += tree.l0.len(); + // Track the largest single tree: backpressure is driven by `segment_max_l0_sst_count` + // because `l0_max_ssts` is enforced per-tree. + segment_max_l0_ssts = segment_max_l0_ssts.max(tree.l0.len()); + sorted_runs += tree.compacted.len(); + let all_views = tree + .l0 + .iter() + .chain(tree.compacted.iter().flat_map(|run| run.sst_views.iter())); + for view in all_views { + sst_views += 1; + // Dedupe by physical SST id: a range clone/rescale can project one SST into + // several views, so `sst_count <= sst_view_count`. + distinct_ssts.insert(view.sst.id); + } + } + self.db.db_stats.l0_sst_count.set(l0_ssts as i64); + self.db + .db_stats + .segment_max_l0_sst_count + .set(segment_max_l0_ssts as i64); + self.db.db_stats.sorted_run_count.set(sorted_runs as i64); + self.db.db_stats.sst_view_count.set(sst_views as i64); + self.db.db_stats.sst_count.set(distinct_ssts.len() as i64); + self.db + .db_stats + .external_db_count + .set(cow.manifest.value.external_dbs.len() as i64); + } + async fn write_checkpoint_safely( &mut self, options: &CheckpointOptions, diff --git a/website/src/content/docs/docs/operations/metrics.mdx b/website/src/content/docs/docs/operations/metrics.mdx index dad0e0647..09481f8fb 100644 --- a/website/src/content/docs/docs/operations/metrics.mdx +++ b/website/src/content/docs/docs/operations/metrics.mdx @@ -100,6 +100,10 @@ All metric names use dot-separated notation: `slatedb..`. | `slatedb.db.total_mem_size_bytes` | gauge | | Total memory usage | | `slatedb.db.l0_sst_count` | gauge | | L0 SST count summed across all segment trees | | `slatedb.db.segment_max_l0_sst_count` | gauge | | Maximum L0 SST count across all segments (useful for detecting backpressure on specific segments) | +| `slatedb.db.sorted_run_count` | gauge | | Number of sorted runs across all trees (root tree plus each named segment tree per RFC-0024) | +| `slatedb.db.sst_view_count` | gauge | | Total number of SST views including L0 views and every sorted-run SST view across all trees | +| `slatedb.db.sst_count` | gauge | | Number of distinct physical SSTables (deduplicates sst_view_count by SsTableId, so sst_count is at most sst_view_count) | +| `slatedb.db.external_db_count` | gauge | | Number of external databases referenced in the manifest | | `slatedb.db.sst_filter_false_positive_count` | counter | | Bloom filter false positives | | `slatedb.db.sst_filter_positive_count` | counter | | Bloom filter positives | | `slatedb.db.sst_filter_negative_count` | counter | | Bloom filter negatives | From 5604c9b85e9234af27b13240f58a7a26067c4234 Mon Sep 17 00:00:00 2001 From: Rohan Date: Thu, 16 Jul 2026 20:01:04 -0400 Subject: [PATCH 27/63] wal refactor k/N: simplify backpressure to use flushed events only (#1930) --- slatedb/src/db.rs | 11 +-- slatedb/src/wal_buffer.rs | 201 +++++++++++++++----------------------- 2 files changed, 84 insertions(+), 128 deletions(-) diff --git a/slatedb/src/db.rs b/slatedb/src/db.rs index f159e54f8..00df3a121 100644 --- a/slatedb/src/db.rs +++ b/slatedb/src/db.rs @@ -379,7 +379,7 @@ impl DbInner { let await_flush_wal = self .wal_observer - .wait_until_wal_released(wal_status.last_purged_wal_id); + .wait_until_wal_released(wal_status.last_flushed_wal_id); let timeout_fut = self.system_clock.sleep(Duration::from_secs(30)); let await_closed = async { @@ -2076,10 +2076,7 @@ impl DbWalObserver { let (status_tx, status_rx) = tokio::sync::watch::channel(wrapped.status()); wrapped .subscribe(Arc::new(move |event| { - let status = match event { - WalEvent::WalFlushed(status) => status, - WalEvent::MemoryReleased(status) => status, - }; + let WalEvent::WalFlushed(status) = event; if let Some(seq) = status.last_flushed_seq { oracle.advance_durable_seq(seq); } @@ -2116,8 +2113,8 @@ impl DbWalObserver { } /// Waits until the wal a given wal id is released by the wal writer - async fn wait_until_wal_released(&self, last_purged_wal_id: u64) -> Result<(), SlateDBError> { - self.wait_on_condition(|status| status.last_purged_wal_id > last_purged_wal_id) + async fn wait_until_wal_released(&self, last_flushed_wal_id: u64) -> Result<(), SlateDBError> { + self.wait_on_condition(|status| status.last_flushed_wal_id > last_flushed_wal_id) .await } } diff --git a/slatedb/src/wal_buffer.rs b/slatedb/src/wal_buffer.rs index ea4ba6e2b..be96aea0b 100644 --- a/slatedb/src/wal_buffer.rs +++ b/slatedb/src/wal_buffer.rs @@ -15,7 +15,7 @@ use crate::utils::{format_bytes_si, WatchableOnceCell, WatchableOnceCellReader}; use crate::wal_buffer_stats::WalBufferStats; use async_trait::async_trait; use futures::{stream::BoxStream, StreamExt}; -use log::{error, trace}; +use log::{error, trace, warn}; use slatedb_common::metrics::MetricsRecorderHelper; use tokio::{runtime::Handle, sync::oneshot}; use tracing::instrument; @@ -80,8 +80,6 @@ struct WalBufferManagerInner { last_flushed_wal_id: u64, /// The last seq that was flushed to the WAL. This value will be None until the first flush. last_flushed_seq: Option, - /// The last wal id that was deallocated from the buffer - last_purged_wal_id: u64, } /// Stores entries to the write-ahead log (WAL) in memory. @@ -126,7 +124,6 @@ impl WalBufferManager { immutable_wals, flush_epoch: 1, last_flushed_wal_id, - last_purged_wal_id: last_flushed_wal_id, next_wal_id: last_flushed_wal_id + 1, last_flushed_seq: None, }; @@ -287,11 +284,9 @@ impl WalBufferManagerInner { /// Returns the list of immutable WALs that need to be flushed. /// Used by the handler to determine which WALs to write to storage. fn flushing_wals(&self) -> Vec<(u64, Arc)> { - let mut flushing_wals = Vec::new(); - for (wal_id, wal) in self.immutable_wals.iter() { - if *wal_id > self.last_flushed_wal_id { - flushing_wals.push((*wal_id, wal.clone())); - } + let flushing_wals: Vec<_> = self.immutable_wals.iter().cloned().collect(); + for (wal_id, _wal) in flushing_wals.iter() { + assert!(*wal_id > self.last_flushed_wal_id); } flushing_wals } @@ -318,7 +313,6 @@ impl WalBufferManagerInner { WalStatus { estimated_bytes: self.estimated_bytes(table_store), last_flushed_wal_id: self.last_flushed_wal_id, - last_purged_wal_id: self.last_purged_wal_id, last_flushed_seq: self.last_flushed_seq, buffered_wal_entries_count, } @@ -336,48 +330,28 @@ impl WalBufferManagerInner { .push_back((next_wal_id, Arc::new(current_wal))); } - fn record_flushed_wal(&mut self, wal_id: u64, wal: &Arc) { - self.last_flushed_wal_id = wal_id; - if let Some(seq) = wal.last_seq() { + fn record_flushed_wal(&mut self, flushed_wal_id: u64, flushed_wal: &Arc) { + let (front_wal_id, front_wal_buffer) = self + .immutable_wals + .pop_front() + .expect("no immutable wals found to pop"); + assert_eq!(front_wal_id, flushed_wal_id); + assert!(Arc::ptr_eq(&front_wal_buffer, flushed_wal)); + assert_eq!( + flushed_wal_id, + self.last_flushed_wal_id + 1, + "flushed wal id {} not next wal id after previous flushed {}", + flushed_wal_id, + self.last_flushed_wal_id + ); + self.last_flushed_wal_id = flushed_wal_id; + if let Some(seq) = flushed_wal.last_seq() { if let Some(last_flushed_seq) = self.last_flushed_seq { assert!(seq >= last_flushed_seq); } self.last_flushed_seq = Some(seq); } } - - fn maybe_release_immutable_wals(&mut self) -> usize { - let last_flushed_seq = self.last_flushed_seq; - - let mut releaseable_count = 0; - for (_id, wal) in self.immutable_wals.iter() { - if wal - .last_seq() - .map(|seq| seq <= last_flushed_seq.unwrap_or(0)) - .unwrap_or(false) - { - releaseable_count += 1; - } else { - break; - } - } - - if releaseable_count > 0 { - trace!( - "draining immutable wals [releaseable_count={}]", - releaseable_count - ); - let last_purged = self - .immutable_wals - .drain(..releaseable_count) - .map(|(id, _wal)| id) - .max(); - if let Some(last_purged) = last_purged { - self.last_purged_wal_id = last_purged; - } - } - releaseable_count - } } impl WalBuffer { @@ -493,8 +467,8 @@ impl WalFlushHandler { inner.flushing_wals() }; - for (wal_id, wal) in flushing_wals.iter() { - let result = self.do_flush_one_wal(*wal_id, wal.clone()).await; + for (wal_id, wal) in flushing_wals { + let result = self.do_flush_one_wal(wal_id, wal.clone()).await; if let Err(e) = &result { // a WAL buffer can be retried to flush multiple times, but WatchableOnceCell is only set once. // we do NOT call `wal.notify_durable` as soon as encountered any error here, but notify @@ -506,31 +480,27 @@ impl WalFlushHandler { // increment the last flushed wal id, and last flushed seq let status = { let mut inner = self.inner.write(); - inner.record_flushed_wal(*wal_id, wal); + inner.record_flushed_wal(wal_id, &wal); inner.status(&self.table_store) }; + // we notify the listener first since that updates the oracle, and then notify + // the table waiters. blocked writes wait on the table, so we have to update the oracle + // first to preserve read-your-writes. This does mean that there is a small window + // after notifying flushed before the wal memory is actually released. + // TODO: once we change writes to block on the durable seq num from the oracle we + // can simplify this and fully drop the wal before notifying listeners self.notify_listener(WalEvent::WalFlushed(status)); wal.notify_durable(result.clone()); + if Arc::strong_count(&wal) > 1 { + warn!("outstanding references to wal id {} after flushing", wal_id); + } + drop(wal); } - self.maybe_release_immutable_wals(); - Ok(()) } - fn maybe_release_immutable_wals(&self) { - let (status, released_wals) = { - let mut inner = self.inner.write(); - let released_wals = inner.maybe_release_immutable_wals(); - let status = inner.status(&self.table_store); - (status, released_wals) - }; - if released_wals > 0 { - self.notify_listener(WalEvent::MemoryReleased(status)); - } - } - async fn do_flush_one_wal(&self, wal_id: u64, wal: Arc) -> Result<(), SlateDBError> { self.stats.flushes.increment(1); @@ -639,8 +609,6 @@ pub(crate) struct WalStatus { pub(crate) last_flushed_wal_id: u64, /// The last sequence number that was durably flushed pub(crate) last_flushed_seq: Option, - /// The last WAL file id whose memory was released - pub(crate) last_purged_wal_id: u64, /// The number of writes currently buffered #[allow(dead_code)] pub(crate) buffered_wal_entries_count: usize, @@ -652,8 +620,6 @@ pub(crate) enum WalEvent { /// Emitted when a WAL file is durably flushed to storage. On receipt of this event, SlateDB /// notifies write tasks blocked on [`crate::config::WriteOptions::await_durable`] WalFlushed(WalStatus), - /// Emitted when `WalBufferManager` releases buffer memory. - MemoryReleased(WalStatus), } impl WalObserver { @@ -722,6 +688,7 @@ mod tests { lookup_metric, DefaultMetricsRecorder, MetricLevel, MetricsRecorderHelper, }; use slatedb_common::MockSystemClock; + use std::sync::atomic::AtomicBool; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -951,9 +918,7 @@ mod tests { observer .subscribe(Arc::new(move |status| { (*listener)(status.clone()); - let WalEvent::WalFlushed(status) = status else { - return; - }; + let WalEvent::WalFlushed(status) = status; oracle.advance_durable_seq(status.last_flushed_seq.unwrap_or(0)) })) .unwrap(); @@ -1042,26 +1007,6 @@ mod tests { assert_eq!(wal_buffer.inner.read().immutable_wals.len(), 0); } - #[tokio::test] - async fn test_immutable_wal_reclaim_with_flush_check() { - let (wal_buffer, _, _, _) = setup_wal_buffer_with_flush_interval(Duration::MAX).await; - - // Append entries to create multiple WALs - for i in 0..100 { - let seq = i + 1; - let entry = make_entry(&format!("key{}", i), &format!("value{}", i), seq, None); - wal_buffer.append(&[entry]).unwrap(); - wal_buffer.inner.write().freeze_current_wal(); - } - // simulate flushing just some of the wals - wal_buffer.inner.write().last_flushed_seq = Some(50); - - let released = wal_buffer.inner.write().maybe_release_immutable_wals(); - - assert_eq!(released, 50); - assert_eq!(wal_buffer.inner.read().immutable_wals.len(), 50); - } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_maybe_trigger_flush_spams_flush_requests() { let (wal_buffer, _, _, recorder) = @@ -1127,12 +1072,9 @@ mod tests { let recorded = events.lock().unwrap().clone(); let mut flushed: Vec<_> = recorded .iter() - .filter_map(|e| { - if let WalEvent::WalFlushed(status) = e { - Some(status) - } else { - None - } + .map(|e| { + let WalEvent::WalFlushed(status) = e; + status }) .collect(); assert_eq!(flushed.len(), 1); @@ -1142,34 +1084,51 @@ mod tests { } #[tokio::test] - async fn test_listener_notified_when_flush_task_releases_wal() { + async fn test_listener_notified_before_table_waiters() { // given: - let (listener, events) = recording_listener(); - let (wal_buffer, _, _, _) = setup_wal_buffer_with_args(Duration::MAX, listener).await; - - // when: - wal_buffer + let object_store: Arc = Arc::new(InMemory::new()); + let table_store = Arc::new(TableStore::new( + ObjectStores::new(object_store, None), + SsTableFormat::default(), + Path::from("/root"), + None, + TableStoreKind::Main, + )); + let system_clock = Arc::new(DefaultSystemClock::new()); + let status_manager = DbStatusManager::new(0); + let recorder = Arc::new(DefaultMetricsRecorder::new()); + let helper = MetricsRecorderHelper::new(recorder.clone(), MetricLevel::default()); + let mut wal_buffer = WalBufferManager::new( + status_manager.clone(), + &helper, + 0, + table_store.clone(), + 1000, + Some(Duration::MAX), + ); + let task_executor = Arc::new(MessageHandlerExecutor::new( + Arc::new(status_manager), + system_clock.clone(), + )); + wal_buffer.init(task_executor.clone()).await.unwrap(); + task_executor + .monitor_on(&Handle::current()) + .expect("failed to monitor executor"); + let waiter = wal_buffer .append(&[make_entry("key1", "value1", 1, None)]) .unwrap(); - wal_buffer.flush().unwrap().await.unwrap().unwrap(); - // The flush should have released the immutable wal from memory. - assert_eq!(wal_buffer.inner.read().immutable_wals.len(), 0); + let called = Arc::new(AtomicBool::new(false)); + let this_called = called.clone(); + let listener = move |event| { + this_called.store(true, Ordering::SeqCst); + assert!(matches!(event, WalEvent::WalFlushed(_))); + // verifies that the table is not yet notified + assert!(waiter.read().is_none()) + }; + wal_buffer.observer().subscribe(Arc::new(listener)).unwrap(); - // The listener should have been notified that wal 1 was purged. - // then: the listener should have been notified that wal 1 was flushed. - let recorded = events.lock().unwrap().clone(); - let mut flushed: Vec<_> = recorded - .iter() - .filter_map(|e| { - if let WalEvent::MemoryReleased(status) = e { - Some(status) - } else { - None - } - }) - .collect(); - assert_eq!(flushed.len(), 1); - let status = flushed.pop().unwrap(); - assert_eq!(status.last_purged_wal_id, 1); + // when/then: + wal_buffer.flush().unwrap().await.unwrap().unwrap(); + assert!(called.load(Ordering::SeqCst)); } } From 5ea9afb03788660f68dcc6ac441ece1c63756bd3 Mon Sep 17 00:00:00 2001 From: Tyson Trautmann Date: Thu, 16 Jul 2026 17:03:28 -0700 Subject: [PATCH 28/63] Fix stale external SSTs after manifest merge (#1936) --- slatedb/src/compactor_state.rs | 26 +++++++++ slatedb/src/compactor_state_protocols.rs | 68 +++++++++++++++++++----- slatedb/src/db_state.rs | 24 +++++++++ 3 files changed, 104 insertions(+), 14 deletions(-) diff --git a/slatedb/src/compactor_state.rs b/slatedb/src/compactor_state.rs index a95d1aaf3..0c47dce2c 100644 --- a/slatedb/src/compactor_state.rs +++ b/slatedb/src/compactor_state.rs @@ -970,6 +970,7 @@ impl CompactorState { sequence_tracker: remote_manifest.value.core.sequence_tracker, }; remote_manifest.value.core = merged; + remote_manifest.value.prune_external_sst_ids(); self.manifest = remote_manifest; } @@ -1717,6 +1718,31 @@ mod tests { assert_eq!(expected_merged_l0s, merged_l0s); } + #[test] + fn test_merge_remote_manifest_reestablishes_external_sst_invariant() { + let manifest = new_dirty_manifest(); + let compactions = new_dirty_compactions(manifest.value.compactor_epoch); + let mut state = CompactorState::new(manifest, compactions); + let stale_id = SsTableId::Compacted(Ulid::new()); + let mut remote = new_dirty_manifest(); + remote.value.external_dbs = vec![crate::manifest::ExternalDb { + path: "/parent/db".to_string(), + source_checkpoint_id: uuid::Uuid::new_v4(), + final_checkpoint_id: Some(uuid::Uuid::new_v4()), + sst_ids: vec![stale_id], + }]; + + state.merge_remote_manifest(remote); + + let external = &state.manifest().value.external_dbs; + assert_eq!(external.len(), 1, "detach metadata must be retained"); + assert!( + external[0].sst_ids.is_empty(), + "IDs absent from the merged tree must not be resurrected" + ); + assert!(external[0].final_checkpoint_id.is_some()); + } + #[test] fn test_should_merge_db_state_correctly() { // given: diff --git a/slatedb/src/compactor_state_protocols.rs b/slatedb/src/compactor_state_protocols.rs index 47b24d66c..142b58fce 100644 --- a/slatedb/src/compactor_state_protocols.rs +++ b/slatedb/src/compactor_state_protocols.rs @@ -344,8 +344,9 @@ mod tests { use crate::error::SlateDBError; use crate::format::sst::SST_FORMAT_VERSION_LATEST; use crate::manifest::store::{ManifestStore, StoredManifest}; - use crate::manifest::{Manifest, ManifestCore, VersionedManifest}; + use crate::manifest::{ExternalDb, Manifest, ManifestCore, VersionedManifest}; use crate::subcompaction::Subcompaction; + use crate::test_utils::GatedObjectStore; use bytes::Bytes; use object_store::memory::InMemory; use object_store::path::Path; @@ -978,7 +979,9 @@ mod tests { #[tokio::test] async fn write_manifest_safely_retries_on_version_conflict() { - let object_store: Arc = Arc::new(InMemory::new()); + let inner_store: Arc = Arc::new(InMemory::new()); + let gated_store = Arc::new(GatedObjectStore::new(Arc::clone(&inner_store))); + let object_store: Arc = gated_store.clone(); let manifest_store = Arc::new(ManifestStore::new( &Path::from(ROOT), Arc::clone(&object_store), @@ -989,13 +992,24 @@ mod tests { )); let system_clock: Arc = Arc::new(DefaultSystemClock::new()); - StoredManifest::create_new_db( + let mut stored_manifest = StoredManifest::create_new_db( manifest_store.clone(), ManifestCore::new(), system_clock.clone(), ) .await .unwrap(); + let stale_sst_id = SsTableId::Compacted(Ulid::new()); + let source_checkpoint_id = uuid::Uuid::new_v4(); + let final_checkpoint_id = uuid::Uuid::new_v4(); + let mut dirty = stored_manifest.prepare_dirty().unwrap(); + dirty.value.external_dbs = vec![ExternalDb { + path: "/parent/db".to_string(), + source_checkpoint_id, + final_checkpoint_id: Some(final_checkpoint_id), + sst_ids: vec![stale_sst_id], + }]; + stored_manifest.update(dirty).await.unwrap(); let options = CompactorOptions::default(); let rand = Arc::new(DbRand::new(7)); @@ -1013,25 +1027,51 @@ mod tests { // Record the version after fencing. let start_id = manifest_store.read_latest_manifest().await.unwrap().id; - // Simulate an external writer creating a checkpoint of the manifest and updating it. - let admin = AdminBuilder::new(ROOT, object_store.clone()).build(); - admin + // Allow write_manifest's checkpoint through, then block its manifest update. + // The safe path has already loaded and pruned local state at that boundary. + let baseline_puts = gated_store.put_opts_gate.arrivals(); + gated_store.put_opts_gate.close(); + gated_store.put_opts_gate.admit(1); + let write_task = tokio::spawn(async move { writer.write_manifest_safely().await }); + gated_store + .put_opts_gate + .wait_for_arrivals(baseline_puts + 2) + .await; + + // Race a checkpoint into the exact version intended by the blocked update. + let admin = AdminBuilder::new(ROOT, inner_store).build(); + let remote_checkpoint = admin .create_detached_checkpoint(&CheckpointOptions::default()) .await .expect("create checkpoint failed"); let conflicting_id = manifest_store.read_latest_manifest().await.unwrap().id; - assert_eq!(conflicting_id, start_id + 1); - - // This should retry on conflict and succeed with a new version. - writer.write_manifest_safely().await.unwrap(); - - let final_id = manifest_store.read_latest_manifest().await.unwrap().id; + assert_eq!(conflicting_id, start_id + 2); + + // Reloading and retrying must neither resurrect the pruned SST nor lose + // checkpoint metadata from either the external DB or the racing writer. + gated_store.put_opts_gate.release(); + write_task.await.unwrap().unwrap(); + + let final_manifest = manifest_store.read_latest_manifest().await.unwrap(); + let external = &final_manifest.manifest.external_dbs[0]; + assert!(external.sst_ids.is_empty()); + assert_eq!(external.source_checkpoint_id, source_checkpoint_id); + assert_eq!(external.final_checkpoint_id, Some(final_checkpoint_id)); + assert!(final_manifest + .manifest + .core + .checkpoints + .iter() + .any(|checkpoint| checkpoint.id == remote_checkpoint.id)); + + let final_id = final_manifest.id; // write_manifest_safely now bumps the manifest twice per successful call because write_manifest // writes a checkpoint first: // - write_manifest() calls self.manifest.write_checkpoint(...) to create the checkpoint, then // - write_manifest() calls self.manifest.update(...) to update the manifest - // So we do +1 for the external update and +2 for the successful write_manifest_safely call. - assert_eq!(final_id, start_id + 3); + // So we do +1 for the first checkpoint, +1 for the external update, and +2 for + // the successful retry. + assert_eq!(final_id, start_id + 4); } } diff --git a/slatedb/src/db_state.rs b/slatedb/src/db_state.rs index 9deefb946..dffa54812 100644 --- a/slatedb/src/db_state.rs +++ b/slatedb/src/db_state.rs @@ -822,6 +822,7 @@ impl<'a> StateModifier<'a> { checkpoints: remote_manifest.value.core.checkpoints, wal_object_store_uri: my_db_state.wal_object_store_uri.clone(), }; + remote_manifest.value.prune_external_sst_ids(); self.state.manifest = remote_manifest; } @@ -879,6 +880,29 @@ mod tests { assert_eq!(vec![checkpoint], db_state.state.core().checkpoints); } + #[test] + fn test_merge_remote_manifest_reestablishes_external_sst_invariant() { + let mut db_state = DbState::new(new_dirty_manifest()); + let stale_id = SsTableId::Compacted(ulid::Ulid::new()); + let mut remote = new_dirty_manifest(); + remote.value.external_dbs = vec![crate::manifest::ExternalDb { + path: "/parent/db".to_string(), + source_checkpoint_id: uuid::Uuid::new_v4(), + final_checkpoint_id: Some(uuid::Uuid::new_v4()), + sst_ids: vec![stale_id], + }]; + + db_state.merge_remote_manifest(remote); + + let external = &db_state.state.manifest.value.external_dbs; + assert_eq!(external.len(), 1, "detach metadata must be retained"); + assert!( + external[0].sst_ids.is_empty(), + "IDs absent from the merged tree must not be resurrected" + ); + assert!(external[0].final_checkpoint_id.is_some()); + } + #[test] fn test_should_merge_db_state_with_l0s_up_to_last_compacted() { // given: From 57c0965eac401d76151546887faafef7d5fd9bb2 Mon Sep 17 00:00:00 2001 From: Rui Fan <1996fanrui@gmail.com> Date: Fri, 17 Jul 2026 02:04:14 +0200 Subject: [PATCH 29/63] [1938] Count active memtable in total_mem_size_bytes (fix 0 with WAL disabled) (#1939) --- slatedb/src/db.rs | 89 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 66 insertions(+), 23 deletions(-) diff --git a/slatedb/src/db.rs b/slatedb/src/db.rs index 00df3a121..fd9f762d4 100644 --- a/slatedb/src/db.rs +++ b/slatedb/src/db.rs @@ -58,6 +58,7 @@ use crate::db_stats::DbStats; use crate::error::SlateDBError; use crate::iter::IterationOrder; use crate::manifest::{Manifest, VersionedManifest}; +use crate::mem_table::KVTableMetadata; use crate::memtable_flusher::{FlushResult, FlushTarget, MemtableFlusher}; use crate::merge_operator::{instrument_merge_operator, MergeOperatorType}; use crate::oracle::{DbOracle, Oracle}; @@ -314,46 +315,46 @@ impl DbInner { pub(crate) async fn maybe_apply_backpressure(&self) -> Result<(), SlateDBError> { loop { self.check_closed()?; - let (wal_status, imm_memtable_size_bytes) = { - let wal_status = self.wal_observer.status(); - let imm_memtable_size_bytes = { - let guard = self.state.read(); - // Exclude active memtable to avoid a write lock. - guard - .state() - .imm_memtable - .iter() - .map(|imm| { - let metadata = imm.table().metadata(); - self.table_store.estimate_encoded_size_compacted( - metadata.entry_num, - metadata.entries_size_in_bytes, - ) - }) - .sum::() + let wal_status = self.wal_observer.status(); + let (active_memtable_size_bytes, imm_memtable_size_bytes) = { + let guard = self.state.read(); + let estimate = |metadata: KVTableMetadata| { + self.table_store.estimate_encoded_size_compacted( + metadata.entry_num, + metadata.entries_size_in_bytes, + ) }; - (wal_status, imm_memtable_size_bytes) + let active_memtable_size_bytes = estimate(guard.memtable().table().metadata()); + let imm_memtable_size_bytes = guard + .state() + .imm_memtable + .iter() + .map(|imm| estimate(imm.table().metadata())) + .sum::(); + (active_memtable_size_bytes, imm_memtable_size_bytes) }; - let total_mem_size_bytes = wal_status.estimated_bytes + imm_memtable_size_bytes; + let total_mem_size_bytes = active_memtable_size_bytes + imm_memtable_size_bytes; self.db_stats .total_mem_size_bytes .set(total_mem_size_bytes as i64); trace!( - "checking backpressure [total_mem_size_bytes={}, wal_size_bytes={}, imm_memtable_size_bytes={}, max_unflushed_bytes={}]", + "checking backpressure [total_mem_size_bytes={}, active_memtable_size_bytes={}, imm_memtable_size_bytes={}, wal_size_bytes={}, max_unflushed_bytes={}]", format_bytes_si(total_mem_size_bytes as u64), - format_bytes_si(wal_status.estimated_bytes as u64), + format_bytes_si(active_memtable_size_bytes as u64), format_bytes_si(imm_memtable_size_bytes as u64), + format_bytes_si(wal_status.estimated_bytes as u64), format_bytes_si(self.settings.max_unflushed_bytes as u64), ); if total_mem_size_bytes >= self.settings.max_unflushed_bytes { self.db_stats.backpressure_count.increment(1); warn!( - "unflushed memtable size exceeds max_unflushed_bytes. applying backpressure. [total_mem_size_bytes={}, wal_size_bytes={}, imm_memtable_size_bytes={}, max_unflushed_bytes={}]", + "unflushed memtable size exceeds max_unflushed_bytes. applying backpressure. [total_mem_size_bytes={}, active_memtable_size_bytes={}, imm_memtable_size_bytes={}, wal_size_bytes={}, max_unflushed_bytes={}]", format_bytes_si(total_mem_size_bytes as u64), - format_bytes_si(wal_status.estimated_bytes as u64), + format_bytes_si(active_memtable_size_bytes as u64), format_bytes_si(imm_memtable_size_bytes as u64), + format_bytes_si(wal_status.estimated_bytes as u64), format_bytes_si(self.settings.max_unflushed_bytes as u64), ); @@ -9525,6 +9526,48 @@ mod tests { db.close().await.unwrap(); } + #[cfg(feature = "wal_disable")] + #[tokio::test] + async fn test_should_record_total_mem_size_bytes_with_wal_disabled() { + // given: WAL disabled, so writes land only in the active memtable. + let object_store: Arc = Arc::new(InMemory::new()); + let metrics_recorder = Arc::new(DefaultMetricsRecorder::new()); + let mut opts = test_db_options(0, 1024, None); + opts.flush_interval = None; + opts.max_unflushed_bytes = 1024 * 1024; + opts.wal_enabled = false; + let db = Db::builder( + "/tmp/test_should_record_total_mem_size_bytes_with_wal_disabled", + object_store, + ) + .with_settings(opts) + .with_metrics_recorder(metrics_recorder.clone()) + .build() + .await + .unwrap(); + + // when: two writes (the second triggers maybe_apply_backpressure for the first's bytes) + let write_opts = WriteOptions { + await_durable: false, + ..Default::default() + }; + db.put_with_options(b"k1", b"v1", &PutOptions::default(), &write_opts) + .await + .unwrap(); + db.put_with_options(b"k2", b"v2", &PutOptions::default(), &write_opts) + .await + .unwrap(); + + // then: total_mem_size_bytes reflects the active memtable even with the WAL off + let mem_size = lookup_metric(&metrics_recorder, crate::db_stats::TOTAL_MEM_SIZE_BYTES); + assert!( + mem_size.is_some_and(|v| v > 0), + "expected total_mem_size_bytes > 0 with WAL disabled, got {:?}", + mem_size + ); + db.close().await.unwrap(); + } + #[tokio::test] async fn test_should_record_total_mem_size_bytes() { // given: From 0f39324c5ab53dd7d1bd3cc53609598f291105b8 Mon Sep 17 00:00:00 2001 From: Rohan Date: Thu, 16 Jul 2026 22:00:32 -0400 Subject: [PATCH 30/63] rename wait_until_wal_released->wait_until_wal_flushed (#1946) --- slatedb/src/db.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/slatedb/src/db.rs b/slatedb/src/db.rs index fd9f762d4..11698c929 100644 --- a/slatedb/src/db.rs +++ b/slatedb/src/db.rs @@ -380,7 +380,7 @@ impl DbInner { let await_flush_wal = self .wal_observer - .wait_until_wal_released(wal_status.last_flushed_wal_id); + .wait_until_wal_flushed(wal_status.last_flushed_wal_id); let timeout_fut = self.system_clock.sleep(Duration::from_secs(30)); let await_closed = async { @@ -2114,7 +2114,7 @@ impl DbWalObserver { } /// Waits until the wal a given wal id is released by the wal writer - async fn wait_until_wal_released(&self, last_flushed_wal_id: u64) -> Result<(), SlateDBError> { + async fn wait_until_wal_flushed(&self, last_flushed_wal_id: u64) -> Result<(), SlateDBError> { self.wait_on_condition(|status| status.last_flushed_wal_id > last_flushed_wal_id) .await } From f3af22f767dbeaa9b4a2fb3650651f0f19be03b8 Mon Sep 17 00:00:00 2001 From: Rui Fan <1996fanrui@gmail.com> Date: Fri, 17 Jul 2026 22:08:21 +0200 Subject: [PATCH 31/63] [1943] Add Settings::validate to reject conflicting configuration (#1950) --- slatedb-dst/src/utils.rs | 3 +- slatedb/src/config.rs | 87 ++++++++++++++++++++++++++++++++++++++ slatedb/src/db.rs | 33 +++++++++------ slatedb/src/db/builder.rs | 12 +----- slatedb/src/db_snapshot.rs | 4 +- slatedb/src/error.rs | 4 ++ slatedb/tests/db.rs | 4 +- 7 files changed, 118 insertions(+), 29 deletions(-) diff --git a/slatedb-dst/src/utils.rs b/slatedb-dst/src/utils.rs index 690c554b8..930fde63b 100644 --- a/slatedb-dst/src/utils.rs +++ b/slatedb-dst/src/utils.rs @@ -43,7 +43,8 @@ pub async fn build_settings(rand: &DbRand) -> Settings { let l0_sst_size_bytes = rng.random_range(MIB_1..MIB_500); let l0_max_ssts = rng.random_range(4..8); let l0_max_ssts_per_key = l0_max_ssts; - let max_unflushed_bytes = rng.random_range(MIB_1..GIB_2); + // Keep `max_unflushed_bytes` strictly greater than `l0_sst_size_bytes`. + let max_unflushed_bytes = rng.random_range((l0_sst_size_bytes + 1)..GIB_2); let compression_codec_idx = rng.random_range(0..COMPRESSION_CODECS.len()); let compression_codec = if let Some(compression_codec) = COMPRESSION_CODECS[compression_codec_idx] { diff --git a/slatedb/src/config.rs b/slatedb/src/config.rs index 55cd2331a..860c9800e 100644 --- a/slatedb/src/config.rs +++ b/slatedb/src/config.rs @@ -820,6 +820,39 @@ impl Settings { serde_json::to_string(self) } + /// Validates that the settings are internally consistent, rejecting field + /// combinations that would deadlock or fail at runtime. + /// + /// # Errors + /// + /// Returns an [`crate::Error`] with [`crate::ErrorKind::Invalid`] describing + /// the first invalid setting or combination encountered. + pub fn validate(&self) -> Result<(), crate::Error> { + if self.l0_flush_parallelism == 0 { + return Err(SlateDBError::InvalidConfiguration( + "l0_flush_parallelism must be at least 1".into(), + ) + .into()); + } + if self.max_wal_flushes_before_l0_flush < 4096 { + return Err(SlateDBError::InvalidConfiguration( + "max_wal_flushes_before_l0_flush must be at least 4096".into(), + ) + .into()); + } + // `max_unflushed_bytes` (the backpressure threshold) must exceed + // `l0_sst_size_bytes` (the memtable freeze threshold) so that memory can + // hold a memtable up to the freeze point before backpressure kicks in. + if self.max_unflushed_bytes <= self.l0_sst_size_bytes { + return Err(SlateDBError::InvalidConfiguration(format!( + "max_unflushed_bytes ({}) must be greater than l0_sst_size_bytes ({})", + self.max_unflushed_bytes, self.l0_sst_size_bytes, + )) + .into()); + } + Ok(()) + } + /// Loads Settings from a file. /// /// This function attempts to read and parse a configuration file to create a Settings instance. @@ -2007,4 +2040,58 @@ object_store_cache_options: assert_eq!(ts2, Some(99999)); assert_eq!(ts3, Some(99999)); } + + #[test] + fn test_validate_accepts_default_settings() { + assert!(Settings::default().validate().is_ok()); + } + + #[test] + fn test_validate_rejects_zero_l0_flush_parallelism() { + let settings = Settings { + l0_flush_parallelism: 0, + ..Settings::default() + }; + let err = settings.validate().expect_err("expected invalid settings"); + assert!(err.to_string().contains("l0_flush_parallelism")); + } + + #[test] + fn test_validate_rejects_low_max_wal_flushes_before_l0_flush() { + let settings = Settings { + max_wal_flushes_before_l0_flush: 4095, + ..Settings::default() + }; + let err = settings.validate().expect_err("expected invalid settings"); + assert!(err.to_string().contains("max_wal_flushes_before_l0_flush")); + } + + #[test] + fn test_validate_rejects_max_unflushed_bytes_not_greater_than_l0_sst_size() { + // Equal is invalid: must be strictly greater. + let equal = Settings { + l0_sst_size_bytes: 64 * 1024 * 1024, + max_unflushed_bytes: 64 * 1024 * 1024, + ..Settings::default() + }; + let err = equal.validate().expect_err("expected invalid settings"); + assert!(err.to_string().contains("max_unflushed_bytes")); + + let smaller = Settings { + l0_sst_size_bytes: 64 * 1024 * 1024, + max_unflushed_bytes: 32 * 1024 * 1024, + ..Settings::default() + }; + assert!(smaller.validate().is_err()); + } + + #[test] + fn test_validate_accepts_max_unflushed_bytes_greater_than_l0_sst_size() { + let settings = Settings { + l0_sst_size_bytes: 64 * 1024 * 1024, + max_unflushed_bytes: 64 * 1024 * 1024 + 1, + ..Settings::default() + }; + assert!(settings.validate().is_ok()); + } } diff --git a/slatedb/src/db.rs b/slatedb/src/db.rs index 11698c929..e893c846e 100644 --- a/slatedb/src/db.rs +++ b/slatedb/src/db.rs @@ -4278,7 +4278,7 @@ mod tests { let object_store: Arc = Arc::new(InMemory::new()); let path = "/tmp/test_flush_memtable_max_wal_flushes"; - let mut settings = test_db_options(0, usize::MAX, None); + let mut settings = test_db_options(0, 64 * 1024 * 1024, None); settings.flush_interval = None; // Disable flushing settings.max_wal_flushes_before_l0_flush = MAX_WAL_FLUSHES_BEFORE_L0_FLUSH; @@ -4987,7 +4987,9 @@ mod tests { let object_store: Arc = Arc::new(InMemory::new()); let path = Path::from("/tmp/test_kv_store"); let mut options = test_db_options(0, 1, None); - options.max_unflushed_bytes = 1; + // Must stay above l0_sst_size_bytes (1) but small enough that a single + // write exceeds it and triggers backpressure. + options.max_unflushed_bytes = 2; let metrics_recorder = Arc::new(DefaultMetricsRecorder::new()); let db = Db::builder(path, object_store.clone()) .with_settings(options) @@ -5062,12 +5064,15 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_backpressure_waiter_exits_when_db_is_fenced() { - // Build a DB whose WAL will not flush on a timer and whose backpressure - // threshold is low enough for one write to exceed it. + // Pause the L0 upload so a frozen memtable can't drain, keeping unflushed + // bytes above the backpressure threshold indefinitely. let object_store: Arc = Arc::new(InMemory::new()); - let mut options = test_db_options(0, 1024 * 1024, None); + let fp_registry = Arc::new(FailPointRegistry::new()); + fail_parallel::cfg(fp_registry.clone(), "write-compacted-sst-io-error", "pause").unwrap(); + + let mut options = test_db_options(0, 4 * 1024, None); options.flush_interval = None; - options.max_unflushed_bytes = 1; + options.max_unflushed_bytes = 8 * 1024; // Use a metrics recorder so the test can observe when the spawned task // has actually entered maybe_apply_backpressure(). @@ -5077,6 +5082,7 @@ mod tests { object_store, ) .with_settings(options) + .with_fp_registry(fp_registry.clone()) .with_metrics_recorder(metrics_recorder.clone()) .build() .await @@ -5086,13 +5092,10 @@ mod tests { ..Default::default() }; - // Write enough data to leave bytes buffered in the WAL while avoiding - // any automatic WAL or memtable flush. - let large_value = vec![b'x'; 8 * 1024]; + let large_value = vec![b'x'; 16 * 1024]; db.put_with_options(b"key1", &large_value, &PutOptions::default(), &write_opts) .await .unwrap(); - assert_eq!(db.inner.wal_observer.status().buffered_wal_entries_count, 1); // Start backpressure on a cloned inner handle. This parks the task on // the same wait path used by writers before they enqueue a batch. @@ -5128,7 +5131,11 @@ mod tests { backpressure_task.abort(); let _ = backpressure_task.await; } - db.close().await.unwrap(); + + // Resume the L0 upload so the pending memtable can drain and close can + // complete cleanly. + fail_parallel::cfg(fp_registry.clone(), "write-compacted-sst-io-error", "off").unwrap(); + let _ = db.close().await; // Assert that the waiter exits with the terminal fenced error, not a // successful write path or some unrelated task failure. @@ -7046,8 +7053,8 @@ mod tests { let path = "/tmp/test_recent_snapshot_min_seq_monotonic"; let object_store = Arc::new(InMemory::new()); let settings = Settings { - l0_sst_size_bytes: 4 * 1024, // Smaller to trigger flush more easily - max_unflushed_bytes: 2 * 1024, // Smaller to trigger flush more easily + l0_sst_size_bytes: 2 * 1024, // Smaller to trigger flush more easily + max_unflushed_bytes: 4 * 1024, // Smaller to trigger flush more easily min_filter_keys: 0, flush_interval: Some(Duration::from_millis(100)), ..Default::default() diff --git a/slatedb/src/db/builder.rs b/slatedb/src/db/builder.rs index 8f8cee183..7cd3d173b 100644 --- a/slatedb/src/db/builder.rs +++ b/slatedb/src/db/builder.rs @@ -409,17 +409,7 @@ impl> DbBuilder

{ /// Builds and opens the database. pub async fn build(self) -> Result { - if self.settings.l0_flush_parallelism == 0 { - return Err(crate::Error::invalid( - "invalid configuration: l0_flush_parallelism must be at least 1".into(), - )); - } - if self.settings.max_wal_flushes_before_l0_flush < 4096 { - return Err(crate::Error::invalid( - "invalid configuration: max_wal_flushes_before_l0_flush must be at least 4096" - .into(), - )); - } + self.settings.validate()?; let path = self.path.into(); // TODO: proper URI generation, for now it works just as a flag diff --git a/slatedb/src/db_snapshot.rs b/slatedb/src/db_snapshot.rs index 440e333bd..90834c5a4 100644 --- a/slatedb/src/db_snapshot.rs +++ b/slatedb/src/db_snapshot.rs @@ -302,7 +302,7 @@ mod tests { scheduler_options: Default::default(), ..Default::default() }), - max_unflushed_bytes: 16 * 1024, + max_unflushed_bytes: 8 * 4096, min_filter_keys: 0, l0_sst_size_bytes: 4 * 4096, ..Default::default() @@ -740,7 +740,7 @@ mod tests { scheduler_options: Default::default(), ..Default::default() }), - max_unflushed_bytes: 16 * 1024, + max_unflushed_bytes: 8 * 4096, min_filter_keys: 0, l0_sst_size_bytes: 4 * 4096, ..Default::default() diff --git a/slatedb/src/error.rs b/slatedb/src/error.rs index b61eb326b..7ca1364ed 100644 --- a/slatedb/src/error.rs +++ b/slatedb/src/error.rs @@ -233,6 +233,9 @@ pub(crate) enum SlateDBError { #[error("invalid sst batch size. size=`{0}`")] InvalidSSTBatchSize(usize), + #[error("invalid configuration: {0}")] + InvalidConfiguration(String), + #[error("cannot seek to a key outside the iterator range. key=`{key:?}`, start_key=`{start_key:?}`, end_key=`{end_key:?}`")] SeekKeyOutOfKeyRange { key: Vec, @@ -645,6 +648,7 @@ impl From for Error { SlateDBError::InvalidObjectStorePath(_) => Error::invalid(msg), SlateDBError::UnknownConfigurationFormat(_) => Error::invalid(msg), SlateDBError::InvalidSSTBatchSize(_) => Error::invalid(msg), + SlateDBError::InvalidConfiguration(_) => Error::invalid(msg), SlateDBError::InvalidCheckpointLifetime(_) => Error::invalid(msg), SlateDBError::InvalidManifestPollInterval(_) => Error::invalid(msg), SlateDBError::CheckpointLifetimeTooShort { .. } => Error::invalid(msg), diff --git a/slatedb/tests/db.rs b/slatedb/tests/db.rs index be4d0a8bc..2d82e717b 100644 --- a/slatedb/tests/db.rs +++ b/slatedb/tests/db.rs @@ -186,8 +186,8 @@ async fn test_concurrent_writers_and_readers() { flush_interval: Some(Duration::from_millis(100)), manifest_poll_interval: Duration::from_millis(100), manifest_update_timeout: Duration::from_secs(300), - // Allow 16KB of unflushed data - max_unflushed_bytes: 16 * 1024, + // Allow 32KB of unflushed data (must exceed l0_sst_size_bytes) + max_unflushed_bytes: 8 * 4096, min_filter_keys: 0, // Allow up to four 4096-byte blocks per-SST l0_sst_size_bytes: 4 * 4096, From 0470faada16699387677277c3773e369cdcc0f51 Mon Sep 17 00:00:00 2001 From: Roman Date: Sat, 18 Jul 2026 14:25:03 +0200 Subject: [PATCH 32/63] fix: prune SST IDs from external_dbs that were projected out (#1949) --- slatedb/src/manifest/mod.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/slatedb/src/manifest/mod.rs b/slatedb/src/manifest/mod.rs index 6d2ed2f5a..fd7cf73cb 100644 --- a/slatedb/src/manifest/mod.rs +++ b/slatedb/src/manifest/mod.rs @@ -1067,13 +1067,8 @@ impl Manifest { } projected.core.segments = kept; - // Drop unused external_dbs based on the surviving SST set across - // every tree (unsegmented + segments). - let used_sst_ids: HashSet = - projected.core.all_sst_views().map(|v| v.sst.id).collect(); - projected - .external_dbs - .retain(|e| e.sst_ids.iter().any(|id| used_sst_ids.contains(id))); + projected.prune_external_sst_ids(); + projected.external_dbs.retain(|e| !e.sst_ids.is_empty()); Ok(projected) } @@ -3156,6 +3151,10 @@ mod tests { assert_eq!(projected.external_dbs.len(), 1); assert_eq!(projected.external_dbs[0].path, "/path/to/db1"); + // The retained entry must have its out-of-range ID (sst_id_2) trimmed. + // Carrying it forward would keep the parent SST pinned in detach GC even + // though the projected tree no longer references it. + assert_eq!(projected.external_dbs[0].sst_ids, vec![sst_id_1]); } #[test] From 2e7684a19a72a4e94ab75a204cf91d1d53a31cda Mon Sep 17 00:00:00 2001 From: Chris Date: Sun, 19 Jul 2026 13:46:42 -0700 Subject: [PATCH 33/63] fix: avoid retaining full cache parts for range reads (#1953) --- .../src/cached_object_store/object_store.rs | 43 +++++++++++++++++-- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/slatedb/src/cached_object_store/object_store.rs b/slatedb/src/cached_object_store/object_store.rs index 0f8f71c7f..7e521951d 100644 --- a/slatedb/src/cached_object_store/object_store.rs +++ b/slatedb/src/cached_object_store/object_store.rs @@ -534,7 +534,7 @@ impl CachedObjectStore { // Cache miss, so we need to fetch from the object store. // Read Part — deduplicate concurrent fetches of the same part. // The SingleFlight fetches the full part and saves it to cache; each - // caller then slices out their own range_in_part. + // caller then copies out their own range_in_part. let bytes = this .part_flights .call((location.clone(), part_id), || async { @@ -565,7 +565,10 @@ impl CachedObjectStore { }) .await?; - Ok((bytes.slice(range_in_part), ReadResultSource::Upstream)) + Ok(( + Bytes::copy_from_slice(&bytes[range_in_part]), + ReadResultSource::Upstream, + )) }) } @@ -1009,7 +1012,7 @@ mod tests { use std::sync::Arc; use std::time::Duration; - use super::CachedObjectStore; + use super::{CachedObjectStore, ReadResultSource}; use crate::cached_object_store::policy::CachePutConfig; use crate::cached_object_store::stats::CachedObjectStoreStats; use crate::cached_object_store::storage::{LocalCacheStorage, PartID}; @@ -1035,6 +1038,13 @@ mod tests { } fn new_cached_store(object_store: Arc) -> Arc { + new_cached_store_with_part_size(object_store, 1024) + } + + fn new_cached_store_with_part_size( + object_store: Arc, + part_size_bytes: usize, + ) -> Arc { let test_cache_folder = new_test_cache_folder(); let recorder = MetricsRecorderHelper::noop(); let stats = Arc::new(CachedObjectStoreStats::new(&recorder)); @@ -1050,13 +1060,38 @@ mod tests { CachedObjectStore::new( object_store, cache_storage, - 1024, + part_size_bytes, CachePutConfig::default(), stats, ) .unwrap() } + #[tokio::test] + async fn test_upstream_part_range_does_not_retain_full_part() { + let part_size = 4 * 1024 * 1024; + let part = Bytes::from(vec![7_u8; part_size]); + let range = 1024..5120; + let source_range_ptr = part[range.clone()].as_ptr(); + let location = Path::from("test"); + let object_store = Arc::new(object_store::memory::InMemory::new()); + object_store + .put(&location, PutPayload::from_bytes(part)) + .await + .unwrap(); + let cached_store = new_cached_store_with_part_size(object_store, part_size); + + let (copied, source) = cached_store + .read_part(&location, 0, range, false) + .await + .unwrap(); + + assert!(matches!(source, ReadResultSource::Upstream)); + assert_eq!(copied.len(), 4096); + assert!(copied.iter().all(|byte| *byte == 7)); + assert_ne!(copied.as_ptr(), source_range_ptr); + } + #[tokio::test] async fn test_save_result_not_aligned() -> object_store::Result<()> { let payload = gen_rand_bytes(1024 * 3 + 32); From 208c32c1c2825cb6a734c449ab6d661f31ab83bb Mon Sep 17 00:00:00 2001 From: Rohan Date: Tue, 21 Jul 2026 12:45:13 -0400 Subject: [PATCH 34/63] add RFC for custom WAL implementations (#1911) --- rfcs/0030-pluggable-wal.md | 786 +++++++++++++++++++++++++++++++++++++ 1 file changed, 786 insertions(+) create mode 100644 rfcs/0030-pluggable-wal.md diff --git a/rfcs/0030-pluggable-wal.md b/rfcs/0030-pluggable-wal.md new file mode 100644 index 000000000..ae340121f --- /dev/null +++ b/rfcs/0030-pluggable-wal.md @@ -0,0 +1,786 @@ +# Pluggable WAL + +Table of Contents: + + + + + +Status: Draft + +Authors: + +* [Rohan Desai](https://github.com/rodesai) + +## Summary + +This RFC proposes (1) an interface to decouple SlateDB from the WAL and (2) adding the ability +for users to plug in their own WAL implementations. + +## Motivation + +SlateDB is designed to use object store for all storage, including its WAL. This brings many +benefits (cheap storage, no xfer costs, simple operations, share-ability). On the other hand, +object storage forces inherent latency/cost/durability tradeoffs for writes. You can reduce +latency by tuning the WAL's flush interval down, but this drives up the cost from PUTs. There's +also a floor for this tradeoff as object store PUTs themselves take 10s of ms on average. +Alternatively you can opt not to write with `await_durable` but then you're not guaranteed that +the write is durable. + +Users also use the WAL for CDC and to populate readers. As with writes, an object-store based +WAL imposes a limit on end-to-end latency. Readers/CDC have to poll for new updates. +Object stores charge for each GET, and GETs themselves can take 10s of ms. + +These tradeoffs are appropriate for many systems that are not latency-sensitive. For those that +are, this RFC proposes allowing plugging in WAL implementations that use alternative backing stores. +Latency-sensitive systems can use an alternate WAL to get low-latency writes and cdc while still +reaping many of the benefits of SlateDB's object-store native architecture. + +## Goals + +- Define a set of traits that users can implement to use alternative WALs in lieu of SlateDB's + native WAL. +- Add apis to the various builders to enable using alternative WALs. +- Support streaming updates from the WAL to readers before regular manifest polls/updates. + +## Non-Goals + +- Expose SlateDB's native wal implementation publicly (e.g. for use outside of SlateDB). +- Add alternative WAL implementations to the SlateDB project. +- Exposing a way to source events other than WAL writes (e.g. tree changes for manifest warming) + +## Design + +

+Status Quo + +### Background/Status Quo + +Lets first take a look at how the WAL works today and how SlateDB uses it. At a high level, +SlateDb uses the WAL to quickly/cheaply persist new writes, and to recover persisted writes that +have not made their way into L0. + +Internally, the WAL is a sequence of Wal Files. Each Wal File stores a sequence of +writes (the writes corresponding to some range of sequence numbers) in an object in object +storage. Each WAL File has a WAL File ID that is exactly one greater than the last WAL File ID. The +WAL Files are strictly ordered, so reading WAL Files in order yields a total order of writes to +SlateDB. + +SlateDB interacts with the WAL in a few places: + +**Fencing** + +When SlateDB starts, it fences older writers by fencing both the Manifest and the WAL +(`WriterFencer`). Both structures need to be fenced as SlateDB does not/can not transactionally +read-modify-write across the two. SlateDB fences the WAL by writing a so-called Fencing WAL File +(with no rows) to the next WAL ID. WAL File PUTs use `If-None-Match`, so older writers fail with +an object store collision on the next WAL write. The specifics of the protocol are more involved, +but for now its sufficient to understand that the WAL and Manifest must both be fenced. + +**Recovery** + +Next, the db replays any writes that have not yet made it into L0 (`WalReplayIterator`). The +manifest specifies a WAL ID that L0 is guaranteed to fully cover (`replay_after_wal_id`), along with +the last sequence number written to L0 (`last_l0_seq`). `replay_after_wal_id` gives the db the +start of the WAL File range it needs to read. Fencing establishes the end of the range. The db +replays these WAL files into memtables, filtering out any rows with sequence numbers at or below +`last_l0_seq`. + +**Writes** + +Once it's recovered persisted writes, the db hands the WAL (`WalBufferManager`) off to the +Batch Writer task. This task serializes all writes and buffers them in `WalBufferManager`, +which periodically flushes the writes to a new WAL file. `WalBufferManager` notifies blocked +write tasks when writes are durably flushed. + +**Memtable/L0 Flushing** + +The Batch Writer task adds writes to the memtable once they've been buffered in +`WalBufferManager`. It "freezes" memtables once they cross the memtable size threshold and +annotates the frozen memtable with a `replay_after_wal_id` which holds the ID of some WAL File +whose writes are fully covered by the memtable (in the current implementation this is the last +durably flushed WAL File). The frozen memtables are picked up by a separate Manifest Writer task, +which stores `replay_after_wal_id` in the manifest alongside the change that commits the +corresponding L0 file. + +**Db Flushing** + +SlateDB client can explicitly request flushes by calling either `flush` or `create_checkpoint`. +Flushes can request a flush of just the WAL, or a flush of the Memtable. Flushes also go through +the Batch Write task, which either flushes the WAL or uses the memtable freeze mechanics +described above depending on what the user requested. + +**Checkpoints** + +When `WalBufferManager` durably persists a WAL File, it notifies the db, which updates +`last_seen_wal_id` in the manifest with the flushed WAL ID. `DbReader` uses this field to +determine the range of WAL Files that should be read for a checkpoint. + +**Garbage Collection** + +GC (`WalGcTask`) is responsible for cleaning up old WAL Files. The GC first resolves all live +Manifests, and then uses these to determine the set of referenced WAL Files. For a given Manifest +`M` that is not the current manifest, its referenced WAL Files are those with range +`M.replay_after_wal_id..=M.last_seen_wal_id`. For the current Manifest `M_c`, its referenced +WAL Files are those with range `M.replay_after_wal_id..`. The GC then deletes any WAL files W that +meet the following conditions: +- W is not referenced. +- W is not a fencing WAL +- W is older than `min_age` + +**Reader Maintenance** + +`DbReader` reads from the WAL to populate its memtables. When loading a user-provided checkpoint, +`DbReader` replays the WAL range specified from the Manifest. When the user does not specify a +checkpoint, `DbReader` periodically polls the Manifest and replays WALs referenced by the +current Manifest. + +**CDC** +The `wal_reader` module defines a low-level interface to support CDC. Users create a `WalReader` +to list/get `WalFile` instances. `WalFile#iterator` is used to read the contents of the file. +Polling/batching is left up to the caller. + +
+ +### Model + +The basic model between SlateDB and the WAL is perfectly reasonable, and we don't propose +fundamentally changing it in this RFC. The WAL remains a log of sequenced writes written into a +series of WAL Files. The former is a fundamental requirement, and the latter choice allows for +implementations to reference the underlying storage structure for recovery and garbage +collection without maintaining an index that maps from SlateDB's sequence number. Storing writes +in a sequence of "files" is a natural structure that should map well to any backing storage. For +example a Kafka WAL could store (batches of) write batches in a kafka record, so each kafka +record is a WAL File whose offset is its WAL ID). + +The Alternatives section discusses a couple of alternative levels of abstraction and the +associated challenges. + +### Proposed Interfaces + +We will add the following traits which WAL implementations implement and which SlateDB calls +when accessing the WAL: + +```rust +/// A range of WAL File IDs +pub struct WalFileRange(pub Bound, pub Bound); + +/// Defines the types of errors that can be returned by WAL implementations. +#[derive(Debug, Clone)] +pub enum WalError { + /// The WAL writer was fenced + Fenced, + /// IO error writing/reading the WAL + IoError(Arc), + /// Fatal error indicating that the WAL is in some unexpected/unrecoverable state. + InternalError(Arc), + /// A WalIterator observed that the tail of the WAL was truncated while iterating. + WalTruncated, + /// Operation against wal after it was closed + Closed, +} + +/// The writer's manifest after fencing. [`crate::Db`] creates this after fencing the manifest +/// and passes it into [`WriterInit::fence_and_init`] +pub struct WriterManifest { + manifest: FenceableManifest, +} + +impl WriterManifest { + /// Returns the current manifest. + pub fn manifest(&self) -> VersionedManifest { + let (id, manifest) = self.manifest.manifest(); + VersionedManifest::from_manifest(id, manifest.clone()) + } + + /// Returns the WAL ID up to which SlateDB has guaranteed to have stored all data in the + /// LSM tree. + pub fn replay_after_wal_id(&self) -> u64 { + self.manifest().core().replay_after_wal_id + } + + /// Returns the writer's epoch + pub fn epoch(&self) -> u64 { + self.manifest().writer_epoch() + } + + /// Refreshes the current manifest. Implementations of `WriterInit::fence_and_init` can + /// use this to detect whether the manifest has been fenced while executing the fencing + /// protocol. SlateDB will call this after calling [`WriterInit::fence_and_init`] + pub async fn refresh(&mut self) -> Result<(), WalError> { + self.manifest.refresh().await?; + Ok(()) + } +} + +/// The result returned by [`WriterInit::fence_and_init`] +pub struct WriterInitResult { + /// An iterator that returns writes that must be replayed before starting SlateDB to recover + /// data from the WAL. + pub replay_iterator: Box, + /// The WAL writer that will be used to append new writes to the WAL + pub wal_writer: Box, +} + +/// API for fencing and initializing a new WAL writer for use by [`crate::db::Db`]. SlateDB requires +/// WAL implementations to execute a fencing protocol that guarantees (1) that earlier writers no +/// longer write to the db and (2) all rows present in the WAL but not in the LSM tree (L0 and +/// sorted runs) are recovered. +/// +/// Every [`crate::db::Db`] instance is assigned a unique `u64` epoch. The epoch is assigned when +/// fencing the Manifest. A given Db instance writes both the WAL and its Manifest (e.g. with new +/// SSTs) independently. The fencing protocol that yields epoch E must ensure that: +/// (1) After the first write to the Manifest with epoch E, there are no further writes to either +/// the Manifest or WAL with epoch E' < E. Note that write here excludes the manifest bump +/// itself. +/// (2) After the first write to the WAL with epoch E, there are no further writes to either the +/// Manifest or WAL with epoch E' < E +/// (3) All rows from the WAL from writers with epoch E' < E that are not present in L0/SRs are +/// replayed before serving reads/writes. +/// +/// `[WriterInit::fence_and_init]` is responsible for +/// (1) Fencing the WAL such that no writers with an epoch earlier than [`WriterManifest::epoch`] +/// (2) Constructing a [`WalWriter`] instance that the writer uses to append new WAL entries. +/// (3) Resolving the end of the WAL and constructing a [`WalReplayIterator`] that returns all +/// rows in WAL files between [`WriterManifest::replay_after_wal_id`] (exclusive) and the +/// current end of the WAL. +#[async_trait] +pub trait WriterInit { + /// Returns the name of the WAL implementation. Will be used to stamp the initial db manifest + /// and to validate that Dbs use the correct WAL implementation. + fn name(&self) -> String; + + /// Fences the WAL and returns a [`WriterInitResult`] with a [`WalWriter`] and + /// [`WalReplayIterator`] used to recover writes that have not yet been flushed to the tree. + async fn fence_and_init( + &self, + manifest: &mut WriterManifest, + ) -> Result; +} + +/// Describes the current status of the WAL +#[derive(Debug, Clone)] +pub struct WalStatus { + /// Set to Some if the WAL has permanently shut down, along with the reason. The reason should + /// be [`WalError::Closed`] on a normal shutdown, and some other [`WalError`] variant on + /// failure. + pub closed_reason: Option, + /// The estimated in-memory bytes used by the WAL to buffer unflushed writes. Used by + /// SlateDB to apply backpressure. + pub estimated_bytes: usize, + /// The id of the last WAL file that was durably flushed + pub last_flushed_wal_id: u64, + /// The last sequence number that was durably flushed + pub last_flushed_seq: Option, + /// The number of writes currently buffered + #[allow(dead_code)] + pub buffered_wal_entries_count: usize, +} + +/// An event emitted by a [`WalWriter`] to subscribers. +#[derive(Debug, Clone)] +pub enum WalEvent { + /// Emitted when a WAL file is durably flushed to storage. On receipt of this event, SlateDB + /// notifies write tasks blocked on [`crate::config::WriteOptions::await_durable`]. SlateDB + /// also uses this to apply backpressure if the implementation sets + /// [`WalStatus::estimated_bytes`]. Implementers should update this to reflect clearing the + /// memory used for buffering the Wal File before emitting this event. + WalFlushed(WalStatus), + /// Emitted when the WAL has closed with the final wal status containing the closed reason + WalClosed(WalStatus), +} + +/// A listener that's called back on WAL events. +pub type WalStatusListener = Arc; + +/// An observer that can read the current [`WalStatus`] and subscribe to event callbacks. +#[async_trait] +pub trait WalObserver: Send + Sync + 'static { + /// Returns the current [`WalStatus`]. + fn status(&self) -> Result; + + /// Adds a listener that subscribes to event callbacks. On success, returns an initial + /// [`WalStatus`]. The listener receives all updates after this initial status. + fn subscribe(&self, listener: WalStatusListener) -> Result; +} + +/// A future that yields the result of flushing the WAL. Returned by [`WalWriter::flush`] +pub type FlushResultFuture = BoxFuture<'static, Result<(), WalError>>; + +/// The WAL's write API. Used by SlateDB to append new WAL writes. Is returned by +/// [`WalWriterInit::fence_and_init_writer`]. +/// +/// Each call to [`WalWriter::append`] takes a single SlateDB write batch, where all rows share +/// the same sequence number ([`RowEntry::seq`]). [`WalWriter`] (optionally accumulates/buffers +/// rows and) writes consecutive write batches into consecutive WAL Files, where each WAL File +/// contains some rows from the total sequence of rows. Specifically: +/// - WAL Files must have a total order and each WAL File must have a u64 id that is greater than +/// all earlier WAL Files. +/// - Reading WAL Files in order should yield rows in sequence order. +/// - The writes in a given write batch must be written to WAL files atomically. That is, a +/// [`WalIterator`] should either observe all the writes with a given sequence number or none +/// of them. +#[async_trait] +pub trait WalWriter: Send { + /// Append a write batch to the WAL. + async fn append(&mut self, write_batch: &[RowEntry]) -> Result<(), WalError>; + + /// Triggers a flush of all appended write batches to durable storage. Returns a + /// future that receives the result of the flush once it completes. + async fn flush(&mut self) -> Result; + + /// Returns a `WalObserver` for reading [`WalStatus`] and subscribing to events. + fn observer(&self) -> Box; + + /// Returns the current `WalStatus`. If the [`WalWriter`] has failed, then returns Err with the + /// final [`WalStatus`] and the reason for the failure in [`WalStatus::closed_reason`] + fn status(&self) -> Result; + + /// Close the `WalWriter` and release resources + async fn close(&mut self) -> Result<(), WalError>; +} + +/// Rows returned by [`WalIterator`] +pub struct WalRows { + /// The rows read from the WAL File. All the rows with a given sequence number must be present + /// in th same [`WalRows`]. + pub rows: Vec, + /// The id of the last WAL File containing rows from `rows`. There may still be rows with higher + /// sequence numbers in the WAL File with this id. + pub last_wal_file_id: u64, + /// True when this batch is the last one in its WAL file. This is an + /// optimization, so its harmless to always set to false. Callers can already infer that a + /// file is fully applied when they see a batch from a later file, but this flag lets them + /// advance their WAL watermark over the current file without waiting for the next one. + pub last_in_file: bool, +} + +/// An iterator over rows in some range of the WAL +#[async_trait] +pub trait WalIterator: Send + 'static { + /// Returns the next set of rows. Rows must be returned in sequence and WAL File order. + /// Returns None when iterator's range is exhausted. Iterators created using an unbounded + /// end range that have exhausted the current WAL block until new rows are appended and never + /// return `None`. + /// Returns [`WalError::WalTruncated`] if the iterator observes that the WAL was truncated + /// while iterating. + async fn next(&mut self) -> Result, WalError>; +} + +/// API for reading from the WAL. Used by the Reader/ +#[async_trait] +pub trait WalReader { + /// Returns the name of the WAL implementation + fn name(&self) -> String, + + /// Returns an iterator over the specified range of WAL File IDs. The start of the range must + /// not be `Unbounded`. If the end of the range is `Unbounded` then the returned iterator + /// continues returning writes as new writes are appended to the WAL. Otherwise, it returns + /// `None` upon reaching the end of he range. + async fn iterator( + &self, + wal_file_id_range: WalFileRange, + ) -> Result, WalError>; +} + +/// API for plugging into WAL GC +#[async_trait] +pub trait WalGC { + /// Hook for garbage collecting the WAL. Takes a list of ranges of WAL Files that are currently + /// referenced by some active Manifest. The implementation may delete any WAL File that is not + /// included in the ranges in this list. + async fn collect( + &self, + referenced_ranges: Vec, + ) -> Result<(), WalError>; +} +``` + +Users can configure a custom WAL for the writer and reader using the db Builder: +```rust +impl> DbBuilder

{ + /// Sets the `[WalWriterInit]` used to initialize a `[WalWriter]` to append new + /// entries to the WAL. Use this to plug in custom WAL implementations to SlateDB. + /// By default, SlateDB uses its own object-store based WAL. + pub fn with_wal_writer(mut self, wal_writer_init: Box) { + self.wal_writer_init = Some(writer_init); + } +} + +impl> DbReaderBuilder

{ + /// Sets the `[WalReader]` used to create `[WalIterator]`s to replay rows from the WAL + /// Use this to plug in custom WAL implementations to the reader. By default, SlateDB uses + /// its own object-store based WAL. + pub fn with_wal_reader(mut self, wal_reader: Box) { + self.wal_reader = Some(wal_reader); + } +} + +impl > GarbageCollectorBuilder

{ + /// Sets the collector for cleaning up WAL Files. Use this if you are using a custom + /// WAL implementation and want SlateDB to coordinate GC. + pub fn with_wal_gc(mut self, wal_gc: Arc) -> Self { + self.wal_gc = Some(wal_gc); + } +} +``` + +### Manifest Changes + +We'll add the WAL name to the manifest and validate that the provided `WalInit` and `WalReader` +match when starting `Db`/`DbReader`. The field is only set if the user provides a custom WAL +implementation. Otherwise, it is left unset. It is an error to use a custom WAL implementation +with an unset `wal_name` or to use an implementation whose name does not match the set name. + +``` +table ManifestV2 { + ... + /// Name of the WAL implementation to use. The value is initially set based on the value + /// returned by `WalInit::name`. If the DB is built without a custom WAL implementation then + /// this field is left unset. + wal_name: string; +} +``` + +### SlateDB Integration +Let's look at how SlateDB will use these interfaces from the various WAL touch-points. + +#### Fencing + +Every [`crate::db::Db`] instance is assigned a unique `u64` epoch. The epoch is assigned when +fencing the Manifest. A given Db instance writes both the WAL and its Manifest (e.g. with new +SSTs) independently. The fencing protocol that yields epoch `E` must ensure that: +1. After the first write to the Manifest with epoch E, there are no further writes to either + the Manifest or WAL with epoch `E' < E`. Note that the write here excludes the epoch bump + itself. Instead, it refers to the set of writes (pushing new L0 files, updating the various + sequence trackers, updating wal trackers, etc) protected by the epoch. +2. After the first write to the WAL with epoch `E`, there are no further writes to either the + Manifest or WAL with epoch `E' < E` +3. All rows from the WAL from writers with epoch `E'` < E that are not present in L0/SRs are + replayed before serving reads/writes. + +The fencing protocol must fence both the Manifest and the WAL. However, this means that the +protocol must be able to deal with the case where another writer `W'` completes the protocol +while a given writer `W` is between the two fencing operations, e.g.: + +``` +t1: W fences Manifest +t2: W' fences Manifest +t3: W' fences WAL and resolves replay range +t3: W' updates WAL/Manifest +t4: W fences WAL and resolves replay range +``` + +It's not safe for `W` to write new WAL entries as it breaks the requirements above. In +practice this is problematic because it has not observed `W'`'s Manifest updates. Further, +if fencing the WAL depends on reading the Manifest (e.g. SlateDB's WAL protocol), `W`'s fencing +operation is operating on a stale Manifest view. + +Take the inverse: +``` +t1: W fences WAL and resolves replay range +t2: W' fences WAL and resolves replay range +t3: W' fences Manifest +t3: W' updates WAL/Manifest +t4: W fences Manifest +``` + +It's not safe for `W` to update the Manifest or serve reads as it has not observed `W'`'s WAL writes. + +There are 2 approaches you can take to solve this, depending on the isolation primitives that +are available (or practical) on your backing store: +- Fencing: Your backing store allows you to fence existing writers such that they can not + append new writes. For example, the basic Kafka transaction protocol. SlateDB's native + WAL also falls into this category (it also has other constraints imposed by the fact + that fencing depends on reading the Manifest, but the solution is the same). +- Transactions: Your backing store allows you to transactionally read then conditionally append. + Examples include any database with transactions, or the Kafka transaction protocol if you + store the epoch in a Kafka topic and read the value after initializing the transactional + producer and before appending new writes. + +If your backing store supports Transactions, the protocol is simple - you simply make each WAL +write conditional on the writer's epoch. + +If your backing store only supports Fencing, then the protocol must fence one resource then the +other, then check that the first resource is still fenced. For example: +1. Fence Manifest +2. Fence WAL +3. Check Manifest is fenced. + +SlateDB will execute this protocol on behalf of the WAL implementation by first fencing the +manifest, then calling `WalWriterInit::fence_and_init`, and then refreshing its `FenceableManifest` +to ensure the db is still fenced. This is a minor change to `WriterFencer` to delegate WAL fencing +to the trait implementation. + +#### Recovery + +`WalWriterInit::fence_and_init` returns a `WriterInitResult` with a `replay_iterator` that +iterates over the section of the WAL that must be replayed. The DB replays from this iterator. +This requires a small refactor of `WalReplayIterator` to iterate over `WalIterator` rather than +directly iterating over WAL Files. + +#### Writes + +Writes mostly stay the same. The batch writer task takes ownership of `WalWriter` and uses it to +append new writes via `WalWriter::append`. + +The WAL no longer maintains a durability watcher for each WAL file. Instead, a write that +awaits durable blocks on `DbStatus` until the durable sequence number is greater than or equal to +the write's sequence number. When the WAL is enabled, slatedb propagates updates to the durable +sequence number via the `WalObserver` subscription. + +**Backpressure** + +The WAL needs a mechanism to apply backpressure to incoming writes. If writes arrive faster than +the WAL can flush them to new WAL Files, they accumulate in the WAL's buffer and use more and +more memory. SlateDB's native WAL backpressure is integrated into SlateDB's api-level +backpressure mechanism via `WalStatus::estimated_bytes` and propagation of +`WalEvent::MemoryReleased` events when the WAL releases memory. + +There's some tension between SlateDB's native WAL and custom WAL implementations here. On the +one hand, SlateDB's existing mechanics mean that its WAL does not need its own backpressure and +users get a single config for applying a memory cap to the write path (`max_unflushed_bytes`). +On the other hand, custom WALs may prefer/need to use their own backpressure. Take a Kafka-backed +WAL for example. The Kafka producer has its own in-built backpressure mechanism that blocks writes +when too many records are buffered. There isn't a straightforward way to observe the buffer +memory or be notified when its released. + +We propose allowing WAL implementations to opt into the SlateDB backpressure mechanism but not +require it. To opt in, the implementation must set `WalStatus::estimated_bytes` to a non-zero value +and emit `WalEvent::MemoryReleased` when memory is released. Alternatively, custom `WalWriter` +implementations can apply backpressure internally by blocking calls to `append`. **To accommodate +this SlateDB needs to account for memory used by writes waiting in the batch writer's channel when +computing the current unflushed bytes when deciding whether to pause a write.** + +This avoids adding a new memory-management config for the bulk of SlateDB users while still +allowing custom WALs to apply backpressure. + +#### Flushing + +Memtable and Db flushing stay the same. The Batch Writer task annotates each immutable memtable +with a safe replay point using `WalStatus::last_flushed_wal_id`, and flushes the WAL using +`WalWriter::flush`. + +#### Garbage Collection + +`WalGcTask` lists manifests to determine the set of referenced WAL File IDs and then delegates +cleanup to `WalGc`. The native implementation of `WalGc` prunes wal files based on max-age and +then deletes them. + +#### Readers + +`DbReaderBuilder` initializes `DbReader` with a `WalReader` that it uses to construct iterators +for replaying the WAL when loading a checkpoint. + +`DbReader` will now also continually stream WAL updates when configured to track the latest +writes. It does this by creating its `WalIterator` with an unbounded end range and blocking on +`next` from its background polling task. If the reader observes a `WalError::WalTruncated` then it +immediately refreshes the manifest. + +#### CDC + +We'll deprecate/remove the current CDC API. Users can use the `WalReader`/`WalIterator` proposed +in this RFC. SlateDB's native `WalReader` will take a buffer size and a poll interval to use when +tailing the current WAL: + +```rust +struct ObjectStoreWalReader { + // ... +} + +impl ObjectStoreWalReader { + pub fn new>( + path: P, + object_store: Arc, + /// The number of WAL Files to prefetch and buffer when streaming the WAL + buffered_files: usize, + /// The interval at which the next WAL file will be polled when streaming the latest updates + poll_interval: Duration + ) { + todo!() + } +} + +impl WalReader for ObjectStoreWalReader { + // ... +} +``` + +#### Error Handling + +Custom WAL implementations are expected to manage the lifecycle of any background tasks and +propagate errors via their regular apis rather than have SlateDB expose its task management +framework. + +### Example Alternative Implementations + +#### Kafka + +The prototype branch has an example implementation of the WAL traits that writes records to +a Kafka topic and implements fencing using Kafka transactions: +https://github.com/slatedb/slatedb/tree/wal-rfc-prototype/slatedb/src/wal/kafka + +It also includes a benchmark test that steadily writes rows to a `Db`. Every 100ms it samples +the last written row and measures how long it took to become available on the reader. With +a Kafka backed WAL it sees the following distribution for latency between durably writing +and being available to read on the reader: +- p50: 11.1ms +- p90: 12.1ms +- p99: 15.8ms + +## Impact Analysis + +SlateDB features and components that this RFC interacts with. Check all that apply. + +### Core API & Query Semantics + +- [x] Basic KV API (`get`/`put`/`delete`) +- [x] Range queries, iterators, seek semantics +- [ ] Range deletions +- [x] Error model, API errors + +### Consistency, Isolation, and Multi-Versioning + +- [ ] Transactions +- [ ] Snapshots +- [x] Sequence numbers + +### Time, Retention, and Derived State + +- [ ] Time to live (TTL) +- [ ] Compaction filters +- [ ] Merge operator +- [x] Change Data Capture (CDC) + +### Metadata, Coordination, and Lifecycles + +- [ ] Manifest format +- [ ] Checkpoints +- [ ] Clones +- [x] Garbage collection +- [ ] Database splitting and merging +- [ ] Multi-writer + +### Compaction + +- [ ] Compaction state persistence +- [ ] Compaction filters +- [ ] Compaction strategies +- [ ] Distributed compaction +- [ ] Compactions format + +### Storage Engine Internals + +- [x] Write-ahead log (WAL) +- [ ] Block cache +- [ ] Object store cache +- [ ] Indexing (bloom filters, metadata) +- [ ] SST format or block format + +### Ecosystem & Operations + +- [ ] CLI tools +- [ ] Language bindings (Go/Python/etc) +- [ ] Observability (metrics/logging/tracing) + +## Operations + +### Performance & Cost + +- Switching the `await_durable` mechanism to block on the durable sequence number means that + there will likely be more spurious wakeups where a blocked write task is woken up, observes + that the sequence number hasn't advanced sufficiently and goes back to sleep. I don't expect + this to add meaningful overhead. + +### Observability + +None. Custom WAL implementations are expected to expose their own metrics/configuration. + +### Compatibility + +No breaking changes. + +## Testing + +**Correctness** + +We will expose a conformance test suite that WAL implementers can use to validate that their WAL +implementation is correct. Some important test cases we'll cover (non-exhaustive): +- `WalWriterInit::fence_and_init` prevents existing writers from writing to the WAL. +- `WalWriterInit::fence_and_init` returns an iterator that replays unflushed writes. +- `WalWriter` flushes WAL rows durably (so they can be observed by `WalReader`) +- `WalWriter` emits events when rows are durably stored. +- `WalIterator` always iterates over writes in sequence order +- `WalIterator` always returns full write batches in `WalRows` +- `WalIterator` tracks the WAL file id in `WalRows` correctly (TODO: this probably needs some + test interfaces in the reader for listing/reading wal files) + +**Performance** + +SlateDB already exposes a benchmarking utility, `DbBench`. WAL implementers can write their own +benchmark tools that instantiate `DbBench` with a db configured to use a custom WAL. + +## Rollout + +- Phase 1 (in-progress): refactor existing WAL to align with the traits proposed here +- Phase 2: introduce traits and pluggability +- Phase 3: add conformance test harnesses and an example implementation + +## Packaging + +The WAL traits and conformance tests will reside in a new crate called `slatedb-wal`. The native +WAL implementation remains in `slatedb`. + +## Alternatives + +List the serious alternatives and why they were rejected (including “status quo”). Include +trade-offs and risks. + +**Status Quo** +Users that want a custom WAL can opt out of using SlateDB's WAL and write their own WAL in front +of SlateDB. This puts a lot of burden on the user. On the writer side you need to implement your +own write serialization, sequencing, replay tracking, and transaction system. On the reader side +you need to write your own layer that buffers WAL records and merges them with rows returned by +the reader. + +**Plug in at ObjectStore Layer** +We could support alternative stores by plugging in at the object store layer. This is just a very +awkward integration point. The Object Store trait is object-store specific and the primitives +don't map very well to other stores you may want to use for a WAL. For example it assumes key-based +access, compare-and-swaps, etc. It also will likely require implementations to understand the +internals of SlateDB's native WAL, which is brittle. + +**More Flexible Fencing API** +The proposed interface for fencing forces the Fence Manifest, Fence WAL, Check Manifest +structure. Custom WAL implementations may not need this (for example if your store supports +transactions). We could instead just have a more generic `fence_and_init` API that takes (some +wrapper over) a `StoredManifest` and is expected to implement the full fence protocol. It feels +too complicated to expect custom WAL implementations to do this. For now we assume minimal +fencing semantics from the custom WAL to keep the expectations from the trait simpler at the +cost of an extra manifest check. + +**Pure Row/Sequence Based Abstraction** +We could have the abstraction track WAL position just using row sequence numbers. This requires +each WAL to have some way to efficiently read starting from a SlateDB sequence number, which is +not always practical. Expecting the WAL to group rows into a series of "files" feels pretty +reasonable/general. + +**Iterate over WAL Files** +We could have `WalReader`/`WalIterator` iterate over WAL Files which in turn support row-based +iteration (similar to the CDC `WalReader`). I don't really see the benefit of imposing the extra +layering. It also forces implementations to map each write batch to a single WAL File. + +## Open Questions + +- ~~This RFC proposes an API for streaming new writes via `WalReader`/`WalIterator`. Should this be + used for CDC in lieu of the existing `WalReader`/`WalFile` API? Does it make sense to retain + both?~~ +- ~~Should we put the traits and conformance tests in a separate `slatedb-wal` crate?~~ + +## References + +- https://github.com/slatedb/slatedb/issues/1768 + +## Updates + +Log major changes to this RFC over time (optional). From ca6067b30cffa551b7f549f56dc3bec1e1bb0ad3 Mon Sep 17 00:00:00 2001 From: nomiero Date: Tue, 21 Jul 2026 11:56:36 -0700 Subject: [PATCH 35/63] Update README with database split/merge status (#1959) --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index f46c74a2e..a15dc595d 100644 --- a/README.md +++ b/README.md @@ -138,8 +138,7 @@ Visit [slatedb.io](https://slatedb.io) to learn more. - [x] Clones ([#49](https://github.com/slatedb/slatedb/issues/49)) - [ ] Range deletions ([#577](https://github.com/slatedb/slatedb/issues/577)) - [x] Change data capture (CDC) ([#249](https://github.com/slatedb/slatedb/issues/249)) -- [ ] Database splitting -- [ ] Database merging +- [x] Database split/merge ([RFC](https://github.com/slatedb/slatedb/blob/main/rfcs/0004-checkpoints.md#manifest-projection-and-union)) ## Projects From 75c1394c97d292e54a503cb2b8ee48cf619fcd5c Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 21 Jul 2026 19:20:19 -0700 Subject: [PATCH 36/63] Freeze large memtables on WAL replay (#1955) --- slatedb/src/db.rs | 72 ++++++++++++++++++++++++++++++++++++++++ slatedb/src/db_common.rs | 23 +++++++++++++ 2 files changed, 95 insertions(+) diff --git a/slatedb/src/db.rs b/slatedb/src/db.rs index e893c846e..c047b89ec 100644 --- a/slatedb/src/db.rs +++ b/slatedb/src/db.rs @@ -551,6 +551,7 @@ impl DbInner { // ensure the assertion holds true. assert!(self.oracle.last_remote_persisted_seq() <= replayed_table.last_seq); self.oracle.advance_durable_seq(replayed_table.last_seq); + self.maybe_freeze_memtable(current_memtable_wal_id); self.maybe_apply_backpressure().await?; let replayed_table_last_wal_id = replayed_table.last_wal_id; self.replay_memtable(current_memtable_wal_id, replayed_table)?; @@ -9937,6 +9938,77 @@ mod tests { } } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn test_wal_replay_flushes_oversized_active_memtable_before_backpressure() { + let object_store: Arc = Arc::new(InMemory::new()); + let path = "/tmp/test_wal_replay_flushes_oversized_active_memtable"; + + // Leave a single WAL SST whose replayed table is smaller than the + // source's freeze threshold. Keeping the source open simulates a crash: + // the recovery writer fences it without first flushing its memtable to L0. + let mut source_settings = test_db_options(0, 64 * 1024, None); + source_settings.flush_interval = None; + let source = Db::builder(path, object_store.clone()) + .with_settings(source_settings) + .build() + .await + .unwrap(); + let value = vec![b'x'; 16 * 1024]; + source + .put_with_options( + b"oversized-replay-value", + &value, + &PutOptions::default(), + &WriteOptions { + await_durable: false, + ..Default::default() + }, + ) + .await + .unwrap(); + source + .flush_with_options(FlushOptions { + flush_type: FlushType::Wal, + }) + .await + .unwrap(); + + // On recovery, one complete WAL SST exceeds both thresholds. Replay + // must freeze it before backpressure; otherwise open spins forever with + // an oversized active memtable and nothing for the flusher to drain. + let mut replay_settings = test_db_options(0, 1024, None); + replay_settings.flush_interval = None; + replay_settings.max_unflushed_bytes = 2 * 1024; + let recovered = tokio::time::timeout( + Duration::from_secs(5), + Db::builder(path, object_store) + .with_settings(replay_settings) + .build(), + ) + .await + .expect("WAL replay deadlocked on an oversized active memtable") + .expect("failed to recover database"); + + assert_eq!( + recovered.get(b"oversized-replay-value").await.unwrap(), + Some(Bytes::from(value)) + ); + recovered + .put_with_options( + b"write-after-replay", + b"value", + &PutOptions::default(), + &WriteOptions { + await_durable: false, + ..Default::default() + }, + ) + .await + .expect("write after oversized WAL replay should succeed"); + + recovered.close().await.unwrap(); + } + /// RFC-0024: WAL replay through a conforming extractor preserves the /// keys and lets segment-aware writes resume after the next open. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/slatedb/src/db_common.rs b/slatedb/src/db_common.rs index 80c75dc57..8b32b5aeb 100644 --- a/slatedb/src/db_common.rs +++ b/slatedb/src/db_common.rs @@ -26,6 +26,29 @@ pub(crate) fn extract_segment_prefix( } impl DbInner { + /// Freezes the active memtable when its estimated encoded size reaches + /// [`Settings::max_unflushed_bytes`](crate::config::Settings::max_unflushed_bytes). + /// + /// The frozen table is stamped with `replay_after_wal_id` and announced to + /// the memtable flusher. The caller must therefore pass the highest WAL ID + /// fully represented by the active memtable. This method does nothing when + /// the active memtable is below the threshold. + /// + /// # Arguments + /// + /// * `replay_after_wal_id` - Durable WAL boundary for the active memtable. + pub(crate) fn maybe_freeze_memtable(&self, replay_after_wal_id: u64) { + let mut guard = self.state.write(); + let metadata = guard.memtable().table().metadata(); + let estimated_bytes = self + .table_store + .estimate_encoded_size_compacted(metadata.entry_num, metadata.entries_size_in_bytes); + + if estimated_bytes >= self.settings.max_unflushed_bytes { + self.freeze_current_memtable_with_state_guard(&mut guard, replay_after_wal_id); + } + } + pub(crate) fn replay_memtable( &self, current_memtable_wal_id: u64, From f363a3943a324b96db85962b07d04db477918a53 Mon Sep 17 00:00:00 2001 From: Kaivalya Apte Date: Wed, 22 Jul 2026 19:39:25 +0200 Subject: [PATCH 37/63] (RFC-0029) Move L0 SST id allocation to dispatch (#1957) --- .../src/memtable_flusher/manifest_writer.rs | 43 +++++- slatedb/src/memtable_flusher/tracker.rs | 139 +++++++++++++++++- slatedb/src/memtable_flusher/uploader.rs | 127 +++++++++++++--- 3 files changed, 278 insertions(+), 31 deletions(-) diff --git a/slatedb/src/memtable_flusher/manifest_writer.rs b/slatedb/src/memtable_flusher/manifest_writer.rs index 09e81414a..569305f1b 100644 --- a/slatedb/src/memtable_flusher/manifest_writer.rs +++ b/slatedb/src/memtable_flusher/manifest_writer.rs @@ -502,10 +502,10 @@ impl ManifestWriterHandler { // still advances. (This is true with or without an // extractor configured.) for segment in &uploaded.segments { - let view = SsTableView::new( - self.db.rand.rng().gen_ulid(self.db.system_clock.as_ref()), - segment.sst_handle.clone(), - ); + // Identity view: the view id is the physical SST ULID, so + // the timestamp `last_compacted_l0_sst_view_id` reads + // equals the one GC deletion reads (RFC-0029). + let view = SsTableView::identity(segment.sst_handle.clone()); let tree = if segmented { // Extractor configured — every flush handle, including // any with empty prefix, is routed into `segments`. @@ -1909,6 +1909,41 @@ mod tests { assert_eq!(core.segments[1].tree.l0.len(), 1); assert_eq!(core.segments[0].tree.l0[0].sst.id, aaa_id); assert_eq!(core.segments[1].tree.l0[0].sst.id, bbb_id); + // Newly flushed L0s are identity views: the view id equals the + // physical SST ULID (RFC-0029). + assert_eq!(core.segments[0].tree.l0[0].id, aaa_id.unwrap_compacted_id()); + assert_eq!(core.segments[1].tree.l0[0].id, bbb_id.unwrap_compacted_id()); + + started.shutdown().await; + } + + #[tokio::test] + async fn should_create_identity_l0_view_on_flush() { + let harness = setup_harness( + "/tmp/test_manifest_writer_identity_l0_view", + Arc::new(FailPointRegistry::new()), + ) + .await; + let inner = Arc::clone(&harness.inner); + let started = start_manifest_writer( + Arc::clone(&inner), + harness.manifest, + Duration::from_secs(3600), + ); + + let uploaded = next_uploaded_memtable(&inner, b"k1", b"v1").await; + let physical_id = uploaded.segments[0].sst_handle.id; + started.notify_uploaded(uploaded).await.unwrap(); + let _ = expect_flushed(&started.tracker_rx).await; + + // The published L0 view id must equal the physical SST ULID so the + // timestamp `last_compacted_l0_sst_view_id` reads matches the one GC + // deletion reads (RFC-0029). + let core = inner.state.read().state().core().clone(); + assert_eq!(core.tree.l0.len(), 1); + let view = &core.tree.l0[0]; + assert_eq!(view.sst.id, physical_id); + assert_eq!(view.id, physical_id.unwrap_compacted_id()); started.shutdown().await; } diff --git a/slatedb/src/memtable_flusher/tracker.rs b/slatedb/src/memtable_flusher/tracker.rs index 360222a65..ac8509eaa 100644 --- a/slatedb/src/memtable_flusher/tracker.rs +++ b/slatedb/src/memtable_flusher/tracker.rs @@ -23,13 +23,16 @@ use crate::config::CheckpointOptions; use crate::db::DbInner; use crate::dispatcher::MessageHandler; use crate::error::SlateDBError; +use crate::mem_table::ImmutableMemtable; use crate::memtable_flusher::manifest_writer::{FlushResult, ManifestWriter}; use crate::memtable_flusher::uploader::{UploadJob, UploadedMemtable, Uploader}; use crate::memtable_flusher::FlushTarget; +use crate::utils::IdGenerator; use fail_parallel::fail_point; -use std::collections::VecDeque; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::sync::Arc; use tokio::sync::oneshot; +use ulid::Ulid; macro_rules! memtable_flush_stat_name { ($suffix:expr) => { @@ -352,7 +355,12 @@ impl FlushTracker { tracked.first_seq, last_seq ); - self.uploader.submit(UploadJob::new(imm_memtable))?; + // Allocate physical SST ids here, in seqno-ordered dispatch, so + // their ULID timestamps never fall below an earlier-dispatched + // (and thus earlier-published) L0's (RFC-0029). + let segment_sst_ids = allocate_segment_sst_ids(&self.inner, &imm_memtable); + self.uploader + .submit(UploadJob::new(imm_memtable, segment_sst_ids))?; } } @@ -383,6 +391,25 @@ impl FlushTracker { } } +/// Allocate one physical SST id per segment `imm` will flush to, keyed by +/// segment prefix. +/// +/// Without an extractor the sole segment is the compatibility-encoded +/// `prefix=""` segment; with one, the segments are the imm's touched prefixes. +/// A segment that retention later prunes to empty simply leaves its id unused. +fn allocate_segment_sst_ids(inner: &DbInner, imm: &ImmutableMemtable) -> BTreeMap { + let prefixes: BTreeSet = if inner.segment_extractor.is_some() { + imm.touched_segments() + } else { + BTreeSet::from([Bytes::new()]) + }; + let mut rng = inner.rand.rng(); + prefixes + .into_iter() + .map(|prefix| (prefix, rng.gen_ulid(inner.system_clock.as_ref()))) + .collect() +} + struct TrackedImm { first_seq: u64, last_seq: u64, @@ -532,11 +559,14 @@ mod tests { use crate::format::sst::{SsTableFormat, SST_FORMAT_VERSION_LATEST}; use crate::manifest::store::{FenceableManifest, ManifestStore, StoredManifest}; use crate::manifest::ManifestCore; + use crate::mem_table::{ImmutableMemtable, WritableKVTable}; use crate::memtable_flusher::uploader::Uploader; use crate::memtable_flusher::{FlushTarget, MemtableFlusher}; use crate::object_stores::ObjectStores; use crate::paths::PathResolver; + use crate::prefix_extractor::PrefixExtractor; use crate::tablestore::{TableStore, TableStoreKind}; + use crate::test_utils::FixedThreeBytePrefixExtractor; use crate::types::RowEntry; use crate::utils::{SafeSender, WatchableOnceCell}; use crate::wal_buffer::WalBufferManager; @@ -545,7 +575,7 @@ mod tests { use object_store::memory::InMemory; use object_store::path::Path; use object_store::ObjectStore; - use slatedb_common::clock::{DefaultSystemClock, SystemClock}; + use slatedb_common::clock::{DefaultSystemClock, MockSystemClock, SystemClock}; use slatedb_common::metrics::{ lookup_metric_with_labels, DefaultMetricsRecorder, MetricLevel, MetricsRecorder, MetricsRecorderHelper, @@ -568,8 +598,15 @@ mod tests { settings: Settings, fp_registry: Arc, ) -> TestHarness { - setup_harness_with_recorder(path, settings, fp_registry, MetricsRecorderHelper::noop()) - .await + setup_harness_with_recorder( + path, + settings, + fp_registry, + MetricsRecorderHelper::noop(), + None, + Arc::new(DefaultSystemClock::new()), + ) + .await } async fn setup_harness_with_recorder( @@ -577,10 +614,11 @@ mod tests { settings: Settings, fp_registry: Arc, db_metrics: MetricsRecorderHelper, + segment_extractor: Option>, + system_clock: Arc, ) -> TestHarness { let object_store: Arc = Arc::new(InMemory::new()); let path = path.to_string(); - let system_clock: Arc = Arc::new(DefaultSystemClock::new()); let rand = Arc::new(DbRand::new(42)); let manifest_store = Arc::new(ManifestStore::new( &Path::from(path.clone()), @@ -628,7 +666,7 @@ mod tests { fp_registry, None, status_manager, - None, + segment_extractor, ) .await .unwrap(), @@ -1189,6 +1227,8 @@ mod tests { settings, Arc::new(FailPointRegistry::new()), helper, + None, + Arc::new(DefaultSystemClock::new()), ) .await; set_local_l0_len(&harness, 1); @@ -1487,6 +1527,8 @@ mod tests { settings, Arc::new(FailPointRegistry::new()), helper, + None, + Arc::new(DefaultSystemClock::new()), ) .await; let ranges: &[(&[u8], &[u8])] = &[(b"aaa", b"zzz")]; @@ -1573,6 +1615,89 @@ mod tests { assert!(result.is_ok() || result.is_err()); } + fn imm_with_touched(entries: &[(&[u8], &[u8], u64)], touched: &[&[u8]]) -> ImmutableMemtable { + let table = WritableKVTable::new(); + for (key, value, seq) in entries { + table.put(RowEntry::new_value(key, value, *seq)); + } + if !touched.is_empty() { + table.record_touched_segments( + touched.iter().map(|p| Bytes::copy_from_slice(p)).collect(), + ); + } + ImmutableMemtable::new(table, 0) + } + + #[tokio::test] + async fn allocate_segment_sst_ids_without_extractor_uses_empty_prefix() { + let harness = setup_harness( + "/tmp/test_allocate_segment_sst_ids_empty_prefix", + Settings::default(), + Arc::new(FailPointRegistry::new()), + ) + .await; + let imm = imm_with_touched(&[(b"k1", b"v1", 1)], &[]); + + let ids = super::allocate_segment_sst_ids(&harness.inner, &imm); + + assert_eq!(ids.len(), 1); + assert!(ids.contains_key(&Bytes::new())); + } + + #[tokio::test] + async fn allocate_segment_sst_ids_with_extractor_covers_touched_segments() { + let harness = setup_harness_with_recorder( + "/tmp/test_allocate_segment_sst_ids_segments", + Settings::default(), + Arc::new(FailPointRegistry::new()), + MetricsRecorderHelper::noop(), + Some(Arc::new(FixedThreeBytePrefixExtractor)), + Arc::new(DefaultSystemClock::new()), + ) + .await; + let imm = imm_with_touched( + &[(b"aaa-1", b"v1", 1), (b"bbb-1", b"v2", 2)], + &[b"aaa", b"bbb"], + ); + + let ids = super::allocate_segment_sst_ids(&harness.inner, &imm); + + let prefixes: Vec<&[u8]> = ids.keys().map(|k| k.as_ref()).collect(); + assert_eq!(prefixes, vec![&b"aaa"[..], &b"bbb"[..]]); + } + + /// RFC-0029 ordering guarantee at the allocation level: ids minted for a + /// later-dispatched memtable never carry an earlier ULID timestamp than an + /// earlier-dispatched one, so `newest_l0` cannot advance past a pending L0. + #[tokio::test] + async fn allocate_segment_sst_ids_do_not_regress_across_dispatch() { + let clock = Arc::new(MockSystemClock::new()); + let harness = setup_harness_with_recorder( + "/tmp/test_allocate_segment_sst_ids_ordering", + Settings::default(), + Arc::new(FailPointRegistry::new()), + MetricsRecorderHelper::noop(), + None, + clock.clone(), + ) + .await; + + let imm1 = imm_with_touched(&[(b"k1", b"v1", 1)], &[]); + let first = + super::allocate_segment_sst_ids(&harness.inner, &imm1)[&Bytes::new()].timestamp_ms(); + + clock.advance(Duration::from_millis(10)).await; + + let imm2 = imm_with_touched(&[(b"k2", b"v2", 2)], &[]); + let second = + super::allocate_segment_sst_ids(&harness.inner, &imm2)[&Bytes::new()].timestamp_ms(); + + assert!( + second > first, + "later dispatch must not regress: first={first}, second={second}" + ); + } + mod frontier_tests { use crate::mem_table::{ImmutableMemtable, WritableKVTable}; use crate::memtable_flusher::tracker::{TrackedImmFrontier, TrackedImmState}; diff --git a/slatedb/src/memtable_flusher/uploader.rs b/slatedb/src/memtable_flusher/uploader.rs index e25b0a27c..7f4f5a6ec 100644 --- a/slatedb/src/memtable_flusher/uploader.rs +++ b/slatedb/src/memtable_flusher/uploader.rs @@ -19,23 +19,29 @@ use crate::dispatcher::{MessageHandler, MessageHandlerExecutor}; use crate::error::SlateDBError; use crate::flush::EncodedSegmentSst; use crate::mem_table::ImmutableMemtable; -use crate::utils::{IdGenerator, SafeSender}; +use crate::utils::SafeSender; use async_trait::async_trait; use bytes::Bytes; use futures::stream::BoxStream; use futures::StreamExt; use log::{info, warn}; +use std::collections::BTreeMap; use std::sync::Arc; use std::time::Duration; use tokio::runtime::Handle; +use ulid::Ulid; const UPLOADER_TASK_NAME: &str = "l0_sst_uploader"; -/// One immutable-memtable upload request submitted to the uploader. The -/// worker allocates SST ids for each segment internally. +/// One immutable-memtable upload request submitted to the uploader. Physical +/// SST ids are allocated at dispatch (in sequence order) and carried here, so +/// the parallel upload workers never mint ids out of publish order (RFC-0029). pub(crate) struct UploadJob { /// Immutable memtable to build into one or more SSTs. pub(crate) imm_memtable: Arc, + /// Pre-allocated physical SST id per segment prefix. A segment that + /// retention prunes to empty simply leaves its id unused. + pub(crate) segment_sst_ids: BTreeMap, } impl std::fmt::Debug for UploadJob { @@ -45,9 +51,15 @@ impl std::fmt::Debug for UploadJob { } impl UploadJob { - /// Creates a new upload job. - pub(crate) fn new(imm_memtable: Arc) -> Self { - Self { imm_memtable } + /// Creates a new upload job with pre-allocated segment SST ids. + pub(crate) fn new( + imm_memtable: Arc, + segment_sst_ids: BTreeMap, + ) -> Self { + Self { + imm_memtable, + segment_sst_ids, + } } } @@ -195,11 +207,22 @@ impl UploadHandler { // Upload all segment SSTs concurrently. `try_join_all` short-circuits // on the first fatal error and drops the remaining futures; sibling // uploads that already landed before the abort are left for the - // garbage collector to reclaim, since the worker allocates ids - // internally and they are not visible here for explicit cleanup. - let segments = - futures::future::try_join_all(built.iter().map(|sst| self.upload_segment_sst(sst))) - .await?; + // garbage collector to reclaim. + let segments = futures::future::try_join_all(built.iter().map(|sst| { + // Ids are pre-allocated at dispatch keyed by segment prefix. Every + // built prefix is a subset of the dispatched touched set, so a + // missing id is an internal invariant violation. + let sst_id = job + .segment_sst_ids + .get(&sst.prefix) + .copied() + .map(SsTableId::Compacted); + async move { + let sst_id = sst_id.ok_or(SlateDBError::InvalidDBState)?; + self.upload_segment_sst(sst, sst_id).await + } + })) + .await?; Ok(UploadedMemtable { imm_memtable: Arc::clone(&job.imm_memtable), @@ -209,15 +232,15 @@ impl UploadHandler { }) } - /// Upload a single segment SST with retry. Each retry reuses the + /// Upload a single segment SST with retry, writing it to the id + /// pre-allocated for its segment at dispatch. Each retry reuses the /// already-encoded SST so the upload loop never rebuilds from the /// memtable. async fn upload_segment_sst( &self, sst: &EncodedSegmentSst, + sst_id: SsTableId, ) -> Result { - let sst_id = - SsTableId::Compacted(self.db.rand.rng().gen_ulid(self.db.system_clock.as_ref())); let written_bytes = sst.encoded.remaining_len() as u64; loop { match self.db.upload_sst(&sst_id, &sst.encoded, true).await { @@ -277,12 +300,13 @@ mod tests { use super::{TrackerMessage, UploadJob, Uploader}; use crate::config::Settings; use crate::db::DbInner; - use crate::db_state::SsTableView; + use crate::db_state::{SsTableId, SsTableView}; use crate::db_status::{ClosedResultWriter, DbStatusManager}; use crate::error::SlateDBError; use crate::format::sst::SsTableFormat; use crate::iter::RowEntryIterator; use crate::manifest::ManifestCore; + use crate::mem_table::ImmutableMemtable; use crate::object_stores::ObjectStores; use crate::paths::PathResolver; use crate::sst_iter::{SstIterator, SstIteratorOptions}; @@ -299,10 +323,24 @@ mod tests { use slatedb_common::clock::{DefaultSystemClock, SystemClock}; use slatedb_common::metrics::{DefaultMetricsRecorder, MetricLevel, MetricsRecorderHelper}; use slatedb_common::DbRand; + use std::collections::BTreeMap; use std::sync::Arc; use std::time::Duration; use tokio::runtime::Handle; use tokio::time::timeout; + use ulid::Ulid; + + /// Build a pre-allocated id map for a test job, mirroring dispatch-time + /// allocation: one id per segment prefix, falling back to the empty prefix + /// when no extractor recorded segments. The tracker owns the real + /// allocation path; tests only need a valid map covering the built SSTs. + fn preallocate_ids(imm: &ImmutableMemtable) -> BTreeMap { + let mut prefixes = imm.touched_segments(); + if prefixes.is_empty() { + prefixes.insert(Bytes::new()); + } + prefixes.into_iter().map(|p| (p, Ulid::new())).collect() + } async fn setup_db(path: &str, fp_registry: Arc) -> Arc { setup_db_with_extractor(path, fp_registry, None).await @@ -387,7 +425,8 @@ mod tests { fn next_upload_job(db: &DbInner, key: &[u8], value: &[u8], seq: u64) -> UploadJob { let imm_memtable = freeze_imm(db, key, value, seq); - UploadJob::new(imm_memtable) + let segment_sst_ids = preallocate_ids(&imm_memtable); + UploadJob::new(imm_memtable, segment_sst_ids) } struct TestUploader { @@ -498,6 +537,40 @@ mod tests { test.shutdown().await; } + #[tokio::test] + async fn should_write_sst_to_preallocated_id() { + // The worker must write each segment SST to the id allocated at + // dispatch (carried in the job), not mint a fresh one (RFC-0029). + let db = setup_db( + "/tmp/test_parallel_l0_flush_uploader_preallocated_id", + Arc::new(FailPointRegistry::new()), + ) + .await; + let job = next_upload_job(&db, b"key", b"value", 1); + let expected_id = *job + .segment_sst_ids + .get(&Bytes::new()) + .expect("empty-prefix id should be pre-allocated"); + + let test = start_test_uploader(&db); + test.submit(job).unwrap(); + + let msg = timeout(Duration::from_secs(5), test.tracker_rx.recv()) + .await + .unwrap() + .unwrap(); + let TrackerMessage::UploadComplete(event) = msg else { + panic!("expected UploadComplete"); + }; + assert_eq!(event.segments.len(), 1); + assert_eq!( + event.segments[0].sst_handle.id, + SsTableId::Compacted(expected_id) + ); + + test.shutdown().await; + } + #[tokio::test] async fn should_retry_upload_failures_until_success() { let fp_registry = Arc::new(FailPointRegistry::new()); @@ -550,7 +623,8 @@ mod tests { .front() .cloned() .unwrap(); - let job = UploadJob::new(imm_memtable); + let segment_sst_ids = preallocate_ids(&imm_memtable); + let job = UploadJob::new(imm_memtable, segment_sst_ids); let test = start_test_uploader(&db); test.submit(job).unwrap(); @@ -623,7 +697,8 @@ mod tests { .front() .cloned() .unwrap(); - let bad_job = UploadJob::new(imm_memtable); + let segment_sst_ids = preallocate_ids(&imm_memtable); + let bad_job = UploadJob::new(imm_memtable, segment_sst_ids); let test = start_test_uploader(&db); test.submit(bad_job).unwrap(); @@ -716,7 +791,8 @@ mod tests { .front() .cloned() .unwrap(); - let job = UploadJob::new(imm_memtable); + let segment_sst_ids = preallocate_ids(&imm_memtable); + let job = UploadJob::new(imm_memtable, segment_sst_ids); let test = start_test_uploader(&db); test.submit(job).unwrap(); @@ -782,6 +858,16 @@ mod tests { ] { guard.memtable().put(RowEntry::new_value(key, value, seq)); } + // The production write path stamps these inline; this test + // bypasses that, so record explicitly. + guard + .memtable() + .table() + .record_touched_segments(std::collections::BTreeSet::from([ + Bytes::from_static(b"aaa"), + Bytes::from_static(b"bbb"), + Bytes::from_static(b"ccc"), + ])); guard.freeze_memtable(0); } let imm_memtable = db @@ -792,7 +878,8 @@ mod tests { .front() .cloned() .unwrap(); - let job = UploadJob::new(imm_memtable); + let segment_sst_ids = preallocate_ids(&imm_memtable); + let job = UploadJob::new(imm_memtable, segment_sst_ids); let test = start_test_uploader(&db); test.submit(job).unwrap(); From 76a132430e86d02126c53fb1ac7dfa28e4f52814 Mon Sep 17 00:00:00 2001 From: nomiero Date: Thu, 23 Jul 2026 10:48:12 -0700 Subject: [PATCH 38/63] RFC 0031 - pluggable block cache policy (#1913) --- rfcs/0031-block-cache-policy.md | 326 ++++++++++++++++++++++++++++++++ 1 file changed, 326 insertions(+) create mode 100644 rfcs/0031-block-cache-policy.md diff --git a/rfcs/0031-block-cache-policy.md b/rfcs/0031-block-cache-policy.md new file mode 100644 index 000000000..b27945d85 --- /dev/null +++ b/rfcs/0031-block-cache-policy.md @@ -0,0 +1,326 @@ +# Block Cache Policy + +Table of Contents: + + + +- [Summary](#summary) +- [Motivation](#motivation) +- [Goals](#goals) +- [Non-Goals](#non-goals) +- [Design](#design) + - [Public API](#public-api) + - [Compaction Output Behavior](#compaction-output-behavior) + - [Compaction Input Behavior](#compaction-input-behavior) + - [Embedded Compactor](#embedded-compactor) +- [Impact Analysis](#impact-analysis) +- [Operations](#operations) +- [Testing](#testing) +- [Alternatives](#alternatives) +- [Open Questions](#open-questions) +- [References](#references) + + + +Status: Accepted. + +Authors: + +* [Hussein Nomier](https://github.com/nomiero) + +## Summary + +This RFC adds a `BlockCachePolicy` to `DbBuilder`. The policy controls: + +- Which decoded SST components are requested for insertion into `DbCache` when + a memtable flush or compaction produces an SST. +- Whether the embedded compactor probes existing decoded cache entries for L0 + or sorted-run inputs. + +## Motivation + +Several internal operations could benefit from configurable block-cache +behavior. Examples include: + +- If L0 blocks are already cached, the embedded compactor can avoid rereading + and decoding them. +- Some workloads may keep indexes and filters in memory to reduce object-store + requests for point gets against newly compacted SSTs. +- Workloads using a hybrid block cache may cache compaction output on disk to + avoid later object-store reads. + +This policy lets users configure those behaviors explicitly. + +## Goals + +- Let users choose which SST components enter the block cache on flush and on + compaction output. +- Let users choose whether compaction reads probe the block cache. + +## Non-Goals + +- Change foreground block cache behavior under the default policy. It stays per + request in `ReadOptions::cache_blocks` and `ScanOptions::cache_blocks`. +- Unify policies across `CachedObjectStore` and `DbCache`. + +## Design + +### Public API + +The policy is a concrete struct value. Components are selected with the +existing `CacheTarget` enum used by `DbCacheManagerOps` (RFC-0023).: + +```rust +/// Block-cache policy for controlling block cache behavior during flush and +/// compaction. +#[derive(Clone, Debug)] +pub struct BlockCachePolicy { + flush_targets: Vec, + compaction_output_targets: Vec, + l0_compaction_cache_probe: bool, + sorted_run_compaction_cache_probe: bool, +} + +impl BlockCachePolicy { + pub fn with_flush_targets( + self, + targets: Vec, + ) -> Self {} + + pub fn with_compaction_output_targets( + self, + targets: Vec, + ) -> Self {} + + pub fn with_l0_compaction_cache_probe(self, enabled: bool) -> Self {} + + pub fn with_sorted_run_compaction_cache_probe(self, enabled: bool) -> Self {} +} + +impl Default for BlockCachePolicy { + fn default() -> Self { + Self { + flush_targets: vec![ + CacheTarget::data::<&[u8], _>(..), + CacheTarget::Index, + CacheTarget::Filters, + ], + compaction_output_targets: vec![ + CacheTarget::Index, + CacheTarget::Filters, + ], + l0_compaction_cache_probe: false, + sorted_run_compaction_cache_probe: false, + } + } +} +``` + + +`DbBuilder` gains: + +```rust +pub fn with_block_cache_policy(self, policy: BlockCachePolicy) -> Self; +``` + +### Compaction Output Behavior + +- Compaction output data is inserted as it is produced by the streaming writer. + When `CacheTarget::Data` carries a bounded key range, only the data blocks + that overlap the range are inserted. Each block streamed to the writer + will carry its first and last key, so the writer can decide overlap with the + configured range per block without waiting for the SST index. +- Metadata components are inserted when they become available at writer close. +- If a compaction write fails after entries have been inserted, a best-effort +cleanup removes the inserted entries from the cache. Entries that survive the +cleanup remain until normal eviction or restart. This is safe because the +failed SST is not visible through the manifest. + +### Compaction Input Behavior + +- Compaction probes existing entries but does not insert misses because + compaction inputs are short-lived and large scans could pollute the cache. +- The L0 and sorted-run settings independently control whether each input type + probes the cache. + +### Embedded Compactor + +`DbBuilder` passes the same scoped `DbCacheWrapper` to the main and embedded- +compactor `TableStore`s so compaction can reuse entries inserted by main table +store and vice versa. + +## Impact Analysis + +SlateDB features and components that this RFC interacts with. Check all that apply. + +### Core API & Query Semantics + +- [ ] Basic KV API (`get`/`put`/`delete`) +- [ ] Range queries, iterators, seek semantics +- [ ] Range deletions +- [ ] Error model, API errors + +### Consistency, Isolation, and Multi-Versioning + +- [ ] Transactions +- [ ] Snapshots +- [ ] Sequence numbers + +### Time, Retention, and Derived State + +- [ ] Time to live (TTL) +- [ ] Compaction filters +- [ ] Merge operator +- [ ] Change Data Capture (CDC) + +### Metadata, Coordination, and Lifecycles + +- [ ] Manifest format +- [ ] Checkpoints +- [ ] Clones +- [ ] Garbage collection +- [ ] Database splitting and merging +- [ ] Multi-writer + +### Compaction + +- [ ] Compaction state persistence +- [ ] Compaction filters +- [ ] Compaction strategies +- [ ] Distributed compaction +- [ ] Compactions format + +Compaction execution I/O is affected, but compaction selection, strategy, and +output semantics are unchanged. + +### Storage Engine Internals + +- [ ] Write-ahead log (WAL) +- [x] Block cache +- [ ] Object store cache +- [x] Indexing (bloom filters, metadata) +- [ ] SST format or block format + +### Ecosystem & Operations + +- [ ] CLI tools +- [x] Language bindings (Go/Python/etc) +- [x] Observability (metrics/logging/tracing) + +## Operations + +### Performance & Cost + +- The default policy keeps current flush behavior. It also inserts the index + and filters of compaction output SSTs, which reduces object-store requests + for point gets against newly compacted SSTs at the cost of the cache space + those entries occupy. +- A non-default policy impacts read performance and, when it caches SST + components on write, write performance. + +### Configuration + +- New configuration: `BlockCachePolicy` on `DbBuilder`. + +### Metrics + +- Block-cache hit and miss metrics gain a `TableStoreKind` label to distinguish + between main and compactor table-store reads. + +### Compatibility + +- The API is additive, so no compatibility impact. + +## Testing + +- Unit tests. +- Performance tests for different use cases. + +## Alternatives + +**Status quo.** +Rejected because of the use cases mentioned in Motivation. + + +**Trait-based policy (previous design).** +Exposed a user-implemented `BlockCachePolicy` trait along with read and write +source and action types. The types were: + +```rust +/// The operation that produced an SST. +pub enum WriteSource { + /// A memtable flush writing an L0 SST. + Flush, + /// Compaction writing an output SST. + CompactionOutput, +} + +/// The operation issuing a read. +pub enum ReadSource { + /// A foreground get or scan, carrying the per-request cache_blocks + /// option from ReadOptions or ScanOptions. + Foreground { cache_blocks: bool }, + /// A compaction read of an L0 input SST. + CompactionL0Input, + /// A compaction read of a sorted run input SST. + CompactionSortedRunInput, + /// Writer startup replay. + WalReplay, + /// DbReader WAL replay, which re-reads the same WAL SSTs when a + /// partially failed replay retries on the next poll. + WalTail, +} + +/// How a written component interacts with the block cache. +pub enum CacheWriteMode { + /// Insert the component into the block cache. + Cache, + /// Do not insert the component. + Skip, +} + +/// How a read interacts with the block cache. +pub enum CacheReadMode { + /// No lookup, no insert. + Bypass, + /// Serve a hit; on a miss, read from the object store without inserting. + Probe, + /// Serve a hit; on a miss, read from the object store and insert. + ReadThrough, +} + +pub trait BlockCachePolicy: Send + Sync + 'static { + /// How `target` of an SST written by `source` interacts with the + /// block cache. + fn write_mode( + &self, + source: WriteSource, + target: CacheTarget, + ) -> CacheWriteMode; + + /// How a read of `target` issued by `source` interacts with the + /// block cache. + fn read_mode(&self, source: ReadSource, target: CacheTarget) -> CacheReadMode; +} +``` + +This design allows more dynamic control, but it exposes more types and gives +control to all possible uses of the block cache without clear use cases. + +The proposed policy can grow with focused builder methods when concrete use +cases arise. + +**Coarse knobs.** +Five booleans (`cache_blocks_on_flush`, `cache_metadata_on_flush`, and so on) +or a `CachedSections { None, MetadataOnly, All }` enum per write source. +Rejected because it introduces many knobs, and new scenarios would add more. + +## Open Questions + +None. + +## References + +- [Issue #1799: Use block cache for L0 compaction if compactor is running on writer](https://github.com/slatedb/slatedb/issues/1799) +- [RFC-0023: Cache Manager](./0023-cache-manager.md) +- [RFC-0027: Decoupled Pluggable Object Store Cache](./0027-decoupled-object-store-cache.md) From 7574f293cc25fba0e065063e4d2f5a9c150368d3 Mon Sep 17 00:00:00 2001 From: nomiero Date: Thu, 23 Jul 2026 17:01:13 -0700 Subject: [PATCH 39/63] Apply ruff check --fix with ruff 0.16 for python bindings (#1965) --- bindings/python/slatedb/uniffi/__init__.py | 2 +- bindings/python/tests/conftest.py | 7 ++++--- bindings/python/tests/test_admin.py | 6 +++--- bindings/python/tests/test_db.py | 4 ++-- bindings/python/tests/test_logging.py | 2 +- bindings/python/tests/test_metrics.py | 2 +- bindings/python/tests/test_reader.py | 23 +++++++++++----------- bindings/python/tests/test_wal_reader.py | 2 +- 8 files changed, 24 insertions(+), 24 deletions(-) diff --git a/bindings/python/slatedb/uniffi/__init__.py b/bindings/python/slatedb/uniffi/__init__.py index 59fa010f1..907f97396 100644 --- a/bindings/python/slatedb/uniffi/__init__.py +++ b/bindings/python/slatedb/uniffi/__init__.py @@ -3,6 +3,6 @@ from importlib import import_module _generated = import_module("._slatedb_uniffi.slatedb", __name__) -from ._slatedb_uniffi import * # noqa: E402,F403 +from ._slatedb_uniffi import * __all__ = _generated.__all__ diff --git a/bindings/python/tests/conftest.py b/bindings/python/tests/conftest.py index 1dd28f612..9ddd245f3 100644 --- a/bindings/python/tests/conftest.py +++ b/bindings/python/tests/conftest.py @@ -4,8 +4,9 @@ import inspect import threading import uuid +from collections.abc import Callable from contextlib import asynccontextmanager -from typing import Any, Callable +from typing import Any from slatedb.uniffi import ( DbBuilder, @@ -23,8 +24,8 @@ PrefixExtractor, PrefixTarget, PutOptions, - ReadOptions, ReaderOptions, + ReadOptions, RowEntry, RowEntryKind, ScanOptions, @@ -166,7 +167,7 @@ async def wait_until( if await _maybe_await(check()): return last_error = None - except Exception as error: # pragma: no cover - helper for polling assertions + except Exception as error: # noqa: BLE001 # pragma: no cover - helper for polling assertions last_error = error if asyncio.get_running_loop().time() >= deadline: diff --git a/bindings/python/tests/test_admin.py b/bindings/python/tests/test_admin.py index f65b1c628..a2ab51df6 100644 --- a/bindings/python/tests/test_admin.py +++ b/bindings/python/tests/test_admin.py @@ -4,8 +4,8 @@ import time import pytest - from conftest import new_memory_store, open_db, open_reader, unique_path, wait_until + from slatedb.uniffi import ( AdminBuilder, CheckpointOptions, @@ -227,7 +227,7 @@ async def test_admin_clone() -> None: for i in range(3): path = unique_path(f"admin-clone-original-{i}") async with open_db(store, path=path) as db: - await db.put(f"k{i}".encode("utf-8"), f"v{i}".encode("utf-8")) + await db.put(f"k{i}".encode(), f"v{i}".encode()) await db.flush() sources.append(CloneSourceSpec(path=path, checkpoint=None, projection_range=None)) @@ -241,7 +241,7 @@ async def test_admin_clone() -> None: async with open_db(store, path=clone_path) as db: for i in range(3): - assert await db.get(f"k{i}".encode("utf-8")) == f"v{i}".encode("utf-8") + assert await db.get(f"k{i}".encode()) == f"v{i}".encode() @pytest.mark.asyncio diff --git a/bindings/python/tests/test_db.py b/bindings/python/tests/test_db.py index 4c2f18a20..8d969e123 100644 --- a/bindings/python/tests/test_db.py +++ b/bindings/python/tests/test_db.py @@ -1,11 +1,10 @@ from __future__ import annotations import pytest - from conftest import ( + TEST_DB_PATH, ConcatMergeOperator, FixedThreeByteSegmentExtractor, - TEST_DB_PATH, drain_iterator, merge_options, new_memory_store, @@ -16,6 +15,7 @@ scan_options, write_options, ) + from slatedb.uniffi import ( CloseReason, DbBuilder, diff --git a/bindings/python/tests/test_logging.py b/bindings/python/tests/test_logging.py index e0bd89ecf..11ca614a7 100644 --- a/bindings/python/tests/test_logging.py +++ b/bindings/python/tests/test_logging.py @@ -1,8 +1,8 @@ from __future__ import annotations import pytest - from conftest import LogCollector, new_memory_store, open_db, unique_path, wait_until + from slatedb.uniffi import Error, LogLevel, init_logging diff --git a/bindings/python/tests/test_metrics.py b/bindings/python/tests/test_metrics.py index c6d75bb7f..ffde369b5 100644 --- a/bindings/python/tests/test_metrics.py +++ b/bindings/python/tests/test_metrics.py @@ -1,8 +1,8 @@ from __future__ import annotations import pytest - from conftest import new_memory_store, open_db, open_reader + from slatedb.uniffi import ( Counter, DefaultMetricsRecorder, diff --git a/bindings/python/tests/test_reader.py b/bindings/python/tests/test_reader.py index 7142882f2..5567c1b95 100644 --- a/bindings/python/tests/test_reader.py +++ b/bindings/python/tests/test_reader.py @@ -1,10 +1,9 @@ from __future__ import annotations import pytest - from conftest import ( - ConcatMergeOperator, TEST_DB_PATH, + ConcatMergeOperator, drain_iterator, new_memory_store, open_db, @@ -15,6 +14,7 @@ scan_options, wait_until, ) + from slatedb.uniffi import ( CloseReason, DbReaderBuilder, @@ -198,18 +198,17 @@ async def has_refreshed() -> bool: async def test_reader_default_mode_replays_new_wal_data() -> None: store = new_memory_store() - async with open_db(store) as db: - async with open_reader( - store, - configure=lambda builder: builder.with_options(reader_options(False)), - ) as reader: - await db.put(b"wal-key", b"wal-value") - await db.flush_with_options(FlushOptions(flush_type=FlushType.WAL)) + async with open_db(store) as db, open_reader( + store, + configure=lambda builder: builder.with_options(reader_options(False)), + ) as reader: + await db.put(b"wal-key", b"wal-value") + await db.flush_with_options(FlushOptions(flush_type=FlushType.WAL)) - async def has_wal_value() -> bool: - return await reader.get(b"wal-key") == b"wal-value" + async def has_wal_value() -> bool: + return await reader.get(b"wal-key") == b"wal-value" - await wait_until(has_wal_value) + await wait_until(has_wal_value) @pytest.mark.asyncio diff --git a/bindings/python/tests/test_wal_reader.py b/bindings/python/tests/test_wal_reader.py index db138a595..381d21d09 100644 --- a/bindings/python/tests/test_wal_reader.py +++ b/bindings/python/tests/test_wal_reader.py @@ -1,7 +1,6 @@ from __future__ import annotations import pytest - from conftest import ( TEST_DB_PATH, drain_wal_iterator, @@ -9,6 +8,7 @@ require_wal_row, seed_wal_files, ) + from slatedb.uniffi import Error, RowEntryKind, WalReader From 5b494cc46af838ee7c278f647bc65a581a76eb7a Mon Sep 17 00:00:00 2001 From: Chris Date: Sat, 25 Jul 2026 08:14:56 -0700 Subject: [PATCH 40/63] Account for WAL bytes in write backpressure (#1967) --- slatedb/src/db.rs | 64 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 10 deletions(-) diff --git a/slatedb/src/db.rs b/slatedb/src/db.rs index c047b89ec..861f85504 100644 --- a/slatedb/src/db.rs +++ b/slatedb/src/db.rs @@ -330,10 +330,12 @@ impl DbInner { .imm_memtable .iter() .map(|imm| estimate(imm.table().metadata())) - .sum::(); + .fold(0usize, |total, size| total.saturating_add(size)); (active_memtable_size_bytes, imm_memtable_size_bytes) }; - let total_mem_size_bytes = active_memtable_size_bytes + imm_memtable_size_bytes; + let total_mem_size_bytes = active_memtable_size_bytes + .saturating_add(imm_memtable_size_bytes) + .saturating_add(wal_status.estimated_bytes); self.db_stats .total_mem_size_bytes .set(total_mem_size_bytes as i64); @@ -350,7 +352,7 @@ impl DbInner { if total_mem_size_bytes >= self.settings.max_unflushed_bytes { self.db_stats.backpressure_count.increment(1); warn!( - "unflushed memtable size exceeds max_unflushed_bytes. applying backpressure. [total_mem_size_bytes={}, active_memtable_size_bytes={}, imm_memtable_size_bytes={}, wal_size_bytes={}, max_unflushed_bytes={}]", + "unflushed WAL and memtable size exceeds max_unflushed_bytes. applying backpressure. [total_mem_size_bytes={}, active_memtable_size_bytes={}, imm_memtable_size_bytes={}, wal_size_bytes={}, max_unflushed_bytes={}]", format_bytes_si(total_mem_size_bytes as u64), format_bytes_si(active_memtable_size_bytes as u64), format_bytes_si(imm_memtable_size_bytes as u64), @@ -363,9 +365,10 @@ impl DbInner { guard.state().imm_memtable.back().cloned() }; - // There is a window of time after mem_size_bytes is larger than max_unflushed_bytes - // but before we get the memtable. During that time, if the memtable is fully - // flushed out, we should short circuit to avoid blocking indefinitely. + // There is a window of time after total_mem_size_bytes is larger than + // max_unflushed_bytes but before we get the memtable. During that time, if + // the memtable and WAL are fully flushed out, we should short circuit to + // avoid blocking indefinitely. if maybe_oldest_unflushed_memtable.is_none() && wal_status.estimated_bytes == 0 { continue; } @@ -4988,9 +4991,16 @@ mod tests { let object_store: Arc = Arc::new(InMemory::new()); let path = Path::from("/tmp/test_kv_store"); let mut options = test_db_options(0, 1, None); - // Must stay above l0_sst_size_bytes (1) but small enough that a single - // write exceeds it and triggers backpressure. - options.max_unflushed_bytes = 2; + let first_entry = RowEntry::new_value(b"key1", b"val1", 1).with_create_ts(0); + let sst_format = SsTableFormat { + min_filter_keys: options.min_filter_keys, + ..SsTableFormat::default() + }; + let first_memtable_bytes = + sst_format.estimate_encoded_size_compacted(1, first_entry.estimated_size()); + // Keep the memtable alone below the limit so this test only applies + // backpressure when the WAL estimate is included. + options.max_unflushed_bytes = first_memtable_bytes.saturating_add(1); let metrics_recorder = Arc::new(DefaultMetricsRecorder::new()); let db = Db::builder(path, object_store.clone()) .with_settings(options) @@ -5030,7 +5040,41 @@ mod tests { .await; // Verify that there is now 1 WAL entry in memory. - assert_eq!(db.inner.wal_observer.status().buffered_wal_entries_count, 1); + let wal_status = db.inner.wal_observer.status(); + assert_eq!(wal_status.buffered_wal_entries_count, 1); + + let (active_memtable_size_bytes, imm_memtable_size_bytes) = { + let guard = db.inner.state.read(); + let estimate = |metadata: KVTableMetadata| { + db.inner.table_store.estimate_encoded_size_compacted( + metadata.entry_num, + metadata.entries_size_in_bytes, + ) + }; + let active_memtable_size_bytes = estimate(guard.memtable().table().metadata()); + let imm_memtable_size_bytes = guard + .state() + .imm_memtable + .iter() + .map(|imm| estimate(imm.table().metadata())) + .fold(0usize, |total, size| total.saturating_add(size)); + (active_memtable_size_bytes, imm_memtable_size_bytes) + }; + let memtable_size_bytes = + active_memtable_size_bytes.saturating_add(imm_memtable_size_bytes); + let total_mem_size_bytes = memtable_size_bytes.saturating_add(wal_status.estimated_bytes); + assert!( + memtable_size_bytes < db.inner.settings.max_unflushed_bytes, + "test requires memtable bytes ({memtable_size_bytes}) to remain below \ + max_unflushed_bytes ({})", + db.inner.settings.max_unflushed_bytes + ); + assert!( + total_mem_size_bytes >= db.inner.settings.max_unflushed_bytes, + "test requires memtable plus WAL bytes ({total_mem_size_bytes}) to reach \ + max_unflushed_bytes ({})", + db.inner.settings.max_unflushed_bytes + ); // Put another WAL entry, which should trigger backpressure. Do this in a separate // task since the put() is blocked until the WAL is flushed, which isn't happening From ac19c5c8bf8b5001852c99362304da610913ea4c Mon Sep 17 00:00:00 2001 From: nomiero Date: Mon, 27 Jul 2026 09:40:38 -0700 Subject: [PATCH 41/63] RFC - 0031 impl: Block cache policy - writes (#1962) --- slatedb/src/block_cache_policy.rs | 136 +++++ slatedb/src/checkpoint.rs | 2 + slatedb/src/compaction_execute_bench.rs | 3 + slatedb/src/compaction_worker.rs | 2 + slatedb/src/compactor.rs | 147 +++++- slatedb/src/compactor_executor.rs | 27 +- slatedb/src/db.rs | 12 +- slatedb/src/db/builder.rs | 34 +- slatedb/src/db_cache/mod.rs | 41 ++ slatedb/src/db_cache_manager.rs | 35 +- slatedb/src/db_reader.rs | 5 +- slatedb/src/fence.rs | 2 + slatedb/src/flush.rs | 21 +- slatedb/src/format/sst.rs | 14 +- slatedb/src/garbage_collector.rs | 10 +- slatedb/src/garbage_collector/compacted_gc.rs | 38 +- slatedb/src/lib.rs | 4 +- .../src/memtable_flusher/manifest_writer.rs | 9 +- slatedb/src/memtable_flusher/tracker.rs | 2 + slatedb/src/memtable_flusher/uploader.rs | 56 +- slatedb/src/ops.rs | 5 +- slatedb/src/reader.rs | 4 +- slatedb/src/sorted_run_iterator.rs | 26 +- slatedb/src/sst_builder.rs | 114 +++- slatedb/src/sst_iter.rs | 54 +- slatedb/src/sst_reader.rs | 2 + slatedb/src/tablestore.rs | 489 +++++++++++++++--- slatedb/src/utils.rs | 6 +- slatedb/src/wal_buffer.rs | 5 +- slatedb/src/wal_reader.rs | 2 + slatedb/src/wal_replay.rs | 10 +- .../src/content/docs/docs/design/caching.mdx | 31 ++ 32 files changed, 1113 insertions(+), 235 deletions(-) create mode 100644 slatedb/src/block_cache_policy.rs diff --git a/slatedb/src/block_cache_policy.rs b/slatedb/src/block_cache_policy.rs new file mode 100644 index 000000000..a36fec464 --- /dev/null +++ b/slatedb/src/block_cache_policy.rs @@ -0,0 +1,136 @@ +//! Block cache policy for controlling which SST components are cached. + +use std::ops::Bound; + +use bytes::Bytes; + +use crate::bytes_range::BytesRange; +use crate::db_cache::CacheTarget; + +/// Whether any [`CacheTarget::Data`] range in `targets` overlaps a data +/// block whose first and last key are `key_span`. +pub(crate) fn should_cache_data_block(targets: &[CacheTarget], key_span: &(Bytes, Bytes)) -> bool { + targets.iter().any(|target| { + let CacheTarget::Data(range) = target else { + return false; + }; + let (first_key, last_key) = key_span; + let Some(range) = BytesRange::try_new(range.0.clone(), range.1.clone()) else { + return false; + }; + let span = BytesRange::new( + Bound::Included(first_key.clone()), + Bound::Included(last_key.clone()), + ); + range.intersect(&span).is_some() + }) +} + +/// Controls block-cache insertion for memtable flush and compaction output. +/// +// TODO: add control over when reads go through the block cache, e.g. for +// probing during compaction. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct BlockCachePolicy { + flush_targets: Vec, + compaction_output_targets: Vec, +} + +impl BlockCachePolicy { + /// Sets the targets requested for insertion after a memtable flush. + /// An empty slice disables insertion. + pub fn with_flush_targets(mut self, targets: &[CacheTarget]) -> Self { + self.flush_targets = targets.to_vec(); + self + } + + /// Sets the targets requested for insertion as compaction output is + /// written. An empty slice disables insertion. + pub fn with_compaction_output_targets(mut self, targets: &[CacheTarget]) -> Self { + self.compaction_output_targets = targets.to_vec(); + self + } + + pub(crate) fn flush_targets(&self) -> &[CacheTarget] { + &self.flush_targets + } + + pub(crate) fn compaction_output_targets(&self) -> &[CacheTarget] { + &self.compaction_output_targets + } +} + +/// The default policy inserts data, index, and filter blocks after a memtable +/// flush, and inserts index and filter blocks as compaction output is written. +impl Default for BlockCachePolicy { + fn default() -> Self { + Self { + flush_targets: vec![ + CacheTarget::data::<&[u8], _>(..), + CacheTarget::Index, + CacheTarget::Filters, + ], + compaction_output_targets: vec![CacheTarget::Index, CacheTarget::Filters], + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_policy_caches_on_flush_and_metadata_on_compaction_output() { + let policy = BlockCachePolicy::default(); + + assert_eq!( + policy.flush_targets(), + &[ + CacheTarget::data::<&[u8], _>(..), + CacheTarget::Index, + CacheTarget::Filters + ] + ); + assert_eq!( + policy.compaction_output_targets(), + &[CacheTarget::Index, CacheTarget::Filters] + ); + } + + #[test] + fn should_cache_data_block_by_key_span_overlap() { + let targets = [CacheTarget::data(b"c".as_slice()..b"f".as_slice())]; + let span = |first: &[u8], last: &[u8]| { + (Bytes::copy_from_slice(first), Bytes::copy_from_slice(last)) + }; + + assert!(should_cache_data_block(&targets, &span(b"a", b"c"))); + assert!(should_cache_data_block(&targets, &span(b"d", b"e"))); + assert!(should_cache_data_block(&targets, &span(b"e", b"z"))); + // end bound is exclusive + assert!(!should_cache_data_block(&targets, &span(b"f", b"z"))); + assert!(!should_cache_data_block(&targets, &span(b"a", b"b"))); + + assert!(!should_cache_data_block( + &[CacheTarget::Index], + &span(b"d", b"e") + )); + assert!(should_cache_data_block( + &[CacheTarget::data::<&[u8], _>(..)], + &span(b"a", b"b") + )); + } + + #[test] + fn setters_replace_targets() { + let policy = BlockCachePolicy::default() + .with_flush_targets(&[CacheTarget::Stats]) + .with_compaction_output_targets(&[CacheTarget::Index, CacheTarget::Filters]); + + assert_eq!(policy.flush_targets, &[CacheTarget::Stats]); + assert_eq!( + policy.compaction_output_targets, + &[CacheTarget::Index, CacheTarget::Filters] + ); + } +} diff --git a/slatedb/src/checkpoint.rs b/slatedb/src/checkpoint.rs index 7b2b3de20..21a6dee91 100644 --- a/slatedb/src/checkpoint.rs +++ b/slatedb/src/checkpoint.rs @@ -51,6 +51,7 @@ impl Db { #[cfg(test)] mod tests { use crate::admin::AdminBuilder; + use crate::block_cache_policy::BlockCachePolicy; use crate::checkpoint::Checkpoint; use crate::checkpoint::CheckpointCreateResult; use crate::config::{CheckpointOptions, CheckpointScope, Settings}; @@ -444,6 +445,7 @@ mod tests { path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let sst_handle = SsTableView::identity(table_store.open_sst(table_id).await.unwrap()); diff --git a/slatedb/src/compaction_execute_bench.rs b/slatedb/src/compaction_execute_bench.rs index 88c83bc87..d5e145bb8 100644 --- a/slatedb/src/compaction_execute_bench.rs +++ b/slatedb/src/compaction_execute_bench.rs @@ -14,6 +14,7 @@ use tokio::runtime::Handle; use tokio::task::JoinHandle; use ulid::Ulid; +use crate::block_cache_policy::BlockCachePolicy; use crate::bytes_generator::OrderedBytesGenerator; use crate::compaction_worker::WorkerMessage; use crate::compactor::stats::{CompactionStats, WorkerStats}; @@ -79,6 +80,7 @@ impl CompactionExecuteBench { self.path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let num_keys = sst_bytes / (val_bytes + key_bytes); let mut key_start = vec![0u8; key_bytes - mem::size_of::()]; @@ -331,6 +333,7 @@ impl CompactionExecuteBench { self.path.clone(), None, TableStoreKind::Compactor, + BlockCachePolicy::default(), )); let (tx, rx) = async_channel::unbounded(); let worker_options = CompactionWorkerOptions::default(); diff --git a/slatedb/src/compaction_worker.rs b/slatedb/src/compaction_worker.rs index 5cf86efd3..3b58f3790 100644 --- a/slatedb/src/compaction_worker.rs +++ b/slatedb/src/compaction_worker.rs @@ -776,6 +776,7 @@ mod tests { use std::time::Duration; use super::*; + use crate::block_cache_policy::BlockCachePolicy; use crate::bytes_range::BytesRange; use crate::compactor_state::{Compaction, CompactionSpec, SourceId}; use crate::db_state::{SsTableHandle, SsTableId, SsTableInfo, SsTableView}; @@ -1321,6 +1322,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Compactor, + BlockCachePolicy::default(), )); let manifest_store = Arc::new(ManifestStore::new(&root_path, inner.clone())); let compactions_store = Arc::new(CompactionsStore::new(&root_path, inner.clone())); diff --git a/slatedb/src/compactor.rs b/slatedb/src/compactor.rs index c322496d8..265bb0545 100644 --- a/slatedb/src/compactor.rs +++ b/slatedb/src/compactor.rs @@ -1429,6 +1429,7 @@ pub mod stats { mod tests { use std::collections::{HashMap, VecDeque}; use std::future::Future; + use std::ops::Range; use std::sync::Arc; use std::time::{Duration, SystemTime}; @@ -1437,10 +1438,12 @@ mod tests { use object_store::ObjectStore; use parking_lot::Mutex; use rand::RngCore; + use rstest::rstest; use slatedb_common::MockSystemClock; use ulid::Ulid; use super::*; + use crate::block_cache_policy::BlockCachePolicy; use crate::compaction_worker::WorkerMessage; use crate::compactions_store::{FenceableCompactions, StoredCompactions}; use crate::compactor::stats::CompactionStats; @@ -1454,9 +1457,11 @@ mod tests { use crate::compactor_state::{SourceId, WorkerSpec}; use crate::config::{ CompactionWorkerOptions, FlushOptions, FlushType, MergeOptions, PutOptions, Settings, - SizeTieredCompactionSchedulerOptions, Ttl, WriteOptions, + SizeTieredCompactionSchedulerOptions, SstBlockSize, Ttl, WriteOptions, }; use crate::db::Db; + use crate::db_cache::test_utils::TestCache; + use crate::db_cache::CacheTarget; use crate::db_state::{SortedRun, SsTableHandle, SsTableId, SsTableInfo, SsTableView}; use crate::error::SlateDBError; use crate::format::sst::{SsTableFormat, SST_FORMAT_VERSION_LATEST}; @@ -1710,6 +1715,145 @@ mod tests { assert!(expected.is_empty()); } + /// An entry expected in the block cache for a compaction output SST. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum ExpectedEntry { + Index, + Filter, + Stats, + /// The data block at this position in the SST index. + DataBlock(usize), + } + + fn data_blocks(positions: Range) -> Vec { + positions.map(ExpectedEntry::DataBlock).collect() + } + + #[rstest] + #[case::filters_only( + BlockCachePolicy::default().with_compaction_output_targets(&[CacheTarget::Filters]), + vec![ExpectedEntry::Filter] + )] + #[case::default_policy( + BlockCachePolicy::default(), + vec![ExpectedEntry::Index, ExpectedEntry::Filter] + )] + #[case::cache_everything( + BlockCachePolicy::default().with_compaction_output_targets(&[ + CacheTarget::data::<&[u8], _>(..), + CacheTarget::Index, + CacheTarget::Filters, + CacheTarget::Stats, + ]), + [ + vec![ExpectedEntry::Index, ExpectedEntry::Filter, ExpectedEntry::Stats], + data_blocks(0..4), + ].concat() + )] + #[case::data_range( + BlockCachePolicy::default().with_compaction_output_targets(&[ + CacheTarget::data(b"b".as_slice()..=b"c".as_slice()), + CacheTarget::Index, + ]), + vec![ExpectedEntry::Index, ExpectedEntry::DataBlock(1), ExpectedEntry::DataBlock(2)] + )] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_compactor_applies_output_cache_policy( + #[case] policy: BlockCachePolicy, + #[case] expected_entries: Vec, + ) { + let os = Arc::new(InMemory::new()); + let system_clock = Arc::new(MockSystemClock::new()); + let cache = Arc::new(TestCache::new()); + let mut options = db_options(Some(compactor_options())); + options.flush_interval = None; + // Ensure that filter is built. + options.min_filter_keys = 1; + options + .compactor_options + .as_mut() + .expect("compactor options must be set") + .scheduler_options = SizeTieredCompactionSchedulerOptions { + // Compact even a single L0 so one flush is enough to trigger. + min_compaction_sources: 1, + ..Default::default() + } + .into(); + + let db = Db::builder(PATH, os.clone()) + .with_settings(options) + // One data block per entry, so a data range selects a subset. + .with_sst_block_size(SstBlockSize::Other(1)) + .with_system_clock(system_clock.clone()) + .with_db_cache(cache.clone()) + .with_block_cache_policy(policy) + .build() + .await + .unwrap(); + + for key in [b"a", b"b", b"c", b"d"] { + db.put_with_options( + key, + b"value", + &PutOptions::default(), + &WriteOptions { + await_durable: false, + ..Default::default() + }, + ) + .await + .unwrap(); + } + db.flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .unwrap(); + + let db_state = await_compaction(&db, os.clone(), Some(system_clock)) + .await + .expect("db was not compacted"); + + let output_ssts: Vec<_> = db_state + .tree + .compacted + .iter() + .flat_map(|sr| &sr.sst_views) + .collect(); + assert_eq!(output_ssts.len(), 1); + let view = output_ssts[0]; + let info = &view.sst.info; + + let (_, _, table_store) = build_test_stores(os); + let index = table_store.read_index(&view.sst, false).await.unwrap(); + let block_metas = index.borrow().block_meta(); + assert_eq!(block_metas.len(), 4); + + let mut expected_ids: Vec = expected_entries + .iter() + .map(|entry| match entry { + ExpectedEntry::Index => info.index_offset, + ExpectedEntry::Filter => info.filter_offset, + ExpectedEntry::Stats => info.stats_offset, + ExpectedEntry::DataBlock(position) => block_metas.get(*position).offset(), + }) + .collect(); + expected_ids.sort(); + + // Entries written by the flush carry the L0 SST's id, so filtering on + // the output id leaves only compaction output entries. + let mut cached_ids: Vec = cache + .keys() + .iter() + .filter(|key| key.sst_id == view.sst.id) + .map(|key| key.block_id) + .collect(); + cached_ids.sort(); + + assert_eq!(cached_ids, expected_ids); + db.close().await.unwrap(); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_compactor_compacts_only_target_segment() { let os = Arc::new(InMemory::new()); @@ -5925,6 +6069,7 @@ mod tests { Path::from(PATH), None, TableStoreKind::Compactor, + BlockCachePolicy::default(), )); (manifest_store, compactions_store, table_store) } diff --git a/slatedb/src/compactor_executor.rs b/slatedb/src/compactor_executor.rs index 54f1ea9fa..f52630569 100644 --- a/slatedb/src/compactor_executor.rs +++ b/slatedb/src/compactor_executor.rs @@ -1010,6 +1010,7 @@ impl TokioCompactionExecutorInner { #[cfg(test)] mod tests { use super::*; + use crate::block_cache_policy::BlockCachePolicy; use crate::bytes_range::BytesRange; use crate::format::sst::SsTableFormat; use crate::manifest::ManifestCore; @@ -1470,6 +1471,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Compactor, + BlockCachePolicy::default(), )); let manifest_store = Arc::new(ManifestStore::new(&root_path, object_store.clone())); StoredManifest::create_new_db(manifest_store.clone(), ManifestCore::new(), clock.clone()) @@ -1691,7 +1693,7 @@ mod tests { }, root_path.clone(), None, - TableStoreKind::Compactor)); + TableStoreKind::Compactor, BlockCachePolicy::default())); let manifest_store = Arc::new(ManifestStore::new(&root_path, object_store.clone())); StoredManifest::create_new_db( manifest_store.clone(), @@ -1926,6 +1928,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Compactor, + BlockCachePolicy::default(), )); let manifest_store = Arc::new(ManifestStore::new(&root_path, object_store.clone())); StoredManifest::create_new_db(manifest_store.clone(), ManifestCore::new(), clock.clone()) @@ -2486,6 +2489,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Compactor, + BlockCachePolicy::default(), )); let manifest_store = Arc::new(ManifestStore::new(&root_path, object_store.clone())); StoredManifest::create_new_db(manifest_store.clone(), ManifestCore::new(), clock.clone()) @@ -2619,6 +2623,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Compactor, + BlockCachePolicy::default(), )); let manifest_store = Arc::new(ManifestStore::new(&root_path, object_store.clone())); StoredManifest::create_new_db(manifest_store.clone(), ManifestCore::new(), clock.clone()) @@ -2909,10 +2914,7 @@ mod tests { .unwrap(); let encoded_sst = sst_builder.build().await.unwrap(); let id = SsTableId::Compacted(Ulid::new()); - let l0 = table_store - .write_sst(&id, &encoded_sst, false) - .await - .unwrap(); + let l0 = table_store.write_sst(&id, &encoded_sst).await.unwrap(); let retention_min_seq_num = 2; let result = ctx @@ -3054,10 +3056,7 @@ mod tests { .unwrap(); let encoded_sst = sst_builder.build().await.unwrap(); let id = SsTableId::Compacted(Ulid::new()); - let l0 = table_store - .write_sst(&id, &encoded_sst, false) - .await - .unwrap(); + let l0 = table_store.write_sst(&id, &encoded_sst).await.unwrap(); let result = ctx.run_compaction(vec![l0], true, None).await.unwrap(); @@ -3155,10 +3154,7 @@ mod tests { .unwrap(); let encoded_sst = sst_builder.build().await.unwrap(); let id = SsTableId::Compacted(Ulid::new()); - let l0 = table_store - .write_sst(&id, &encoded_sst, false) - .await - .unwrap(); + let l0 = table_store.write_sst(&id, &encoded_sst).await.unwrap(); let result = ctx.run_compaction(vec![l0], true, None).await; @@ -3221,10 +3217,7 @@ mod tests { .unwrap(); let encoded_sst = sst_builder.build().await.unwrap(); let id = SsTableId::Compacted(Ulid::new()); - let l0 = table_store - .write_sst(&id, &encoded_sst, false) - .await - .unwrap(); + let l0 = table_store.write_sst(&id, &encoded_sst).await.unwrap(); let result = ctx.run_compaction(vec![l0], true, None).await; diff --git a/slatedb/src/db.rs b/slatedb/src/db.rs index 861f85504..a883bce36 100644 --- a/slatedb/src/db.rs +++ b/slatedb/src/db.rs @@ -22,7 +22,8 @@ pub use crate::db_status::{DbStatus, SegmentPrefix}; -use crate::db_cache_manager::{self, CacheTarget}; +use crate::db_cache::CacheTarget; +use crate::db_cache_manager; use std::ops::Range; use std::sync::Arc; @@ -2127,6 +2128,7 @@ impl DbWalObserver { #[cfg(test)] mod tests { use super::*; + use crate::block_cache_policy::BlockCachePolicy; use crate::config::DurabilityLevel::{Memory, Remote}; use crate::config::MetricLevel; use crate::config::{ @@ -4123,6 +4125,7 @@ mod tests { path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let db = Db::builder(path.clone(), object_store.clone()) .with_settings(options) @@ -4224,6 +4227,7 @@ mod tests { path, None, TableStoreKind::Main, + BlockCachePolicy::default(), )); // Write data a few times such that each loop results in a memtable flush @@ -4411,6 +4415,7 @@ mod tests { path, None, TableStoreKind::Main, + BlockCachePolicy::default(), )); // Write some data to populate the memtable @@ -6008,6 +6013,7 @@ mod tests { path, None, TableStoreKind::Main, + BlockCachePolicy::default(), )); // Get the next WAL SST ID based on what's currently in the object store @@ -6317,6 +6323,7 @@ mod tests { path, None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let mut w1_paused = false; for _ in 0..600 { @@ -6413,6 +6420,7 @@ mod tests { path, None, TableStoreKind::Main, + BlockCachePolicy::default(), ); wait_for_wal_sst_count( &probe_table_store, @@ -6491,6 +6499,7 @@ mod tests { path, None, TableStoreKind::Main, + BlockCachePolicy::default(), ); wait_for_wal_sst_count( &probe_table_store, @@ -7931,6 +7940,7 @@ mod tests { path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let compacted_ssts = table_store .list_compacted_ssts(..) diff --git a/slatedb/src/db/builder.rs b/slatedb/src/db/builder.rs index 7cd3d173b..1a8bd1070 100644 --- a/slatedb/src/db/builder.rs +++ b/slatedb/src/db/builder.rs @@ -117,6 +117,7 @@ use tokio::runtime::Handle; use crate::admin::Admin; use crate::batch_write::WriteBatchEventHandler; use crate::batch_write::WRITE_BATCH_TASK_NAME; +use crate::block_cache_policy::BlockCachePolicy; use crate::cached_object_store::CachedObjectStore; use crate::clone::{SegmentFilterFn, SegmentProjectionFn}; #[cfg(feature = "compaction_filters")] @@ -179,6 +180,7 @@ pub struct DbBuilder> { main_object_store: Arc, wal_object_store: Option>, db_cache: Option>, + block_cache_policy: BlockCachePolicy, system_clock: Option>, gc_runtime: Option, compactor_builder: Option>, @@ -202,6 +204,7 @@ impl> DbBuilder

{ settings: Settings::default(), wal_object_store: None, db_cache: default_db_cache(), + block_cache_policy: BlockCachePolicy::default(), system_clock: None, gc_runtime: None, compactor_builder: None, @@ -276,6 +279,13 @@ impl> DbBuilder

{ self } + /// Sets the policy for inserting flush and compaction output into the + /// decoded block cache. + pub fn with_block_cache_policy(mut self, policy: BlockCachePolicy) -> Self { + self.block_cache_policy = policy; + self + } + /// Sets the system clock to use for the database. System timestamps are used for /// scheduling operations such as compaction and garbage collection. pub fn with_system_clock(mut self, clock: Arc) -> Self { @@ -532,6 +542,13 @@ impl> DbBuilder

{ // Create path resolver and table store let path_resolver = PathResolver::new_with_external_ssts(path.clone(), external_ssts); + let db_cache = self.db_cache.as_ref().map(|cache| { + Arc::new(DbCacheWrapper::new( + cache.clone(), + &recorder, + system_clock.clone(), + )) as Arc + }); let table_store = Arc::new(TableStore::new_with_fp_registry( ObjectStores::new( maybe_cached_main_object_store.clone(), @@ -540,14 +557,9 @@ impl> DbBuilder

{ sst_format.clone(), path_resolver.clone(), self.fp_registry.clone(), - self.db_cache.as_ref().map(|c| { - Arc::new(DbCacheWrapper::new( - c.clone(), - &recorder, - system_clock.clone(), - )) as Arc - }), + db_cache.clone(), TableStoreKind::Main, + self.block_cache_policy.clone(), )); // Initialize the database @@ -706,8 +718,9 @@ impl> DbBuilder

{ sst_format.clone(), path_resolver.clone(), self.fp_registry.clone(), - None, + db_cache.clone(), TableStoreKind::Compactor, + self.block_cache_policy.clone(), )); let compactor_handlers = builder .build_handler( @@ -760,6 +773,7 @@ impl> DbBuilder

{ self.fp_registry.clone(), None, TableStoreKind::GC, + BlockCachePolicy::default(), )); let gc = gc_builder .with_system_clock(system_clock.clone()) @@ -1049,6 +1063,7 @@ impl> GarbageCollectorBuilder

{ path, None, // no need for cache in GC TableStoreKind::GC, + BlockCachePolicy::default(), )); GarbageCollector::new( manifest_store, @@ -1273,6 +1288,7 @@ impl> CompactorBuilder

{ path, None, TableStoreKind::Compactor, + BlockCachePolicy::default(), )); let scheduler_supplier = self @@ -1490,6 +1506,7 @@ impl> CompactionWorkerBuilder

{ path, None, TableStoreKind::Compactor, + BlockCachePolicy::default(), )); let recorder = MetricsRecorderHelper::new( self.metrics_recorder, @@ -1829,6 +1846,7 @@ impl> DbReaderBuilder

{ Arc::new(FailPointRegistry::new()), wrapped_cache, TableStoreKind::Reader, + BlockCachePolicy::default(), )); let reader = DbReader::open_internal( diff --git a/slatedb/src/db_cache/mod.rs b/slatedb/src/db_cache/mod.rs index 6316e9668..1831702c8 100644 --- a/slatedb/src/db_cache/mod.rs +++ b/slatedb/src/db_cache/mod.rs @@ -11,10 +11,12 @@ //! //! To use the cache, you need to configure the [DbOptions](crate::config::DbOptions) with the desired cache implementation. +use std::ops::{Bound, RangeBounds}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; use async_trait::async_trait; +use bytes::Bytes; use chrono::{DateTime, TimeDelta, Utc}; use futures::future::BoxFuture; use log::{debug, error, trace}; @@ -238,6 +240,41 @@ pub trait DbCache: Send + Sync { } } +/// An SST component that can be inserted into the block cache, either by +/// warming an existing SST via +/// [`DbCacheManagerOps::warm_sst`](crate::DbCacheManagerOps::warm_sst) or as +/// the SST is written via [`BlockCachePolicy`](crate::BlockCachePolicy). +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub enum CacheTarget { + /// All filter blocks on the SST, if any exist. + Filters, + /// The SST index. + Index, + /// The SST stats block, if one exists. + Stats, + /// Data blocks whose key span overlaps the supplied key range. + Data((Bound, Bound)), +} + +impl CacheTarget { + /// Convenience constructor for [`CacheTarget::Data`] that accepts any + /// [`RangeBounds`], mirroring the `Db::scan` signature. Pass `..` to + /// select all data blocks. + pub fn data(range: T) -> Self + where + K: AsRef<[u8]>, + T: RangeBounds, + { + let start = range + .start_bound() + .map(|b| Bytes::copy_from_slice(b.as_ref())); + let end = range + .end_bound() + .map(|b| Bytes::copy_from_slice(b.as_ref())); + CacheTarget::Data((start, end)) + } +} + /// A key used to identify a cached entry. /// /// The key is composed of a scope ID (set per [`DbCacheWrapper`] instance), an SSTable ID, @@ -1062,6 +1099,10 @@ pub(crate) mod test_utils { items: Mutex::new(HashMap::new()), } } + + pub(crate) fn keys(&self) -> Vec { + self.items.lock().unwrap().keys().cloned().collect() + } } #[async_trait] diff --git a/slatedb/src/db_cache_manager.rs b/slatedb/src/db_cache_manager.rs index 12a3b0559..e9929ba62 100644 --- a/slatedb/src/db_cache_manager.rs +++ b/slatedb/src/db_cache_manager.rs @@ -6,6 +6,7 @@ use log::{debug, warn}; use tokio::sync::OnceCell; use crate::bytes_range::BytesRange; +use crate::db_cache::CacheTarget; use crate::db_state::{SsTableHandle, SsTableId}; use crate::error::SlateDBError; use crate::flatbuffer_types::SsTableIndexOwned; @@ -13,40 +14,6 @@ use crate::manifest::VersionedManifest; use crate::partitioned_keyspace::partitions_covering_range; use crate::tablestore::TableStore; -/// Cache content that [`DbCacheManagerOps::warm_sst`](crate::DbCacheManagerOps::warm_sst) should populate. -#[derive(Clone, Debug)] -pub enum CacheTarget { - /// Warm all filters on the SST, if any exist. - Filters, - /// Warm the SST index. - Index, - /// Warm the SST stats block, if one exists. - Stats, - /// Warm the SST data blocks that overlap the supplied key range. - /// - /// Also warms the SST index, since block planning depends on it. - Data((Bound, Bound)), -} - -impl CacheTarget { - /// Convenience constructor for [`CacheTarget::Data`] that accepts any - /// [`RangeBounds`], mirroring the `Db::scan` signature. Pass `..` to - /// warm all data blocks. - pub fn data(range: T) -> Self - where - K: AsRef<[u8]>, - T: RangeBounds, - { - let start = range - .start_bound() - .map(|b| Bytes::copy_from_slice(b.as_ref())); - let end = range - .end_bound() - .map(|b| Bytes::copy_from_slice(b.as_ref())); - CacheTarget::Data((start, end)) - } -} - pub(crate) async fn warm_sst_impl( table_store: &Arc, manifest: &VersionedManifest, diff --git a/slatedb/src/db_reader.rs b/slatedb/src/db_reader.rs index f588e5a21..f7f7592c2 100644 --- a/slatedb/src/db_reader.rs +++ b/slatedb/src/db_reader.rs @@ -2,7 +2,8 @@ use crate::bytes_range::{ByteRangeBounds, BytesRange}; use crate::cached_object_store::CachedObjectStore; use crate::clock::MonotonicClock; use crate::config::{CheckpointOptions, DbReaderOptions, ReadOptions, ScanOptions}; -use crate::db_cache_manager::{self, CacheTarget}; +use crate::db_cache::CacheTarget; +use crate::db_cache_manager; use crate::db_common::extract_segment_prefix; use crate::db_state::{collect_touched_segments, SsTableId}; use crate::db_stats::DbStats; @@ -1386,6 +1387,7 @@ fn has_not_found_object_store_error(err: &(dyn std::error::Error + 'static)) -> #[cfg(test)] mod tests { use super::{DbReaderMessage, ManifestPoller, ReaderState}; + use crate::block_cache_policy::BlockCachePolicy; use crate::clock::MonotonicClock; use crate::config::{ CheckpointOptions, CheckpointScope, FlushOptions, FlushType, MergeOptions, PutOptions, @@ -3277,6 +3279,7 @@ mod tests { Arc::clone(&self.fp_registry), None, TableStoreKind::Reader, + BlockCachePolicy::default(), )) } diff --git a/slatedb/src/fence.rs b/slatedb/src/fence.rs index b1cc40147..fb2fb9e74 100644 --- a/slatedb/src/fence.rs +++ b/slatedb/src/fence.rs @@ -126,6 +126,7 @@ impl WriterFencer { #[cfg(test)] mod tests { + use crate::block_cache_policy::BlockCachePolicy; use crate::compactions_store::CompactionsStore; use crate::config::{ FlushOptions, FlushType, GarbageCollectorDirectoryOptions, GarbageCollectorOptions, @@ -178,6 +179,7 @@ mod tests { path, None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let stored_manifest = StoredManifest::create_new_db( manifest_store.clone(), diff --git a/slatedb/src/flush.rs b/slatedb/src/flush.rs index 4ae6c47c5..2563f2a60 100644 --- a/slatedb/src/flush.rs +++ b/slatedb/src/flush.rs @@ -147,12 +147,8 @@ impl DbInner { &self, id: &db_state::SsTableId, encoded_sst: &EncodedSsTable, - write_cache: bool, ) -> Result { - let handle = self - .table_store - .write_sst(id, encoded_sst, write_cache) - .await?; + let handle = self.table_store.write_sst(id, encoded_sst).await?; Ok(handle) } @@ -166,7 +162,6 @@ impl DbInner { pub(crate) async fn flush_l0_for_test( &self, imm_table: Arc, - write_cache: bool, ) -> Result, SlateDBError> { use crate::utils::IdGenerator; // Tests that construct an `imm_table` outside the write path @@ -178,7 +173,7 @@ impl DbInner { let id = db_state::SsTableId::Compacted( self.rand.rng().gen_ulid(self.system_clock.as_ref()), ); - let handle = self.upload_sst(&id, &sst.encoded, write_cache).await?; + let handle = self.upload_sst(&id, &sst.encoded).await?; handles.push(handle); } Ok(handles) @@ -497,7 +492,7 @@ mod tests { // When let handles = db .inner - .flush_l0_for_test(table.table().clone(), false) + .flush_l0_for_test(table.table().clone()) .await .unwrap(); @@ -542,7 +537,7 @@ mod tests { ); db.inner - .flush_l0_for_test(table.table().clone(), false) + .flush_l0_for_test(table.table().clone()) .await .unwrap(); @@ -571,7 +566,7 @@ mod tests { // When db.inner - .flush_l0_for_test(table.table().clone(), false) + .flush_l0_for_test(table.table().clone()) .await .map_or_else( |err| match err { @@ -594,7 +589,7 @@ mod tests { // When db.inner - .flush_l0_for_test(table.table().clone(), false) + .flush_l0_for_test(table.table().clone()) .await .unwrap(); } @@ -689,7 +684,7 @@ mod tests { let handles = db .inner - .flush_l0_for_test(table.table().clone(), false) + .flush_l0_for_test(table.table().clone()) .await .unwrap(); let sst_handle = handles.into_iter().next().expect("expected single SST"); @@ -838,7 +833,7 @@ mod tests { ]; for (sst, entries) in ssts.into_iter().zip(expected.into_iter()) { let id = SsTableId::Compacted(Ulid::new()); - let handle = db.inner.upload_sst(&id, &sst.encoded, false).await.unwrap(); + let handle = db.inner.upload_sst(&id, &sst.encoded).await.unwrap(); verify_sst(&db, &handle, &entries).await; } db.close().await.unwrap(); diff --git a/slatedb/src/format/sst.rs b/slatedb/src/format/sst.rs index 4a8ee3a9e..069f34dfc 100644 --- a/slatedb/src/format/sst.rs +++ b/slatedb/src/format/sst.rs @@ -230,6 +230,9 @@ pub(crate) struct EncodedSsTableBlock { pub(crate) block: Arc, /// compressed and transformed block pub(crate) encoded_bytes: Bytes, + /// first and last key of the block. None when the producer does not track + /// keys (WAL blocks, whose index tracks sequence numbers instead) + pub(crate) key_span: Option<(Bytes, Bytes)>, } impl EncodedSsTableBlock { @@ -244,6 +247,8 @@ pub(crate) struct EncodedSsTableBlockBuilder { block_builder: BlockBuilder, /// offset of the block within the SST offset: u64, + /// first and last key of the block + key_span: Option<(Bytes, Bytes)>, /// codec for compressing the data block compression_codec: Option, /// transformer for transforming the data block (e.g. encryption) @@ -255,11 +260,18 @@ impl EncodedSsTableBlockBuilder { Self { block_builder, offset, + key_span: None, compression_codec: None, block_transformer: None, } } + /// Sets the first and last key of the block + pub(crate) fn with_key_span(mut self, first_key: Bytes, last_key: Bytes) -> Self { + self.key_span = Some((first_key, last_key)); + self + } + /// Sets the compression codec for compressing the data block pub(crate) fn with_compression_codec(mut self, codec: CompressionCodec) -> Self { self.compression_codec = Some(codec); @@ -287,6 +299,7 @@ impl EncodedSsTableBlockBuilder { offset: self.offset, block: Arc::new(block), encoded_bytes: Bytes::from(compressed_and_transformed_block), + key_span: self.key_span, }) } } @@ -484,7 +497,6 @@ pub(crate) struct EncodedSsTable { pub(crate) info: SsTableInfo, pub(crate) index: SsTableIndexOwned, pub(crate) filters: Arc<[NamedFilter]>, - #[allow(dead_code)] pub(crate) stats: Option, pub(crate) unconsumed_blocks: VecDeque, pub(crate) footer: Bytes, diff --git a/slatedb/src/garbage_collector.rs b/slatedb/src/garbage_collector.rs index bfc0ea9dd..6f7018b99 100644 --- a/slatedb/src/garbage_collector.rs +++ b/slatedb/src/garbage_collector.rs @@ -445,6 +445,7 @@ impl GarbageCollector { #[cfg(test)] mod tests { use super::*; + use crate::block_cache_policy::BlockCachePolicy; use crate::tablestore::TableStoreKind; use std::collections::HashSet; @@ -926,7 +927,7 @@ mod tests { let mut sst = table_store.table_builder(); sst.add(RowEntry::new_value(b"key", b"value", 0)).await?; let table1 = sst.build().await?; - table_store.write_sst(table_id, &table1, false).await?; + table_store.write_sst(table_id, &table1).await?; Ok(()) } @@ -1063,7 +1064,7 @@ mod tests { .unwrap(); let table1 = sst1.build().await.unwrap(); - table_store.write_sst(&id1, &table1, false).await.unwrap(); + table_store.write_sst(&id1, &table1).await.unwrap(); let id2 = SsTableId::Wal(2); let mut sst2 = table_store.table_builder(); @@ -1071,7 +1072,7 @@ mod tests { .await .unwrap(); let table2 = sst2.build().await.unwrap(); - table_store.write_sst(&id2, &table2, false).await.unwrap(); + table_store.write_sst(&id2, &table2).await.unwrap(); // Set the both WAL SST file to be a day old let now_minus_24h_1 = set_modified( @@ -1669,6 +1670,7 @@ mod tests { path, None, TableStoreKind::GC, + BlockCachePolicy::default(), )); ( @@ -1692,7 +1694,7 @@ mod tests { .await .unwrap(); let table = sst.build().await.unwrap(); - table_store.write_sst(&sst_id, &table, false).await.unwrap() + table_store.write_sst(&sst_id, &table).await.unwrap() } /// Set the modified time of a file to be a certain number of seconds ago. diff --git a/slatedb/src/garbage_collector/compacted_gc.rs b/slatedb/src/garbage_collector/compacted_gc.rs index 6cd3597ba..de1570f66 100644 --- a/slatedb/src/garbage_collector/compacted_gc.rs +++ b/slatedb/src/garbage_collector/compacted_gc.rs @@ -269,6 +269,7 @@ impl GcTask for CompactedGcTask { #[cfg(test)] mod tests { use super::*; + use crate::block_cache_policy::BlockCachePolicy; use crate::cached_object_store::policy::CachePutConfig; use crate::cached_object_store::stats::CachedObjectStoreStats; use crate::cached_object_store::{CachedObjectStore, FsCacheStorage}; @@ -300,6 +301,7 @@ mod tests { Path::from("/root"), None, TableStoreKind::GC, + BlockCachePolicy::default(), )); // Manifest store and initial manifest @@ -340,15 +342,15 @@ mod tests { let sst_active_recent = build_test_sst(&format, 1).await; table_store - .write_sst(&id_to_delete, &sst_to_delete, false) + .write_sst(&id_to_delete, &sst_to_delete) .await .unwrap(); table_store - .write_sst(&id_within_min_age, &sst_within_min_age, false) + .write_sst(&id_within_min_age, &sst_within_min_age) .await .unwrap(); let active_handle = table_store - .write_sst(&id_active_recent, &sst_active_recent, false) + .write_sst(&id_active_recent, &sst_active_recent) .await .unwrap(); @@ -406,6 +408,7 @@ mod tests { Path::from("/root"), None, TableStoreKind::GC, + BlockCachePolicy::default(), )); // Manifest store and initial manifest @@ -446,17 +449,14 @@ mod tests { let sst_newer = build_test_sst(&format, 1).await; table_store - .write_sst(&id_to_delete, &sst_to_delete, false) + .write_sst(&id_to_delete, &sst_to_delete) .await .unwrap(); let manifest_handle = table_store - .write_sst(&id_manifest, &sst_manifest, false) - .await - .unwrap(); - table_store - .write_sst(&id_newer, &sst_newer, false) + .write_sst(&id_manifest, &sst_manifest) .await .unwrap(); + table_store.write_sst(&id_newer, &sst_newer).await.unwrap(); // Mark id_manifest as the only active SST in the manifest so that // most_recent_sst_dt is 3_000ms, which becomes the cutoff. @@ -514,6 +514,7 @@ mod tests { Path::from("/root"), None, TableStoreKind::GC, + BlockCachePolicy::default(), )); // Manifest store with empty DB @@ -539,15 +540,15 @@ mod tests { let sst_barrier = build_test_sst(&format, 1).await; let sst_to_newer = build_test_sst(&format, 1).await; table_store - .write_sst(&id_to_delete, &sst_to_delete, false) + .write_sst(&id_to_delete, &sst_to_delete) .await .unwrap(); table_store - .write_sst(&id_barrier, &sst_barrier, false) + .write_sst(&id_barrier, &sst_barrier) .await .unwrap(); let active_handle = table_store - .write_sst(&id_to_newer, &sst_to_newer, false) + .write_sst(&id_to_newer, &sst_to_newer) .await .unwrap(); @@ -618,6 +619,7 @@ mod tests { Path::from("/root"), None, TableStoreKind::GC, + BlockCachePolicy::default(), )); // Manifest with an L0 newer than the compaction output. @@ -645,7 +647,7 @@ mod tests { // Newest L0 in the manifest has a later timestamp (9_000ms). let l0_id = SsTableId::Compacted(ulid::Ulid::from_parts(9_000, 0)); let l0_handle = table_store - .write_sst(&l0_id, &build_test_sst(&format, 1).await, false) + .write_sst(&l0_id, &build_test_sst(&format, 1).await) .await .unwrap(); let mut dirty_manifest = stored_manifest.prepare_dirty().unwrap(); @@ -658,11 +660,7 @@ mod tests { // output SST (6_000ms), but hasn't updated the manifest yet. let compaction_output_id = SsTableId::Compacted(ulid::Ulid::from_parts(6_000, 0)); table_store - .write_sst( - &compaction_output_id, - &build_test_sst(&format, 1).await, - false, - ) + .write_sst(&compaction_output_id, &build_test_sst(&format, 1).await) .await .unwrap(); @@ -881,6 +879,7 @@ mod tests { Path::from("/root"), None, TableStoreKind::GC, + BlockCachePolicy::default(), )); let main_table_store = Arc::new(TableStore::new( ObjectStores::new(cached_store.clone(), None), @@ -888,13 +887,14 @@ mod tests { Path::from("/root"), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); // Written through the Main store so cache_on_flush admits it. let id_to_delete = SsTableId::Compacted(ulid::Ulid::from_parts(1_000, 0)); let sst = build_test_sst(&format, 1).await; main_table_store - .write_sst(&id_to_delete, &sst, false) + .write_sst(&id_to_delete, &sst) .await .unwrap(); diff --git a/slatedb/src/lib.rs b/slatedb/src/lib.rs index bb37dc97d..1a64426a5 100644 --- a/slatedb/src/lib.rs +++ b/slatedb/src/lib.rs @@ -33,6 +33,7 @@ pub use fail_parallel; pub use object_store; pub use batch::WriteBatch; +pub use block_cache_policy::BlockCachePolicy; pub use bytes_range::ByteRangeBounds; pub use cached_object_store::stats as cached_object_store_stats; pub use checkpoint::{Checkpoint, CheckpointCreateResult}; @@ -48,7 +49,7 @@ pub use config::{Settings, SstBlockSize}; pub use db::builder::{CloneSourceSpec, CompactionWorkerBuilder}; pub use db::{Db, DbBuilder, DbReaderBuilder, DbStatus, SegmentPrefix, WriteHandle}; pub use db_cache::stats as db_cache_stats; -pub use db_cache_manager::CacheTarget; +pub use db_cache::CacheTarget; pub use db_iter::{DbIterator, DbRecencyIterator}; pub use db_reader::{DbReader, DbReaderMode}; pub use db_snapshot::DbSnapshot; @@ -98,6 +99,7 @@ mod batch; pub use batch::benches as write_batch_benches; mod batch_write; mod blob; +mod block_cache_policy; mod block_iterator; mod block_iterator_v2; #[cfg(feature = "bench-internal")] diff --git a/slatedb/src/memtable_flusher/manifest_writer.rs b/slatedb/src/memtable_flusher/manifest_writer.rs index 569305f1b..ae4645620 100644 --- a/slatedb/src/memtable_flusher/manifest_writer.rs +++ b/slatedb/src/memtable_flusher/manifest_writer.rs @@ -894,6 +894,7 @@ impl crate::dispatcher::Notifier for DurableSeqNotifier { #[cfg(test)] mod tests { use super::{ManifestWriter, ManifestWriterCommand, ManifestWriterHandler, TrackerMessage}; + use crate::block_cache_policy::BlockCachePolicy; use crate::config::{CheckpointOptions, Settings}; use crate::db::DbInner; use crate::db_status::{ClosedResultWriter, DbStatusManager}; @@ -1064,6 +1065,7 @@ mod tests { Arc::clone(&fp_registry), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let status_manager = DbStatusManager::new(0); let (write_tx, _) = @@ -1205,10 +1207,7 @@ mod tests { value: &[u8], ) -> UploadedMemtable { let imm_memtable = freeze_imm(inner, key, value); - let handles = inner - .flush_l0_for_test(imm_memtable.table(), true) - .await - .unwrap(); + let handles = inner.flush_l0_for_test(imm_memtable.table()).await.unwrap(); let sst_handle = handles.into_iter().next().expect("expected single SST"); let first_seq = imm_memtable.table().first_seq().unwrap(); let last_seq = imm_memtable.table().last_seq().unwrap(); @@ -1842,7 +1841,7 @@ mod tests { let id = crate::db_state::SsTableId::Compacted( inner.rand.rng().gen_ulid(inner.system_clock.as_ref()), ); - let sst_handle = inner.upload_sst(&id, &encoded_sst, false).await.unwrap(); + let sst_handle = inner.upload_sst(&id, &encoded_sst).await.unwrap(); segments.push(SegmentedSstHandle { prefix: Bytes::copy_from_slice(prefix), sst_handle, diff --git a/slatedb/src/memtable_flusher/tracker.rs b/slatedb/src/memtable_flusher/tracker.rs index ac8509eaa..e7971d4df 100644 --- a/slatedb/src/memtable_flusher/tracker.rs +++ b/slatedb/src/memtable_flusher/tracker.rs @@ -546,6 +546,7 @@ enum TrackedImmState { #[cfg(test)] mod tests { use crate::batch_write::BatchWriterMessage; + use crate::block_cache_policy::BlockCachePolicy; use crate::config::{CheckpointOptions, Settings}; use crate::db::DbInner; use crate::db_state::{ @@ -638,6 +639,7 @@ mod tests { Arc::clone(&fp_registry), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let status_manager = DbStatusManager::new(0); let (write_tx, _) = diff --git a/slatedb/src/memtable_flusher/uploader.rs b/slatedb/src/memtable_flusher/uploader.rs index 7f4f5a6ec..6e2fcf3d9 100644 --- a/slatedb/src/memtable_flusher/uploader.rs +++ b/slatedb/src/memtable_flusher/uploader.rs @@ -243,7 +243,7 @@ impl UploadHandler { ) -> Result { let written_bytes = sst.encoded.remaining_len() as u64; loop { - match self.db.upload_sst(&sst_id, &sst.encoded, true).await { + match self.db.upload_sst(&sst_id, &sst.encoded).await { Ok(sst_handle) => { self.db.db_stats.l0_flush_bytes.increment(written_bytes); return Ok(SegmentedSstHandle { @@ -298,8 +298,12 @@ impl MessageHandler for UploadHandler { #[cfg(test)] mod tests { use super::{TrackerMessage, UploadJob, Uploader}; + use crate::block_cache_policy::BlockCachePolicy; use crate::config::Settings; use crate::db::DbInner; + use crate::db_cache::test_utils::TestCache; + use crate::db_cache::CacheTarget; + use crate::db_cache::{CachedKey, DbCache}; use crate::db_state::{SsTableId, SsTableView}; use crate::db_status::{ClosedResultWriter, DbStatusManager}; use crate::error::SlateDBError; @@ -350,6 +354,23 @@ mod tests { path: &str, fp_registry: Arc, segment_extractor: Option>, + ) -> Arc { + setup_db_with_cache_policy( + path, + fp_registry, + segment_extractor, + None, + BlockCachePolicy::default(), + ) + .await + } + + async fn setup_db_with_cache_policy( + path: &str, + fp_registry: Arc, + segment_extractor: Option>, + cache: Option>, + block_cache_policy: BlockCachePolicy, ) -> Arc { let object_store: Arc = Arc::new(InMemory::new()); let settings = Settings::default(); @@ -372,8 +393,9 @@ mod tests { SsTableFormat::default(), PathResolver::new(Path::from(path)), fp_registry.clone(), - None, + cache, TableStoreKind::Main, + block_cache_policy, )); let status_manager = DbStatusManager::new(0); let (write_tx, _) = @@ -571,6 +593,36 @@ mod tests { test.shutdown().await; } + #[tokio::test] + async fn should_apply_flush_cache_policy() { + let cache = Arc::new(TestCache::new()); + let db = setup_db_with_cache_policy( + "/tmp/test_parallel_l0_flush_cache_policy", + Arc::new(FailPointRegistry::new()), + None, + Some(cache.clone()), + BlockCachePolicy::default().with_flush_targets(&[CacheTarget::Filters]), + ) + .await; + let job = next_upload_job(&db, b"key", b"value", 1); + let test = start_test_uploader(&db); + + test.submit(job).unwrap(); + let msg = timeout(Duration::from_secs(5), test.tracker_rx.recv()) + .await + .unwrap() + .unwrap(); + let TrackerMessage::UploadComplete(event) = msg else { + panic!("expected UploadComplete"); + }; + let handle = &event.segments[0].sst_handle; + let filter_key: CachedKey = (handle.id, handle.info.filter_offset).into(); + + assert!(cache.get_filter(&filter_key).await.unwrap().is_some()); + assert_eq!(cache.entry_count(), 1); + test.shutdown().await; + } + #[tokio::test] async fn should_retry_upload_failures_until_success() { let fp_registry = Arc::new(FailPointRegistry::new()); diff --git a/slatedb/src/ops.rs b/slatedb/src/ops.rs index f6cfbd079..dcdd1cd40 100644 --- a/slatedb/src/ops.rs +++ b/slatedb/src/ops.rs @@ -7,7 +7,7 @@ use crate::config::{ FlushOptions, MergeOptions, PutOptions, ReadOptions, ScanOptions, WriteOptions, }; use crate::db::WriteHandle; -use crate::db_cache_manager::CacheTarget; +use crate::db_cache::CacheTarget; use crate::db_state::SsTableId; use crate::db_status::DbStatus; use crate::manifest::VersionedManifest; @@ -601,6 +601,9 @@ pub trait DbCacheManagerOps { /// `FuturesUnordered`) to get the concurrency they want. Per-target /// outcomes are reflected in cache-manager metrics, not the return value. /// + /// Warming [`CacheTarget::Data`] also warms the SST index, since block + /// planning depends on it. + /// /// Returns `Err` on the first failing target. If no block cache is /// configured, or if the SST is not reachable from the current manifest, /// the call is a no-op that returns `Ok(())`. diff --git a/slatedb/src/reader.rs b/slatedb/src/reader.rs index e3c3a5122..ad4b95ac0 100644 --- a/slatedb/src/reader.rs +++ b/slatedb/src/reader.rs @@ -468,6 +468,7 @@ mod tests { .as_ref() .map(|wb| WriteBatchIterator::new(wb, range.clone(), order, u64::MAX, None, None)) } + use crate::block_cache_policy::BlockCachePolicy; use crate::db_state::{SortedRun, SsTableHandle, SsTableId}; use crate::db_status::DbStatusManager; use crate::format::sst::SsTableFormat; @@ -522,6 +523,7 @@ mod tests { Path::from("/test"), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); Self { @@ -602,7 +604,7 @@ mod tests { let encoded = builder.build().await?; let id = SsTableId::Compacted(Ulid::new()); - self.table_store.write_sst(&id, &encoded, false).await + self.table_store.write_sst(&id, &encoded).await } } diff --git a/slatedb/src/sorted_run_iterator.rs b/slatedb/src/sorted_run_iterator.rs index 0a28975f6..8e7646393 100644 --- a/slatedb/src/sorted_run_iterator.rs +++ b/slatedb/src/sorted_run_iterator.rs @@ -248,6 +248,7 @@ impl RowEntryIterator for SortedRunIterator<'_> { #[cfg(test)] mod tests { use super::*; + use crate::block_cache_policy::BlockCachePolicy; use crate::bytes_generator::OrderedBytesGenerator; use crate::db_state::{SsTableHandle, SsTableId}; use crate::format::sst::SsTableFormat; @@ -281,6 +282,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let mut builder = table_store.table_builder(); builder @@ -297,7 +299,7 @@ mod tests { .unwrap(); let encoded = builder.build().await.unwrap(); let id = SsTableId::Compacted(ulid::Ulid::new()); - let handle = table_store.write_sst(&id, &encoded, false).await.unwrap(); + let handle = table_store.write_sst(&id, &encoded).await.unwrap(); let sr = SortedRun { id: 0, sst_views: vec![SsTableView::identity(handle)], @@ -339,6 +341,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let mut builder = table_store.table_builder(); builder @@ -351,7 +354,7 @@ mod tests { .unwrap(); let encoded = builder.build().await.unwrap(); let id1 = SsTableId::Compacted(ulid::Ulid::new()); - let handle1 = table_store.write_sst(&id1, &encoded, false).await.unwrap(); + let handle1 = table_store.write_sst(&id1, &encoded).await.unwrap(); let mut builder = table_store.table_builder(); builder .add_value(b"key3", b"value3", Some(3), None) @@ -359,7 +362,7 @@ mod tests { .unwrap(); let encoded = builder.build().await.unwrap(); let id2 = SsTableId::Compacted(ulid::Ulid::new()); - let handle2 = table_store.write_sst(&id2, &encoded, false).await.unwrap(); + let handle2 = table_store.write_sst(&id2, &encoded).await.unwrap(); let sr = SortedRun { id: 0, sst_views: vec![ @@ -406,6 +409,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let mut builder = table_store.table_builder(); for i in 1..=4 { @@ -418,7 +422,7 @@ mod tests { } let encoded = builder.build().await.unwrap(); let id1 = SsTableId::Compacted(ulid::Ulid::new()); - let handle1 = table_store.write_sst(&id1, &encoded, false).await.unwrap(); + let handle1 = table_store.write_sst(&id1, &encoded).await.unwrap(); let mut builder = table_store.table_builder(); for i in 5..=8 { let key = format!("key{i}"); @@ -430,7 +434,7 @@ mod tests { } let encoded = builder.build().await.unwrap(); let id2 = SsTableId::Compacted(ulid::Ulid::new()); - let handle2 = table_store.write_sst(&id2, &encoded, false).await.unwrap(); + let handle2 = table_store.write_sst(&id2, &encoded).await.unwrap(); let sr = SortedRun { id: 0, sst_views: vec![ @@ -491,6 +495,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let key_gen = OrderedBytesGenerator::new_with_byte_range(&[b'a'; 16], b'a', b'z'); let mut test_case_key_gen = key_gen.clone(); @@ -536,6 +541,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let key_gen = OrderedBytesGenerator::new_with_byte_range(&[b'a'; 16], b'a', b'z'); let mut expected_key_gen = key_gen.clone(); @@ -575,6 +581,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let key_gen = OrderedBytesGenerator::new_with_byte_range(&[b'a'; 16], b'a', b'z'); let val_gen = OrderedBytesGenerator::new_with_byte_range(&[0u8; 16], 0u8, 26u8); @@ -602,6 +609,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let mut rng = proptest_util::rng::new_test_rng(None); @@ -667,7 +675,7 @@ mod tests { let encoded = builder.build().await.unwrap(); let id = SsTableId::Compacted(ulid::Ulid::new()); - let handle = table_store.write_sst(&id, &encoded, false).await.unwrap(); + let handle = table_store.write_sst(&id, &encoded).await.unwrap(); ssts.push(SsTableView::identity(handle)); } @@ -717,7 +725,7 @@ mod tests { } let encoded = builder.build().await.unwrap(); let id = SsTableId::Compacted(ulid::Ulid::new()); - table_store.write_sst(&id, &encoded, false).await.unwrap() + table_store.write_sst(&id, &encoded).await.unwrap() } async fn build_sst_v2( @@ -731,7 +739,7 @@ mod tests { } let encoded = builder.build().await.unwrap(); let id = SsTableId::Compacted(ulid::Ulid::new()); - table_store.write_sst(&id, &encoded, false).await.unwrap() + table_store.write_sst(&id, &encoded).await.unwrap() } #[tokio::test] @@ -749,6 +757,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), )); // Build a sorted run with v1, v2, v1, v2 SSTs @@ -821,6 +830,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), )); // Build a sorted run with v1, v2, v1, v2 SSTs diff --git a/slatedb/src/sst_builder.rs b/slatedb/src/sst_builder.rs index 344e250b4..0da29e62d 100644 --- a/slatedb/src/sst_builder.rs +++ b/slatedb/src/sst_builder.rs @@ -129,6 +129,7 @@ pub(crate) struct EncodedSsTableBuilder { first_key: Option>>, sst_first_key: Option, sst_last_key: Option, + current_block_first_key: Option, current_block_max_key: Option, block_meta: Vec>>, current_len: u64, @@ -163,6 +164,7 @@ impl EncodedSsTableBuilder { first_key: None, sst_first_key: None, sst_last_key: None, + current_block_first_key: None, current_block_max_key: None, block_size, block_format: BlockFormat::Latest, @@ -223,7 +225,7 @@ impl EncodedSsTableBuilder { self.stats.raw_key_size += entry.key.len() as u64; self.stats.raw_val_size += entry.value.len() as u64; - let index_key = compute_index_key(self.current_block_max_key.take(), &entry.key); + let index_key = compute_index_key(self.current_block_max_key.clone(), &entry.key); let is_sst_first_key = self.sst_first_key.is_none(); let mut block_size = None; @@ -241,6 +243,9 @@ impl EncodedSsTableBuilder { self.sst_first_key = Some(entry.key.clone()); } self.sst_last_key = Some(entry.key.clone()); + if self.builder.is_empty() { + self.current_block_first_key = Some(entry.key.clone()); + } self.current_block_max_key = Some(entry.key.clone()); self.builder.add(entry)?; @@ -285,6 +290,13 @@ impl EncodedSsTableBuilder { let old_builder = std::mem::replace(&mut self.builder, new_builder); let (builder, block_stats) = old_builder.into_parts(); let mut block_builder = EncodedSsTableBlockBuilder::new(builder, self.current_len); + if let Some((first_key, last_key)) = self + .current_block_first_key + .take() + .zip(self.current_block_max_key.take()) + { + block_builder = block_builder.with_key_span(first_key, last_key); + } if let Some(codec) = self.compression_codec { block_builder = block_builder.with_compression_codec(codec); } @@ -424,6 +436,7 @@ mod tests { use super::*; use crate::blob::ReadOnlyBlob; + use crate::block_cache_policy::BlockCachePolicy; use crate::block_iterator::{BlockIteratorLatest, BlockLike}; use crate::bytes_range::BytesRange; use crate::db_state::{SsTableId, SsTableView}; @@ -487,6 +500,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let path_resolver = PathResolver::new(root_path); @@ -543,7 +557,7 @@ mod tests { let encoded = builder.build().await.unwrap(); let compacted_id = SsTableId::Compacted(ulid::Ulid::new()); table_store - .write_sst(&compacted_id, &encoded, false) + .write_sst(&compacted_id, &encoded) .await .unwrap(); report( @@ -560,10 +574,7 @@ mod tests { } let wal_encoded = wal_builder.build().await.unwrap(); let wal_id = SsTableId::Wal(1); - table_store - .write_sst(&wal_id, &wal_encoded, false) - .await - .unwrap(); + table_store.write_sst(&wal_id, &wal_encoded).await.unwrap(); report( "wal", format.estimate_encoded_size_wal(num_entries, estimated_entries_size), @@ -593,6 +604,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = table_store.table_builder(); builder @@ -649,6 +661,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = table_store.table_builder(); builder @@ -729,6 +742,46 @@ mod tests { } } + #[tokio::test] + async fn test_builder_should_track_block_key_spans() { + // one entry per block + let format = SsTableFormat { + block_size: 32, + ..SsTableFormat::default() + }; + let mut builder = format.table_builder(); + for i in 0..4u8 { + builder + .add_value(&[b'a' + i; 16], &[i; 16], None, None) + .await + .unwrap(); + } + let sst = builder.build().await.unwrap(); + assert_eq!(sst.unconsumed_blocks.len(), 4); + for (i, block) in sst.unconsumed_blocks.iter().enumerate() { + let key = Bytes::copy_from_slice(&[b'a' + i as u8; 16]); + assert_eq!(block.key_span, Some((key.clone(), key))); + } + + // all entries in one block + let mut builder = SsTableFormat::default().table_builder(); + for i in 0..4u8 { + builder + .add_value(&[b'a' + i; 16], &[i; 16], None, None) + .await + .unwrap(); + } + let sst = builder.build().await.unwrap(); + assert_eq!(sst.unconsumed_blocks.len(), 1); + assert_eq!( + sst.unconsumed_blocks[0].key_span, + Some(( + Bytes::copy_from_slice(&[b'a'; 16]), + Bytes::copy_from_slice(&[b'd'; 16]) + )) + ); + } + #[rstest] #[case::default_sst(SsTableFormat::default(), 0, true)] #[case::sst_with_no_filter(SsTableFormat { min_filter_keys: 9, ..SsTableFormat::default() }, 0, false)] @@ -748,6 +801,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = table_store.table_builder(); for k in 1..=8 { @@ -773,7 +827,7 @@ mod tests { // write sst and validate that the handle returned has the correct content. let sst_handle = table_store - .write_sst(&SsTableId::Wal(wal_id), &encoded, false) + .write_sst(&SsTableId::Wal(wal_id), &encoded) .await .unwrap(); assert_eq!(encoded_info, sst_handle.info); @@ -833,6 +887,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = table_store.table_builder(); builder @@ -846,7 +901,7 @@ mod tests { let encoded = builder.build().await.unwrap(); let encoded_info = encoded.info.clone(); table_store - .write_sst(&SsTableId::Wal(0), &encoded, false) + .write_sst(&SsTableId::Wal(0), &encoded) .await .unwrap(); let sst_handle = table_store.open_sst(&SsTableId::Wal(0)).await.unwrap(); @@ -901,6 +956,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = table_store.table_builder(); builder @@ -914,7 +970,7 @@ mod tests { let encoded = builder.build().await.unwrap(); let encoded_info = encoded.info.clone(); table_store - .write_sst(&SsTableId::Wal(0), &encoded, false) + .write_sst(&SsTableId::Wal(0), &encoded) .await .unwrap(); @@ -929,6 +985,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let sst_handle = table_store.open_sst(&SsTableId::Wal(0)).await.unwrap(); let index = table_store.read_index(&sst_handle, true).await.unwrap(); @@ -988,6 +1045,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = table_store.table_builder(); builder @@ -1041,6 +1099,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = table_store.table_builder(); builder @@ -1056,7 +1115,7 @@ mod tests { // write sst and validate that the handle returned has the correct content. let sst_handle = table_store - .write_sst(&SsTableId::Wal(0), &encoded, false) + .write_sst(&SsTableId::Wal(0), &encoded) .await .unwrap(); assert_eq!(encoded_info, sst_handle.info); @@ -1110,6 +1169,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = table_store.table_builder(); builder @@ -1166,6 +1226,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let mut builder = table_store.table_builder(); for key in 'a'..='z' { @@ -1175,9 +1236,8 @@ mod tests { let encoded = builder.build().await?; let sst_id = SsTableId::Wal(0); - let sst_handle = - SsTableView::identity(table_store.write_sst(&sst_id, &encoded, false).await?) - .with_visible_range(BytesRange::from_ref("c"..="f")); + let sst_handle = SsTableView::identity(table_store.write_sst(&sst_id, &encoded).await?) + .with_visible_range(BytesRange::from_ref("c"..="f")); let expected_entries = vec![ RowEntry::new_value(b"c", b"value", 0), @@ -1287,6 +1347,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = table_store.table_builder(); builder @@ -1300,7 +1361,7 @@ mod tests { let encoded = builder.build().await.unwrap(); let encoded_info = encoded.info.clone(); table_store - .write_sst(&SsTableId::Wal(0), &encoded, false) + .write_sst(&SsTableId::Wal(0), &encoded) .await .unwrap(); @@ -1343,6 +1404,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = table_store.table_builder(); builder @@ -1355,7 +1417,7 @@ mod tests { .unwrap(); let encoded = builder.build().await.unwrap(); table_store - .write_sst(&SsTableId::Wal(0), &encoded, false) + .write_sst(&SsTableId::Wal(0), &encoded) .await .unwrap(); @@ -1431,6 +1493,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = table_store .table_builder() @@ -1449,7 +1512,7 @@ mod tests { } let encoded = builder.build().await.unwrap(); let sst_handle = table_store - .write_sst(&SsTableId::Wal(0), &encoded, false) + .write_sst(&SsTableId::Wal(0), &encoded) .await .unwrap(); @@ -1498,6 +1561,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = table_store.table_builder(); let mut expected = Vec::new(); @@ -1514,7 +1578,7 @@ mod tests { } let encoded = builder.build().await.unwrap(); let sst_handle = table_store - .write_sst(&SsTableId::Wal(1), &encoded, false) + .write_sst(&SsTableId::Wal(1), &encoded) .await .unwrap(); @@ -1562,6 +1626,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = table_store.table_builder(); @@ -1612,7 +1677,7 @@ mod tests { let encoded = builder.build().await.unwrap(); let sst_handle = table_store - .write_sst(&SsTableId::Wal(0), &encoded, false) + .write_sst(&SsTableId::Wal(0), &encoded) .await .unwrap(); let stats = table_store @@ -1670,6 +1735,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = table_store.table_builder(); builder @@ -1682,7 +1748,7 @@ mod tests { .unwrap(); let encoded = builder.build().await.unwrap(); let sst_handle = table_store - .write_sst(&SsTableId::Wal(0), &encoded, false) + .write_sst(&SsTableId::Wal(0), &encoded) .await .unwrap(); let stats = table_store @@ -1713,6 +1779,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = table_store.table_builder(); builder @@ -1725,7 +1792,7 @@ mod tests { .unwrap(); let encoded = builder.build().await.unwrap(); let sst_handle = table_store - .write_sst(&SsTableId::Wal(0), &encoded, false) + .write_sst(&SsTableId::Wal(0), &encoded) .await .unwrap(); let stats = table_store @@ -1759,6 +1826,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = table_store.table_builder(); // Block 0: put @@ -1791,7 +1859,7 @@ mod tests { let encoded = builder.build().await.unwrap(); let sst_handle = table_store - .write_sst(&SsTableId::Wal(0), &encoded, false) + .write_sst(&SsTableId::Wal(0), &encoded) .await .unwrap(); let stats = table_store @@ -1857,6 +1925,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), ); // Write keys whose 3-byte prefix is "key". @@ -1870,7 +1939,7 @@ mod tests { } let encoded = builder.build().await.unwrap(); table_store - .write_sst(&SsTableId::Wal(0), &encoded, false) + .write_sst(&SsTableId::Wal(0), &encoded) .await .unwrap(); let handle = table_store.open_sst(&SsTableId::Wal(0)).await.unwrap(); @@ -1914,6 +1983,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let handle_partial = store_partial.open_sst(&SsTableId::Wal(0)).await.unwrap(); let partial = store_partial diff --git a/slatedb/src/sst_iter.rs b/slatedb/src/sst_iter.rs index 1f1a3f683..8f158b7c4 100644 --- a/slatedb/src/sst_iter.rs +++ b/slatedb/src/sst_iter.rs @@ -1060,6 +1060,7 @@ impl RowEntryIterator for SstIterator<'_> { #[cfg(test)] mod tests { use super::*; + use crate::block_cache_policy::BlockCachePolicy; use crate::bytes_generator::OrderedBytesGenerator; use crate::db_cache::test_utils::TestCache; use crate::db_cache::DbCache; @@ -1099,6 +1100,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let mut builder = table_store.table_builder(); builder @@ -1119,7 +1121,7 @@ mod tests { .unwrap(); let encoded = builder.build().await.unwrap(); table_store - .write_sst(&SsTableId::Wal(0), &encoded, false) + .write_sst(&SsTableId::Wal(0), &encoded) .await .unwrap(); let sst_handle = table_store.open_sst(&SsTableId::Wal(0)).await.unwrap(); @@ -1349,6 +1351,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = writer.table_builder(); builder @@ -1363,7 +1366,6 @@ mod tests { .write_sst( &SsTableId::Compacted(ulid::Ulid::new()), &builder.build().await.unwrap(), - false, ) .await .unwrap(); @@ -1381,6 +1383,7 @@ mod tests { root_path, Some(cache), TableStoreKind::Main, + BlockCachePolicy::default(), )); let filter_key = (handle.sst.id, handle.sst.info.filter_offset).into(); @@ -1436,6 +1439,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = writer.table_builder(); builder @@ -1450,7 +1454,6 @@ mod tests { .write_sst( &SsTableId::Compacted(ulid::Ulid::new()), &builder.build().await.unwrap(), - false, ) .await .unwrap(); @@ -1468,6 +1471,7 @@ mod tests { root_path, Some(cache), TableStoreKind::Main, + BlockCachePolicy::default(), )); let index_key = (handle.sst.id, handle.sst.info.index_offset).into(); @@ -1523,6 +1527,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), )) } @@ -1537,7 +1542,7 @@ mod tests { } let encoded = builder.build().await.unwrap(); let id = SsTableId::Compacted(ulid::Ulid::new()); - SsTableView::identity(table_store.write_sst(&id, &encoded, false).await.unwrap()) + SsTableView::identity(table_store.write_sst(&id, &encoded).await.unwrap()) } #[tokio::test] @@ -1559,6 +1564,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let mut builder = table_store.table_builder(); @@ -1576,7 +1582,7 @@ mod tests { let encoded = builder.build().await.unwrap(); table_store - .write_sst(&SsTableId::Wal(0), &encoded, false) + .write_sst(&SsTableId::Wal(0), &encoded) .await .unwrap(); let sst_handle = table_store.open_sst(&SsTableId::Wal(0)).await.unwrap(); @@ -1636,6 +1642,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let first_key = [b'a'; 16]; let key_gen = OrderedBytesGenerator::new_with_byte_range(&first_key, b'a', b'z'); @@ -1687,6 +1694,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let first_key = [b'b'; 16]; let key_gen = OrderedBytesGenerator::new_with_byte_range(&first_key, b'a', b'y'); @@ -1732,6 +1740,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let first_key = [b'b'; 16]; let key_gen = OrderedBytesGenerator::new_with_byte_range(&first_key, b'a', b'y'); @@ -1774,6 +1783,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); // Build SST with specified format (keys 0-99) @@ -1798,8 +1808,7 @@ mod tests { let encoded = builder.build().await.unwrap(); let id = SsTableId::Compacted(ulid::Ulid::new()); - let sst_handle = - SsTableView::identity(table_store.write_sst(&id, &encoded, false).await.unwrap()); + let sst_handle = SsTableView::identity(table_store.write_sst(&id, &encoded).await.unwrap()); // Initialize iterator in descending order with full range let mut iter = SstIterator::new_borrowed_initialized( @@ -1851,6 +1860,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let first_key = [b'b'; 16]; let key_gen = OrderedBytesGenerator::new_with_byte_range(&first_key, b'a', b'y'); @@ -1962,6 +1972,7 @@ mod tests { root_path.clone(), Some(split_cache.clone()), TableStoreKind::Main, + BlockCachePolicy::default(), )); let mut builder = table_store.table_builder(); @@ -1983,7 +1994,7 @@ mod tests { .unwrap(); let encoded = builder.build().await.unwrap(); let id = SsTableId::Compacted(ulid::Ulid::new()); - table_store.write_sst(&id, &encoded, false).await.unwrap(); + table_store.write_sst(&id, &encoded).await.unwrap(); let sst_handle = table_store.open_sst(&id).await.unwrap(); let sst_iter_options = SstIteratorOptions { @@ -2062,7 +2073,7 @@ mod tests { } let encoded = builder.build().await.unwrap(); let id = SsTableId::Compacted(ulid::Ulid::new()); - SsTableView::identity(table_store.write_sst(&id, &encoded, false).await.unwrap()) + SsTableView::identity(table_store.write_sst(&id, &encoded).await.unwrap()) } #[tokio::test] @@ -2080,6 +2091,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let keys_and_values = vec![ @@ -2129,6 +2141,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let keys_and_values = vec![ @@ -2180,6 +2193,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), )); // Create keys with shared prefixes to exercise prefix compression @@ -2199,8 +2213,7 @@ mod tests { let encoded = builder.build().await.unwrap(); let id = SsTableId::Compacted(ulid::Ulid::new()); - let sst_handle = - SsTableView::identity(table_store.write_sst(&id, &encoded, false).await.unwrap()); + let sst_handle = SsTableView::identity(table_store.write_sst(&id, &encoded).await.unwrap()); // when: iterating over all keys let sst_iter_options = SstIteratorOptions { @@ -2244,6 +2257,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), )); // Create keys that will span multiple blocks @@ -2263,7 +2277,7 @@ mod tests { let encoded = builder.build().await.unwrap(); let id = SsTableId::Compacted(ulid::Ulid::new()); - let sst_handle = table_store.write_sst(&id, &encoded, false).await.unwrap(); + let sst_handle = table_store.write_sst(&id, &encoded).await.unwrap(); // Verify we have multiple blocks let index = table_store.read_index(&sst_handle, true).await.unwrap(); @@ -2313,6 +2327,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let mut builder = table_store @@ -2331,8 +2346,7 @@ mod tests { let encoded = builder.build().await.unwrap(); let id = SsTableId::Compacted(ulid::Ulid::new()); - let sst_handle = - SsTableView::identity(table_store.write_sst(&id, &encoded, false).await.unwrap()); + let sst_handle = SsTableView::identity(table_store.write_sst(&id, &encoded).await.unwrap()); // when: searching for a non-existent key (odd number) let mut iter = SstIterator::for_key_with_stats_initialized( @@ -2367,6 +2381,7 @@ mod tests { root_path, None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let mut builder = table_store @@ -2384,8 +2399,7 @@ mod tests { let encoded = builder.build().await.unwrap(); let id = SsTableId::Compacted(ulid::Ulid::new()); - let sst_handle = - SsTableView::identity(table_store.write_sst(&id, &encoded, false).await.unwrap()); + let sst_handle = SsTableView::identity(table_store.write_sst(&id, &encoded).await.unwrap()); // when: seeking past the last key let iter = SstIterator::new_borrowed_initialized( @@ -2427,6 +2441,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); // Build an SST with enough keys to span multiple blocks @@ -2445,7 +2460,7 @@ mod tests { } let encoded = builder.build().await.unwrap(); let id = SsTableId::Compacted(ulid::Ulid::new()); - table_store.write_sst(&id, &encoded, false).await.unwrap(); + table_store.write_sst(&id, &encoded).await.unwrap(); let sst_handle = table_store.open_sst(&id).await.unwrap(); let index = table_store.read_index(&sst_handle, true).await.unwrap(); @@ -2566,6 +2581,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); // Build an SST with enough data for multiple blocks @@ -2583,7 +2599,7 @@ mod tests { } let encoded = builder.build().await.unwrap(); let id = SsTableId::Compacted(ulid::Ulid::new()); - table_store.write_sst(&id, &encoded, false).await.unwrap(); + table_store.write_sst(&id, &encoded).await.unwrap(); let sst_handle = table_store.open_sst(&id).await.unwrap(); let index = table_store.read_index(&sst_handle, true).await.unwrap(); @@ -2636,6 +2652,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let mut writer = table_store.table_writer(SsTableId::Wal(0)); @@ -2733,6 +2750,7 @@ mod tests { root_path.clone(), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); // Keys spaced by 10: key_000, key_010, key_020, ..., key_190. diff --git a/slatedb/src/sst_reader.rs b/slatedb/src/sst_reader.rs index 5153de863..0acffdc62 100644 --- a/slatedb/src/sst_reader.rs +++ b/slatedb/src/sst_reader.rs @@ -52,6 +52,7 @@ use object_store::path::Path; use object_store::ObjectStore; use ulid::Ulid; +use crate::block_cache_policy::BlockCachePolicy; use crate::block_iterator::DataBlockIterator; use crate::db_cache::DbCache; use crate::db_state::{SsTableHandle, SsTableId, SsTableInfo}; @@ -95,6 +96,7 @@ impl SstReader { root_path.into(), cache, TableStoreKind::Reader, + BlockCachePolicy::default(), )); Self { table_store } } diff --git a/slatedb/src/tablestore.rs b/slatedb/src/tablestore.rs index 98ae8ab5f..1111a7be5 100644 --- a/slatedb/src/tablestore.rs +++ b/slatedb/src/tablestore.rs @@ -17,14 +17,15 @@ use tokio::io::AsyncWriteExt; use ulid::Ulid; use crate::blob::ReadOnlyBlob; +use crate::block_cache_policy::{should_cache_data_block, BlockCachePolicy}; +use crate::db_cache::CacheTarget; use crate::db_cache::{CacheLoader, CachedEntry, CachedKey, DbCache, EncodedCachedFilter}; -use crate::db_cache_manager::CacheTarget; use crate::db_state::{SsTableHandle, SsTableId, SstType}; use crate::error::SlateDBError; use crate::filter_policy::NamedFilter; use crate::flatbuffer_types::SsTableIndexOwned; use crate::format::block::Block; -use crate::format::sst::{EncodedSsTable, SsTableFormat}; +use crate::format::sst::{EncodedSsTable, EncodedSsTableBlock, SsTableFormat}; use crate::object_store_tag::ObjectStoreCallTag; pub(crate) use crate::object_store_tag::TableStoreKind; use crate::object_stores::{ObjectStoreType, ObjectStores}; @@ -40,8 +41,10 @@ pub(crate) struct TableStore { path_resolver: PathResolver, #[allow(dead_code)] fp_registry: Arc, - /// In-memory cache for data blocks, indices, and filters + /// In-memory cache for data blocks and SST metadata. cache: Option>, + /// Selects which components to insert into the cache. + block_cache_policy: BlockCachePolicy, /// Which component owns this store. Tagged on compacted-SST calls. kind: TableStoreKind, } @@ -125,6 +128,7 @@ impl TableStore { root_path: P, block_cache: Option>, kind: TableStoreKind, + block_cache_policy: BlockCachePolicy, ) -> Self { Self::new_with_fp_registry( object_stores, @@ -133,6 +137,7 @@ impl TableStore { Arc::new(FailPointRegistry::new()), block_cache, kind, + block_cache_policy, ) } @@ -143,6 +148,7 @@ impl TableStore { fp_registry: Arc, cache: Option>, kind: TableStoreKind, + block_cache_policy: BlockCachePolicy, ) -> Self { Self { object_stores, @@ -150,6 +156,7 @@ impl TableStore { path_resolver, fp_registry, cache, + block_cache_policy, kind, } } @@ -349,7 +356,6 @@ impl TableStore { &self, id: &SsTableId, encoded_sst: &EncodedSsTable, - write_cache: bool, ) -> Result { fail_point!( self.fp_registry.clone(), @@ -392,35 +398,82 @@ impl TableStore { } } - if let Some(ref cache) = self.cache { - if write_cache { - for block in &encoded_sst.unconsumed_blocks { - cache - .insert( - (*id, block.offset).into(), - CachedEntry::with_block(Arc::clone(&block.block)), - ) - .await; - } + self.cache_on_sst_write(*id, encoded_sst).await; + Ok(SsTableHandle::new( + *id, + encoded_sst.format_version, + encoded_sst.info.clone(), + )) + } + + /// Targets a write of the SST inserts into the block cache. + fn targets_to_cache(&self, id: &SsTableId) -> &[CacheTarget] { + match (id, self.kind) { + (SsTableId::Wal(_), _) => &[], + (SsTableId::Compacted(_), TableStoreKind::Compactor) => { + self.block_cache_policy.compaction_output_targets() + } + (SsTableId::Compacted(_), TableStoreKind::Main) => { + self.block_cache_policy.flush_targets() + } + // We only cache from the main store (flush) and the compactor + // (compaction output) right now. + (SsTableId::Compacted(_), _) => &[], + } + } + + /// Inserts the targets selected by the block cache policy for + /// `sst_table_id` into the block cache. + /// + /// Data blocks come from `unconsumed_blocks`, so a caller that already + /// streamed blocks out (the streaming writer) only has the blocks it has + /// not drained yet, and caches the rest itself as it drains them. + async fn cache_on_sst_write(&self, sst_table_id: SsTableId, encoded_sst: &EncodedSsTable) { + let Some(cache) = &self.cache else { + return; + }; + let targets = self.targets_to_cache(&sst_table_id); + for block in &encoded_sst.unconsumed_blocks { + // Blocks without a tracked key span (WAL blocks) are never cached. + let Some(key_span) = &block.key_span else { + continue; + }; + if !should_cache_data_block(targets, key_span) { + continue; + } + cache + .insert( + (sst_table_id, block.offset).into(), + CachedEntry::with_block(Arc::clone(&block.block)), + ) + .await; + } + if targets.contains(&CacheTarget::Index) { + cache + .insert( + (sst_table_id, encoded_sst.info.index_offset).into(), + CachedEntry::with_sst_index(Arc::new(encoded_sst.index.clone())), + ) + .await; + } + if targets.contains(&CacheTarget::Filters) && !encoded_sst.filters.is_empty() { + cache + .insert( + (sst_table_id, encoded_sst.info.filter_offset).into(), + CachedEntry::with_filters(encoded_sst.filters.clone()), + ) + .await; + } + if targets.contains(&CacheTarget::Stats) { + if let Some(stats) = &encoded_sst.stats { cache .insert( - (*id, encoded_sst.info.index_offset).into(), - CachedEntry::with_sst_index(Arc::new(encoded_sst.index.clone())), + (sst_table_id, encoded_sst.info.stats_offset).into(), + CachedEntry::with_sst_stats(Arc::new(stats.clone())), ) .await; } } - self.cache_filters( - *id, - encoded_sst.info.filter_offset, - encoded_sst.filters.clone(), - ) - .await; - Ok(SsTableHandle::new( - *id, - encoded_sst.format_version, - encoded_sst.info.clone(), - )) } /// Writes a zero-byte WAL object as a fencing marker. @@ -442,23 +495,12 @@ impl TableStore { .await } - async fn cache_filters(&self, sst: SsTableId, id: u64, filters: Arc<[NamedFilter]>) { - let Some(ref cache) = self.cache else { - return; - }; - if !filters.is_empty() { - cache - .insert((sst, id).into(), CachedEntry::with_filters(filters)) - .await; - } - } - /// Decodes an `EncodedCachedFilter` slice into a fully-decoded /// `Arc<[NamedFilter]>` and overwrites the cache entry under `cache_key` /// with the decoded form so subsequent hits bypass the decode step. /// Entries whose policy name has no match in the configured policies are /// dropped. - async fn decode_and_refresh( + async fn decode_and_refresh_filter( &self, cache: &Arc, cache_key: CachedKey, @@ -601,7 +643,7 @@ impl TableStore { return Ok(Arc::from([])); } let cache_key: CachedKey = (handle.id, handle.info.filter_offset).into(); - if let Some(ref cache) = self.cache { + if let Some(cache) = self.cache_for_reads() { // cache_blocks=true: dedup-aware fetch; concurrent callers collapse onto // one loader. cache_blocks=false: read-only lookup that won't pollute the // cache on miss. Cache errors fall through to a best-effort direct load; @@ -627,7 +669,9 @@ impl TableStore { // Encoded form from disk-cache deserialize. Decode and overwrite // the cache entry with the decoded form. if let Some(encoded) = entry.encoded_filters() { - return Ok(self.decode_and_refresh(cache, cache_key, &encoded).await); + return Ok(self + .decode_and_refresh_filter(cache, cache_key, &encoded) + .await); } } } @@ -650,7 +694,7 @@ impl TableStore { return Ok(None); } let cache_key = (handle.id, handle.info.stats_offset).into(); - if let Some(ref cache) = self.cache { + if let Some(cache) = self.cache_for_reads() { // See `read_filters` for the rationale on the fall-through path. let entry = if cache_blocks { cache @@ -681,7 +725,7 @@ impl TableStore { cache_blocks: bool, ) -> Result, SlateDBError> { let cache_key = (handle.id, handle.info.index_offset).into(); - if let Some(ref cache) = self.cache { + if let Some(cache) = self.cache_for_reads() { // See `read_filters` for the rationale on the fall-through path. let entry = if cache_blocks { cache @@ -846,7 +890,7 @@ impl TableStore { // run of uncached blocks. Cache errors fall through to the direct load, // which produces the authoritative error if any. if cache_blocks && blocks.len() == 1 { - if let Some(ref cache) = self.cache { + if let Some(cache) = self.cache_for_reads() { let block_num = blocks.start; let offset = index.borrow().block_meta().get(block_num).offset(); let cache_key: CachedKey = (handle.id, offset).into(); @@ -868,7 +912,7 @@ impl TableStore { let mut uncached_ranges = Vec::new(); // If block cache is available, try to retrieve cached blocks - if let Some(ref cache) = self.cache { + if let Some(cache) = self.cache_for_reads() { let index_borrow = index.borrow(); // Attempt to get all requested blocks from cache concurrently let cached_blocks = join_all(blocks.clone().map(|block_num| async move { @@ -951,7 +995,7 @@ impl TableStore { } // Cache the newly read blocks if caching is enabled - if let Some(ref cache) = self.cache { + if let Some(cache) = self.cache_for_reads() { if !blocks_to_cache.is_empty() { join_all(blocks_to_cache.into_iter().map(|(id, offset, block)| { cache.insert((id, offset).into(), CachedEntry::with_block(block)) @@ -1004,6 +1048,20 @@ impl TableStore { self.cache.as_ref() } + /// The block cache to probe for read operations, gated based on the table + /// store kind. + /// + /// compactor reads bypass it so compaction input does not pollute the + /// cache. + // TODO: revisit this when the read side of BlockCachePolicy is implemented. + fn cache_for_reads(&self) -> Option<&Arc> { + if self.kind == TableStoreKind::Compactor { + None + } else { + self.cache.as_ref() + } + } + /// Best-effort removal of all cache entries associated with the given SST: /// data blocks, index, filters, and stats. Returns the offsets whose /// cache removal was attempted. @@ -1177,15 +1235,21 @@ impl EncodedSsTableWriter { } pub(crate) async fn close(mut self) -> Result { - let mut encoded_sst = self.builder.build().await?; - while let Some(block) = encoded_sst.unconsumed_blocks.pop_front() { + let encoded_sst = self.builder.build().await?; + for block in &encoded_sst.unconsumed_blocks { self.writer.write_all(block.encoded_bytes.as_ref()).await?; } - self.writer.write_all(encoded_sst.footer.as_ref()).await?; self.writer.shutdown().await?; + + // Cache inserts happen after writer shutdown so an SST whose upload + // fails contributes no metadata entries. + // + // Blocks drained while entries were added are cached by `write_block`, + // so the only data block left for `cache_on_sst_write` is the tail + // block that `build` finished. self.table_store - .cache_filters(self.id, encoded_sst.info.filter_offset, encoded_sst.filters) + .cache_on_sst_write(self.id, &encoded_sst) .await; Ok(SsTableHandle::new( self.id, @@ -1196,7 +1260,7 @@ impl EncodedSsTableWriter { async fn drain_blocks(&mut self) -> Result<(), SlateDBError> { while let Some(block) = self.builder.next_block() { - self.writer.write_all(block.encoded_bytes.as_ref()).await?; + self.write_block(block).await?; #[cfg(test)] { self.blocks_written += 1; @@ -1205,6 +1269,22 @@ impl EncodedSsTableWriter { Ok(()) } + async fn write_block(&mut self, block: EncodedSsTableBlock) -> Result<(), SlateDBError> { + self.writer.write_all(block.encoded_bytes.as_ref()).await?; + if let (Some(cache), Some(key_span)) = (&self.table_store.cache, &block.key_span) { + let targets = self.table_store.targets_to_cache(&self.id); + if should_cache_data_block(targets, key_span) { + cache + .insert( + (self.id, block.offset).into(), + CachedEntry::with_block(block.block), + ) + .await; + } + } + Ok(()) + } + pub(crate) fn is_drained(&self) -> bool { self.builder.is_drained() } @@ -1237,9 +1317,11 @@ mod tests { use std::collections::VecDeque; use std::sync::Arc; + use crate::block_cache_policy::BlockCachePolicy; use crate::db_cache::test_utils::TestCache; + use crate::db_cache::CacheTarget; use crate::db_cache::SplitCache; - use crate::db_cache::{DbCache, DbCacheWrapper}; + use crate::db_cache::{CachedKey, DbCache, DbCacheWrapper}; use crate::error; use crate::format::block::Block; use crate::format::sst::SsTableFormat; @@ -1387,6 +1469,7 @@ mod tests { Path::from(ROOT), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let id = SsTableId::Compacted(ulid::Ulid::new()); @@ -1461,6 +1544,7 @@ mod tests { Path::from(ROOT), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let id = SsTableId::Wal(123); @@ -1531,6 +1615,7 @@ mod tests { Path::from(ROOT), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let wal_id = SsTableId::Wal(1); @@ -1540,7 +1625,7 @@ mod tests { .await .unwrap(); let table = sst1.build().await.unwrap(); - ts.write_sst(&wal_id, &table, false).await.unwrap(); + ts.write_sst(&wal_id, &table).await.unwrap(); let mut sst2 = ts.table_builder(); sst2.add(RowEntry::new_value(b"key", b"value", 0)) @@ -1549,7 +1634,7 @@ mod tests { let table2 = sst2.build().await.unwrap(); // write another wal sst with the same id. - let result = ts.write_sst(&wal_id, &table2, false).await; + let result = ts.write_sst(&wal_id, &table2).await; assert!(matches!(result, Err(error::SlateDBError::Fenced))); } @@ -1562,6 +1647,7 @@ mod tests { Path::from(ROOT), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); ts.write_wal_fence(1).await.unwrap(); @@ -1579,6 +1665,7 @@ mod tests { Path::from(ROOT), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); ts.write_wal_fence(1).await.unwrap(); @@ -1595,6 +1682,7 @@ mod tests { Path::from(ROOT), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); ts.write_wal_fence(1).await.unwrap(); @@ -1604,7 +1692,7 @@ mod tests { .await .unwrap(); let table = sst.build().await.unwrap(); - let result = ts.write_sst(&SsTableId::Wal(1), &table, false).await; + let result = ts.write_sst(&SsTableId::Wal(1), &table).await; assert!(matches!(result, Err(error::SlateDBError::Fenced))); } @@ -1627,6 +1715,7 @@ mod tests { Path::from(ROOT), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let id = SsTableId::Compacted(ulid::Ulid::new()); @@ -1644,7 +1733,7 @@ mod tests { let sst = builder.build().await.unwrap(); // when: - ts.write_sst(&id, &sst, false).await.unwrap(); + ts.write_sst(&id, &sst).await.unwrap(); // then: assert_eq!(os.put_attempts(), 0); @@ -1671,6 +1760,7 @@ mod tests { Path::from(ROOT), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let wal_id = SsTableId::Wal(1); @@ -1688,7 +1778,7 @@ mod tests { let sst = builder.build().await.unwrap(); // when: - let result = ts.write_sst(&wal_id, &sst, false).await; + let result = ts.write_sst(&wal_id, &sst).await; // then: assert!(matches!( @@ -1713,6 +1803,7 @@ mod tests { Path::from(ROOT), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let id = SsTableId::Compacted(ulid::Ulid::new()); @@ -1781,6 +1872,7 @@ mod tests { Path::from("/root"), Some(wrapper.clone()), TableStoreKind::Main, + BlockCachePolicy::default(), )); // Create and write SST @@ -1909,6 +2001,7 @@ mod tests { Path::from(ROOT), None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = writer.table_builder(); @@ -1922,7 +2015,7 @@ mod tests { .unwrap(); let id = SsTableId::Compacted(ulid::Ulid::new()); let handle = writer - .write_sst(&id, &builder.build().await.unwrap(), false) + .write_sst(&id, &builder.build().await.unwrap()) .await .unwrap(); @@ -1938,6 +2031,7 @@ mod tests { Path::from(ROOT), Some(cache), TableStoreKind::Main, + BlockCachePolicy::default(), ); assert_eq!(meta_cache.entry_count(), 0); @@ -1969,6 +2063,7 @@ mod tests { Path::from(ROOT), None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = writer.table_builder(); @@ -1982,7 +2077,7 @@ mod tests { .unwrap(); let id = SsTableId::Compacted(ulid::Ulid::new()); let handle = writer - .write_sst(&id, &builder.build().await.unwrap(), false) + .write_sst(&id, &builder.build().await.unwrap()) .await .unwrap(); @@ -1998,6 +2093,7 @@ mod tests { Path::from(ROOT), Some(cache), TableStoreKind::Main, + BlockCachePolicy::default(), ); assert_eq!(meta_cache.entry_count(), 0); @@ -2027,6 +2123,7 @@ mod tests { Path::from(ROOT), None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = writer.table_builder(); @@ -2040,7 +2137,7 @@ mod tests { .unwrap(); let id = SsTableId::Compacted(ulid::Ulid::new()); let handle = writer - .write_sst(&id, &builder.build().await.unwrap(), false) + .write_sst(&id, &builder.build().await.unwrap()) .await .unwrap(); assert!(handle.info.stats_len > 0); @@ -2057,6 +2154,7 @@ mod tests { Path::from(ROOT), Some(cache), TableStoreKind::Main, + BlockCachePolicy::default(), ); assert_eq!(meta_cache.entry_count(), 0); @@ -2101,13 +2199,14 @@ mod tests { Path::from("/root"), Some(wrapper.clone()), TableStoreKind::Main, + BlockCachePolicy::default(), )); let id = SsTableId::Compacted(ulid::Ulid::new()); let sst = build_test_sst(&ts.sst_format, 3).await; let sst_bytes = sst.remaining_as_bytes(); let sst_info = sst.info.clone(); - ts.write_sst(&id, &sst, true).await.unwrap(); + ts.write_sst(&id, &sst).await.unwrap(); let index = ts .sst_format @@ -2147,13 +2246,14 @@ mod tests { Path::from("/root"), Some(wrapper), TableStoreKind::Main, + BlockCachePolicy::default().with_flush_targets(&[CacheTarget::Filters]), )); let id = SsTableId::Compacted(ulid::Ulid::new()); let sst = build_test_sst(&ts.sst_format, 3).await; let sst_bytes = sst.remaining_as_bytes(); let sst_info = sst.info.clone(); - ts.write_sst(&id, &sst, false).await.unwrap(); + ts.write_sst(&id, &sst).await.unwrap(); let index = ts .sst_format @@ -2171,6 +2271,241 @@ mod tests { } } + #[rstest] + #[case::filters_only(&[CacheTarget::Filters])] + #[case::index_and_filters(&[CacheTarget::Index, CacheTarget::Filters])] + #[case::all(&[ + CacheTarget::data::<&[u8], _>(..), + CacheTarget::Filters, + CacheTarget::Index, + CacheTarget::Stats, + ])] + #[tokio::test] + async fn write_sst_should_cache_only_selected_components(#[case] selected: &[CacheTarget]) { + let cache = Arc::new(TestCache::new()); + let ts = Arc::new(TableStore::new( + ObjectStores::new(Arc::new(InMemory::new()), None), + SsTableFormat::default(), + Path::from("/root"), + Some(cache.clone()), + TableStoreKind::Main, + BlockCachePolicy::default().with_flush_targets(selected), + )); + let id = SsTableId::Compacted(ulid::Ulid::new()); + let sst = build_test_sst(&ts.sst_format, 3).await; + let data_key: CachedKey = (id, sst.unconsumed_blocks[0].offset).into(); + let index_key: CachedKey = (id, sst.info.index_offset).into(); + let filter_key: CachedKey = (id, sst.info.filter_offset).into(); + let stats_key: CachedKey = (id, sst.info.stats_offset).into(); + + ts.write_sst(&id, &sst).await.unwrap(); + + assert_eq!( + cache.get_block(&data_key).await.unwrap().is_some(), + selected.iter().any(|c| matches!(c, CacheTarget::Data(_))) + ); + assert_eq!( + cache.get_index(&index_key).await.unwrap().is_some(), + selected.contains(&CacheTarget::Index) + ); + assert_eq!( + cache.get_filter(&filter_key).await.unwrap().is_some(), + selected.contains(&CacheTarget::Filters) + ); + assert_eq!( + cache.get_stats(&stats_key).await.unwrap().is_some(), + selected.contains(&CacheTarget::Stats) + ); + } + + #[tokio::test] + async fn streaming_writer_should_use_block_cache_but_skip_compactor_reads() { + let os = Arc::new(InMemory::new()); + let cache = Arc::new(TestCache::new()); + let format = SsTableFormat { + block_size: 32, + min_filter_keys: 1, + ..SsTableFormat::default() + }; + let ts = Arc::new(TableStore::new( + ObjectStores::new(os.clone(), None), + format, + Path::from("/root"), + Some(cache.clone()), + TableStoreKind::Compactor, + BlockCachePolicy::default().with_compaction_output_targets(&[ + CacheTarget::data::<&[u8], _>(..), + CacheTarget::Index, + CacheTarget::Stats, + ]), + )); + let id = SsTableId::Compacted(ulid::Ulid::new()); + let mut writer = ts.table_writer(id); + for i in 0..4 { + writer + .add(RowEntry::new_value(&[b'a' + i; 16], &[i; 16], 0)) + .await + .unwrap(); + } + + let handle = writer.close().await.unwrap(); + let index_key: CachedKey = (id, handle.info.index_offset).into(); + let filter_key: CachedKey = (id, handle.info.filter_offset).into(); + let stats_key: CachedKey = (id, handle.info.stats_offset).into(); + let index = cache + .get_index(&index_key) + .await + .unwrap() + .unwrap() + .sst_index() + .unwrap(); + + assert!(ts.cache().is_some()); + assert!(cache.get_filter(&filter_key).await.unwrap().is_none()); + assert!(cache.get_stats(&stats_key).await.unwrap().is_some()); + for block_meta in index.borrow().block_meta().iter() { + let data_key: CachedKey = (id, block_meta.offset()).into(); + assert!(cache.get_block(&data_key).await.unwrap().is_some()); + } + + // Delete the SST from the object store and verify that the cache won't + // be used and reading the index will just return an error. + os.delete(&ts.path(&id)).await.unwrap(); + assert!(ts.read_index(&handle, false).await.is_err()); + } + + #[tokio::test] + async fn write_sst_should_cache_only_blocks_in_data_range() { + let cache = Arc::new(TestCache::new()); + let format = SsTableFormat { + block_size: 32, + min_filter_keys: 1, + ..SsTableFormat::default() + }; + let ts = Arc::new(TableStore::new( + ObjectStores::new(Arc::new(InMemory::new()), None), + format, + Path::from("/root"), + Some(cache.clone()), + TableStoreKind::Main, + BlockCachePolicy::default().with_flush_targets(&[CacheTarget::data( + [b'b'; 16].as_slice()..=[b'c'; 16].as_slice(), + )]), + )); + // single-entry blocks for keys aa.., bb.., cc.., dd.. + let mut builder = ts.table_builder(); + for i in 0..4 { + builder + .add(RowEntry::new_value(&[b'a' + i; 16], &[i; 16], 0)) + .await + .unwrap(); + } + let sst = builder.build().await.unwrap(); + assert_eq!(sst.unconsumed_blocks.len(), 4); + let id = SsTableId::Compacted(ulid::Ulid::new()); + + ts.write_sst(&id, &sst).await.unwrap(); + + for (block, expected) in sst.unconsumed_blocks.iter().zip([false, true, true, false]) { + let data_key: CachedKey = (id, block.offset).into(); + assert_eq!( + cache.get_block(&data_key).await.unwrap().is_some(), + expected + ); + } + } + + #[tokio::test] + async fn streaming_writer_should_cache_only_blocks_in_data_range() { + let cache = Arc::new(TestCache::new()); + let format = SsTableFormat { + block_size: 32, + min_filter_keys: 1, + ..SsTableFormat::default() + }; + let ts = Arc::new(TableStore::new( + ObjectStores::new(Arc::new(InMemory::new()), None), + format, + Path::from("/root"), + Some(cache.clone()), + TableStoreKind::Compactor, + BlockCachePolicy::default().with_compaction_output_targets(&[ + CacheTarget::data([b'b'; 16].as_slice()..=[b'c'; 16].as_slice()), + CacheTarget::Index, + ]), + )); + let id = SsTableId::Compacted(ulid::Ulid::new()); + // single-entry blocks for keys aa.., bb.., cc.., dd.. + let mut writer = ts.table_writer(id); + for i in 0..4 { + writer + .add(RowEntry::new_value(&[b'a' + i; 16], &[i; 16], 0)) + .await + .unwrap(); + } + + let handle = writer.close().await.unwrap(); + + let index_key: CachedKey = (id, handle.info.index_offset).into(); + let index = cache + .get_index(&index_key) + .await + .unwrap() + .unwrap() + .sst_index() + .unwrap(); + let block_metas = index.borrow().block_meta(); + assert_eq!(block_metas.len(), 4); + for (i, expected) in [false, true, true, false].into_iter().enumerate() { + let data_key: CachedKey = (id, block_metas.get(i).offset()).into(); + assert_eq!( + cache.get_block(&data_key).await.unwrap().is_some(), + expected + ); + } + } + + #[tokio::test] + async fn streaming_writer_should_cache_only_index_and_filters_for_compaction_output() { + let cache = Arc::new(TestCache::new()); + // The default policy requests only the index and filter + // compaction-output components; stats stay uncached, and data blocks + // are streamed out before close so they can never be inserted. + let ts = Arc::new(TableStore::new( + ObjectStores::new(Arc::new(InMemory::new()), None), + SsTableFormat::default(), + Path::from("/root"), + Some(cache.clone()), + TableStoreKind::Compactor, + BlockCachePolicy::default(), + )); + let id = SsTableId::Compacted(ulid::Ulid::new()); + let mut writer = ts.table_writer(id); + writer + .add(RowEntry::new_value(b"key", b"value", 0)) + .await + .unwrap(); + + let handle = writer.close().await.unwrap(); + + assert_eq!(cache.entry_count(), 2); + assert!(cache + .get_index(&(id, handle.info.index_offset).into()) + .await + .unwrap() + .is_some()); + assert!(cache + .get_filter(&(id, handle.info.filter_offset).into()) + .await + .unwrap() + .is_some()); + assert!(cache + .get_stats(&(id, handle.info.stats_offset).into()) + .await + .unwrap() + .is_none()); + } + #[allow(dead_code)] async fn assert_blocks(blocks: &VecDeque>, expected: &[(Vec, ValueDeletable)]) { let mut block_iter = blocks.iter(); @@ -2208,6 +2543,7 @@ mod tests { Path::from(ROOT), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); // Create id1, id2, and i3 as three random UUIDs that have been sorted ascending. @@ -2278,6 +2614,7 @@ mod tests { Path::from(ROOT), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let id1 = SsTableId::Wal(1); @@ -2352,6 +2689,7 @@ mod tests { Path::from(ROOT), None, TableStoreKind::Main, + BlockCachePolicy::default(), )) } @@ -2410,6 +2748,7 @@ mod tests { Path::from(ROOT), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); // Build an SST and compute expected bytes @@ -2418,7 +2757,7 @@ mod tests { let expected_bytes = sst.remaining_as_bytes(); // When writing via TableStore (should retry once) - ts.write_sst(&id, &sst, false).await.unwrap(); + ts.write_sst(&id, &sst).await.unwrap(); // Then: a retry happened assert!(flaky.put_attempts() >= 2); @@ -2448,6 +2787,7 @@ mod tests { Path::from(ROOT), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let id1 = SsTableId::Compacted(ulid::Ulid::new()); @@ -2493,6 +2833,7 @@ mod tests { Path::from(ROOT), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let id1 = SsTableId::Wal(123); @@ -2543,6 +2884,7 @@ mod tests { Path::from(ROOT), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let id = SsTableId::Compacted(ulid::Ulid::new()); let path = ts.path(&id); @@ -2568,6 +2910,7 @@ mod tests { Path::from(ROOT), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let id = SsTableId::Wal(42); let path = ts.path(&id); @@ -2592,7 +2935,7 @@ mod tests { let os = Arc::new(InMemory::new()); let format = SsTableFormat { block_size, ..SsTableFormat::default() }; let ts = Arc::new(TableStore::new(ObjectStores::new(os, None), - format, Path::from(ROOT), None, TableStoreKind::Main)); + format, Path::from(ROOT), None, TableStoreKind::Main, BlockCachePolicy::default())); if let Some(bytes) = block_size.checked_mul(num_blocks) { assert_eq!(num_blocks, ts.bytes_to_blocks(bytes)); } @@ -2629,6 +2972,7 @@ mod tests { Path::from(ROOT), None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = writer.table_builder(); builder @@ -2641,7 +2985,7 @@ mod tests { .unwrap(); let id = SsTableId::Compacted(ulid::Ulid::new()); let handle = writer - .write_sst(&id, &builder.build().await.unwrap(), false) + .write_sst(&id, &builder.build().await.unwrap()) .await .unwrap(); @@ -2664,6 +3008,7 @@ mod tests { Path::from(ROOT), Some(cache), TableStoreKind::Main, + BlockCachePolicy::default(), )); // when: task A starts reading the index; its loader will pause inside the @@ -2731,6 +3076,7 @@ mod tests { Path::from(ROOT), None, TableStoreKind::Main, + BlockCachePolicy::default(), ); let mut builder = writer.table_builder(); builder @@ -2743,7 +3089,7 @@ mod tests { .unwrap(); let id = SsTableId::Compacted(ulid::Ulid::new()); let handle = writer - .write_sst(&id, &builder.build().await.unwrap(), false) + .write_sst(&id, &builder.build().await.unwrap()) .await .unwrap(); @@ -2771,6 +3117,7 @@ mod tests { Path::from(ROOT), Some(cache), TableStoreKind::Main, + BlockCachePolicy::default(), )); // when: task A starts a single-block read; its loader will pause inside the @@ -2833,6 +3180,7 @@ mod tests { read_with_validation_retry, ObjectStoreCallTag, TableStoreKind, MAX_VALIDATION_RETRIES, }; use super::{Path, ROOT}; + use crate::block_cache_policy::BlockCachePolicy; use crate::db_state::{SsTableId, SstType}; use crate::error::{RetryReason, SlateDBError}; use crate::format::sst::SsTableFormat; @@ -2859,6 +3207,7 @@ mod tests { Path::from(ROOT), None, kind, + BlockCachePolicy::default(), )); (recording, ts) } @@ -2869,7 +3218,7 @@ mod tests { let (recording, ts) = recording_store(TableStoreKind::Reader); let encoded = build_test_sst(&format(), 4).await; let id = SsTableId::Compacted(ulid::Ulid::new()); - let handle = ts.write_sst(&id, &encoded, false).await.unwrap(); + let handle = ts.write_sst(&id, &encoded).await.unwrap(); recording.clear(); ts.read_index(&handle, false).await.unwrap(); @@ -2899,7 +3248,7 @@ mod tests { let (recording, ts) = recording_store(TableStoreKind::Compactor); let encoded = build_test_sst(&format(), 1).await; let id = SsTableId::Compacted(ulid::Ulid::new()); - ts.write_sst(&id, &encoded, false).await.unwrap(); + ts.write_sst(&id, &encoded).await.unwrap(); recording.clear(); ts.metadata(&id).await.unwrap(); @@ -2922,7 +3271,7 @@ mod tests { let (recording, ts) = recording_store(TableStoreKind::Main); let encoded = build_test_sst(&format(), 1).await; let id = SsTableId::Wal(1); - ts.write_sst(&id, &encoded, false).await.unwrap(); + ts.write_sst(&id, &encoded).await.unwrap(); let kinds = recording.write_kinds(); let sst_types = recording.write_sst_types(); @@ -3028,7 +3377,7 @@ mod tests { let (recording, ts) = recording_store(TableStoreKind::Compactor); let encoded = build_test_sst(&format(), 4).await; let id = SsTableId::Compacted(ulid::Ulid::new()); - ts.write_sst(&id, &encoded, false).await.unwrap(); + ts.write_sst(&id, &encoded).await.unwrap(); let kinds = recording.write_kinds(); let sst_types = recording.write_sst_types(); diff --git a/slatedb/src/utils.rs b/slatedb/src/utils.rs index ab68ea658..3d647ef27 100644 --- a/slatedb/src/utils.rs +++ b/slatedb/src/utils.rs @@ -1109,7 +1109,7 @@ mod tests { .unwrap(); let encoded_sst = sst_builder.build().await.unwrap(); let _sst1 = table_store - .write_sst(&SsTableId::Compacted(Ulid::new()), &encoded_sst, false) + .write_sst(&SsTableId::Compacted(Ulid::new()), &encoded_sst) .await .unwrap(); @@ -1124,7 +1124,7 @@ mod tests { .unwrap(); let encoded_sst = sst_builder.build().await.unwrap(); let sst2 = table_store - .write_sst(&SsTableId::Compacted(Ulid::new()), &encoded_sst, false) + .write_sst(&SsTableId::Compacted(Ulid::new()), &encoded_sst) .await .unwrap(); @@ -1162,7 +1162,7 @@ mod tests { .unwrap(); let encoded_sst = sst_builder.build().await.unwrap(); let sst = table_store - .write_sst(&SsTableId::Compacted(Ulid::new()), &encoded_sst, false) + .write_sst(&SsTableId::Compacted(Ulid::new()), &encoded_sst) .await .unwrap(); diff --git a/slatedb/src/wal_buffer.rs b/slatedb/src/wal_buffer.rs index be96aea0b..d7614218b 100644 --- a/slatedb/src/wal_buffer.rs +++ b/slatedb/src/wal_buffer.rs @@ -513,7 +513,7 @@ impl WalFlushHandler { let encoded_sst = sst_builder.build().await?; let written_bytes = encoded_sst.remaining_len() as u64; self.table_store - .write_sst(&SsTableId::Wal(wal_id), &encoded_sst, false) + .write_sst(&SsTableId::Wal(wal_id), &encoded_sst) .await?; self.stats.flush_bytes.increment(written_bytes); Ok(()) @@ -672,6 +672,7 @@ pub mod stats { #[cfg(test)] mod tests { use super::*; + use crate::block_cache_policy::BlockCachePolicy; use crate::db_status::DbStatusManager; use crate::format::sst::SsTableFormat; use crate::iter::RowEntryIterator; @@ -899,6 +900,7 @@ mod tests { Path::from("/root"), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let test_clock = Arc::new(MockSystemClock::new()); let system_clock = Arc::new(DefaultSystemClock::new()); @@ -1093,6 +1095,7 @@ mod tests { Path::from("/root"), None, TableStoreKind::Main, + BlockCachePolicy::default(), )); let system_clock = Arc::new(DefaultSystemClock::new()); let status_manager = DbStatusManager::new(0); diff --git a/slatedb/src/wal_reader.rs b/slatedb/src/wal_reader.rs index 76483126d..cdd3eb218 100644 --- a/slatedb/src/wal_reader.rs +++ b/slatedb/src/wal_reader.rs @@ -71,6 +71,7 @@ use std::sync::Arc; use object_store::path::Path; use object_store::ObjectStore; +use crate::block_cache_policy::BlockCachePolicy; use crate::db_state::SsTableId; use crate::format::sst::SsTableFormat; use crate::iter::{EmptyIterator, RowEntryIterator}; @@ -200,6 +201,7 @@ impl WalReader { path.into(), None, TableStoreKind::Reader, + BlockCachePolicy::default(), )); Self { table_store } } diff --git a/slatedb/src/wal_replay.rs b/slatedb/src/wal_replay.rs index 3b6741ee3..c98277ab9 100644 --- a/slatedb/src/wal_replay.rs +++ b/slatedb/src/wal_replay.rs @@ -275,6 +275,7 @@ impl WalReplayIterator { #[cfg(test)] mod tests { use super::{WalReplayIterator, WalReplayOptions}; + use crate::block_cache_policy::BlockCachePolicy; use crate::bytes_range::BytesRange; use crate::db_state::SsTableId; use crate::format::sst::SsTableFormat; @@ -368,7 +369,7 @@ mod tests { builder.add(row.clone()).await.unwrap(); let encoded_sst = builder.build().await.unwrap(); table_store - .write_sst(&SsTableId::Wal(2), &encoded_sst, false) + .write_sst(&SsTableId::Wal(2), &encoded_sst) .await .unwrap(); @@ -500,7 +501,7 @@ mod tests { } let encoded_sst = builder.build().await.unwrap(); table_store - .write_sst(&SsTableId::Wal(wal_id as u64 + 1), &encoded_sst, false) + .write_sst(&SsTableId::Wal(wal_id as u64 + 1), &encoded_sst) .await .unwrap(); } @@ -567,7 +568,7 @@ mod tests { } let encoded_sst = builder.build().await.unwrap(); table_store - .write_sst(&SsTableId::Wal(1), &encoded_sst, false) + .write_sst(&SsTableId::Wal(1), &encoded_sst) .await .unwrap(); @@ -626,7 +627,7 @@ mod tests { } let encoded_sst = builder.build().await.unwrap(); table_store - .write_sst(&SsTableId::Wal(1), &encoded_sst, false) + .write_sst(&SsTableId::Wal(1), &encoded_sst) .await .unwrap(); @@ -766,6 +767,7 @@ mod tests { path, None, TableStoreKind::Main, + BlockCachePolicy::default(), )) } diff --git a/website/src/content/docs/docs/design/caching.mdx b/website/src/content/docs/docs/design/caching.mdx index e375be750..11a1e7c4e 100644 --- a/website/src/content/docs/docs/design/caching.mdx +++ b/website/src/content/docs/docs/design/caching.mdx @@ -31,6 +31,37 @@ Reads consult the block cache first. On a miss there, SlateDB fetches bytes thro The defaults reflect the usual access patterns. Point reads default to `cache_blocks = true` because they are more likely to revisit hot data. Scans default to `cache_blocks = false` so a long sequential read does not fill the cache with data blocks that probably will not be reused soon. Scans can still benefit from entries that are already hot. Internal tasks follow the same idea: WAL replay and compaction read SSTs without populating the foreground cache. +Writes fill the block cache as well. [`BlockCachePolicy`](https://docs.rs/slatedb/latest/slatedb/struct.BlockCachePolicy.html) selects which components of an SST are inserted as that SST is written, and you install it with [`DbBuilder::with_block_cache_policy`](https://docs.rs/slatedb/latest/slatedb/db/struct.DbBuilder.html#method.with_block_cache_policy). The policy applies to the block cache only. Admission into the object-store cache is controlled separately, as described below. + +The policy holds one target list per write source. The default inserts data blocks, the index, and the filters after a memtable flush, and inserts only the index and the filters as compaction output is written, so compaction does not evict blocks that reads have made hot. Replace either list with [`BlockCachePolicy::with_flush_targets`](https://docs.rs/slatedb/latest/slatedb/struct.BlockCachePolicy.html#method.with_flush_targets) or [`BlockCachePolicy::with_compaction_output_targets`](https://docs.rs/slatedb/latest/slatedb/struct.BlockCachePolicy.html#method.with_compaction_output_targets). An empty list disables insertion for that source. + +Each list holds [`CacheTarget`](https://docs.rs/slatedb/latest/slatedb/db_cache/enum.CacheTarget.html) values: + +- `CacheTarget::Filters` inserts the SST filter blocks, if any exist. +- `CacheTarget::Index` inserts the SST index. +- `CacheTarget::Stats` inserts the SST stats block, if one exists. +- `CacheTarget::data(range)` inserts the data blocks whose key span overlaps `range`. + +`CacheTarget::data` accepts any key range, so `CacheTarget::data::<&[u8], _>(..)` selects every data block, while `CacheTarget::data(b"user:".as_slice()..b"user;".as_slice())` selects only the blocks covering that key range. This policy caches all four components after a flush and nothing after compaction: + +```rust +use slatedb::{BlockCachePolicy, CacheTarget, Db}; + +let policy = BlockCachePolicy::default() + .with_flush_targets(&[ + CacheTarget::data::<&[u8], _>(..), + CacheTarget::Index, + CacheTarget::Filters, + CacheTarget::Stats, + ]) + .with_compaction_output_targets(&[]); + +let db = Db::builder(path, object_store) + .with_block_cache_policy(policy) + .build() + .await?; +``` + The disk cache stays disabled unless `object_store_cache_options.root_folder` is set. If you want it warm before serving traffic, you can preload it on startup with [`PreloadLevel::L0Sst`](https://docs.rs/slatedb/latest/slatedb/config/enum.PreloadLevel.html#variant.L0Sst) or [`PreloadLevel::AllSst`](https://docs.rs/slatedb/latest/slatedb/config/enum.PreloadLevel.html#variant.AllSst). SlateDB loads recent SSTs, or all SSTs, into the local cache until the cache size limit is reached. By default, writes go straight to the upstream object store and do not populate the object-store cache. Each SST write source has its own admission flag: [`cache_on_flush`](https://docs.rs/slatedb/latest/slatedb/config/struct.ObjectStoreCacheOptions.html#structfield.cache_on_flush) stores SSTs written by memtable flushes locally, and [`cache_on_compaction`](https://docs.rs/slatedb/latest/slatedb/config/struct.ObjectStoreCacheOptions.html#structfield.cache_on_compaction) does the same for compaction output. Enabling them can help if readers are likely to touch freshly written SSTs soon afterward. WAL and manifest writes are never cached. From bb3c30edef219fe0d35a49380a27fedefd9b4bd1 Mon Sep 17 00:00:00 2001 From: nomiero Date: Mon, 27 Jul 2026 10:38:15 -0700 Subject: [PATCH 42/63] Make object store cache pluggable and external (#1947) --- slatedb/src/cached_object_store/mod.rs | 2 +- .../src/cached_object_store/object_store.rs | 305 ++++++++++++++++-- slatedb/src/clone.rs | 23 +- slatedb/src/config.rs | 15 +- slatedb/src/db.rs | 117 +++++-- slatedb/src/db/builder.rs | 239 ++++++++------ slatedb/src/db_reader.rs | 19 +- slatedb/src/error.rs | 12 +- slatedb/src/garbage_collector.rs | 52 +-- slatedb/src/instrumented_object_store.rs | 7 + slatedb/src/lib.rs | 3 +- slatedb/src/manifest/mod.rs | 4 + .../src/memtable_flusher/manifest_writer.rs | 2 +- slatedb/src/memtable_flusher/tracker.rs | 2 +- slatedb/src/memtable_flusher/uploader.rs | 2 +- slatedb/src/object_store_tag.rs | 65 ++-- slatedb/src/paths.rs | 42 ++- slatedb/src/sst_builder.rs | 4 +- slatedb/src/sst_reader.rs | 2 +- slatedb/src/tablestore.rs | 4 +- slatedb/src/utils.rs | 4 +- .../src/content/docs/docs/design/caching.mdx | 36 ++- .../content/docs/docs/operations/metrics.mdx | 14 +- 23 files changed, 732 insertions(+), 243 deletions(-) diff --git a/slatedb/src/cached_object_store/mod.rs b/slatedb/src/cached_object_store/mod.rs index b2ba68c6f..6911fb239 100644 --- a/slatedb/src/cached_object_store/mod.rs +++ b/slatedb/src/cached_object_store/mod.rs @@ -1,4 +1,4 @@ -pub(crate) use object_store::CachedObjectStore; +pub use object_store::{CachedObjectStore, CachedObjectStoreBuilder}; #[allow(unused_imports)] pub use storage::{LocalCacheEntry, LocalCacheHead, LocalCacheStorage, PartID}; pub use storage_fs::FsCacheStorage; diff --git a/slatedb/src/cached_object_store/object_store.rs b/slatedb/src/cached_object_store/object_store.rs index 7e521951d..216a408a3 100644 --- a/slatedb/src/cached_object_store/object_store.rs +++ b/slatedb/src/cached_object_store/object_store.rs @@ -15,7 +15,7 @@ use object_store::{ PutResult, RenameOptions, }; use object_store::{ListResult, MultipartUpload, PutOptions, PutPayload}; -use slatedb_common::clock::SystemClock; +use slatedb_common::clock::{DefaultSystemClock, SystemClock}; use slatedb_common::DbRand; use std::{ops::Range, sync::Arc}; @@ -23,13 +23,38 @@ use crate::single_flight::SingleFlight; use crate::cached_object_store::storage::{LocalCacheStorage, PartID}; use crate::error::SlateDBError; +use crate::utils::build_concurrent; use log::warn; -use crate::utils::build_concurrent; -use slatedb_common::metrics::MetricsRecorderHelper; +use slatedb_common::metrics::{ + MetricLevel, MetricsRecorder, MetricsRecorderHelper, NoopMetricsRecorder, +}; +/// An [`ObjectStore`] wrapper that caches object parts on local disk. +/// +/// The cache splits each object into fixed-size parts and stores them under a +/// root folder. +/// +/// Reads tagged by SlateDB as compacted SST reads are served from +/// disk when present and admitted on a miss. +/// +/// Writes can optionally be admitted via +/// [`CachedObjectStoreBuilder::with_cache_on_flush`] and +/// [`CachedObjectStoreBuilder::with_cache_on_compaction`]. +/// +/// All other calls (manifests, WAL, listings) pass through to the wrapped store. +/// +/// Construct it over the raw backend and pass it to SlateDB as the object +/// store itself: +/// +/// ```ignore +/// let cache = CachedObjectStore::builder("/var/slatedb-cache", backend) +/// .build() +/// .await?; +/// let db = Db::builder(path, cache).build().await?; +/// ``` #[derive(Debug, Clone)] -pub(crate) struct CachedObjectStore { +pub struct CachedObjectStore { object_store: Arc, part_size_bytes: usize, // expected to be aligned with mb or kb pub(crate) cache_storage: Arc, @@ -94,23 +119,6 @@ impl CachedObjectStore { })) } - /// Returns a new handle that reads through the new `object_store` on cache - /// misses while sharing everything else (all fields other than the - /// object_store in the cache are shared by ref-count clones). - /// - /// This lets a component with its own instrumented store (for example - /// the compactor) share the cache while keeping its I/O recorded under - /// its own metric labels. - pub(crate) fn clone_with_new_object_store( - &self, - object_store: Arc, - ) -> Arc { - Arc::new(Self { - object_store, - ..self.clone() - }) - } - pub(crate) async fn start_evictor(&self) { self.cache_storage.start_evictor().await; } @@ -153,14 +161,37 @@ impl CachedObjectStore { Ok(Some(cached)) } - /// Load files into cache up to a maximum number of bytes. - /// This method fetches objects from the provided paths and stores them in the cache - /// until the specified max_bytes limit is reached. - pub(crate) async fn load_files_to_cache( + /// Returns a builder for a `CachedObjectStore` that caches parts of the + /// objects in `object_store` under `root_folder` on the local filesystem. + pub fn builder( + root_folder: impl Into, + object_store: Arc, + ) -> CachedObjectStoreBuilder { + CachedObjectStoreBuilder { + object_store, + options: ObjectStoreCacheOptions { + root_folder: Some(root_folder.into()), + ..ObjectStoreCacheOptions::default() + }, + metrics_recorder: Arc::new(NoopMetricsRecorder::new()), + metric_level: MetricLevel::default(), + } + } + + /// Loads files into the cache up to a maximum number of bytes. + /// + /// Fetches each object's raw bytes from the wrapped store and saves them + /// as cache parts on disk. Can be used to warm up the cache. + /// + /// The `max_bytes` budget is applied in path order: loading stops at the + /// first file that does not fit, so order paths by priority. + /// + /// Fetches are best-effort and failures are logged and skipped. + pub async fn load_files_to_cache( &self, file_paths: Vec, max_bytes: usize, - ) -> Result<(), SlateDBError> { + ) -> Result<(), crate::Error> { if file_paths.is_empty() || max_bytes == 0 { return Ok(()); } @@ -547,17 +578,50 @@ impl CachedObjectStore { .get_opts( &location, GetOptions { - range: Some(GetRange::Bounded(part_range)), + range: Some(GetRange::Bounded(part_range.clone())), ..Default::default() }, ) .await?; - // Save the head and the part to cache for future accesses. - let entry = this.cache_storage.entry(&location, this.part_size_bytes); let meta = get_result.meta.clone(); let attrs = get_result.attributes.clone(); let bytes = get_result.bytes().await?; + + // A truncated but successful ranged body is possible and + // should be caught here because the rest of the code + // assumes the size is correct. + // + // We return a retryable error before anything is saved or + // sliced and the retry layer above will retry. + // + // We also have to take the min of the part range end and + // the object size because the part range may extend beyond + // the object size. + let expected_len = usize::try_from( + meta.size + .min(part_range.end) + .saturating_sub(part_range.start), + ) + .expect("part length exceeds usize"); + if bytes.len() != expected_len { + return Err(object_store::Error::Generic { + store: "cached_object_store", + source: format!( + "part fetch size check failed: {} bytes read, but expected \ + {expected_len} bytes (part range {}..{} truncated at object \ + size {})", + bytes.len(), + part_range.start, + part_range.end, + meta.size + ) + .into(), + }); + } + + // Save the head and the part to cache for future accesses. + let entry = this.cache_storage.entry(&location, this.part_size_bytes); entry.save_head((&meta, &attrs)).await.ok(); entry.save_part(part_id, bytes.clone()).await.ok(); @@ -652,6 +716,100 @@ impl CachedObjectStore { } } +/// Builder for [`CachedObjectStore`]. Created by [`CachedObjectStore::builder`]. +pub struct CachedObjectStoreBuilder { + object_store: Arc, + options: ObjectStoreCacheOptions, + metrics_recorder: Arc, + metric_level: MetricLevel, +} + +impl CachedObjectStoreBuilder { + /// Sets the limit of the cache size in bytes. + /// + /// `None` disables eviction and the default is 16gb. + pub fn with_max_cache_size_bytes(mut self, max_cache_size_bytes: Option) -> Self { + self.options.max_cache_size_bytes = max_cache_size_bytes; + self + } + + /// Sets the size of each part file. Must be a multiple of 1kb. + /// + /// The default is 4mb. + pub fn with_part_size_bytes(mut self, part_size_bytes: usize) -> Self { + self.options.part_size_bytes = part_size_bytes; + self + } + + /// Sets whether compacted SSTs produced by memtable flushes are admitted + /// to the cache on write. + /// + /// The default is false. + pub fn with_cache_on_flush(mut self, cache_on_flush: bool) -> Self { + self.options.cache_on_flush = cache_on_flush; + self + } + + /// Sets whether compacted SSTs produced by compaction are admitted to the + /// cache on write. + /// + /// The default is false. + pub fn with_cache_on_compaction(mut self, cache_on_compaction: bool) -> Self { + self.options.cache_on_compaction = cache_on_compaction; + self + } + + /// Sets the interval at which the cache directory is scanned to rebuild + /// the evictor's in-memory map. + /// + /// `None` scans only once on startup and the default is 1 hour. + pub fn with_scan_interval(mut self, scan_interval: Option) -> Self { + self.options.scan_interval = scan_interval; + self + } + + /// Sets the maximum number of open file handles kept by the part file + /// handle cache. + /// + /// The default is 1000. + pub fn with_max_open_file_handles(mut self, max_open_file_handles: usize) -> Self { + self.options.max_open_file_handles = max_open_file_handles; + self + } + + /// Sets the recorder for the cache's metrics (hit and access counters, + /// cache size gauges, eviction counters). + /// + /// Defaults to a no-op recorder. + pub fn with_metrics_recorder(mut self, metrics_recorder: Arc) -> Self { + self.metrics_recorder = metrics_recorder; + self + } + + /// Sets the metric level for the cache's metrics. + /// + /// Defaults to [`MetricLevel::default`]. + pub fn with_metric_level(mut self, metric_level: MetricLevel) -> Self { + self.metric_level = metric_level; + self + } + + /// Builds the `CachedObjectStore` and starts its evictor. + pub async fn build(self) -> Result, crate::Error> { + let recorder = MetricsRecorderHelper::new(self.metrics_recorder, self.metric_level); + let cached = CachedObjectStore::from_config( + self.object_store, + &self.options, + &recorder, + Arc::new(DefaultSystemClock::new()), + Arc::new(DbRand::default()), + ) + .await + .map_err(crate::Error::from)?; + Ok(cached.expect("builder always sets root_folder")) + } +} + fn head_only_get_result( meta: ObjectMeta, attributes: Attributes, @@ -1019,7 +1177,10 @@ mod tests { use crate::cached_object_store::storage_fs::FsCacheEntry; use crate::cached_object_store::storage_fs::FsCacheStorage; use crate::db_state::SstType; + use crate::instrumented_object_store::{InstrumentedObjectStore, ObjectStoreComponent}; use crate::object_store_tag::{ObjectStoreCallTag, TableStoreKind}; + use crate::object_stores::ObjectStoreType; + use crate::retrying_object_store::RetryingObjectStore; use crate::test_utils::{ gen_rand_bytes, ExtensionMarker, ExtensionObjectStore, FlakyObjectStore, GatedObjectStore, }; @@ -2050,6 +2211,92 @@ mod tests { assert_eq!(&bytes[..], &payload[..512]); } + #[tokio::test] + async fn test_part_fetch_validates_truncated_body_and_retries() { + let part_size = 1024usize; + let payload = gen_rand_bytes(part_size * 3); + let location = Path::from("/data/testfile1"); + let inner: Arc = Arc::new(object_store::memory::InMemory::new()); + inner + .put(&location, PutPayload::from_bytes(payload.clone())) + .await + .unwrap(); + + let test_cache_folder = new_test_cache_folder(); + let recorder = MetricsRecorderHelper::noop(); + let stats = Arc::new(CachedObjectStoreStats::new(&recorder)); + let cache_storage = Arc::new(FsCacheStorage::new( + test_cache_folder.clone(), + None, + None, + stats.clone(), + Arc::new(DefaultSystemClock::new()), + Arc::new(DbRand::default()), + 1000, + )); + let opts = || GetOptions { + range: Some(GetRange::Bounded(0..(part_size as u64 * 3))), + extensions: ObjectStoreCallTag::new(TableStoreKind::Main, SstType::Compacted).into(), + ..Default::default() + }; + + // Prefill the cache through a clean handle, then delete one part file + // so the read below must fill it from the backend lazily. + let prefill = CachedObjectStore::new( + inner.clone(), + cache_storage.clone(), + part_size, + CachePutConfig::default(), + stats.clone(), + ) + .unwrap(); + prefill + .get_opts(&location, opts()) + .await + .unwrap() + .bytes() + .await + .unwrap(); + let part_path = + FsCacheEntry::make_part_path(test_cache_folder.clone(), &location, 1, part_size); + std::fs::remove_file(&part_path).unwrap(); + + // The backend truncates the next ranged body to 1 byte but reports + // success, mimicking a response cut mid-body without a stream error. + let flaky = Arc::new(FlakyObjectStore::new(inner, 0).with_truncate_get_range_bytes(1, 1)); + let cached = CachedObjectStore::new( + flaky.clone(), + cache_storage, + part_size, + CachePutConfig::default(), + stats, + ) + .unwrap(); + let instrumented = Arc::new(InstrumentedObjectStore::new( + cached, + &recorder, + ObjectStoreComponent::Db, + ObjectStoreType::Main, + )); + let retrying = RetryingObjectStore::new( + instrumented, + Arc::new(DbRand::default()), + Arc::new(DefaultSystemClock::new()), + None, + ); + + let got = retrying + .get_opts(&location, opts()) + .await + .unwrap() + .bytes() + .await + .unwrap(); + assert_eq!(got, payload); + // The truncated fill plus the successful fill on the reissued read. + assert_eq!(flaky.get_range_attempts(), 2); + } + #[rstest::rstest] #[case::no_evictor_cached(false, true)] #[case::with_evictor_cached(true, true)] diff --git a/slatedb/src/clone.rs b/slatedb/src/clone.rs index b14c5e5da..7f8a03644 100644 --- a/slatedb/src/clone.rs +++ b/slatedb/src/clone.rs @@ -344,10 +344,10 @@ async fn validate_no_data_wal( continue; } - let path_resolver = PathResolver::new(source.path.clone()); + let path_resolver = PathResolver::from_root(source.path.clone()); let mut has_data_wal = false; for wal_id in (core.replay_after_wal_id + 1)..core.next_wal_sst_id { - let path = path_resolver.table_path(&SsTableId::Wal(wal_id)); + let path = path_resolver.sst_path(&SsTableId::Wal(wal_id)); match wal_object_store.head(&path).await { Ok(meta) => { // Fence WALs are zero-byte `SsTableId::Wal` objects (written via @@ -462,8 +462,8 @@ async fn copy_wal_ssts( clone_path: &Path, #[allow(unused)] fp_registry: Arc, ) -> Result<(), SlateDBError> { - let parent_path_resolver = PathResolver::new(parent_path.clone()); - let clone_path_resolver = PathResolver::new(clone_path.clone()); + let parent_path_resolver = PathResolver::from_root(parent_path.clone()); + let clone_path_resolver = PathResolver::from_root(clone_path.clone()); let mut wal_id = parent_checkpoint_state.replay_after_wal_id + 1; while wal_id < parent_checkpoint_state.next_wal_sst_id { @@ -472,8 +472,8 @@ async fn copy_wal_ssts( )); let id = SsTableId::Wal(wal_id); - let parent_path = parent_path_resolver.table_path(&id); - let clone_path = clone_path_resolver.table_path(&id); + let parent_path = parent_path_resolver.sst_path(&id); + let clone_path = clone_path_resolver.sst_path(&id); object_store .as_ref() .copy(&parent_path, &clone_path) @@ -1317,8 +1317,8 @@ mod tests { manifest.manifest.core.replay_after_wal_id + 1 < manifest.manifest.core.next_wal_sst_id, "expected cloned state to retain WAL-only SSTs" ); - let expected_missing_wal_path = PathResolver::new(Path::from(parent_path)) - .table_path(&SsTableId::Wal( + let expected_missing_wal_path = PathResolver::from_root(Path::from(parent_path)) + .sst_path(&SsTableId::Wal( manifest.manifest.core.replay_after_wal_id + 1, )) .to_string(); @@ -1997,7 +1997,8 @@ mod tests { // Plant the WAL object directly in the object store at the resolved path. use object_store::ObjectStoreExt; - let wal_path = PathResolver::new(path.clone()).table_path(&SsTableId::Wal(planted_wal_id)); + let wal_path = + PathResolver::from_root(path.clone()).sst_path(&SsTableId::Wal(planted_wal_id)); object_store.put(&wal_path, wal_bytes.into()).await.unwrap(); planted_wal_id @@ -2183,8 +2184,8 @@ mod tests { .await; build_plain_wal_disabled_parent(&parent_path_b, object_store.clone(), &table_b).await; - let expected_missing_wal_path = PathResolver::new(parent_path_a.clone()) - .table_path(&SsTableId::Wal({ + let expected_missing_wal_path = PathResolver::from_root(parent_path_a.clone()) + .sst_path(&SsTableId::Wal({ let manifest_store = Arc::new(ManifestStore::new(&parent_path_a, object_store.clone())); let sm = StoredManifest::load(manifest_store, system_clock.clone()) diff --git a/slatedb/src/config.rs b/slatedb/src/config.rs index 860c9800e..9e1ab4640 100644 --- a/slatedb/src/config.rs +++ b/slatedb/src/config.rs @@ -742,7 +742,14 @@ pub struct Settings { /// The compression algorithm to use for SSTables. pub compression_codec: Option, - /// The object store cache options. + /// The object store cache options. When `root_folder` is set, the database + /// wraps its main object store in a + /// [`CachedObjectStore`](crate::cached_object_store::CachedObjectStore) + /// built from these options. To construct and share the cache yourself, + /// build one with + /// [`CachedObjectStore::builder`](crate::cached_object_store::CachedObjectStore::builder) + /// and pass it to [`Db::builder`](crate::Db::builder) instead, leaving + /// these options unset. pub object_store_cache_options: ObjectStoreCacheOptions, /// Configuration options for the garbage collector. @@ -1719,11 +1726,10 @@ where #[cfg(test)] mod tests { + use super::*; use std::collections::HashMap; use std::path::PathBuf; - use super::*; - #[test] fn test_db_options_load_from_env() { figment::Jail::expect_with(|jail| { @@ -1740,7 +1746,6 @@ mod tests { Some(PathBuf::from("/tmp/slatedb-root")), options.object_store_cache_options.root_folder ); - Ok(()) }); } @@ -1810,7 +1815,7 @@ mod tests { { "flush_interval": "1s", "metric_level": "Debug", - "object_store_cache_options": { + "object_store_cache_options": { "root_folder": "/tmp/slatedb-root" } } diff --git a/slatedb/src/db.rs b/slatedb/src/db.rs index a883bce36..cdf81bb46 100644 --- a/slatedb/src/db.rs +++ b/slatedb/src/db.rs @@ -11145,12 +11145,16 @@ mod tests { mod object_store_cache { use super::*; use crate::cached_object_store::stats::{PART_ACCESS_COUNT, PART_HIT_COUNT}; + use crate::cached_object_store::CachedObjectStore; use object_store::ObjectStoreExt; /// Fixture for the object store cache tests. struct ObjectStoreCacheTest { db: Db, upstream: Arc, + /// The typed handle to the cache passed to the db as its object + /// store; `None` when built `without_object_store_cache`. + cache: Option>, cache_root: std::path::PathBuf, db_path: String, should_compact: Option>, @@ -11251,15 +11255,30 @@ mod tests { .unwrap(); let cache_root = temp_dir.keep(); - let mut opts = test_db_options(0, l0_sst_size_bytes, None); - opts.object_store_cache_options.root_folder = - object_store_cache.then(|| cache_root.clone()); - opts.object_store_cache_options.part_size_bytes = part_size; - opts.object_store_cache_options.cache_on_flush = cache_on_flush; - opts.object_store_cache_options.cache_on_compaction = cache_on_compaction; + let opts = test_db_options(0, l0_sst_size_bytes, None); + + // The cache is user-constructed and passed to the db as the + // object store itself. + let cache = if object_store_cache { + Some( + CachedObjectStore::builder(cache_root.clone(), upstream.clone()) + .with_part_size_bytes(part_size) + .with_cache_on_flush(cache_on_flush) + .with_cache_on_compaction(cache_on_compaction) + .build() + .await + .unwrap(), + ) + } else { + None + }; + let main_store: Arc = match &cache { + Some(cache) => cache.clone(), + None => upstream.clone(), + }; let mut builder = - Db::builder(db_path.as_str(), upstream.clone()).with_settings(opts); + Db::builder(db_path.as_str(), main_store.clone()).with_settings(opts); if let Some(recorder) = metrics_recorder { builder = builder.with_metrics_recorder(recorder); } @@ -11269,12 +11288,14 @@ mod tests { let scheduler = Arc::new(OnDemandCompactionSchedulerSupplier::new(Arc::new( move |_state| flag_clone.swap(false, Ordering::SeqCst), ))); - // A different Arc over the same storage; open gates make - // GatedObjectStore a pass-through. + // A custom compactor store bypasses the cache entirely; + // otherwise the compactor shares the db's (possibly + // cached) store. Open gates make GatedObjectStore a + // pass-through. let compactor_store: Arc = if custom_compactor_store { Arc::new(GatedObjectStore::new(upstream.clone())) } else { - upstream.clone() + main_store.clone() }; // One subcompaction writes one output SST, keeping exact // part counts deterministic. @@ -11296,6 +11317,7 @@ mod tests { ObjectStoreCacheTest { db, upstream, + cache, cache_root, db_path, should_compact, @@ -11371,7 +11393,7 @@ mod tests { /// The upstream path of a compacted SST id. fn compacted_sst_path(&self, id: &SsTableId) -> object_store::path::Path { - self.sub_path(&format!("compacted/{}.sst", id.unwrap_compacted_id())) + crate::paths::PathResolver::from_root(self.db_path.as_str()).sst_path(id) } fn l0_ids(&self) -> Vec { @@ -11486,7 +11508,7 @@ mod tests { } #[tokio::test] - async fn test_db_records_remote_object_store_reads_but_not_cache_hits() { + async fn test_db_records_read_calls_into_cached_object_store() { let object_store: Arc = Arc::new(InMemory::new()); let mut opts = test_db_options(0, 1024, None); let temp_dir = tempfile::Builder::new() @@ -11494,15 +11516,18 @@ mod tests { .tempdir() .unwrap(); - opts.object_store_cache_options.root_folder = Some(temp_dir.keep()); - opts.object_store_cache_options.part_size_bytes = 1024; opts.manifest_poll_interval = Duration::from_secs(3600); let metrics_recorder = Arc::new(DefaultMetricsRecorder::new()); - let path = "/tmp/test_db_records_remote_object_store_reads_but_not_cache_hits"; + let path = "/tmp/test_db_records_read_calls_into_cached_object_store"; + let cached_store = CachedObjectStore::builder(temp_dir.keep(), object_store) + .with_part_size_bytes(1024) + .build() + .await + .unwrap(); // Disable the in-memory block cache so reads reach the object store // cache layer (the subject of this test) instead of being served from // decoded blocks in memory. - let kv_store = Db::builder(path, object_store) + let kv_store = Db::builder(path, cached_store) .with_settings(opts) .with_db_cache_disabled() .with_metrics_recorder(metrics_recorder.clone()) @@ -11528,19 +11553,67 @@ mod tests { let requests_after_second = lookup_object_store_op_request_count(&metrics_recorder, "db", "main", "get"); - // The cold read misses the object store cache and fetches the SST part - // from the remote store; the warm read hits the cache and issues no - // remote request. - assert_eq!(requests_after_first, requests_before + 1); + // The instrumented store sits above the object store cache and counts + // logical read calls whether they are served from the cache or the + // remote store. + // A point get reads the single-part SST in three sub-ranges (index, + // filter and block). + assert_eq!(requests_after_first, requests_before + 3); assert_eq!(got, Some(Bytes::from_static(b"test_value"))); - assert_eq!(requests_after_second, requests_after_first); + assert_eq!(requests_after_second, requests_after_first + 3); assert_eq!( lookup_object_store_op_histogram_count(&metrics_recorder, "db", "main", "get"), - requests_after_first as u64 + requests_after_second as u64 ); kv_store.close().await.unwrap(); } + /// Warming a disk cache by enumerating SSTs from the manifest, + /// resolving their paths with `PathResolver`, and loading their raw + /// bytes with `load_files_to_cache`. + #[tokio::test] + async fn test_preload_disk_cache_from_manifest() { + let fixture = + ObjectStoreCacheTest::builder("/tmp/test_preload_disk_cache_from_manifest") + .build() + .await; + + // Two flushed L0 SSTs, not admitted on write. + for (key, value) in [(b"k1", b"v1"), (b"k2", b"v2")] { + fixture.db().put(key, value).await.unwrap(); + fixture.db().flush().await.unwrap(); + fixture + .db() + .flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .unwrap(); + } + + let ids = fixture.l0_ids(); + assert_eq!(ids.len(), 2); + let paths: Vec<_> = ids + .iter() + .map(|id| fixture.compacted_sst_path(id)) + .collect(); + for path in &paths { + fixture.assert_cached(path, 0); + } + + let cache = fixture.cache.as_ref().unwrap(); + cache + .load_files_to_cache(paths.clone(), usize::MAX) + .await + .unwrap(); + + // Each small SST fits in a single 1 KiB part. + for path in &paths { + fixture.assert_cached(path, 1); + } + fixture.close().await; + } + /// A flushed L0 SST is a compacted SST written by the main store, so /// cache_on_flush admits it. The manifest (untagged) and the WAL /// (skipped by policy) are never cached. diff --git a/slatedb/src/db/builder.rs b/slatedb/src/db/builder.rs index 1a8bd1070..2e52b775a 100644 --- a/slatedb/src/db/builder.rs +++ b/slatedb/src/db/builder.rs @@ -197,6 +197,11 @@ pub struct DbBuilder> { impl> DbBuilder

{ /// Creates a new builder for a database at the given path. + /// + /// `main_object_store` may be any [`ObjectStore`], including a wrapper + /// like + /// [`CachedObjectStore`](crate::cached_object_store::CachedObjectStore) + /// built over the raw backend to serve SST reads from a local disk cache. pub fn new(path: P, main_object_store: Arc) -> Self { Self { path, @@ -434,27 +439,51 @@ impl> DbBuilder

{ let recorder = MetricsRecorderHelper::new(self.metrics_recorder, self.settings.metric_level); let max_retries = self.settings.object_store_max_retries; - let retrying_main_object_store = instrumented_retrying_object_store( + + // Wraps a store in a retry and instrumentation layer, recording I/O + // under the given component and store-type metric labels. Each + // component (db, compactor, gc) gets its own layer over the store its + // builder holds. + let wrap_object_store = |store: Arc, + component: ObjectStoreComponent, + store_type: ObjectStoreType| { + instrumented_retrying_object_store( + store, + &recorder, + component, + store_type, + rand.clone(), + system_clock.clone(), + max_retries, + ) + }; + // Set up the object store with optional caching from the settings, + // producing the same layering as a caller-built + // [`CachedObjectStore`] passed to [`DbBuilder::new`]: the cache sits + // under the retry and instrumentation layers, so the same cache + // instance can be shared with the compactor and GC below while each + // component keeps its own layers. + let cached_object_store = CachedObjectStore::from_config( self.main_object_store.clone(), + &self.settings.object_store_cache_options, &recorder, + system_clock.clone(), + rand.clone(), + ) + .await?; + let maybe_cached_main_object_store: Arc = match &cached_object_store { + Some(cached_store) => cached_store.clone(), + None => self.main_object_store.clone(), + }; + + let retrying_main_object_store = wrap_object_store( + maybe_cached_main_object_store, ObjectStoreComponent::Db, ObjectStoreType::Main, - rand.clone(), - system_clock.clone(), - max_retries, ); - let retrying_wal_object_store: Option> = - self.wal_object_store.map(|s| { - instrumented_retrying_object_store( - s, - &recorder, - ObjectStoreComponent::Db, - ObjectStoreType::Wal, - rand.clone(), - system_clock.clone(), - max_retries, - ) - }); + let retrying_wal_object_store: Option> = self + .wal_object_store + .map(|s| wrap_object_store(s, ObjectStoreComponent::Db, ObjectStoreType::Wal)); // Log the database opening if let Ok(settings_json) = self.settings.to_json_string() { @@ -490,21 +519,6 @@ impl> DbBuilder

{ ..SsTableFormat::default() }; - // Setup object store with optional caching - let cached_object_store = CachedObjectStore::from_config( - retrying_main_object_store.clone(), - &self.settings.object_store_cache_options, - &recorder, - system_clock.clone(), - rand.clone(), - ) - .await?; - - let maybe_cached_main_object_store: Arc = match &cached_object_store { - Some(cached_store) => cached_store.clone(), - None => retrying_main_object_store.clone(), - }; - // Setup the manifest store and load latest manifest let manifest_store = Arc::new(ManifestStore::new( &path, @@ -551,7 +565,7 @@ impl> DbBuilder

{ }); let table_store = Arc::new(TableStore::new_with_fp_registry( ObjectStores::new( - maybe_cached_main_object_store.clone(), + retrying_main_object_store.clone(), retrying_wal_object_store.clone(), ), sst_format.clone(), @@ -647,39 +661,22 @@ impl> DbBuilder

{ &tokio_handle, )?; - // Wraps a background component's (compactor, GC) raw main store in - // the component's own retry and instrumentation layer, so its I/O is - // recorded under its own metric labels. Returns (main, uncached). - // - // main: when the component runs against the DB's own store (the auto - // from settings path, or a caller-supplied builder holding a clone of - // the DB's store) and object store caching is configured, the DB's - // cache is shared on top of that layer, so cache fills and evictions - // stay coherent with the DB's. A different caller-supplied store is - // used as given (so a custom compaction reader takes effect instead - // of being silently ignored) and stays cacheless. + // Selects the store a background component (compactor, GC) reads and + // writes through, before the component wraps it in its own retry and + // instrumentation layer. // - // uncached: the same wrapped store without the cache, for I/O that - // must bypass it. - let background_component_stores = - |raw_store: Arc, component: ObjectStoreComponent| { - let retrying = instrumented_retrying_object_store( - raw_store.clone(), - &recorder, - component, - ObjectStoreType::Main, - rand.clone(), - system_clock.clone(), - max_retries, - ); - let main: Arc = match &cached_object_store { - Some(cached) if Arc::ptr_eq(&raw_store, &self.main_object_store) => { - cached.clone_with_new_object_store(retrying.clone()) - } - _ => retrying.clone(), - }; - (main, retrying) - }; + // When the component runs against the DB's own store (the auto from + // settings path, or a caller-supplied builder holding a clone of the + // DB's store) and object store caching is configured, the DB's cache + // instance is shared. A different caller-supplied store is used as + // given (e.g. a custom compaction reader takes effect instead of being + // silently ignored) and stays cacheless. + let background_component_store = |raw_store: Arc| -> Arc { + match &cached_object_store { + Some(cached) if Arc::ptr_eq(&raw_store, &self.main_object_store) => cached.clone(), + _ => raw_store, + } + }; // The compactor reads/writes through the object store held by its // builder: the DB's own store on the auto from settings path, or the @@ -706,9 +703,10 @@ impl> DbBuilder

{ } builder = builder.with_fp_registry(self.fp_registry.clone()); - let (compactor_main_object_store, _) = background_component_stores( - builder.main_object_store.clone(), + let compactor_main_object_store = wrap_object_store( + background_component_store(builder.main_object_store.clone()), ObjectStoreComponent::Compactor, + ObjectStoreType::Main, ); let compactor_table_store = Arc::new(TableStore::new_with_fp_registry( ObjectStores::new( @@ -762,12 +760,13 @@ impl> DbBuilder

{ .options .metric_level .or(Some(self.settings.metric_level)); - let (gc_main_object_store, gc_object_store) = background_component_stores( - gc_builder.main_object_store.clone(), + let gc_object_store = wrap_object_store( + background_component_store(gc_builder.main_object_store.clone()), ObjectStoreComponent::Gc, + ObjectStoreType::Main, ); let gc_table_store = Arc::new(TableStore::new_with_fp_registry( - ObjectStores::new(gc_main_object_store, retrying_wal_object_store.clone()), + ObjectStores::new(gc_object_store.clone(), retrying_wal_object_store.clone()), sst_format.clone(), path_resolver.clone(), self.fp_registry.clone(), @@ -1757,8 +1756,24 @@ impl> DbReaderBuilder

{ ); // TODO: proper URI generation, for now it works just as a flag let wal_object_store_uri = self.wal_object_store.as_ref().map(|_| String::new()); + // Set up the object store with optional caching from the reader + // options, with the cache under the retry and instrumentation layers, + // matching a caller-built [`CachedObjectStore`] passed to the reader. + let maybe_cached = CachedObjectStore::from_config( + self.object_store.clone(), + &self.options.object_store_cache_options, + &recorder, + self.system_clock.clone(), + self.rand.clone(), + ) + .await?; + let maybe_cached_object_store: Arc = match &maybe_cached { + Some(cached) => Arc::clone(cached) as Arc, + None => self.object_store, + }; + let retrying_object_store = instrumented_retrying_object_store( - self.object_store, + maybe_cached_object_store, &recorder, ObjectStoreComponent::Reader, ObjectStoreType::Main, @@ -1780,23 +1795,8 @@ impl> DbReaderBuilder

{ ) }); - // Setup object store with optional caching - let maybe_cached = CachedObjectStore::from_config( - retrying_object_store.clone(), - &self.options.object_store_cache_options, - &recorder, - self.system_clock.clone(), - self.rand.clone(), - ) - .await?; - - let object_store: Arc = match &maybe_cached { - Some(cached) => Arc::clone(cached) as Arc, - None => retrying_object_store.clone(), - }; - // Validate WAL object store configuration. - let manifest_store = Arc::new(ManifestStore::new(&path, retrying_object_store)); + let manifest_store = Arc::new(ManifestStore::new(&path, retrying_object_store.clone())); let latest_manifest = StoredManifest::try_load(Arc::clone(&manifest_store), self.system_clock.clone()) .await?; @@ -1840,7 +1840,7 @@ impl> DbReaderBuilder

{ }; let path_resolver = PathResolver::new_with_external_ssts(path.clone(), external_ssts); let table_store = Arc::new(TableStore::new_with_fp_registry( - ObjectStores::new(object_store, retrying_wal_object_store), + ObjectStores::new(retrying_object_store, retrying_wal_object_store), sst_format, path_resolver, Arc::new(FailPointRegistry::new()), @@ -2130,6 +2130,7 @@ pub(crate) fn default_meta_cache() -> Option> { #[cfg(test)] mod tests { + use crate::cached_object_store::CachedObjectStore; use crate::compactions_store::{CompactionsStore, StoredCompactions}; use crate::config::{CompactorOptions, GarbageCollectorOptions, MetricLevel, Settings}; use crate::error::ErrorKind; @@ -2386,25 +2387,83 @@ mod tests { .tempdir() .expect("failed to create cache dir"); let cache_path = cache_dir.path().to_path_buf(); + let settings = Settings { + garbage_collector_options: None, + ..Settings::default() + }; + let cached_store = CachedObjectStore::builder(cache_path.clone(), object_store) + .with_part_size_bytes(1024) + .build() + .await + .expect("failed to build cached store"); + + let db = crate::Db::builder(path.clone(), cached_store) + .with_settings(settings) + .with_metrics_recorder(metrics_recorder.clone()) + .build() + .await + .expect("failed to build db"); + + let cached_db_path = cache_path.join(path.as_ref()); + assert!(!cached_db_path.join("manifest").exists()); + assert!(!cached_db_path.join("compactions").exists()); + assert!(!cached_db_path.join("gc").exists()); + + db.close().await.expect("failed to close db"); + } + + #[tokio::test] + async fn test_settings_configured_object_store_cache() { + let object_store: Arc = Arc::new(InMemory::new()); + let path = Path::from("test_settings_configured_object_store_cache"); + + // Seed an L0 SST without any cache configured. + let db = crate::Db::builder(path.clone(), object_store.clone()) + .with_settings(Settings { + garbage_collector_options: None, + ..Settings::default() + }) + .build() + .await + .expect("failed to build db"); + db.put(b"k1", b"v1").await.expect("failed to put"); + db.flush().await.expect("failed to flush"); + db.close().await.expect("failed to close db"); + + // Reopen with the cache configured through Settings and preload on. + let cache_dir = tempfile::Builder::new() + .prefix("settings_cache_test_") + .tempdir() + .expect("failed to create cache dir"); + let cache_path = cache_dir.path().to_path_buf(); let mut settings = Settings { garbage_collector_options: None, ..Settings::default() }; settings.object_store_cache_options.root_folder = Some(cache_path.clone()); settings.object_store_cache_options.part_size_bytes = 1024; + settings + .object_store_cache_options + .preload_disk_cache_on_startup = Some(crate::config::PreloadLevel::AllSst); let db = crate::Db::builder(path.clone(), object_store) .with_settings(settings) - .with_metrics_recorder(metrics_recorder.clone()) .build() .await .expect("failed to build db"); + // The preload populated the cache with the compacted SST's parts. let cached_db_path = cache_path.join(path.as_ref()); + let cached_compacted = std::fs::read_dir(cached_db_path.join("compacted")) + .expect("expected cached compacted dir") + .count(); + assert!(cached_compacted > 0); assert!(!cached_db_path.join("manifest").exists()); - assert!(!cached_db_path.join("compactions").exists()); - assert!(!cached_db_path.join("gc").exists()); + assert_eq!( + db.get(b"k1").await.expect("failed to get").as_deref(), + Some(b"v1".as_ref()) + ); db.close().await.expect("failed to close db"); } } diff --git a/slatedb/src/db_reader.rs b/slatedb/src/db_reader.rs index f7f7592c2..5f0aed5ce 100644 --- a/slatedb/src/db_reader.rs +++ b/slatedb/src/db_reader.rs @@ -3200,22 +3200,27 @@ mod tests { db.flush().await.unwrap(); db.close().await.unwrap(); - // Open a DbReader with disk caching enabled + // Open a DbReader over a user-constructed cached store let cache_dir = tempfile::Builder::new() .prefix("dbreader_cache_test_") .tempdir() .unwrap(); let cache_path = cache_dir.keep(); - let mut reader_opts = DbReaderOptions::default(); - reader_opts.object_store_cache_options.root_folder = Some(cache_path.clone()); - reader_opts.object_store_cache_options.part_size_bytes = 1024; + let cached_store = crate::cached_object_store::CachedObjectStore::builder( + cache_path.clone(), + Arc::clone(&object_store), + ) + .with_part_size_bytes(1024) + .build() + .await + .unwrap(); let reader = DbReader::open( path.clone(), - Arc::clone(&object_store), + cached_store, DbReaderMode::ManagedCheckpoint, - reader_opts, + DbReaderOptions::default(), ) .await .unwrap(); @@ -3275,7 +3280,7 @@ mod tests { Arc::new(TableStore::new_with_fp_registry( ObjectStores::new(Arc::clone(&self.object_store), None), SsTableFormat::default(), - PathResolver::new(self.path.clone()), + PathResolver::from_root(self.path.clone()), Arc::clone(&self.fp_registry), None, TableStoreKind::Reader, diff --git a/slatedb/src/error.rs b/slatedb/src/error.rs index 7ca1364ed..674a8689b 100644 --- a/slatedb/src/error.rs +++ b/slatedb/src/error.rs @@ -500,12 +500,18 @@ impl std::fmt::Display for ErrorKind { /// Why a recoverable SST read is being reissued (the reason it failed validation /// the first time). /// -/// Carried on the reissued read's tag so a caching wrapper can try a different -/// strategy on the retry. +/// Carried on the reissued read's +/// [`ObjectStoreCallTag`](crate::object_store_tag::ObjectStoreCallTag) so a +/// caching wrapper can drop its local copy and refetch instead of serving the +/// same bytes again. +#[non_exhaustive] #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum RetryReason { +pub enum RetryReason { + /// The read bytes failed a checksum validation. CrcMismatch, + /// The read bytes could not be decoded as a block. BlockDecodeError, + /// The read bytes could not be decompressed. #[cfg(any( feature = "snappy", feature = "zlib", diff --git a/slatedb/src/garbage_collector.rs b/slatedb/src/garbage_collector.rs index 6f7018b99..426e235dc 100644 --- a/slatedb/src/garbage_collector.rs +++ b/slatedb/src/garbage_collector.rs @@ -934,7 +934,7 @@ mod tests { #[tokio::test] async fn test_collect_garbage_wal_ssts() { let (manifest_store, compactions_store, table_store, local_object_store) = build_objects(); - let path_resolver = PathResolver::new("/"); + let path_resolver = PathResolver::from_root("/"); // write a wal sst let id1 = SsTableId::Wal(1); @@ -946,7 +946,7 @@ mod tests { // Set the first WAL SST file to be a day old let now_minus_24h = set_modified( local_object_store.clone(), - &path_resolver.table_path(&SsTableId::Wal(1)), + &path_resolver.sst_path(&SsTableId::Wal(1)), 86400, ); @@ -993,7 +993,7 @@ mod tests { #[tokio::test] async fn test_do_not_remove_wals_referenced_by_active_checkpoints() { let (manifest_store, compactions_store, table_store, local_object_store) = build_objects(); - let path_resolver = PathResolver::new("/"); + let path_resolver = PathResolver::from_root("/"); let id1 = SsTableId::Wal(1); write_sst(table_store.clone(), &id1).await.unwrap(); @@ -1029,7 +1029,7 @@ mod tests { for i in 1..=3 { set_modified( local_object_store.clone(), - &path_resolver.table_path(&SsTableId::Wal(i)), + &path_resolver.sst_path(&SsTableId::Wal(i)), 86400, ); } @@ -1054,7 +1054,7 @@ mod tests { #[tokio::test] async fn test_collect_garbage_wal_ssts_and_keep_expired_last_compacted() { let (manifest_store, compactions_store, table_store, local_object_store) = build_objects(); - let path_resolver = PathResolver::new("/"); + let path_resolver = PathResolver::from_root("/"); // write a wal sst let id1 = SsTableId::Wal(1); @@ -1077,12 +1077,12 @@ mod tests { // Set the both WAL SST file to be a day old let now_minus_24h_1 = set_modified( local_object_store.clone(), - &path_resolver.table_path(&SsTableId::Wal(1)), + &path_resolver.sst_path(&SsTableId::Wal(1)), 86400, ); let now_minus_24h_2 = set_modified( local_object_store.clone(), - &path_resolver.table_path(&SsTableId::Wal(2)), + &path_resolver.sst_path(&SsTableId::Wal(2)), 86400, ); @@ -1130,7 +1130,7 @@ mod tests { #[tokio::test] async fn test_regular_wal_gc_does_not_delete_wal_fences() { let (manifest_store, compactions_store, table_store, local_object_store) = build_objects(); - let path_resolver = PathResolver::new("/"); + let path_resolver = PathResolver::from_root("/"); let fence_id = SsTableId::Wal(1); table_store.write_wal_fence(1).await.unwrap(); @@ -1143,7 +1143,7 @@ mod tests { for id in [fence_id, regular_wal_id] { set_modified( local_object_store.clone(), - &path_resolver.table_path(&id), + &path_resolver.sst_path(&id), 86400, ); } @@ -1195,7 +1195,7 @@ mod tests { #[tokio::test] async fn test_wal_fence_gc_deletes_old_fences() { let (manifest_store, compactions_store, table_store, local_object_store) = build_objects(); - let path_resolver = PathResolver::new("/"); + let path_resolver = PathResolver::from_root("/"); let old_fence_id = SsTableId::Wal(1); table_store.write_wal_fence(1).await.unwrap(); @@ -1211,7 +1211,7 @@ mod tests { for id in [old_fence_id, regular_wal_id, newer_fence_id] { set_modified( local_object_store.clone(), - &path_resolver.table_path(&id), + &path_resolver.sst_path(&id), 86400, ); } @@ -1273,13 +1273,13 @@ mod tests { #[tokio::test] async fn test_wal_fence_gc_deletes_single_old_fence() { let (manifest_store, compactions_store, table_store, local_object_store) = build_objects(); - let path_resolver = PathResolver::new("/"); + let path_resolver = PathResolver::from_root("/"); let fence_id = SsTableId::Wal(1); table_store.write_wal_fence(1).await.unwrap(); set_modified( local_object_store, - &path_resolver.table_path(&fence_id), + &path_resolver.sst_path(&fence_id), 86400, ); @@ -1328,7 +1328,7 @@ mod tests { #[tokio::test] async fn test_regular_and_wal_fence_gc_run_independently() { let (manifest_store, compactions_store, table_store, local_object_store) = build_objects(); - let path_resolver = PathResolver::new("/"); + let path_resolver = PathResolver::from_root("/"); let old_fence_id = SsTableId::Wal(1); table_store.write_wal_fence(1).await.unwrap(); @@ -1354,7 +1354,7 @@ mod tests { ] { set_modified( local_object_store.clone(), - &path_resolver.table_path(&id), + &path_resolver.sst_path(&id), 86400, ); } @@ -2163,14 +2163,14 @@ mod tests { #[tokio::test] async fn test_should_record_gc_wal_deleted_count() { let (manifest_store, compactions_store, table_store, local_object_store) = build_objects(); - let path_resolver = PathResolver::new("/"); + let path_resolver = PathResolver::from_root("/"); // given: two WAL SSTs, first one old enough to GC let id1 = SsTableId::Wal(1); write_sst(table_store.clone(), &id1).await.unwrap(); let id2 = SsTableId::Wal(2); write_sst(table_store.clone(), &id2).await.unwrap(); - set_modified(local_object_store, &path_resolver.table_path(&id1), 86400); + set_modified(local_object_store, &path_resolver.sst_path(&id1), 86400); let mut state = ManifestCore::new(); state.replay_after_wal_id = id2.unwrap_wal_id(); @@ -2332,7 +2332,7 @@ mod tests { #[tokio::test] async fn test_gc_filter_can_reject_all_directory_gc_deletes() { let (manifest_store, compactions_store, table_store, local_object_store) = build_objects(); - let path_resolver = PathResolver::new("/"); + let path_resolver = PathResolver::from_root("/"); let now = DefaultSystemClock::default().now(); let expired_ms = (now - TimeDelta::seconds(7200)).timestamp_millis() as u64; let unexpired_ms = (now - TimeDelta::seconds(1800)).timestamp_millis() as u64; @@ -2349,12 +2349,12 @@ mod tests { set_modified( local_object_store.clone(), - &path_resolver.table_path(&old_wal_id), + &path_resolver.sst_path(&old_wal_id), 86400, ); set_modified( local_object_store.clone(), - &path_resolver.table_path(&old_fence_id), + &path_resolver.sst_path(&old_fence_id), 86400, ); @@ -2485,7 +2485,7 @@ mod tests { #[tokio::test] async fn test_gc_filter_allows_subset_and_stats_count_successful_deletes() { let (manifest_store, compactions_store, table_store, local_object_store) = build_objects(); - let path_resolver = PathResolver::new("/"); + let path_resolver = PathResolver::from_root("/"); // Create three old WALs below the replay boundary so all would be eligible // without a filter. The middle one is the only filter-approved delete. @@ -2500,7 +2500,7 @@ mod tests { write_sst(table_store.clone(), &id).await.unwrap(); set_modified( local_object_store.clone(), - &path_resolver.table_path(&id), + &path_resolver.sst_path(&id), 86400, ); } @@ -2543,7 +2543,7 @@ mod tests { &helper, Arc::new(DefaultSystemClock::default()), Some(Arc::new(LocationGcFilter { - allowed_locations: HashSet::from([path_resolver.table_path(&allowed_wal_id)]), + allowed_locations: HashSet::from([path_resolver.sst_path(&allowed_wal_id)]), })), ); @@ -2572,7 +2572,7 @@ mod tests { #[tokio::test] async fn test_dry_run_skips_directory_gc_deletes() { let (manifest_store, compactions_store, table_store, local_object_store) = build_objects(); - let path_resolver = PathResolver::new("/"); + let path_resolver = PathResolver::from_root("/"); let now = DefaultSystemClock::default().now(); let expired_ms = (now - TimeDelta::seconds(7200)).timestamp_millis() as u64; let unexpired_ms = (now - TimeDelta::seconds(1800)).timestamp_millis() as u64; @@ -2588,12 +2588,12 @@ mod tests { set_modified( local_object_store.clone(), - &path_resolver.table_path(&old_wal_id), + &path_resolver.sst_path(&old_wal_id), 86400, ); set_modified( local_object_store.clone(), - &path_resolver.table_path(&old_fence_id), + &path_resolver.sst_path(&old_fence_id), 86400, ); diff --git a/slatedb/src/instrumented_object_store.rs b/slatedb/src/instrumented_object_store.rs index 1528a1191..822111e47 100644 --- a/slatedb/src/instrumented_object_store.rs +++ b/slatedb/src/instrumented_object_store.rs @@ -15,6 +15,13 @@ //! so each `InstrumentedObjectStore` instance is constructed with one //! specific (component, type) pair. The cross-product of these two //! dimensions lets operators slice metrics by either axis. +//! +//! Note: if the wrapped `ObjectStore` is itself a wrapper like +//! `CachedObjectStore`, the metrics count the calls into that wrapper, not +//! the traffic it generates against the underlying store. A cache hit is +//! counted as one request, and requests the cache makes internally to fill a +//! miss are not counted at all. + // `Instant` is intentionally used here for monotonic elapsed-time measurement. // SlateDB's clock abstraction is for wall-clock timestamps, not request timing. #![allow(clippy::disallowed_methods, clippy::disallowed_types)] diff --git a/slatedb/src/lib.rs b/slatedb/src/lib.rs index 1a64426a5..686ef1e04 100644 --- a/slatedb/src/lib.rs +++ b/slatedb/src/lib.rs @@ -67,6 +67,7 @@ pub use iter::IterationOrder; pub use manifest::VersionedManifest; pub use merge_operator::{MergeOperator, MergeOperatorError}; pub use ops::{DbCacheManagerOps, DbMetadataOps, DbReadOps, DbTransactionOps, DbWriteOps}; +pub use paths::PathResolver; pub use prefix_extractor::{PrefixExtractor, PrefixTarget}; pub use slatedb_common::{DbRand, IdentifiedObjectMetadata, ObjectMetadata}; #[cfg(test)] @@ -90,6 +91,7 @@ pub mod config; pub mod db_cache; pub mod db_stats; pub mod manifest; +pub mod object_store_tag; pub mod prefix_extractor; pub mod seq_tracker; pub mod size_tiered_compaction; @@ -143,7 +145,6 @@ mod mem_table; mod memtable_flusher; mod merge_iterator; mod merge_operator; -mod object_store_tag; mod object_stores; mod ops; mod oracle; diff --git a/slatedb/src/manifest/mod.rs b/slatedb/src/manifest/mod.rs index fd7cf73cb..1995d9e33 100644 --- a/slatedb/src/manifest/mod.rs +++ b/slatedb/src/manifest/mod.rs @@ -935,6 +935,10 @@ impl VersionedManifest { &self.manifest.core } + pub(crate) fn external_ssts(&self) -> HashMap { + self.manifest.external_ssts() + } + /// The named segments configured in this manifest (RFC-0024), in prefix /// order. Empty when no segment extractor is configured. The unsegmented /// default tree is accessed via [`Self::l0`] / [`Self::compacted`] / diff --git a/slatedb/src/memtable_flusher/manifest_writer.rs b/slatedb/src/memtable_flusher/manifest_writer.rs index ae4645620..a29f4b486 100644 --- a/slatedb/src/memtable_flusher/manifest_writer.rs +++ b/slatedb/src/memtable_flusher/manifest_writer.rs @@ -1061,7 +1061,7 @@ mod tests { let table_store = Arc::new(TableStore::new_with_fp_registry( ObjectStores::new(Arc::clone(&object_store), None), SsTableFormat::default(), - PathResolver::new(Path::from(path.clone())), + PathResolver::from_root(Path::from(path.clone())), Arc::clone(&fp_registry), None, TableStoreKind::Main, diff --git a/slatedb/src/memtable_flusher/tracker.rs b/slatedb/src/memtable_flusher/tracker.rs index e7971d4df..656997f29 100644 --- a/slatedb/src/memtable_flusher/tracker.rs +++ b/slatedb/src/memtable_flusher/tracker.rs @@ -635,7 +635,7 @@ mod tests { let table_store = Arc::new(TableStore::new_with_fp_registry( ObjectStores::new(Arc::clone(&object_store), None), SsTableFormat::default(), - PathResolver::new(Path::from(path.clone())), + PathResolver::from_root(Path::from(path.clone())), Arc::clone(&fp_registry), None, TableStoreKind::Main, diff --git a/slatedb/src/memtable_flusher/uploader.rs b/slatedb/src/memtable_flusher/uploader.rs index 6e2fcf3d9..1aae23f90 100644 --- a/slatedb/src/memtable_flusher/uploader.rs +++ b/slatedb/src/memtable_flusher/uploader.rs @@ -391,7 +391,7 @@ mod tests { let table_store = Arc::new(TableStore::new_with_fp_registry( ObjectStores::new(Arc::clone(&object_store), None), SsTableFormat::default(), - PathResolver::new(Path::from(path)), + PathResolver::from_root(Path::from(path)), fp_registry.clone(), cache, TableStoreKind::Main, diff --git a/slatedb/src/object_store_tag.rs b/slatedb/src/object_store_tag.rs index f5f09e50a..eb4da45ed 100644 --- a/slatedb/src/object_store_tag.rs +++ b/slatedb/src/object_store_tag.rs @@ -1,22 +1,45 @@ -//! The per-call tag SlateDB attaches to the object store calls which the -//! [`TableStore`](crate::tablestore::TableStore) issues for an SST. +//! The per-call tag SlateDB attaches to the object store calls it issues for +//! an SST (through its internal TableStore component). //! -//! The tag is part of the [`object_store::Extensions`] on every `GetOptions`, -//! `PutOptions`, and `PutMultipartOptions` the TableStore builds for an SST. -//! The TableStore is the only writer of the tag; a caching object store wrapper -//! is the reader. +//! This module is the contract between SlateDB and a caching +//! [`ObjectStore`](object_store::ObjectStore) wrapper, whether the bundled +//! [`CachedObjectStore`](crate::cached_object_store::CachedObjectStore) or an +//! external implementation passed to +//! [`Db::builder`](crate::Db::builder) as the object store. +//! +//! SlateDB inserts an [`ObjectStoreCallTag`] into the +//! [`object_store::Extensions`] of every `GetOptions`, `PutOptions`, and +//! `PutMultipartOptions` the TableStore builds for an SST. A wrapper reads it +//! back with one lookup: +//! +//! ```ignore +//! if let Some(tag) = ObjectStoreCallTag::from_extensions(&options.extensions) { +//! // classify by tag.kind, tag.sst_type, tag.retry +//! } +//! ``` +//! +//! Manifest reads and writes, compaction state, garbage collector listings, +//! and other coordination I/O carry no tag. +//! +//! When decoding a read fails with a recoverable validation error, SlateDB +//! reissues the read once with `retry` set. A caching wrapper must not serve +//! the same locally cached bytes for a retry-tagged read: drop the cached +//! entry for the path and refetch from the wrapped store, otherwise the +//! caller keeps receiving the corrupt bytes and the read fails permanently. use object_store::Extensions; -use crate::db_state::SstType; -use crate::error::RetryReason; +pub use crate::db_state::SstType; +pub use crate::error::RetryReason; -/// Identifies the component whose [`TableStore`](crate::tablestore::TableStore) -/// issued an object store call (the call source). Tagged on every SST read and -/// write alongside the [`SstType`]. A caching wrapper combines it with the call -/// type (get vs put) to decide admission. +/// Identifies the component whose TableStore issued an object store call (the +/// call source). Tagged on every SST read and write alongside the +/// [`SstType`]. +/// +/// A caching wrapper combines it with the call type (get vs put) +/// to decide admission. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum TableStoreKind { +pub enum TableStoreKind { /// The primary database store: foreground reads and memtable flush writes. Main, /// A read-only store. @@ -30,23 +53,23 @@ pub(crate) enum TableStoreKind { /// The tag carried on every TableStore SST object store call via /// [`object_store::Extensions`]. /// -/// An `ObjectStore` wrapper (such as the bundled object store cache) reads the -/// tag to decide the action for the call. +/// An `ObjectStore` wrapper (such as an object store cache) reads the tag to +/// decide the action for the call. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct ObjectStoreCallTag { +pub struct ObjectStoreCallTag { /// The source of the call, to distinguish main store, compactor, etc. - pub(crate) kind: TableStoreKind, + pub kind: TableStoreKind, /// The kind of SST the call is targeting (WAL vs compacted). - pub(crate) sst_type: SstType, + pub sst_type: SstType, /// The reason for retry if this call is reissued after a validation failure /// on a read. - pub(crate) retry: Option, + pub retry: Option, } impl ObjectStoreCallTag { /// A tag with no retry reason: the common case (a read sets the retry reason /// itself on a reissue). - pub(crate) fn new(kind: TableStoreKind, sst_type: SstType) -> Self { + pub fn new(kind: TableStoreKind, sst_type: SstType) -> Self { Self { kind, sst_type, @@ -55,7 +78,7 @@ impl ObjectStoreCallTag { } /// Reads the tag back from an extensions map, if present. - pub(crate) fn from_extensions(extensions: &Extensions) -> Option { + pub fn from_extensions(extensions: &Extensions) -> Option { extensions.get::().copied() } } diff --git a/slatedb/src/paths.rs b/slatedb/src/paths.rs index 9194d7909..a909a9615 100644 --- a/slatedb/src/paths.rs +++ b/slatedb/src/paths.rs @@ -9,14 +9,39 @@ use ulid::Ulid; const WAL_PATH: &str = "wal"; const COMPACTED_PATH: &str = "compacted"; +/// Resolves the object store paths of a SlateDB database's files from the +/// database's root path. +/// +/// Useful outside the database handle, for example to map the SST ids in a +/// [`VersionedManifest`](crate::VersionedManifest) to paths in Object Store +/// +/// Can be used when preloading `CachedObjectStore`, or for administrative +/// tooling that inspects a database's objects directly. +/// +/// Constructed from a manifest via [`Self::new`]. The manifest is required +/// because it can reference SSTs owned by another database (external SSTs, +/// from cloning), which live outside this database's root path and cannot be +/// resolved from the root path alone. #[derive(Clone, Debug)] -pub(crate) struct PathResolver { +pub struct PathResolver { root_path: Path, external_ssts: HashMap, } impl PathResolver { - pub(crate) fn new>(root_path: P) -> Self { + /// Creates a resolver for the database rooted at `root_path`, resolving + /// the external SSTs referenced by `manifest` to their owning database's + /// path. + pub fn new>(root_path: P, manifest: &crate::manifest::VersionedManifest) -> Self { + Self::new_with_external_ssts(root_path.into(), manifest.external_ssts()) + } + + /// Creates a resolver for the database rooted at `root_path`. + /// + /// Internal only: without a manifest, external SSTs resolve to wrong + /// paths under `root_path`, so callers must only use this where external + /// SSTs cannot appear (for example WAL paths). + pub(crate) fn from_root>(root_path: P) -> Self { Self { root_path: root_path.into(), external_ssts: HashMap::new(), @@ -66,7 +91,8 @@ impl PathResolver { } } - pub(crate) fn table_path(&self, table_id: &SsTableId) -> Path { + /// Returns the path of the SST with the given id. + pub fn sst_path(&self, table_id: &SsTableId) -> Path { let root_path = match self.external_ssts.get(table_id) { Some(external_path) => external_path, None => &self.root_path, @@ -99,9 +125,9 @@ mod tests { fn should_serialize_and_deserialize_wal_paths( wal_id in any::(), ) { - let path_resolver = PathResolver::new(Path::from(ROOT)); + let path_resolver = PathResolver::from_root(Path::from(ROOT)); let table_id = SsTableId::Wal(wal_id); - let path = path_resolver.table_path(&table_id); + let path = path_resolver.sst_path(&table_id); let parsed_table_id = path_resolver.parse_table_id(&path).unwrap(); assert_eq!(Some(table_id), parsed_table_id); } @@ -110,9 +136,9 @@ mod tests { fn should_serialize_and_deserialize_compacted_paths( compacted_id in any::(), ) { - let path_resolver = PathResolver::new(Path::from(ROOT)); + let path_resolver = PathResolver::from_root(Path::from(ROOT)); let table_id = SsTableId::Compacted(Ulid::from(compacted_id)); - let path = path_resolver.table_path(&table_id); + let path = path_resolver.sst_path(&table_id); let parsed_table_id = path_resolver.parse_table_id(&path).unwrap(); assert_eq!(Some(table_id), parsed_table_id); } @@ -120,7 +146,7 @@ mod tests { #[test] fn test_parse_id() { - let path_resolver = PathResolver::new(Path::from(ROOT)); + let path_resolver = PathResolver::from_root(Path::from(ROOT)); let path = Path::from("/root/wal/00000000000000000003.sst"); let id = path_resolver.parse_table_id(&path).unwrap(); assert_eq!(id, Some(SsTableId::Wal(3))); diff --git a/slatedb/src/sst_builder.rs b/slatedb/src/sst_builder.rs index 0da29e62d..edc5853e8 100644 --- a/slatedb/src/sst_builder.rs +++ b/slatedb/src/sst_builder.rs @@ -502,7 +502,7 @@ mod tests { TableStoreKind::Main, BlockCachePolicy::default(), ); - let path_resolver = PathResolver::new(root_path); + let path_resolver = PathResolver::from_root(root_path); // 16-byte keys/values, no timestamps. Keys are spread across the // keyspace (bit-reversed counter in the leading bytes) so adjacent keys @@ -535,7 +535,7 @@ mod tests { let actual_size = |id: &SsTableId| { let object_store = object_store.clone(); - let path = path_resolver.table_path(id); + let path = path_resolver.sst_path(id); async move { object_store.head(&path).await.unwrap().size as usize } }; diff --git a/slatedb/src/sst_reader.rs b/slatedb/src/sst_reader.rs index 0acffdc62..6d852fe1b 100644 --- a/slatedb/src/sst_reader.rs +++ b/slatedb/src/sst_reader.rs @@ -492,7 +492,7 @@ mod tests { assert!(metadata.metadata.size > 0); assert_eq!( metadata.metadata.location, - PathResolver::new(path).table_path(&view.sst.id) + PathResolver::from_root(path).sst_path(&view.sst.id) ); } diff --git a/slatedb/src/tablestore.rs b/slatedb/src/tablestore.rs index 1111a7be5..7a0717264 100644 --- a/slatedb/src/tablestore.rs +++ b/slatedb/src/tablestore.rs @@ -133,7 +133,7 @@ impl TableStore { Self::new_with_fp_registry( object_stores, sst_format, - PathResolver::new(root_path), + PathResolver::from_root(root_path), Arc::new(FailPointRegistry::new()), block_cache, kind, @@ -1023,7 +1023,7 @@ impl TableStore { } fn path(&self, id: &SsTableId) -> Path { - self.path_resolver.table_path(id) + self.path_resolver.sst_path(id) } pub(crate) fn estimate_encoded_size_compacted( diff --git a/slatedb/src/utils.rs b/slatedb/src/utils.rs index 3d647ef27..d89338250 100644 --- a/slatedb/src/utils.rs +++ b/slatedb/src/utils.rs @@ -662,7 +662,7 @@ pub(crate) async fn preload_cache_from_manifest( Some(PreloadLevel::AllSst) => { let all_sst_paths: Vec = core .all_sst_views() - .map(|view| path_resolver.table_path(&view.sst.id)) + .map(|view| path_resolver.sst_path(&view.sst.id)) .collect(); if !all_sst_paths.is_empty() { if let Err(e) = cached_obj_store @@ -677,7 +677,7 @@ pub(crate) async fn preload_cache_from_manifest( let l0_sst_paths: Vec = core .trees() .flat_map(|tree| tree.l0.iter()) - .map(|view| path_resolver.table_path(&view.sst.id)) + .map(|view| path_resolver.sst_path(&view.sst.id)) .collect(); if !l0_sst_paths.is_empty() { if let Err(e) = cached_obj_store diff --git a/website/src/content/docs/docs/design/caching.mdx b/website/src/content/docs/docs/design/caching.mdx index 11a1e7c4e..3dcf798f9 100644 --- a/website/src/content/docs/docs/design/caching.mdx +++ b/website/src/content/docs/docs/design/caching.mdx @@ -17,11 +17,29 @@ You can replace the cache with [`DbBuilder::with_db_cache`](https://docs.rs/slat ## Object-Store Cache -[`ObjectStoreCacheOptions`](https://docs.rs/slatedb/latest/slatedb/config/struct.ObjectStoreCacheOptions.html) enables a second cache layer for raw object-store bytes. When [`root_folder`](https://docs.rs/slatedb/latest/slatedb/config/struct.ObjectStoreCacheOptions.html#structfield.root_folder) is set, SlateDB wraps the configured object store in a local cache. It splits each object into fixed-size parts, stores those parts under the local root, and can serve later `GET` and `HEAD` requests from those local files when the needed parts are already present. +[`CachedObjectStore`](https://docs.rs/slatedb/latest/slatedb/cached_object_store/struct.CachedObjectStore.html) is a second cache layer for raw object-store bytes. It splits each object into fixed-size parts, stores those parts under a local root folder, and can serve later `GET` and `HEAD` requests from those local files when the needed parts are already present. + +There are two ways to enable it. The first is configuration: set [`root_folder`](https://docs.rs/slatedb/latest/slatedb/config/struct.ObjectStoreCacheOptions.html#structfield.root_folder) in [`ObjectStoreCacheOptions`](https://docs.rs/slatedb/latest/slatedb/config/struct.ObjectStoreCacheOptions.html), and SlateDB builds the cache from those options and wraps the object store you passed to the builder in it. This is the only way to enable the cache from a settings file. + +The second is to build the cache yourself with [`CachedObjectStore::builder`](https://docs.rs/slatedb/latest/slatedb/cached_object_store/struct.CachedObjectStore.html#method.builder) and pass it to SlateDB as the object store: + +```rust +let cache = CachedObjectStore::builder("/var/slatedb-cache", object_store) + .with_cache_on_flush(true) + .with_cache_on_compaction(true) + .with_max_cache_size_bytes(Some(16 * 1024 * 1024 * 1024)) + .build() + .await?; +let db = Db::builder(path, cache).build().await?; +``` + +Building it yourself lets you hold on to the cache instance, share it across several `Db` and `DbReader` instances in the same process, and warm it directly. Leave [`root_folder`](https://docs.rs/slatedb/latest/slatedb/config/struct.ObjectStoreCacheOptions.html#structfield.root_folder) unset when you do this, otherwise SlateDB wraps your cache in a second one. + +Either way the cache ends up as the innermost layer, closest to the object store you provided. SlateDB adds its own retry and metrics handling above it, so a read served from the cache is still counted as an object-store request. See [Metrics](/docs/operations/metrics) for what that means for the `slatedb.object_store.*` metrics. This cache stores object-store bytes, not decoded SST blocks. It helps when the block cache is cold because it can avoid a remote read even though SlateDB still needs to read from local disk and decode the block afterward. On a miss, SlateDB aligns the requested range to the configured part size, fetches that larger range from the object store, and saves the returned parts locally. The default part size is 4 MiB. -The built-in object-store cache is disk-backed today. [`root_folder`](https://docs.rs/slatedb/latest/slatedb/config/struct.ObjectStoreCacheOptions.html#structfield.root_folder) is a filesystem path, and the current implementation stores parts under that directory. If you want an in-memory cache, use the block cache layer instead. +The built-in object-store cache is disk-backed today. The root folder is a filesystem path, and the current implementation stores parts under that directory. If you want an in-memory cache, use the block cache layer instead. ## How Caches Get Filled @@ -62,17 +80,23 @@ let db = Db::builder(path, object_store) .await?; ``` -The disk cache stays disabled unless `object_store_cache_options.root_folder` is set. If you want it warm before serving traffic, you can preload it on startup with [`PreloadLevel::L0Sst`](https://docs.rs/slatedb/latest/slatedb/config/enum.PreloadLevel.html#variant.L0Sst) or [`PreloadLevel::AllSst`](https://docs.rs/slatedb/latest/slatedb/config/enum.PreloadLevel.html#variant.AllSst). SlateDB loads recent SSTs, or all SSTs, into the local cache until the cache size limit is reached. +The disk cache stays disabled unless you enable it, either by setting `object_store_cache_options.root_folder` or by passing your own `CachedObjectStore` to the builder. + +By default, writes go straight to the upstream object store and do not populate the object-store cache. Each SST write source has its own admission flag: [`cache_on_flush`](https://docs.rs/slatedb/latest/slatedb/config/struct.ObjectStoreCacheOptions.html#structfield.cache_on_flush) stores SSTs written by memtable flushes locally, and [`cache_on_compaction`](https://docs.rs/slatedb/latest/slatedb/config/struct.ObjectStoreCacheOptions.html#structfield.cache_on_compaction) does the same for compaction output. The builder exposes the same two flags as [`with_cache_on_flush`](https://docs.rs/slatedb/latest/slatedb/cached_object_store/struct.CachedObjectStoreBuilder.html#method.with_cache_on_flush) and [`with_cache_on_compaction`](https://docs.rs/slatedb/latest/slatedb/cached_object_store/struct.CachedObjectStoreBuilder.html#method.with_cache_on_compaction). Enabling them can help if readers are likely to touch freshly written SSTs soon afterward. WAL and manifest writes are never cached. + +## Warming the Object-Store Cache + +If you want the disk cache warm before serving traffic, and you configured it through [`ObjectStoreCacheOptions`](https://docs.rs/slatedb/latest/slatedb/config/struct.ObjectStoreCacheOptions.html), set [`preload_disk_cache_on_startup`](https://docs.rs/slatedb/latest/slatedb/config/struct.ObjectStoreCacheOptions.html#structfield.preload_disk_cache_on_startup) to [`PreloadLevel::L0Sst`](https://docs.rs/slatedb/latest/slatedb/config/enum.PreloadLevel.html#variant.L0Sst) or [`PreloadLevel::AllSst`](https://docs.rs/slatedb/latest/slatedb/config/enum.PreloadLevel.html#variant.AllSst). `Db` and `DbReader` load recent SSTs, or all SSTs, into the local cache on open until the cache size limit is reached. -By default, writes go straight to the upstream object store and do not populate the object-store cache. Each SST write source has its own admission flag: [`cache_on_flush`](https://docs.rs/slatedb/latest/slatedb/config/struct.ObjectStoreCacheOptions.html#structfield.cache_on_flush) stores SSTs written by memtable flushes locally, and [`cache_on_compaction`](https://docs.rs/slatedb/latest/slatedb/config/struct.ObjectStoreCacheOptions.html#structfield.cache_on_compaction) does the same for compaction output. Enabling them can help if readers are likely to touch freshly written SSTs soon afterward. WAL and manifest writes are never cached. +That setting only applies to a cache SlateDB built for you. A cache you built yourself is warmed by you. Read the current SST set from [`DbMetadataOps::manifest`](https://docs.rs/slatedb/latest/slatedb/ops/trait.DbMetadataOps.html#tymethod.manifest), resolve each SST id to an object-store path with [`PathResolver::sst_path`](https://docs.rs/slatedb/latest/slatedb/struct.PathResolver.html#method.sst_path), and pass the paths to [`CachedObjectStore::load_files_to_cache`](https://docs.rs/slatedb/latest/slatedb/cached_object_store/struct.CachedObjectStore.html#method.load_files_to_cache). Construct the resolver with [`PathResolver::new`](https://docs.rs/slatedb/latest/slatedb/struct.PathResolver.html#method.new), which takes the manifest as well as the database path because a manifest can reference SSTs owned by another database after a clone. The `max_bytes` budget is applied in path order, so order paths by priority. Fetches are best-effort, and failures are logged and skipped. ## Sharing Between Instances The two cache layers behave differently when you open multiple [`Db`](https://docs.rs/slatedb/latest/slatedb/struct.Db.html) or [`DbReader`](https://docs.rs/slatedb/latest/slatedb/struct.DbReader.html) instances against the same cache. -For the object-store cache, sharing is straightforward. If multiple instances on the same machine use the same [`root_folder`](https://docs.rs/slatedb/latest/slatedb/config/struct.ObjectStoreCacheOptions.html#structfield.root_folder), they reuse the same local cache directory. For the same database path, that lets one instance benefit from parts fetched by another. When you use [`Db::resolve_object_store`](https://docs.rs/slatedb/latest/slatedb/struct.Db.html#method.resolve_object_store) and provide different path arguments to builder methods like [`DbBuilder::new`](https://docs.rs/slatedb/latest/slatedb/db/struct.DbBuilder.html#method.new), SlateDB keeps the cached files under different path prefixes, so they do not clobber one another. Using the same [`part_size_bytes`](https://docs.rs/slatedb/latest/slatedb/config/struct.ObjectStoreCacheOptions.html#structfield.part_size_bytes) gives the best reuse. Different part sizes can coexist, but they will not reuse the same cached part files. +For the object-store cache, sharing is straightforward. If multiple instances on the same machine use the same root folder, whether they set [`root_folder`](https://docs.rs/slatedb/latest/slatedb/config/struct.ObjectStoreCacheOptions.html#structfield.root_folder) or pass it to `CachedObjectStore::builder`, they reuse the same local cache directory. Instances in the same process can also share one `CachedObjectStore` instance directly. For the same database path, that lets one instance benefit from parts fetched by another. When you use [`Db::resolve_object_store`](https://docs.rs/slatedb/latest/slatedb/struct.Db.html#method.resolve_object_store) and provide different path arguments to builder methods like [`DbBuilder::new`](https://docs.rs/slatedb/latest/slatedb/db/struct.DbBuilder.html#method.new), SlateDB keeps the cached files under different path prefixes, so they do not clobber one another. Using the same part size gives the best reuse. Different part sizes can coexist, but they will not reuse the same cached part files. -If you provide an object store directly with `PrefixStore` or other custom wrapping instead of using `Db::resolve_object_store`, you must configure different [`root_folder`](https://docs.rs/slatedb/latest/slatedb/config/struct.ObjectStoreCacheOptions.html#structfield.root_folder) values for each instance to prevent cache collisions. SlateDB no longer automatically resolves root prefixes from metadata locations. +If you provide an object store directly with `PrefixStore` or other custom wrapping instead of using `Db::resolve_object_store`, you must configure different root folder values for each instance to prevent cache collisions. SlateDB no longer automatically resolves root prefixes from metadata locations. The block cache is different. Both [`DbBuilder::with_db_cache`](https://docs.rs/slatedb/latest/slatedb/db/struct.DbBuilder.html#method.with_db_cache) and [`DbReaderBuilder::with_db_cache`](https://docs.rs/slatedb/latest/slatedb/db/struct.DbReaderBuilder.html#method.with_db_cache) let you pass in your own cache object, so you can choose to reuse the same process-local cache implementation across builders. For `Db`, that mainly gives you a shared memory or disk budget, not shared hits: SlateDB scopes each instance's entries so one `Db` does not read another `Db`'s cached blocks by accident. If your main goal is cross-instance warming, the object-store cache is the better fit. diff --git a/website/src/content/docs/docs/operations/metrics.mdx b/website/src/content/docs/docs/operations/metrics.mdx index 09481f8fb..75c832765 100644 --- a/website/src/content/docs/docs/operations/metrics.mdx +++ b/website/src/content/docs/docs/operations/metrics.mdx @@ -170,8 +170,11 @@ All object store metrics carry four labels: | `slatedb.object_store.request_duration_seconds` | histogram | Per-request latency | The instrumented store sits beneath the retrying layer, so each retry attempt -is counted separately. Cache hits that never reach the remote store are not -counted. +is counted separately. It sits above the optional object-store cache, so these metrics +count the calls SlateDB makes into the cache: a cache hit is counted as one +request even though it never reaches the remote store, and the requests the +cache issues against the remote store to fill a miss are not counted. +Use the cache metrics below to tell hits from misses. ### Object store cache (`slatedb.object_store_cache.*`) @@ -184,6 +187,11 @@ counted. | `slatedb.object_store_cache.evicted_keys` | counter | | Evicted keys | | `slatedb.object_store_cache.evicted_bytes` | counter | | Evicted bytes | +When the cache comes from `object_store_cache_options`, it records to the +recorder configured on the database builder. When you build a +`CachedObjectStore` yourself, it records to a no-op recorder unless you pass +one with `with_metrics_recorder`, so these metrics stay empty until you do. + These are passed to `register_histogram` as the `boundaries` parameter. ## Using DefaultMetricsRecorder @@ -246,7 +254,7 @@ impl MetricsRecorder for MetricsRsRecorder { ``` `MetricsRsRecorder` is stateless since the `metrics` facade manages all state -globally. +globally. ## Implementing a Prometheus recorder From 40ef029470706aea8ea496c40a957cd258b7f656 Mon Sep 17 00:00:00 2001 From: Rohan Date: Tue, 28 Jul 2026 02:26:54 -0400 Subject: [PATCH 43/63] [rfc-30 1/N]: refactor writer WAL write path to use new traits (#1961) --- rfcs/0030-pluggable-wal.md | 20 +- slatedb/src/batch_write.rs | 286 ++++++++---- slatedb/src/compactor.rs | 7 +- slatedb/src/db.rs | 322 +++++++++++--- slatedb/src/db/builder.rs | 72 +-- slatedb/src/db_status.rs | 8 + slatedb/src/dispatcher.rs | 107 ++++- slatedb/src/error.rs | 16 + slatedb/src/fence.rs | 132 +++--- slatedb/src/lib.rs | 2 +- slatedb/src/manifest/store.rs | 4 + .../src/memtable_flusher/manifest_writer.rs | 21 +- slatedb/src/memtable_flusher/tracker.rs | 19 +- slatedb/src/memtable_flusher/uploader.rs | 21 +- slatedb/src/oracle.rs | 5 +- slatedb/src/snapshot_manager.rs | 7 +- slatedb/src/transaction_manager.rs | 4 +- slatedb/src/wal/mod.rs | 305 +++++++++++++ slatedb/src/wal/test_utils.rs | 68 +++ slatedb/src/wal/wal_disabled.rs | 22 + slatedb/src/wal/writer_init.rs | 137 ++++++ slatedb/src/wal_buffer.rs | 421 ++++++++---------- 22 files changed, 1474 insertions(+), 532 deletions(-) create mode 100644 slatedb/src/wal/test_utils.rs create mode 100644 slatedb/src/wal/wal_disabled.rs create mode 100644 slatedb/src/wal/writer_init.rs diff --git a/rfcs/0030-pluggable-wal.md b/rfcs/0030-pluggable-wal.md index ae340121f..b18081daa 100644 --- a/rfcs/0030-pluggable-wal.md +++ b/rfcs/0030-pluggable-wal.md @@ -167,17 +167,20 @@ pub struct WalFileRange(pub Bound, pub Bound); /// Defines the types of errors that can be returned by WAL implementations. #[derive(Debug, Clone)] +#[non_exhaustive] pub enum WalError { /// The WAL writer was fenced Fenced, - /// IO error writing/reading the WAL - IoError(Arc), - /// Fatal error indicating that the WAL is in some unexpected/unrecoverable state. - InternalError(Arc), /// A WalIterator observed that the tail of the WAL was truncated while iterating. WalTruncated, /// Operation against wal after it was closed Closed, + /// WAL is unavailable, e.g. due to an I/O error or error in the backing storage system + Unavailable(Arc), + /// WAL implementation detected invalid data/corruption + DataError(Arc), + /// Fatal error indicating that the WAL is in some unexpected/unrecoverable state. + InternalError(Arc), } /// The writer's manifest after fencing. [`crate::Db`] creates this after fencing the manifest @@ -299,9 +302,8 @@ pub trait WalObserver: Send + Sync + 'static { /// Returns the current [`WalStatus`]. fn status(&self) -> Result; - /// Adds a listener that subscribes to event callbacks. On success, returns an initial - /// [`WalStatus`]. The listener receives all updates after this initial status. - fn subscribe(&self, listener: WalStatusListener) -> Result; + /// Adds a listener that subscribes to event callbacks. + fn subscribe(&self, listener: WalStatusListener) -> Result<(), WalError>; } /// A future that yields the result of flushing the WAL. Returned by [`WalWriter::flush`] @@ -371,12 +373,12 @@ pub trait WalIterator: Send + 'static { #[async_trait] pub trait WalReader { /// Returns the name of the WAL implementation - fn name(&self) -> String, + fn name(&self) -> String; /// Returns an iterator over the specified range of WAL File IDs. The start of the range must /// not be `Unbounded`. If the end of the range is `Unbounded` then the returned iterator /// continues returning writes as new writes are appended to the WAL. Otherwise, it returns - /// `None` upon reaching the end of he range. + /// `None` upon reaching the end of the range. async fn iterator( &self, wal_file_id_range: WalFileRange, diff --git a/slatedb/src/batch_write.rs b/slatedb/src/batch_write.rs index bb92715f1..1749177d4 100644 --- a/slatedb/src/batch_write.rs +++ b/slatedb/src/batch_write.rs @@ -28,7 +28,7 @@ use async_trait::async_trait; use fail_parallel::fail_point; use futures::stream::BoxStream; -use futures::StreamExt; +use futures::{FutureExt, StreamExt}; use log::warn; use std::sync::Arc; use std::time::Duration; @@ -43,7 +43,7 @@ use crate::dispatcher::MessageHandler; use crate::mem_table::KVTable; use crate::types::RowEntry; use crate::utils::WatchableOnceCellReader; -use crate::wal_buffer::WalBufferManager; +use crate::wal::{FlushResultFuture, WalWriter}; use crate::{batch::WriteBatch, db::DbInner, db::WriteHandle, error::SlateDBError}; use bytes::Bytes; use parking_lot::RwLockWriteGuard; @@ -52,13 +52,7 @@ use tokio::sync::oneshot; pub(crate) const WRITE_BATCH_TASK_NAME: &str = "writer"; -pub(crate) type WriteBatchResult = Result< - ( - WriteHandle, - WatchableOnceCellReader>, - ), - SlateDBError, ->; +pub(crate) type WriteBatchResult = Result; /// A message processed by the batch writer event loop. #[allow(clippy::large_enum_variant)] @@ -75,7 +69,7 @@ pub(crate) struct BatchWriterFlush { /// Sends a message when the writer has processed the flush message. On successful receipt /// of a message, the caller should wait on the received Receiver to get the result of the /// wal flush. - done: oneshot::Sender>, SlateDBError>>, + done: oneshot::Sender>, } pub(crate) struct WriteBatchRequest { @@ -109,15 +103,15 @@ impl std::fmt::Debug for BatchWriterMessage { pub(crate) struct WriteBatchEventHandler { db_inner: Arc, is_first_write: bool, - wal_buffer: WalBufferManager, + wal_writer: Option>, } impl WriteBatchEventHandler { - pub(crate) fn new(db_inner: Arc, wal_buffer: WalBufferManager) -> Self { + pub(crate) fn new(db_inner: Arc, wal_writer: Option>) -> Self { Self { db_inner, is_first_write: true, - wal_buffer, + wal_writer, } } } @@ -134,22 +128,25 @@ impl MessageHandler for WriteBatchEventHandler { }) => { let result = self .db_inner - .write_batch(batch, &options, txn.as_ref(), &self.wal_buffer) + .write_batch( + batch, + &options, + txn.as_ref(), + self.wal_writer.as_mut(), + self.is_first_write, + ) .await; - // if this is the first write and the WAL is disabled, make sure users are flushing - // their memtables in a timely manner. - if self.is_first_write && !self.db_inner.wal_enabled && options.await_durable { - if let Ok((_, this_watcher)) = &result { - let this_watcher = this_watcher.clone(); - let this_clock = self.db_inner.system_clock.clone(); - tokio::spawn(async move { - monitor_first_write(this_watcher, this_clock).await; - }); + self.is_first_write = false; + match result { + Ok(write_result) => { + let _ = done.send(write_result); + Ok(()) + } + Err(error) => { + let _ = done.send(Err(error.clone())); + Err(error) } } - self.is_first_write = false; - _ = done.send(result); - Ok(()) } BatchWriterMessage::Flush(flush_msg) => { let BatchWriterFlush { @@ -158,9 +155,18 @@ impl MessageHandler for WriteBatchEventHandler { } = flush_msg; let result = self .db_inner - .flush_batch_writer(freeze_memtable, &self.wal_buffer); - let _ = done.send(result); - Ok(()) + .flush_batch_writer(freeze_memtable, self.wal_writer.as_mut()) + .await; + match result { + Ok(flush_result) => { + let _ = done.send(Ok(flush_result)); + Ok(()) + } + Err(error) => { + let _ = done.send(Err(error.clone())); + Err(error) + } + } } } } @@ -185,6 +191,9 @@ impl MessageHandler for WriteBatchEventHandler { } } } + if let Some(wal_writer) = self.wal_writer.as_mut() { + wal_writer.close().await?; + } Ok(()) } } @@ -197,8 +206,9 @@ impl DbInner { batch: WriteBatch, options: &WriteOptions, txn: Option<&DbTransaction>, - wal_buffer: &WalBufferManager, - ) -> WriteBatchResult { + wal_writer: Option<&mut Box>, + is_first_write: bool, + ) -> Result { let _options = options; #[cfg(not(dst))] let now = self.mono_clock.now().await?; @@ -211,10 +221,10 @@ impl DbInner { let commit_seq = if options.seqnum > 0 { let current = self.oracle.last_seq(); if options.seqnum <= current { - return Err(SlateDBError::InvalidSequenceNumber { + return Ok(Err(SlateDBError::InvalidSequenceNumber { provided: options.seqnum, current, - }); + })); } self.oracle.advance_last_seq(options.seqnum); options.seqnum @@ -226,13 +236,13 @@ impl DbInner { // if this batch is part of a transaction. if let Some(txn) = txn { if self.txn_manager.check_has_conflict(&txn.id()) { - return Err(SlateDBError::TransactionConflict); + return Ok(Err(SlateDBError::TransactionConflict)); } } // Count batch-local merge folding on the flush path so DB-side merge // resolution uses one metric for both write batches and memtable flushes. - let (entries, touched_segments, entries_size) = batch + let (entries, touched_segments, entries_size) = match batch .extract_entries( commit_seq, now, @@ -240,30 +250,45 @@ impl DbInner { self.flush_merge_operator.clone(), self.segment_extractor.as_deref(), ) - .await?; + .await + { + Ok(extracted) => extracted, + Err(error) => return Ok(Err(error)), + }; // RFC-0024 route-consistency: when a segment extractor is // configured, every write must extract a prefix that does // not nest with the current segment set. Runs before the // WAL append so a rejected batch produces no durable side // effects. - self.validate_segment_antichain(&touched_segments)?; + if let Err(error) = self.validate_segment_antichain(&touched_segments) { + return Ok(Err(error)); + } - let durable_watcher = if self.wal_enabled { + if let Some(wal_writer) = wal_writer { + assert!(self.wal_enabled); // WAL entries must be appended to the wal buffer atomically. Otherwise, // the WAL buffer might flush the entries in the middle of the batch, which // would violate the guarantee that batches are written atomically. We do // this by appending the entire entry batch in a single call to the WAL buffer, // which holds a write lock during the append. - let wal_watcher = wal_buffer.append(&entries)?; - wal_buffer.maybe_trigger_flush()?; + wal_writer.append(&entries).await?; // TODO: handle sync here, if sync is enabled, we can call `flush` here. let's put this // in another Pull Request. self.write_entries_to_memtable(entries, touched_segments); - wal_watcher } else { + assert!(!self.wal_enabled); // if WAL is disabled, we just write the entries to memtable. - self.write_entries_to_memtable(entries, touched_segments) + let watcher = self.write_entries_to_memtable(entries, touched_segments); + // if this is the first write and the WAL is disabled, make sure users are flushing + // their memtables in a timely manner. + if is_first_write && options.await_durable { + let this_watcher = watcher.clone(); + let this_clock = self.system_clock.clone(); + tokio::spawn(async move { + monitor_first_write(this_watcher, this_clock).await; + }); + } }; // increment memtable_write_bytes by the size of the keys and values inserted into the memtable // after merge operators and overwrites are collapsed @@ -301,18 +326,15 @@ impl DbInner { self.record_memtable_sequence(commit_seq); // maybe freeze the memtable. - self.maybe_freeze_current_memtable(wal_buffer)?; + self.maybe_freeze_current_memtable()?; let write_handle = WriteHandle::new(commit_seq, now); - Ok((write_handle, durable_watcher)) + Ok(Ok(write_handle)) } - fn maybe_freeze_current_memtable( - &self, - wal_buffer: &WalBufferManager, - ) -> Result<(), SlateDBError> { - let replay_after_wal_id = wal_buffer.last_flushed_wal_id(); + fn maybe_freeze_current_memtable(&self) -> Result<(), SlateDBError> { + let replay_after_wal_id = self.wal_observer.status()?.last_flushed_wal_id; let mut guard = self.state.write(); let meta = guard.memtable().metadata(); @@ -340,25 +362,21 @@ impl DbInner { Ok(()) } - fn flush_batch_writer( + async fn flush_batch_writer( &self, freeze_memtable: bool, - wal_buffer: &WalBufferManager, - ) -> Result>, SlateDBError> { - let flush_rx = if self.wal_enabled { - wal_buffer.flush()? + wal_writer: Option<&mut Box>, + ) -> Result { + let flush_rx = if let Some(wal_writer) = wal_writer { + wal_writer.flush().await? } else { - let (flush_tx, flush_rx) = oneshot::channel(); - flush_tx - .send(Ok(())) - .expect("unexpected oneshot send failure"); - flush_rx + async { Ok(()) }.boxed() }; if freeze_memtable { // Note that this likely won't reflect the result of the above flush call as we don't // block until the flush completes. That's fine, as any earlier wal is still a safe // replay point. - let replay_after_wal_id = wal_buffer.last_flushed_wal_id(); + let replay_after_wal_id = self.wal_observer.status()?.last_flushed_wal_id; let mut guard = self.state.write(); self.freeze_current_memtable_with_state_guard(&mut guard, replay_after_wal_id); } @@ -393,7 +411,7 @@ impl DbInner { freeze_memtable, done, }))?; - rx.await??.await? + Ok(rx.await??.await?) } /// RFC-0024 route-consistency check. Verifies that `batch_prefixes`, @@ -522,8 +540,58 @@ async fn monitor_first_write( mod tests { use super::*; use crate::object_store::memory::InMemory; + use crate::wal::test_utils::FakeWalWriter; + use crate::wal::{WalError, WalObserver, WalStatus}; use crate::Db; + enum FailingWalOperation { + Append, + Flush, + } + + struct FailingWalWriter { + inner: FakeWalWriter, + operation: FailingWalOperation, + } + + impl FailingWalWriter { + fn new(operation: FailingWalOperation) -> Self { + Self { + inner: FakeWalWriter::new(0), + operation, + } + } + } + + #[async_trait] + impl WalWriter for FailingWalWriter { + async fn append(&mut self, write_batch: &[RowEntry]) -> Result<(), WalError> { + if matches!(self.operation, FailingWalOperation::Append) { + return Err(WalError::Fenced); + } + self.inner.append(write_batch).await + } + + async fn flush(&mut self) -> Result { + if matches!(self.operation, FailingWalOperation::Flush) { + return Err(WalError::Fenced); + } + self.inner.flush().await + } + + fn observer(&self) -> Box { + self.inner.observer() + } + + fn status(&self) -> Result { + self.inner.status() + } + + async fn close(&mut self) -> Result<(), WalError> { + self.inner.close().await + } + } + /// Build a transaction-less `WriteBatchMessage` and its result receiver, /// keeping the `txn: None` and channel boilerplate out of individual tests. fn test_message( @@ -554,16 +622,9 @@ mod tests { ) .await .unwrap(); - let wal_buffer = WalBufferManager::new( - db.inner.status_manager.clone(), - &db.inner.recorder, - 0, - db.inner.table_store.clone(), - 1024, - None, - ); - let mut handler = WriteBatchEventHandler::new(db.inner.clone(), wal_buffer); + let wal_writer = Box::new(FakeWalWriter::new(0)); + let mut handler = WriteBatchEventHandler::new(db.inner.clone(), Some(wal_writer)); assert!(handler.is_first_write); let mut batch = WriteBatch::new(); @@ -577,22 +638,70 @@ mod tests { assert!(!handler.is_first_write); } + #[tokio::test] + async fn test_append_error_notifies_caller_and_fails_handler() { + let object_store = Arc::new(InMemory::new()); + let db = Db::open( + "/tmp/test_append_error_notifies_caller_and_fails_handler", + object_store, + ) + .await + .unwrap(); + let wal_writer = Box::new(FailingWalWriter::new(FailingWalOperation::Append)); + let mut handler = WriteBatchEventHandler::new(db.inner.clone(), Some(wal_writer)); + + let mut batch = WriteBatch::new(); + batch.put(b"key", b"value"); + let (msg, done_rx) = test_message(batch, WriteOptions::default()); + + let handler_error = handler.handle(msg).await.unwrap_err(); + assert!(matches!(handler_error, SlateDBError::Fenced)); + let caller_error = match done_rx.await.unwrap() { + Ok(_) => panic!("append unexpectedly succeeded"), + Err(error) => error, + }; + assert!(matches!(caller_error, SlateDBError::Fenced)); + assert_eq!(db.get(b"key").await.unwrap(), None); + + db.close().await.unwrap(); + } + + #[tokio::test] + async fn test_flush_error_notifies_caller_and_fails_handler() { + let object_store = Arc::new(InMemory::new()); + let db = Db::open( + "/tmp/test_flush_error_notifies_caller_and_fails_handler", + object_store, + ) + .await + .unwrap(); + let wal_writer = Box::new(FailingWalWriter::new(FailingWalOperation::Flush)); + let mut handler = WriteBatchEventHandler::new(db.inner.clone(), Some(wal_writer)); + let (done, done_rx) = tokio::sync::oneshot::channel(); + let msg = BatchWriterMessage::Flush(BatchWriterFlush { + freeze_memtable: false, + done, + }); + + let handler_error = handler.handle(msg).await.unwrap_err(); + assert!(matches!(handler_error, SlateDBError::Fenced)); + let caller_error = match done_rx.await.unwrap() { + Ok(_) => panic!("flush unexpectedly succeeded"), + Err(error) => error, + }; + assert!(matches!(caller_error, SlateDBError::Fenced)); + + db.close().await.unwrap(); + } + #[tokio::test] async fn test_user_defined_seqnum() { let object_store = Arc::new(InMemory::new()); let db = Db::open("/tmp/test_user_defined_seqnum", object_store) .await .unwrap(); - let wal_buffer = WalBufferManager::new( - db.inner.status_manager.clone(), - &db.inner.recorder, - 0, - db.inner.table_store.clone(), - 1024, - None, - ); - - let mut handler = WriteBatchEventHandler::new(db.inner.clone(), wal_buffer); + let wal_writer = Box::new(FakeWalWriter::new(0)); + let mut handler = WriteBatchEventHandler::new(db.inner.clone(), Some(wal_writer)); // Write with a user-defined seqnum let mut batch = WriteBatch::new(); @@ -605,7 +714,7 @@ mod tests { }, ); handler.handle(msg).await.unwrap(); - let (write_handle, _) = done_rx.await.unwrap().unwrap(); + let write_handle = done_rx.await.unwrap().unwrap(); assert_eq!(write_handle.seqnum(), 42); // Write without a seqnum and verify auto-assigned is > 42 @@ -613,7 +722,7 @@ mod tests { batch.put(b"key2", b"value2"); let (msg, done_rx) = test_message(batch, WriteOptions::default()); handler.handle(msg).await.unwrap(); - let (write_handle, _) = done_rx.await.unwrap().unwrap(); + let write_handle = done_rx.await.unwrap().unwrap(); assert!(write_handle.seqnum() > 42); } @@ -626,23 +735,16 @@ mod tests { ) .await .unwrap(); - let wal_buffer = WalBufferManager::new( - db.inner.status_manager.clone(), - &db.inner.recorder, - 0, - db.inner.table_store.clone(), - 1024, - None, - ); + let wal_writer = Box::new(FakeWalWriter::new(0)); - let mut handler = WriteBatchEventHandler::new(db.inner.clone(), wal_buffer); + let mut handler = WriteBatchEventHandler::new(db.inner.clone(), Some(wal_writer)); // First, do a normal write to advance the oracle let mut batch = WriteBatch::new(); batch.put(b"key1", b"value1"); let (msg, done_rx) = test_message(batch, WriteOptions::default()); handler.handle(msg).await.unwrap(); - let (write_handle, _) = done_rx.await.unwrap().unwrap(); + let write_handle = done_rx.await.unwrap().unwrap(); let first_seq = write_handle.seqnum(); // Try to write with a seqnum <= the current max diff --git a/slatedb/src/compactor.rs b/slatedb/src/compactor.rs index 265bb0545..f5188772b 100644 --- a/slatedb/src/compactor.rs +++ b/slatedb/src/compactor.rs @@ -6102,7 +6102,12 @@ mod tests { let db_state = db.inner.state.read(); let cow_db_state = db_state.state(); ( - db.inner.wal_observer.status().buffered_wal_entries_count == 0, + db.inner + .wal_observer + .status() + .unwrap() + .buffered_wal_entries_count + == 0, db_state.memtable().is_empty() && cow_db_state.imm_memtable.is_empty(), db_state.state().core().clone(), ) diff --git a/slatedb/src/db.rs b/slatedb/src/db.rs index cdf81bb46..13d43a4d9 100644 --- a/slatedb/src/db.rs +++ b/slatedb/src/db.rs @@ -72,7 +72,6 @@ use crate::tablestore::TableStore; use crate::transaction_manager::TransactionManager; use crate::types::KeyValue; use crate::utils::{format_bytes_si, SafeSender, WatchableOnceCellReader}; -use crate::wal_buffer::{WalEvent, WalObserver, WalStatus, WAL_BUFFER_TASK_NAME}; use crate::wal_replay::{WalReplayIterator, WalReplayOptions}; use crate::{DbCacheManagerOps, DbMetadataOps, DbReadOps, DbWriteOps}; use slatedb_common::clock::SystemClock; @@ -81,6 +80,7 @@ use slatedb_common::DbRand; use slatedb_txn_obj::DirtyObject; use crate::db_status::{ClosedResultWriter, DbStatusManager}; +use crate::wal::{WalEvent, WalObserver, WalStatus}; pub use builder::DbBuilder; pub use builder::DbReaderBuilder; @@ -115,7 +115,7 @@ pub(crate) struct DbInner { /// [`txn_manager`] tracks all the live transactions and related metadata. pub(crate) txn_manager: Arc, pub(crate) snapshot_manager: Arc, - pub(crate) status_manager: DbStatusManager, + pub(crate) status_manager: Arc, /// Segment extractor (RFC-0024). When `Some`, the writer routes every /// key through this extractor and groups flush output into per-segment /// L0 SSTs. When `None`, the database is the singleton `prefix=""` @@ -132,11 +132,11 @@ impl DbInner { manifest: DirtyObject, memtable_flusher: Arc, write_notifier: SafeSender, - wal_observer: WalObserver, + wal_observer: Box, recorder: MetricsRecorderHelper, fp_registry: Arc, merge_operator: Option, - status_manager: DbStatusManager, + status_manager: Arc, segment_extractor: Option>, ) -> Result { // both last_seq and last_committed_seq will be updated after WAL replay. @@ -180,7 +180,7 @@ impl DbInner { wal_observer, oracle.clone(), state.clone(), - status_manager.result_reader(), + status_manager.clone(), ); let db_inner = Self { @@ -270,7 +270,7 @@ impl DbInner { } #[allow(unused_variables)] - fn wal_enabled_in_options(settings: &Settings) -> bool { + pub(crate) fn wal_enabled_in_options(settings: &Settings) -> bool { #[cfg(feature = "wal_disable")] return settings.wal_enabled; #[cfg(not(feature = "wal_disable"))] @@ -303,10 +303,23 @@ impl DbInner { // TODO: this can be modified as awaiting the last_durable_seq watermark & fatal error. - let (write_handle, mut durable_watcher) = rx.await??; + let write_handle = rx.await??; if options.await_durable { - durable_watcher.await_value().await?; + let seq = write_handle.seq; + let mut status_subscription = self.status_manager.subscribe(); + let status = status_subscription + .wait_for(|s| s.durable_seq >= seq || s.close_reason.is_some()) + .await + .map_err(|_| SlateDBError::Closed)?; + if status.durable_seq < seq { + self.check_closed()?; + warn!( + "durable seq {} not advanced past write seq {} and db not closed", + status.durable_seq, seq + ); + return Err(SlateDBError::InvalidDBState); + } } Ok(write_handle) @@ -316,7 +329,7 @@ impl DbInner { pub(crate) async fn maybe_apply_backpressure(&self) -> Result<(), SlateDBError> { loop { self.check_closed()?; - let wal_status = self.wal_observer.status(); + let wal_status = self.wal_observer.status()?; let (active_memtable_size_bytes, imm_memtable_size_bytes) = { let guard = self.state.read(); let estimate = |metadata: KVTableMetadata| { @@ -763,10 +776,6 @@ impl Db { warn!("failed to shutdown writer task [error={:?}]", e); } - if let Err(e) = self.task_executor.shutdown_task(WAL_BUFFER_TASK_NAME).await { - warn!("failed to shutdown wal writer task [error={:?}]", e); - } - if let Err(e) = self.inner.table_store.close_cache().await { warn!("failed to close block cache [error={:?}]", e); } @@ -2067,55 +2076,70 @@ impl WriteHandle { /// via a [`tokio::sync::watch`] channel. #[derive(Clone)] pub(crate) struct DbWalObserver { - status_rx: tokio::sync::watch::Receiver, + status_rx: tokio::sync::watch::Receiver>, closed_reader: WatchableOnceCellReader>, - wrapped: WalObserver, + wrapped: Arc, } impl DbWalObserver { fn new( - wrapped: WalObserver, + wrapped: Box, oracle: Arc, db_state: Arc>, - closed_reader: WatchableOnceCellReader>, + closed_writer: Arc, ) -> Self { let (status_tx, status_rx) = tokio::sync::watch::channel(wrapped.status()); + let closed_reader = closed_writer.result_reader(); wrapped .subscribe(Arc::new(move |event| { - let WalEvent::WalFlushed(status) = event; - if let Some(seq) = status.last_flushed_seq { - oracle.advance_durable_seq(seq); - } - let mut guard = db_state.write(); - guard.set_next_wal_id(status.last_flushed_wal_id + 1); - drop(guard); + let status: Result = match event { + WalEvent::WalFlushed(status) => { + if let Some(seq) = status.last_flushed_seq { + oracle.advance_durable_seq(seq); + } + let mut guard = db_state.write(); + guard.set_next_wal_id(status.last_flushed_wal_id + 1); + drop(guard); + Ok(status) + } + WalEvent::WalClosed(status) => { + closed_writer.write_result(Err(status.clone().into())); + Err(status) + } + }; let _ = status_tx.send(status); })) .expect("failed to subscribe to wal"); Self { status_rx, closed_reader, - wrapped, + wrapped: wrapped.into(), } } - pub(crate) fn status(&self) -> WalStatus { + pub(crate) fn status(&self) -> Result { self.wrapped.status() } async fn wait_on_condition( &self, - predicate: impl FnMut(&WalStatus) -> bool, + mut predicate: impl FnMut(&WalStatus) -> bool, ) -> Result<(), SlateDBError> { let mut status_rx = self.status_rx.clone(); - let result = status_rx.wait_for(predicate).await.map(|_| ()); - match result { - Ok(_) => Ok(()), - Err(_) => { - debug!("wal listener tx dropped - wait on db close"); - self.closed_reader.clone().await_value().await - } - } + let result = status_rx + .wait_for(|s| match s { + Err(_) => true, + Ok(s) => predicate(s), + }) + .await; + let Ok(result) = result else { + drop(result); + debug!("wal listener tx dropped - wait on db close"); + return self.closed_reader.clone().await_value().await; + }; + let result = result.clone(); + result?; + Ok(()) } /// Waits until the wal a given wal id is released by the wal writer @@ -2160,6 +2184,7 @@ mod tests { OnDemandCompactionSchedulerSupplier, StringConcatMergeOperator, }; use crate::types::RowEntry; + use crate::wal::WalError; use crate::wal_reader::WalReader; use crate::{proptest_util, test_utils, CloseReason, CompactorBuilder, KeyValue}; use async_trait::async_trait; @@ -3091,11 +3116,27 @@ mod tests { .unwrap(); // a sanity check: the wal contains the most recent write - assert_ne!(kv_store.inner.wal_observer.status().estimated_bytes, 0); + assert_ne!( + kv_store + .inner + .wal_observer + .status() + .unwrap() + .estimated_bytes, + 0 + ); // and a flush() should clear it kv_store.flush().await.unwrap(); - assert_eq!(kv_store.inner.wal_observer.status().estimated_bytes, 0); + assert_eq!( + kv_store + .inner + .wal_observer + .status() + .unwrap() + .estimated_bytes, + 0 + ); } #[tokio::test] @@ -3133,6 +3174,7 @@ mod tests { .inner .wal_observer .status() + .unwrap() .buffered_wal_entries_count, 1 ); @@ -3153,6 +3195,7 @@ mod tests { .inner .wal_observer .status() + .unwrap_err() .buffered_wal_entries_count, 0 ); @@ -3196,20 +3239,24 @@ mod tests { .await .unwrap(); - db.put_with_options( - b"test_key", - b"test_value", - &PutOptions::default(), - &WriteOptions { - await_durable: false, - ..Default::default() - }, - ) - .await - .unwrap(); + let put_seq = db + .put_with_options( + b"test_key", + b"test_value", + &PutOptions::default(), + &WriteOptions { + await_durable: false, + ..Default::default() + }, + ) + .await + .unwrap() + .seq; // Sanity check: WAL has buffered entries before close. - assert_eq!(db.inner.wal_observer.status().buffered_wal_entries_count, 1); + let wal_status = db.inner.wal_observer.status().unwrap(); + assert_eq!(wal_status.buffered_wal_entries_count, 1); + assert!(wal_status.last_flushed_seq.unwrap_or(0) < put_seq); assert_eq!( lookup_metric( &metrics_recorder, @@ -3227,7 +3274,9 @@ mod tests { // close() should succeed but not flush when failed. db.close().await.unwrap(); - assert_eq!(db.inner.wal_observer.status().buffered_wal_entries_count, 1); + let wal_status = db.inner.wal_observer.status().unwrap_err(); + assert!(matches!(wal_status.closed_reason, Some(WalError::Fenced))); + assert!(wal_status.last_flushed_seq.unwrap_or(0) < put_seq); assert_eq!( lookup_metric( &metrics_recorder, @@ -3256,19 +3305,23 @@ mod tests { .await .unwrap(); - db.put_with_options( - b"test_key", - b"test_value", - &PutOptions::default(), - &WriteOptions { - await_durable: false, - ..Default::default() - }, - ) - .await - .unwrap(); + let put_seq = db + .put_with_options( + b"test_key", + b"test_value", + &PutOptions::default(), + &WriteOptions { + await_durable: false, + ..Default::default() + }, + ) + .await + .unwrap() + .seq; - assert_eq!(db.inner.wal_observer.status().buffered_wal_entries_count, 1); + let wal_status = db.inner.wal_observer.status().unwrap(); + assert_eq!(wal_status.buffered_wal_entries_count, 1); + assert!(wal_status.last_flushed_seq.unwrap_or(0) < put_seq); assert_eq!( lookup_metric( &metrics_recorder, @@ -3280,7 +3333,9 @@ mod tests { db.close().await.unwrap(); - assert_eq!(db.inner.wal_observer.status().buffered_wal_entries_count, 0); + let wal_status = db.inner.wal_observer.status().unwrap_err(); + assert!(matches!(wal_status.closed_reason, Some(WalError::Closed))); + assert_eq!(wal_status.last_flushed_seq, Some(put_seq)); assert_eq!( lookup_metric( &metrics_recorder, @@ -3487,6 +3542,14 @@ mod tests { .build() .await .unwrap(); + tokio::time::timeout( + Duration::from_secs(1), + db.task_executor + .join_task(crate::wal_buffer::WAL_BUFFER_TASK_NAME), + ) + .await + .expect("native WAL task should not run when the WAL is disabled") + .unwrap(); let put_options = PutOptions::default(); let write_options = WriteOptions { await_durable: false, @@ -4318,7 +4381,12 @@ mod tests { } // Verify WALs flushes. - let wal_id = kv_store.inner.wal_observer.status().last_flushed_wal_id; + let wal_id = kv_store + .inner + .wal_observer + .status() + .unwrap() + .last_flushed_wal_id; assert_eq!(wal_id, MAX_WAL_FLUSHES_BEFORE_L0_FLUSH); // account for the empty WAL written for fencing // Verify no memtable was frozen or L0 flush happened. @@ -4482,7 +4550,12 @@ mod tests { // Verify that the WAL was also flushed since we guarantee // memtable data is persisted in the WAL prior to L0 flush. - let recent_flushed_wal_id = kv_store.inner.wal_observer.status().last_flushed_wal_id; + let recent_flushed_wal_id = kv_store + .inner + .wal_observer + .status() + .unwrap() + .last_flushed_wal_id; assert_eq!(recent_flushed_wal_id, 2); // Verify that the data is still accessible after flush @@ -4551,6 +4624,7 @@ mod tests { .inner .wal_observer .status() + .unwrap() .buffered_wal_entries_count, 1 ); @@ -4567,6 +4641,7 @@ mod tests { .inner .wal_observer .status() + .unwrap() .buffered_wal_entries_count, 0 ); @@ -4889,7 +4964,12 @@ mod tests { .unwrap(); // Get initial WAL ID to verify flush occurred - let initial_wal_id = kv_store.inner.wal_observer.status().last_flushed_wal_id; + let initial_wal_id = kv_store + .inner + .wal_observer + .status() + .unwrap() + .last_flushed_wal_id; // Flush WAL using flush_with_options - this should succeed without error let flush_result = kv_store @@ -4910,7 +4990,12 @@ mod tests { // Verify that the WAL buffer is in a consistent state after flush // The recent_flushed_wal_id should be at least as high as before - let final_wal_id = kv_store.inner.wal_observer.status().last_flushed_wal_id; + let final_wal_id = kv_store + .inner + .wal_observer + .status() + .unwrap() + .last_flushed_wal_id; assert!( final_wal_id >= initial_wal_id, "WAL ID should not decrease after flush" @@ -5040,12 +5125,12 @@ mod tests { // Wait for put to end up in the WAL buffer let this_wal_buffer = db.inner.wal_observer.clone(); wait_for(Box::new(move || { - this_wal_buffer.status().buffered_wal_entries_count > 0 + this_wal_buffer.status().unwrap().buffered_wal_entries_count > 0 })) .await; // Verify that there is now 1 WAL entry in memory. - let wal_status = db.inner.wal_observer.status(); + let wal_status = db.inner.wal_observer.status().unwrap(); assert_eq!(wal_status.buffered_wal_entries_count, 1); let (active_memtable_size_bytes, imm_memtable_size_bytes) = { @@ -5146,6 +5231,14 @@ mod tests { db.put_with_options(b"key1", &large_value, &PutOptions::default(), &write_opts) .await .unwrap(); + assert_eq!( + db.inner + .wal_observer + .status() + .unwrap() + .buffered_wal_entries_count, + 1 + ); // Start backpressure on a cloned inner handle. This parks the task on // the same wait path used by writers before they enqueue a batch. @@ -5906,7 +5999,10 @@ mod tests { let value1 = [b'b'; 96]; let result = db.put(&key1, &value1).await; assert!(result.is_ok(), "Failed to write key1"); - assert_eq!(db.inner.wal_observer.status().last_flushed_wal_id, 2); + assert_eq!( + db.inner.wal_observer.status().unwrap().last_flushed_wal_id, + 2 + ); // Let background flush attempts fail while WAL durability preserves recovery. // expect to fail as l0 upload is blocked @@ -6065,6 +6161,71 @@ mod tests { .expect_err("close should error out due to WAL IO error"); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_await_durable_write_returns_error_if_db_closes_before_durable() { + let fp_registry = Arc::new(FailPointRegistry::new()); + let object_store: Arc = Arc::new(InMemory::new()); + let mut settings = test_db_options(0, 1024, None); + settings.flush_interval = None; + let db = Arc::new( + Db::builder( + "/tmp/test_await_durable_write_returns_error_if_db_closes_before_durable", + object_store, + ) + .with_settings(settings) + .with_fp_registry(fp_registry.clone()) + .build() + .await + .unwrap(), + ); + // pause writes so that we can force the write to fail on the close status before the + // final flush causes the write to become durable + fail_parallel::cfg(fp_registry.clone(), "write-wal-sst-io-error", "pause").unwrap(); + let write_db = db.clone(); + let write_task = tokio::spawn(async move { + write_db + .put_with_options( + b"foo", + b"bar", + &PutOptions::default(), + &WriteOptions::default(), + ) + .await + }); + tokio::time::timeout(Duration::from_secs(10), async { + loop { + if db + .inner + .wal_observer + .status() + .unwrap() + .buffered_wal_entries_count + == 1 + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("write was not buffered"); + let close_db = db.clone(); + let close_task = tokio::spawn(async move { close_db.close().await }); + + let write_error = tokio::time::timeout(Duration::from_secs(10), write_task) + .await + .expect("timed out waiting for write") + .expect("write task panicked") + .expect_err("write unexpectedly reported success"); + + assert_eq!( + write_error.kind(), + crate::ErrorKind::Closed(CloseReason::Clean) + ); + fail_parallel::cfg(fp_registry, "write-wal-sst-io-error", "off").unwrap(); + let _ = close_task.await.unwrap(); + } + async fn do_test_should_read_compacted_db(mut options: Settings) { let object_store: Arc = Arc::new(InMemory::new()); let path = "/tmp/test_kv_store"; @@ -9895,11 +10056,21 @@ mod tests { .await .unwrap(); if i == 0 { - first_l0_flushed_wal_id = source.inner.wal_observer.status().last_flushed_wal_id; + first_l0_flushed_wal_id = source + .inner + .wal_observer + .status() + .unwrap() + .last_flushed_wal_id; } l0_flushed_seq = write.seqnum(); } - let l0_flushed_boundary_wal_id = source.inner.wal_observer.status().last_flushed_wal_id; + let l0_flushed_boundary_wal_id = source + .inner + .wal_observer + .status() + .unwrap() + .last_flushed_wal_id; assert!(l0_flushed_boundary_wal_id > first_l0_flushed_wal_id); // Write several smaller records, each flushed into a separate WAL. On @@ -9923,7 +10094,12 @@ mod tests { .await .unwrap(); } - let final_source_wal_id = source.inner.wal_observer.status().last_flushed_wal_id; + let final_source_wal_id = source + .inner + .wal_observer + .status() + .unwrap() + .last_flushed_wal_id; assert!(final_source_wal_id >= l0_flushed_boundary_wal_id + 2); // Recover with a much smaller replay target so WAL replay splits into diff --git a/slatedb/src/db/builder.rs b/slatedb/src/db/builder.rs index 2e52b775a..741e9a502 100644 --- a/slatedb/src/db/builder.rs +++ b/slatedb/src/db/builder.rs @@ -161,7 +161,8 @@ use crate::retrying_object_store::RetryingObjectStore; use crate::tablestore::{TableStore, TableStoreKind}; use crate::utils::SafeSender; use crate::utils::WatchableOnceCell; -use crate::wal_buffer::WalBufferManager; +use crate::wal::wal_disabled::DisabledWalObserver; +use crate::wal::WalObserver; use slatedb_common::clock::DefaultSystemClock; use slatedb_common::clock::SystemClock; use slatedb_common::metrics::MetricsRecorder; @@ -594,30 +595,50 @@ impl> DbBuilder

{ } }; - let fencer = WriterFencer::new(table_store.clone(), &self.settings, system_clock.clone()); + let manifest_dirty = stored_manifest.prepare_dirty()?; + let status_manager = Arc::new(DbStatusManager::new_with_initial_values( + manifest_dirty.value.core.last_l0_seq, + manifest_dirty.into(), + BTreeSet::new(), + )); + + let task_executor = Arc::new(MessageHandlerExecutor::new( + status_manager.clone(), + system_clock.clone(), + )); + + let fencer = WriterFencer::new( + status_manager.result_reader(), + recorder.clone(), + table_store.clone(), + &self.settings, + system_clock.clone(), + task_executor.clone(), + ); let WriterFenceResult { manifest, replay_range, + mut wal_writer, } = fencer.fence(stored_manifest).await?; + let (wal_writer, wal_observer) = if DbInner::wal_enabled_in_options(&self.settings) { + let wal_observer = wal_writer.observer(); + (Some(wal_writer), wal_observer) + } else { + wal_writer.close().await.map_err(SlateDBError::from)?; + let Err(final_status) = wal_writer.status() else { + return Err(crate::Error::internal( + "closed wal writer did not return terminal status".to_string(), + )); + }; + let wal_observer = + Box::new(DisabledWalObserver::new(final_status)) as Box; + (None, wal_observer) + }; let manifest_dirty = manifest.prepare_dirty()?; - - // Shared lifecycle state — created before DbInner so it can be shared - // with the executor and future channel construction. - let status_manager = DbStatusManager::new_with_initial_values( + status_manager.report_fence_manifest( manifest_dirty.value.core.last_l0_seq, manifest_dirty.clone().into(), - BTreeSet::new(), - ); - - let recent_flushed_wal_id = replay_range.end - 1; - let mut wal_buffer = WalBufferManager::new( - status_manager.clone(), - &recorder, - recent_flushed_wal_id, - table_store.clone(), - self.settings.l0_sst_size_bytes, - self.settings.flush_interval, ); // Setup communication channels wired to the shared closed state. @@ -625,7 +646,7 @@ impl> DbBuilder

{ let (write_tx, write_rx) = SafeSender::unbounded_channel(reader); // Create the database inner state - let memtable_flusher = Arc::new(MemtableFlusher::new(&status_manager)); + let memtable_flusher = Arc::new(MemtableFlusher::new(status_manager.as_ref())); let inner = Arc::new( DbInner::new( self.settings.clone(), @@ -635,11 +656,11 @@ impl> DbBuilder

{ manifest_dirty, Arc::clone(&memtable_flusher), write_tx, - wal_buffer.observer(), + wal_observer, recorder.clone(), self.fp_registry.clone(), self.merge_operator.clone(), - status_manager.clone(), + status_manager, self.segment_extractor.clone(), ) .await?, @@ -647,16 +668,9 @@ impl> DbBuilder

{ // Setup background tasks let tokio_handle = Handle::current(); - let task_executor = Arc::new(MessageHandlerExecutor::new( - Arc::new(status_manager), - system_clock.clone(), - )); - if inner.wal_enabled { - wal_buffer.init(task_executor.clone()).await?; - }; task_executor.add_handler( WRITE_BATCH_TASK_NAME.to_string(), - Box::new(WriteBatchEventHandler::new(inner.clone(), wal_buffer)), + Box::new(WriteBatchEventHandler::new(inner.clone(), wal_writer)), write_rx, &tokio_handle, )?; @@ -797,7 +811,7 @@ impl> DbBuilder

{ manifest, &tokio_handle, &task_executor, - &inner.status_manager, + inner.status_manager.as_ref(), )?; // Monitor background tasks diff --git a/slatedb/src/db_status.rs b/slatedb/src/db_status.rs index 2508a4e8f..2448db8ce 100644 --- a/slatedb/src/db_status.rs +++ b/slatedb/src/db_status.rs @@ -109,6 +109,14 @@ impl DbStatusManager { } } + pub(crate) fn report_fence_manifest(&self, durable_seq: u64, manifest: VersionedManifest) { + self.tx.send_if_modified(|s| { + s.durable_seq = durable_seq; + s.current_manifest = manifest; + true + }); + } + pub(crate) fn report_durable_seq(&self, seq: u64) { self.tx.send_if_modified(|s| { if seq > s.durable_seq { diff --git a/slatedb/src/dispatcher.rs b/slatedb/src/dispatcher.rs index c68819df4..0dc7aac67 100644 --- a/slatedb/src/dispatcher.rs +++ b/slatedb/src/dispatcher.rs @@ -345,7 +345,7 @@ impl MessageDispatcher { let (run_result, run_maybe_panic) = split_unwind_result(name.clone(), run_unwind_result); if let Err(ref err) = run_result { error!( - "background task panicked unexpectedly. [task_name={}, error={:?}, panic={:?}]", + "background task exited unexpectedly. [task_name={}, error={:?}, panic={:?}]", name, err, run_maybe_panic.map(|p| panic_string(&p)) @@ -842,7 +842,7 @@ impl MessageHandlerExecutor { Ok(()) } - /// Cancels a task and waits for it to complete. + /// Cancels a running task and waits for it to complete. /// /// ## Arguments /// @@ -855,6 +855,32 @@ impl MessageHandlerExecutor { self.cancel_task(name); self.join_task(name).await } + + /// Removes a task that may not have started yet, or cancels a running task and waits for it to + /// complete. + /// + /// ## Arguments + /// + /// * `name`: The name of the task to cancel and wait for. + /// + /// ## Returns + /// + /// [`Some`] with the result of the task if it was started, [`None`] otherwise + pub(crate) async fn shutdown_or_deregister_task( + &self, + name: &str, + ) -> Option> { + { + let mut guard = self.futures.lock(); + if let Some(task_definitions) = guard.as_mut() { + if task_definitions.iter().any(|task| task.name == name) { + task_definitions.retain(|task| task.name != name); + return None; + } + } + } + Some(self.shutdown_task(name).await) + } } #[cfg(all(test, feature = "test-util"))] @@ -1183,6 +1209,83 @@ mod test { ); } + #[tokio::test] + async fn test_shutdown_task_removes_handler_before_monitor_starts() { + let clock = Arc::new(DefaultSystemClock::new()); + let closed_result = Arc::new(WatchableOnceCell::new()); + let task_executor = MessageHandlerExecutor::new(closed_result.clone(), clock.clone()); + let removed_cleanup = WatchableOnceCell::new(); + let retained_cleanup = WatchableOnceCell::new(); + let (removed_tx, removed_rx) = async_channel::unbounded(); + let (_retained_tx, retained_rx) = async_channel::unbounded(); + + task_executor + .add_handler( + "removed".to_string(), + Box::new(TestHandler::new( + Arc::new(Mutex::new(Vec::new())), + removed_cleanup.clone(), + clock.clone(), + )), + removed_rx, + &Handle::current(), + ) + .unwrap(); + task_executor + .add_handler( + "retained".to_string(), + Box::new(TestHandler::new( + Arc::new(Mutex::new(Vec::new())), + retained_cleanup.clone(), + clock, + )), + retained_rx, + &Handle::current(), + ) + .unwrap(); + + assert!(task_executor + .shutdown_or_deregister_task("removed") + .await + .is_none()); + assert!(removed_tx.is_closed()); + assert!(removed_cleanup.result_reader().read().is_none()); + + let monitor = task_executor.monitor_on(&Handle::current()).unwrap(); + assert!(!task_executor.tokens.contains_key("removed")); + assert!(task_executor.tokens.contains_key("retained")); + + task_executor + .shutdown_or_deregister_task("retained") + .await + .unwrap() + .unwrap(); + monitor.await.unwrap(); + assert!(retained_cleanup.result_reader().read().is_some()); + assert!(closed_result.result_reader().read().is_some()); + } + + #[tokio::test] + async fn test_shutdown_task_cancels_when_pending_task_not_found() { + let task_executor = MessageHandlerExecutor::new( + Arc::new(WatchableOnceCell::new()), + Arc::new(DefaultSystemClock::new()), + ); + let running_token = CancellationToken::new(); + let running_result = WatchableOnceCell::new(); + running_result.write(Ok(())); + task_executor + .tokens + .insert("running".to_string(), running_token.clone()); + task_executor + .results + .insert("running".to_string(), running_result); + + task_executor.shutdown_task("running").await.unwrap(); + + assert!(running_token.is_cancelled()); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_dispatcher_prioritizes_messages_over_tickers() { let log = Arc::new(Mutex::new(Vec::<(Phase, TestMessage)>::new())); diff --git a/slatedb/src/error.rs b/slatedb/src/error.rs index 674a8689b..ea5802b35 100644 --- a/slatedb/src/error.rs +++ b/slatedb/src/error.rs @@ -77,6 +77,18 @@ pub(crate) enum SlateDBError { #[error("wal store reconfiguration unsupported")] WalStoreReconfigurationError, + #[error("wal truncated")] + WalTruncated, + + #[error("wal unavailable")] + WalUnavailable(Arc), + + #[error("wal internal error")] + WalInternalError(Arc), + + #[error("wal data error")] + WalDataError(Arc), + #[error("invalid compaction")] InvalidCompaction, @@ -640,6 +652,7 @@ impl From for Error { #[cfg(feature = "foyer")] SlateDBError::FoyerError(err) => Error::unavailable(msg).with_source(Box::new(err)), SlateDBError::TransactionalObjectTimeout { .. } => Error::unavailable(msg), + SlateDBError::WalUnavailable(src) => Error::unavailable(msg).with_source(Box::new(src)), // Invalid errors SlateDBError::InvalidCachePartSize => Error::invalid(msg), @@ -714,6 +727,7 @@ impl From for Error { SlateDBError::CloneExternalDbMissing => Error::data(msg), SlateDBError::CloneIncorrectExternalDbCheckpoint { .. } => Error::data(msg), SlateDBError::CloneIncorrectFinalCheckpoint { .. } => Error::data(msg), + SlateDBError::WalDataError(src) => Error::data(msg).with_source(Box::new(src)), // Internal errors SlateDBError::CompactorExecutorFailed => Error::internal(msg), @@ -728,6 +742,8 @@ impl From for Error { SlateDBError::TransactionalObjectError(err) => { Error::internal(msg).with_source(Box::new(err)) } + SlateDBError::WalTruncated => Error::internal(msg), + SlateDBError::WalInternalError(src) => Error::internal(msg).with_source(Box::new(src)), } } } diff --git a/slatedb/src/fence.rs b/slatedb/src/fence.rs index fb2fb9e74..ead82336e 100644 --- a/slatedb/src/fence.rs +++ b/slatedb/src/fence.rs @@ -1,17 +1,27 @@ +use crate::dispatcher::MessageHandlerExecutor; use crate::error::SlateDBError; use crate::manifest::store::{FenceableManifest, StoredManifest}; use crate::tablestore::TableStore; +use crate::utils::WatchableOnceCellReader; +use crate::wal::writer_init::{WalWriterInit, WalWriterInitOptions}; +use crate::wal::{WalWriter, WriterInit}; use crate::Settings; use fail_parallel::{fail_point_send, FailPointTx}; +use log::error; +use slatedb_common::metrics::MetricsRecorderHelper; use slatedb_common::SystemClock; use std::ops::Range; use std::sync::Arc; use std::time::Duration; pub(crate) struct WriterFencer { + closed_result_reader: WatchableOnceCellReader>, + recorder: MetricsRecorderHelper, + wal_writer_init_options: WalWriterInitOptions, table_store: Arc, manifest_update_timeout: Duration, system_clock: Arc, + task_executor: Arc, #[cfg_attr(not(test), allow(dead_code))] fp_tx: FailPointTx, } @@ -19,27 +29,46 @@ pub(crate) struct WriterFencer { pub(crate) struct WriterFenceResult { pub(crate) manifest: FenceableManifest, pub(crate) replay_range: Range, + pub(crate) wal_writer: Box, } impl WriterFencer { pub(crate) fn new( + closed_result_reader: WatchableOnceCellReader>, + recorder: MetricsRecorderHelper, table_store: Arc, settings: &Settings, system_clock: Arc, + task_executor: Arc, ) -> Self { - Self::new_with_fp_handle(table_store, settings, system_clock, FailPointTx::dummy()) + Self::new_with_fp_handle( + closed_result_reader, + recorder, + table_store, + settings, + system_clock, + task_executor, + FailPointTx::dummy(), + ) } fn new_with_fp_handle( + closed_result_reader: WatchableOnceCellReader>, + recorder: MetricsRecorderHelper, table_store: Arc, settings: &Settings, system_clock: Arc, + task_executor: Arc, fp_tx: FailPointTx, ) -> Self { Self { + closed_result_reader, + recorder, table_store, + wal_writer_init_options: settings.into(), manifest_update_timeout: settings.manifest_update_timeout, system_clock, + task_executor, fp_tx, } } @@ -57,13 +86,18 @@ impl WriterFencer { self, stored_manifest: StoredManifest, ) -> Result { - let mut empty_wal_id = self - .table_store - .next_wal_sst_id(stored_manifest.manifest().core.replay_after_wal_id) - .await?; - self.fail_point_send("LoadEmptyWalId"); + let wal_writer_init = WalWriterInit::load( + self.closed_result_reader.clone(), + self.recorder.clone(), + self.table_store.clone(), + self.wal_writer_init_options, + stored_manifest.manifest(), + self.task_executor.clone(), + self.fp_tx.clone(), + ) + .await?; - let mut manifest = FenceableManifest::init_writer( + let manifest = FenceableManifest::init_writer( stored_manifest, self.manifest_update_timeout, self.system_clock.clone(), @@ -71,56 +105,26 @@ impl WriterFencer { .await?; self.fail_point_send("FenceManifest"); - let mut manifest_dirty = manifest.prepare_dirty()?; - // verify that the empty_wal_id we computed is still valid. Its possible that between - // computing empty_wal_id and fencing the manifest, the fenced writer advanced the gc - // boundary (replay_after_wal_id) - if empty_wal_id <= manifest_dirty.value.core.replay_after_wal_id { - // the wal gc boundary advanced because the old writer finished a flush - recompute - // the next wal id - empty_wal_id = self - .table_store - .next_wal_sst_id(manifest_dirty.value.core.replay_after_wal_id) - .await?; - manifest.refresh().await?; - manifest_dirty = manifest.prepare_dirty()?; - self.fail_point_send("ReloadEmptyWalId"); - // at this point we still hold the epoch, so it should not be possible for the barrier - // to have advanced past the computed empty_wal_id - assert!(empty_wal_id > manifest_dirty.value.core.replay_after_wal_id); - } + let mut manifest = manifest.into(); + let result = wal_writer_init.fence_and_init(&mut manifest).await?; + let mut manifest: FenceableManifest = manifest.into(); - let mut attempt = 0; - loop { - attempt += 1; - let wrote_fence = match self.table_store.write_wal_fence(empty_wal_id).await { - Ok(()) => true, - Err(SlateDBError::Fenced) => false, - Err(err) => return Err(err), - }; - self.fail_point_send(format!("{}:{}", "WriteWalFence", attempt)); - - // Refresh validates that we own the latest epoch still. - manifest.refresh().await?; - let dirty_manifest = manifest.prepare_dirty()?; - let replay_after_wal_id = dirty_manifest.value.core.replay_after_wal_id; - self.fail_point_send(format!("{}:{}", "RefreshManifest", attempt)); - - if wrote_fence { - // this writer is the only writer that could have written replay_after_wal_id, - // so it should not be possible for it to have advanced past the fencing wal. - // older writers would have failed with a stale epoch - assert!(empty_wal_id > replay_after_wal_id); - return Ok(WriterFenceResult { - manifest, - replay_range: replay_after_wal_id + 1..empty_wal_id + 1, - }); - } else { - // The old writer managed to write a WAL before we could write the fencing wal. - // Try the next wal ID - empty_wal_id += 1; + // Refresh validates that we own the latest epoch still. + manifest.refresh().await?; + fail_point_send!(self.fp_tx, "FinalRefreshManifest"); + + let replay_range = match result.replay_range.try_into() { + Ok(replay_range) => replay_range, + Err(_) => { + error!("replay range must use inclusive lower bound and exclusive upper bound"); + return Err(SlateDBError::InvalidDBState); } - } + }; + Ok(WriterFenceResult { + manifest, + wal_writer: result.wal_writer, + replay_range, + }) } } @@ -131,6 +135,7 @@ mod tests { use crate::config::{ FlushOptions, FlushType, GarbageCollectorDirectoryOptions, GarbageCollectorOptions, }; + use crate::dispatcher::MessageHandlerExecutor; use crate::error::SlateDBError; use crate::fence::WriterFencer; use crate::format::sst::SsTableFormat; @@ -140,6 +145,7 @@ mod tests { use crate::memtable_flusher::MANIFEST_REFRESH_COUNT; use crate::object_stores::ObjectStores; use crate::tablestore::{TableStore, TableStoreKind}; + use crate::utils::WatchableOnceCell; use crate::{CloseReason, Db, ErrorKind, Settings}; use bytes::Bytes; use fail_parallel::fail_point_channel; @@ -148,7 +154,9 @@ mod tests { use object_store::path::Path; use object_store::ObjectStore; use rstest::rstest; - use slatedb_common::metrics::{lookup_metric, DefaultMetricsRecorder, MetricsRecorderHelper}; + use slatedb_common::metrics::{ + lookup_metric, DefaultMetricsRecorder, MetricLevel, MetricsRecorderHelper, + }; use slatedb_common::{DefaultSystemClock, SystemClock}; use std::collections::HashMap; use std::sync::Arc; @@ -190,10 +198,22 @@ mod tests { .unwrap(); let fp_registry = Arc::new(FailPointRegistry::new()); let (fp_tx, event_rx) = fail_point_channel(fp_registry.clone()); + let cell = Arc::new(WatchableOnceCell::new()); + let recorder = MetricsRecorderHelper::new( + Arc::new(DefaultMetricsRecorder::new()), + MetricLevel::Info, + ); + let task_executor = Arc::new(MessageHandlerExecutor::new( + cell.clone(), + system_clock.clone(), + )); let fencer = WriterFencer::new_with_fp_handle( + cell.reader(), + recorder, table_store.clone(), &settings, system_clock.clone(), + task_executor.clone(), fp_tx, ); Self { diff --git a/slatedb/src/lib.rs b/slatedb/src/lib.rs index 686ef1e04..f9d05befe 100644 --- a/slatedb/src/lib.rs +++ b/slatedb/src/lib.rs @@ -95,6 +95,7 @@ pub mod object_store_tag; pub mod prefix_extractor; pub mod seq_tracker; pub mod size_tiered_compaction; +pub mod wal; mod batch; #[cfg(feature = "bench-internal")] @@ -173,7 +174,6 @@ mod types; mod utils; mod fence; -mod wal; mod wal_buffer; mod wal_reader; mod wal_replay; diff --git a/slatedb/src/manifest/store.rs b/slatedb/src/manifest/store.rs index 829a97d2a..8707dc04b 100644 --- a/slatedb/src/manifest/store.rs +++ b/slatedb/src/manifest/store.rs @@ -66,6 +66,10 @@ impl FenceableManifest { Ok(Self { inner: fr, clock }) } + pub(crate) fn manifest(&self) -> (u64, &Manifest) { + (self.inner.id().id(), self.inner.object()) + } + pub(crate) fn local_epoch(&self) -> u64 { self.inner.local_epoch() } diff --git a/slatedb/src/memtable_flusher/manifest_writer.rs b/slatedb/src/memtable_flusher/manifest_writer.rs index a29f4b486..a9e4055a9 100644 --- a/slatedb/src/memtable_flusher/manifest_writer.rs +++ b/slatedb/src/memtable_flusher/manifest_writer.rs @@ -908,7 +908,9 @@ mod tests { use crate::tablestore::{TableStore, TableStoreKind}; use crate::types::RowEntry; use crate::utils::WatchableOnceCell; - use crate::wal_buffer::WalBufferManager; + + use crate::wal::test_utils::FakeWalWriter; + use crate::wal::WalWriter; use bytes::Bytes; use fail_parallel::FailPointRegistry; use object_store::memory::InMemory; @@ -916,7 +918,7 @@ mod tests { use object_store::ObjectStore; use slatedb_common::clock::DefaultSystemClock; use slatedb_common::clock::SystemClock; - use slatedb_common::metrics::{DefaultMetricsRecorder, MetricLevel, MetricsRecorderHelper}; + use slatedb_common::metrics::MetricsRecorderHelper; use slatedb_common::DbRand; use std::sync::Arc; use std::time::Duration; @@ -1070,16 +1072,7 @@ mod tests { let status_manager = DbStatusManager::new(0); let (write_tx, _) = crate::utils::SafeSender::unbounded_channel(status_manager.result_reader()); - let recorder = Arc::new(DefaultMetricsRecorder::new()); - let helper = MetricsRecorderHelper::new(recorder, MetricLevel::Info); - let wal_buffer = Arc::new(WalBufferManager::new( - status_manager.clone(), - &helper, - 0, - table_store.clone(), - 1024, - None, - )); + let wal_writer = Box::new(FakeWalWriter::new(0)); let inner = Arc::new( DbInner::new( settings.clone(), @@ -1091,11 +1084,11 @@ mod tests { &WatchableOnceCell::new(), )), write_tx, - wal_buffer.observer(), + wal_writer.observer(), db_metrics, fp_registry, None, - status_manager, + Arc::new(status_manager), segment_extractor, ) .await diff --git a/slatedb/src/memtable_flusher/tracker.rs b/slatedb/src/memtable_flusher/tracker.rs index 656997f29..f608b2238 100644 --- a/slatedb/src/memtable_flusher/tracker.rs +++ b/slatedb/src/memtable_flusher/tracker.rs @@ -570,7 +570,9 @@ mod tests { use crate::test_utils::FixedThreeBytePrefixExtractor; use crate::types::RowEntry; use crate::utils::{SafeSender, WatchableOnceCell}; - use crate::wal_buffer::WalBufferManager; + + use crate::wal::test_utils::FakeWalWriter; + use crate::wal::WalWriter; use bytes::Bytes; use fail_parallel::FailPointRegistry; use object_store::memory::InMemory; @@ -644,16 +646,7 @@ mod tests { let status_manager = DbStatusManager::new(0); let (write_tx, _) = SafeSender::::unbounded_channel(status_manager.result_reader()); - let recorder = Arc::new(DefaultMetricsRecorder::new()); - let helper = MetricsRecorderHelper::new(recorder, MetricLevel::Info); - let wal_buffer = Arc::new(WalBufferManager::new( - status_manager.clone(), - &helper, - 0, - table_store.clone(), - 1024, - None, - )); + let wal_writer = Box::new(FakeWalWriter::new(0)); let inner = Arc::new( DbInner::new( settings, @@ -663,11 +656,11 @@ mod tests { stored_manifest.prepare_dirty().unwrap(), Arc::new(MemtableFlusher::new(&status_manager)), write_tx, - wal_buffer.observer(), + wal_writer.observer(), db_metrics, fp_registry, None, - status_manager, + Arc::new(status_manager), segment_extractor, ) .await diff --git a/slatedb/src/memtable_flusher/uploader.rs b/slatedb/src/memtable_flusher/uploader.rs index 1aae23f90..271466dcd 100644 --- a/slatedb/src/memtable_flusher/uploader.rs +++ b/slatedb/src/memtable_flusher/uploader.rs @@ -318,14 +318,16 @@ mod tests { use crate::test_utils::FixedThreeBytePrefixExtractor; use crate::types::{RowEntry, ValueDeletable}; use crate::utils::WatchableOnceCell; - use crate::wal_buffer::WalBufferManager; + + use crate::wal::test_utils::FakeWalWriter; + use crate::wal::WalWriter; use bytes::Bytes; use fail_parallel::FailPointRegistry; use object_store::memory::InMemory; use object_store::path::Path; use object_store::ObjectStore; use slatedb_common::clock::{DefaultSystemClock, SystemClock}; - use slatedb_common::metrics::{DefaultMetricsRecorder, MetricLevel, MetricsRecorderHelper}; + use slatedb_common::metrics::MetricsRecorderHelper; use slatedb_common::DbRand; use std::collections::BTreeMap; use std::sync::Arc; @@ -400,16 +402,7 @@ mod tests { let status_manager = DbStatusManager::new(0); let (write_tx, _) = crate::utils::SafeSender::unbounded_channel(status_manager.result_reader()); - let recorder = Arc::new(DefaultMetricsRecorder::new()); - let helper = MetricsRecorderHelper::new(recorder, MetricLevel::Info); - let wal_buffer = Arc::new(WalBufferManager::new( - status_manager.clone(), - &helper, - 0, - table_store.clone(), - 1024, - None, - )); + let wal_writer = Box::new(FakeWalWriter::new(0)); Arc::new( DbInner::new( settings, @@ -421,11 +414,11 @@ mod tests { &status_manager, )), write_tx, - wal_buffer.observer(), + wal_writer.observer(), db_metrics, fp_registry, None, - status_manager, + Arc::new(status_manager), segment_extractor, ) .await diff --git a/slatedb/src/oracle.rs b/slatedb/src/oracle.rs index 292ba14a9..446e97069 100644 --- a/slatedb/src/oracle.rs +++ b/slatedb/src/oracle.rs @@ -1,5 +1,6 @@ use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering::SeqCst; +use std::sync::Arc; use crate::db_status::DbStatusManager; @@ -23,7 +24,7 @@ pub(crate) struct DbOracle { last_seq: AtomicU64, last_committed_seq: AtomicU64, last_durable_seq: AtomicU64, - status_reporter: DbStatusManager, + status_reporter: Arc, } impl DbOracle { @@ -31,7 +32,7 @@ impl DbOracle { last_seq: u64, last_committed_seq: u64, last_durable_seq: u64, - status_reporter: DbStatusManager, + status_reporter: Arc, ) -> Self { Self { last_seq: AtomicU64::new(last_seq), diff --git a/slatedb/src/snapshot_manager.rs b/slatedb/src/snapshot_manager.rs index cba2f87d5..f346647a9 100644 --- a/slatedb/src/snapshot_manager.rs +++ b/slatedb/src/snapshot_manager.rs @@ -71,7 +71,12 @@ mod tests { fn new_snapshot_manager(seq: u64) -> SnapshotManager { SnapshotManager::new( - Arc::new(DbOracle::new(seq, seq, seq, DbStatusManager::new(seq))), + Arc::new(DbOracle::new( + seq, + seq, + seq, + Arc::new(DbStatusManager::new(seq)), + )), Arc::new(DbRand::new(0)), ) } diff --git a/slatedb/src/transaction_manager.rs b/slatedb/src/transaction_manager.rs index a9cba127e..e6df1cb07 100644 --- a/slatedb/src/transaction_manager.rs +++ b/slatedb/src/transaction_manager.rs @@ -411,7 +411,7 @@ mod tests { fn create_transaction_manager() -> TransactionManager { let db_rand = Arc::new(DbRand::new(0)); let status_reporter = DbStatusManager::new(0); - let oracle = Arc::new(DbOracle::new(0, 0, 0, status_reporter)); + let oracle = Arc::new(DbOracle::new(0, 0, 0, Arc::new(status_reporter))); TransactionManager::new(oracle, db_rand) } @@ -419,7 +419,7 @@ mod tests { fn test_new_transaction_uses_oracle_seq() { let db_rand = Arc::new(DbRand::new(0)); let status_reporter = DbStatusManager::new(123); - let oracle = Arc::new(DbOracle::new(123, 123, 123, status_reporter)); + let oracle = Arc::new(DbOracle::new(123, 123, 123, Arc::new(status_reporter))); let txn_manager = TransactionManager::new(oracle, db_rand); let (txn_id, seq) = txn_manager.new_transaction(); diff --git a/slatedb/src/wal/mod.rs b/slatedb/src/wal/mod.rs index c7423ac6f..e70922741 100644 --- a/slatedb/src/wal/mod.rs +++ b/slatedb/src/wal/mod.rs @@ -1 +1,306 @@ +use crate::error::SlateDBError; +use crate::manifest::store::FenceableManifest; +use crate::{CloseReason, ErrorKind, RowEntry, VersionedManifest}; +use async_trait::async_trait; +use futures::future::BoxFuture; +use std::error::Error; +use std::fmt::{Display, Formatter}; +use std::ops::{Bound, Range}; +use std::sync::Arc; + +#[cfg(test)] +pub(crate) mod test_utils; +pub(crate) mod wal_disabled; pub(crate) mod wal_sst_builder; +pub(crate) mod writer_init; + +/// A range of WAL File IDs +pub struct WalFileRange(Bound, Bound); + +impl From> for WalFileRange { + fn from(range: Range) -> Self { + WalFileRange(Bound::Included(range.start), Bound::Excluded(range.end)) + } +} + +impl TryFrom for Range { + type Error = (); + + fn try_from(range: WalFileRange) -> Result { + match (range.0, range.1) { + (Bound::Included(start), Bound::Excluded(end)) => Ok(start..end), + _ => Err(()), + } + } +} + +/// Defines the types of errors that can be returned by WAL implementations. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub enum WalError { + /// The WAL writer was fenced + Fenced, + /// A WalIterator observed that the tail of the WAL was truncated while iterating. + WalTruncated, + /// Operation against wal after it was closed + Closed, + /// WAL is unavailable, e.g. due to an I/O error or error in the backing storage system + Unavailable(Arc), + /// WAL implementation detected invalid data/corruption + DataError(Arc), + /// Indicates that the WAL is in some unexpected/unrecoverable state. + InternalError(Arc), +} + +impl Display for WalError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + WalError::Fenced => write!(f, "WAL writer was fenced"), + WalError::WalTruncated => write!(f, "WAL was truncated"), + WalError::Closed => write!(f, "WAL is closed"), + WalError::Unavailable(source) => write!(f, "WAL is unavailable: {source}"), + WalError::DataError(source) => write!(f, "WAL data error: {source}"), + WalError::InternalError(source) => write!(f, "WAL internal error: {source}"), + } + } +} + +impl Error for WalError {} + +/// The writer's manifest after fencing. Created by calling [`ManifestFencer::fence`] +pub struct WriterManifest { + manifest: FenceableManifest, +} + +impl From for FenceableManifest { + fn from(manifest: WriterManifest) -> Self { + manifest.manifest + } +} + +impl From for WriterManifest { + fn from(manifest: FenceableManifest) -> Self { + WriterManifest { manifest } + } +} + +impl WriterManifest { + /// Returns the current manifest. + pub fn manifest(&self) -> VersionedManifest { + let (id, manifest) = self.manifest.manifest(); + VersionedManifest::from_manifest(id, manifest.clone()) + } + + /// Returns the WAL ID up to which SlateDB has guaranteed to have stored all data in the + /// LSM tree. + pub fn replay_after_wal_id(&self) -> u64 { + self.manifest().core().replay_after_wal_id + } + + /// Returns the writer's epoch + pub fn epoch(&self) -> u64 { + self.manifest().writer_epoch() + } + + /// Refreshes the current manifest. Implementations of `WriterInit::fence_and_init` can + /// use this to detect whether the manifest has been fenced while executing the fencing + /// protocol. SlateDB will call this after calling [`WriterInit::fence_and_init`] + pub async fn refresh(&mut self) -> Result<(), WalError> { + self.manifest.refresh().await?; + Ok(()) + } +} + +/// The result returned by [`WriterInit::fence_and_init`] +pub struct WriterInitResult { + // TODO: change me to an iterator + /// An iterator that returns writes that must be replayed before starting SlateDB to recover + /// data from the WAL. + pub replay_range: WalFileRange, + /// The WAL writer that will be used to append new writes to the WAL + pub wal_writer: Box, +} + +/// API for fencing and initializing a new WAL writer for use by [`crate::db::Db`]. SlateDB requires +/// WAL implementations to execute a fencing protocol that guarantees (1) that earlier writers no +/// longer write to the db and (2) all rows present in the WAL but not in the LSM tree (L0 and +/// sorted runs) are recovered. +/// +/// Every [`crate::db::Db`] instance is assigned a unique `u64` epoch. The epoch is assigned when +/// fencing the Manifest. A given Db instance writes both the WAL and its Manifest (e.g. with new +/// SSTs) independently. The fencing protocol that yields epoch E must ensure that: +/// (1) After the first write to the Manifest with epoch E, there are no further writes to either +/// the Manifest or WAL with epoch E' < E +/// (2) After the first write to the WAL with epoch E, there are no further writes to either the +/// Manifest or WAL with epoch E' < E +/// (3) All rows from the WAL from writers with epoch E' < E that are not present in L0/SRs are +/// replayed before serving reads/writes. +/// +/// `[WriterInit::fence_and_init]` is responsible for +/// (1) Fencing the WAL such that no writers with an epoch earlier than [`WriterManifest::epoch`] +/// (2) Constructing a [`WalWriter`] instance that the writer uses to append new WAL entries. +/// (3) Resolving the end of the WAL and constructing a [`WalReplayIterator`] that returns all +/// rows in WAL files between [`WriterManifest::replay_after_wal_id`] (exclusive) and the +/// current end of the WAL. +#[async_trait] +pub trait WriterInit { + /// Fences the WAL and returns a [`WriterInitResult`] with a [`WalWriter`] and + /// [`WalReplayIterator`] used to recover writes that have not yet been flushed to the tree. + async fn fence_and_init( + &self, + manifest: &mut WriterManifest, + ) -> Result; +} + +/// Describes the current status of the WAL +#[derive(Debug, Clone)] +pub struct WalStatus { + /// Set to Some if the WAL has permanently shut down, along with the reason. The reason should + /// be [`WalError::Closed`] on a normal shutdown, and some other [`WalError`] variant on + /// failure. + pub closed_reason: Option, + /// The estimated in-memory bytes used by the WAL to buffer unflushed writes. + pub estimated_bytes: usize, + /// The id of the last WAL file that was durably flushed + pub last_flushed_wal_id: u64, + /// The last sequence number that was durably flushed + pub last_flushed_seq: Option, + /// The number of writes currently buffered + #[allow(dead_code)] + pub buffered_wal_entries_count: usize, +} + +/// An event emitted by a [`WalWriter`] to subscribers. +#[derive(Debug, Clone)] +pub enum WalEvent { + /// Emitted when a WAL file is durably flushed to storage. On receipt of this event, SlateDB + /// notifies write tasks blocked on [`crate::config::WriteOptions::await_durable`] + WalFlushed(WalStatus), + /// Emitted when the WAL has closed with the final wal status containing the closed reason + WalClosed(WalStatus), +} + +/// A listener that's called back on WAL events. +pub type WalStatusListener = Arc; + +/// An observer that can read the current [`WalStatus`] and subscribe to event callbacks. +#[async_trait] +pub trait WalObserver: Send + Sync + 'static { + /// Returns the current [`WalStatus`]. + fn status(&self) -> Result; + + /// Adds a listener that subscribes to event callbacks. + fn subscribe(&self, listener: WalStatusListener) -> Result<(), WalError>; +} + +pub type FlushResultFuture = BoxFuture<'static, Result<(), WalError>>; + +/// The WAL's write API. Used by SlateDB to append new WAL writes. Is returned by +/// [`WalWriterInit::fence_and_init_writer`]. +/// +/// Each call to [`WalWriter::append`] takes a single SlateDB write batch, where all rows share +/// the same sequence number ([`RowEntry::seq`]). [`WalWriter`] (optionally accumulates/buffers +/// rows and) writes consecutive write batches into consecutive WAL Files, where each WAL File +/// contains some rows from the total sequence of rows. Specifically: +/// - WAL Files must have a total order and each WAL File must have a u64 id that is greater than +/// all earlier WAL Files. +/// - Reading WAL Files in order should yield rows in sequence order. +/// - The writes in a given write batch must be written to WAL files atomically. That is, a +/// [`WalIterator`] should either observe all the writes with a given sequence number or none +/// of them. +#[async_trait] +pub trait WalWriter: Send { + /// Append a write batch to the WAL. + async fn append(&mut self, write_batch: &[RowEntry]) -> Result<(), WalError>; + + /// Triggers a flush of all appended write batches to durable storage. Returns a + /// future that receives the result of the flush once it completes. + async fn flush(&mut self) -> Result; + + /// Returns a `WalObserver` for reading [`WalStatus`] and subscribing to events. + fn observer(&self) -> Box; + + /// Returns the current `WalStatus`. If the [`WalWriter`] has failed, then returns Err with the + /// final [`WalStatus`] and the reason for the failure in [`WalStatus::closed_reason`]. This + /// allows callers to observe the final status after the [`WalWriter`] has shut down, e.g. to + /// read the last flushed wal file or sequence number. + fn status(&self) -> Result; + + /// Close the `WalWriter` and release resources + async fn close(&mut self) -> Result<(), WalError>; +} + +impl From for WalError { + fn from(status: WalStatus) -> Self { + status + .closed_reason + .expect("unexpected conversion of wal status with no error") + } +} + +impl From for SlateDBError { + fn from(status: WalStatus) -> Self { + WalError::from(status).into() + } +} + +impl From for WalError { + fn from(value: SlateDBError) -> Self { + { + let public: crate::Error = value.clone().into(); + match public.kind() { + ErrorKind::Closed(CloseReason::Fenced) => WalError::Fenced, + ErrorKind::Closed(CloseReason::Clean) => WalError::Closed, + ErrorKind::Closed(_) => WalError::InternalError(Arc::new(value)), + ErrorKind::Unavailable => WalError::Unavailable(Arc::new(value)), + ErrorKind::Invalid => WalError::InternalError(Arc::new(value)), + ErrorKind::Data => WalError::DataError(Arc::new(value)), + ErrorKind::Internal => WalError::InternalError(Arc::new(value)), + ErrorKind::Transaction => WalError::InternalError(Arc::new(value)), + } + } + } +} + +impl From for SlateDBError { + fn from(value: WalError) -> Self { + match value { + WalError::Fenced => SlateDBError::Fenced, + WalError::WalTruncated => SlateDBError::WalTruncated, + WalError::Closed => SlateDBError::Closed, + WalError::Unavailable(err) => SlateDBError::WalUnavailable(err), + WalError::DataError(err) => SlateDBError::WalDataError(err), + WalError::InternalError(err) => SlateDBError::WalInternalError(err), + } + } +} + +#[cfg(test)] +mod tests { + use super::WalError; + use std::sync::Arc; + + #[test] + fn wal_error_display() { + let source = || { + Arc::new(std::io::Error::other("source error")) + as Arc + }; + + assert_eq!(WalError::Fenced.to_string(), "WAL writer was fenced"); + assert_eq!(WalError::WalTruncated.to_string(), "WAL was truncated"); + assert_eq!(WalError::Closed.to_string(), "WAL is closed"); + assert_eq!( + WalError::Unavailable(source()).to_string(), + "WAL is unavailable: source error" + ); + assert_eq!( + WalError::DataError(source()).to_string(), + "WAL data error: source error" + ); + assert_eq!( + WalError::InternalError(source()).to_string(), + "WAL internal error: source error" + ); + } +} diff --git a/slatedb/src/wal/test_utils.rs b/slatedb/src/wal/test_utils.rs new file mode 100644 index 000000000..afe51e5de --- /dev/null +++ b/slatedb/src/wal/test_utils.rs @@ -0,0 +1,68 @@ +use crate::wal::{ + FlushResultFuture, WalError, WalObserver, WalStatus, WalStatusListener, WalWriter, +}; +use crate::RowEntry; +use futures::FutureExt; + +pub(crate) struct FakeWalWriter { + status: WalStatus, +} + +impl FakeWalWriter { + pub(crate) fn new(last_flushed_wal_id: u64) -> Self { + Self::new_with_closed_reason(last_flushed_wal_id, None) + } + + pub(crate) fn new_with_closed_reason( + last_flushed_wal_id: u64, + closed_reason: Option, + ) -> Self { + let status = WalStatus { + estimated_bytes: 0, + last_flushed_wal_id, + last_flushed_seq: None, + buffered_wal_entries_count: 0, + closed_reason, + }; + Self { status } + } +} + +#[async_trait::async_trait] +impl WalWriter for FakeWalWriter { + async fn append(&mut self, _write_batch: &[RowEntry]) -> Result<(), WalError> { + Ok(()) + } + + async fn flush(&mut self) -> Result { + Ok(async { Ok(()) }.boxed()) + } + + fn observer(&self) -> Box { + Box::new(FakeWalObserver { + status: self.status.clone(), + }) + } + + fn status(&self) -> Result { + Ok(self.status.clone()) + } + + async fn close(&mut self) -> Result<(), WalError> { + Ok(()) + } +} + +pub(crate) struct FakeWalObserver { + status: WalStatus, +} + +impl WalObserver for FakeWalObserver { + fn status(&self) -> Result { + Ok(self.status.clone()) + } + + fn subscribe(&self, _listener: WalStatusListener) -> Result<(), WalError> { + Ok(()) + } +} diff --git a/slatedb/src/wal/wal_disabled.rs b/slatedb/src/wal/wal_disabled.rs new file mode 100644 index 000000000..ecf352d6f --- /dev/null +++ b/slatedb/src/wal/wal_disabled.rs @@ -0,0 +1,22 @@ +use crate::wal::{WalError, WalObserver, WalStatus, WalStatusListener}; + +#[derive(Clone, Debug)] +pub(crate) struct DisabledWalObserver { + status: WalStatus, +} + +impl DisabledWalObserver { + pub(crate) fn new(status: WalStatus) -> Self { + Self { status } + } +} + +impl WalObserver for DisabledWalObserver { + fn status(&self) -> Result { + Ok(self.status.clone()) + } + + fn subscribe(&self, _listener: WalStatusListener) -> Result<(), WalError> { + Ok(()) + } +} diff --git a/slatedb/src/wal/writer_init.rs b/slatedb/src/wal/writer_init.rs new file mode 100644 index 000000000..f98c85ff4 --- /dev/null +++ b/slatedb/src/wal/writer_init.rs @@ -0,0 +1,137 @@ +use crate::dispatcher::MessageHandlerExecutor; +use crate::error::SlateDBError; +use crate::manifest::Manifest; +use crate::tablestore::TableStore; +use crate::utils::WatchableOnceCellReader; +use crate::wal::{WalError, WriterInitResult, WriterManifest}; +use crate::wal_buffer::WalBufferManager; +use crate::{wal, Settings}; +use async_trait::async_trait; +use fail_parallel::{fail_point_send, FailPointTx}; +use slatedb_common::metrics::MetricsRecorderHelper; +use std::sync::Arc; +use std::time::Duration; + +#[derive(Clone, Copy)] +pub(crate) struct WalWriterInitOptions { + max_wal_bytes_size: usize, + max_flush_interval: Option, +} + +impl From<&Settings> for WalWriterInitOptions { + fn from(settings: &Settings) -> Self { + Self { + max_wal_bytes_size: settings.l0_sst_size_bytes, + max_flush_interval: settings.flush_interval, + } + } +} + +pub(crate) struct WalWriterInit { + closed_result_reader: WatchableOnceCellReader>, + recorder: MetricsRecorderHelper, + table_store: Arc, + max_wal_bytes_size: usize, + max_flush_interval: Option, + empty_wal_id: u64, + task_executor: Arc, + #[cfg_attr(not(test), allow(dead_code))] + fp_tx: FailPointTx, +} + +impl WalWriterInit { + pub(crate) async fn load( + closed_result_reader: WatchableOnceCellReader>, + recorder: MetricsRecorderHelper, + table_store: Arc, + options: WalWriterInitOptions, + manifest: &Manifest, + task_executor: Arc, + fp_tx: FailPointTx, + ) -> Result { + let empty_wal_id = table_store + .next_wal_sst_id(manifest.core.replay_after_wal_id) + .await?; + fail_point_send!(fp_tx, "LoadEmptyWalId"); + Ok(Self { + closed_result_reader, + recorder, + table_store, + max_wal_bytes_size: options.max_wal_bytes_size, + max_flush_interval: options.max_flush_interval, + empty_wal_id, + task_executor, + fp_tx, + }) + } +} + +#[async_trait] +impl wal::WriterInit for WalWriterInit { + async fn fence_and_init( + &self, + writer_manifest: &mut WriterManifest, + ) -> Result { + let mut empty_wal_id = self.empty_wal_id; + let mut manifest = writer_manifest.manifest(); + // verify that the empty_wal_id we computed is still valid. Its possible that between + // computing empty_wal_id and fencing the manifest, the fenced writer advanced the gc + // boundary (replay_after_wal_id) + if empty_wal_id <= manifest.core().replay_after_wal_id { + // the wal gc boundary advanced because the old writer finished a flush - recompute + // the next wal id + empty_wal_id = self + .table_store + .next_wal_sst_id(manifest.core().replay_after_wal_id) + .await?; + writer_manifest.refresh().await?; + manifest = writer_manifest.manifest(); + fail_point_send!(self.fp_tx, "ReloadEmptyWalId"); + // at this point we still hold the epoch, so it should not be possible for the barrier + // to have advanced past the computed empty_wal_id + assert!(empty_wal_id > manifest.core().replay_after_wal_id); + } + let manifest = manifest.clone(); + + let mut _attempt = 0; + loop { + _attempt += 1; + let wrote_fence = match self.table_store.write_wal_fence(empty_wal_id).await { + Ok(()) => true, + Err(SlateDBError::Fenced) => false, + Err(err) => return Err(err.into()), + }; + fail_point_send!(self.fp_tx, format!("{}:{}", "WriteWalFence", _attempt)); + + if wrote_fence { + // this writer is the only writer that could have written replay_after_wal_id, + // so it should not be possible for it to have advanced past the fencing wal. + // older writers would have failed with a stale epoch + let replay_after_wal_id = manifest.core().replay_after_wal_id; + assert!(empty_wal_id > replay_after_wal_id); + let wal_writer = WalBufferManager::start_new( + self.closed_result_reader.clone(), + &self.recorder, + empty_wal_id, + self.table_store.clone(), + self.max_wal_bytes_size, + self.max_flush_interval, + self.task_executor.clone(), + ) + .await?; + let result = WriterInitResult { + replay_range: (replay_after_wal_id + 1..empty_wal_id + 1).into(), + wal_writer: Box::new(wal_writer), + }; + return Ok(result); + } else { + // Refresh validates that we own the latest epoch still. + writer_manifest.refresh().await?; + fail_point_send!(self.fp_tx, format!("{}:{}", "RefreshManifest", _attempt)); + // The old writer managed to write a WAL before we could write the fencing wal. + // Try the next wal ID + empty_wal_id += 1; + } + } + } +} diff --git a/slatedb/src/wal_buffer.rs b/slatedb/src/wal_buffer.rs index d7614218b..027af0031 100644 --- a/slatedb/src/wal_buffer.rs +++ b/slatedb/src/wal_buffer.rs @@ -5,16 +5,17 @@ use std::sync::Arc; use std::time::Duration; use crate::db_state::SsTableId; -use crate::db_status::ClosedResultWriter; use crate::dispatcher::{MessageHandler, MessageHandlerExecutor, MessageTickerDef}; use crate::error::SlateDBError; use crate::tablestore::TableStore; use crate::types::RowEntry; use crate::utils::SafeSender; use crate::utils::{format_bytes_si, WatchableOnceCell, WatchableOnceCellReader}; +use crate::wal; +use crate::wal::{FlushResultFuture, WalError, WalEvent, WalStatus, WalWriter}; use crate::wal_buffer_stats::WalBufferStats; use async_trait::async_trait; -use futures::{stream::BoxStream, StreamExt}; +use futures::{stream::BoxStream, FutureExt, StreamExt}; use log::{error, trace, warn}; use slatedb_common::metrics::MetricsRecorderHelper; use tokio::{runtime::Handle, sync::oneshot}; @@ -22,8 +23,6 @@ use tracing::instrument; pub(crate) const WAL_BUFFER_TASK_NAME: &str = "wal_writer"; -pub(crate) type WalStatusListener = Arc; - /// [`WalBufferManager`] buffers write operations in memory before flushing them to persistent storage. /// The flush operation only targets Remote storage right now, later we can add an option to flush to local /// storage. @@ -52,17 +51,12 @@ pub(crate) struct WalBufferManager { stats: Arc, table_store: Arc, max_wal_bytes_size: usize, - max_flush_interval: Option, /// The largest flush_epoch for which a size-triggered flush request has been /// sent. Compared against `flush_epoch` in the inner struct to avoid sending /// redundant flush requests for the same WAL. last_flush_requested_epoch: AtomicU64, - /// The channel to send the flush work to the background worker. - flush_tx: SafeSender, - /// The channel that the flush task waits on to receive work. Will be consumed by init - flush_rx: Option>, /// task executor for the background worker. - task_executor: Option>, + task_executor: Arc, } struct WalBufferManagerInner { @@ -80,6 +74,10 @@ struct WalBufferManagerInner { last_flushed_wal_id: u64, /// The last seq that was flushed to the WAL. This value will be None until the first flush. last_flushed_seq: Option, + /// Set to Some with error reason if the flush task has exited + flush_task_exited_reason: Option, + /// The channel to send the flush work to the background worker. + flush_tx: SafeSender, } /// Stores entries to the write-ahead log (WAL) in memory. @@ -109,16 +107,18 @@ struct WalBufferIterator { } impl WalBufferManager { - pub(crate) fn new( - status_manager: crate::db_status::DbStatusManager, + pub(crate) async fn start_new( + closed_result_reader: WatchableOnceCellReader>, recorder: &MetricsRecorderHelper, last_flushed_wal_id: u64, table_store: Arc, max_wal_bytes_size: usize, max_flush_interval: Option, - ) -> Self { + task_executor: Arc, + ) -> Result { let current_wal = WalBuffer::new(); let immutable_wals = VecDeque::new(); + let (flush_tx, flush_rx) = SafeSender::unbounded_channel(closed_result_reader); let inner = WalBufferManagerInner { current_wal, immutable_wals, @@ -126,73 +126,42 @@ impl WalBufferManager { last_flushed_wal_id, next_wal_id: last_flushed_wal_id + 1, last_flushed_seq: None, - }; - let (flush_tx, flush_rx) = SafeSender::unbounded_channel(status_manager.result_reader()); - Self { - inner: Arc::new(parking_lot::RwLock::new(inner)), - stats: Arc::new(WalBufferStats::new(recorder)), - table_store, - max_wal_bytes_size, - max_flush_interval, - last_flush_requested_epoch: AtomicU64::new(0), + flush_task_exited_reason: None, flush_tx, - flush_rx: Some(flush_rx), - task_executor: None, - } - } - - // todo: consider consolidating with new - pub(crate) async fn init( - &mut self, - task_executor: Arc, - ) -> Result<(), SlateDBError> { - let Some(flush_rx) = self.flush_rx.take() else { - error!("WalBufferManager#init called multiple times"); - return Err(SlateDBError::InvalidDBState); }; - assert!(self.task_executor.is_none()); + let inner = Arc::new(parking_lot::RwLock::new(inner)); + let stats = Arc::new(WalBufferStats::new(recorder)); let wal_flush_handler = WalFlushHandler { - max_flush_interval: self.max_flush_interval, - inner: self.inner.clone(), - table_store: self.table_store.clone(), - stats: self.stats.clone(), + max_flush_interval, + inner: inner.clone(), + table_store: table_store.clone(), + stats: stats.clone(), listener: None, }; - let result = task_executor.add_handler( + task_executor.add_handler( WAL_BUFFER_TASK_NAME.to_string(), Box::new(wal_flush_handler), flush_rx, &Handle::current(), - ); - self.task_executor = Some(task_executor); - result - } - - pub(crate) fn last_flushed_wal_id(&self) -> u64 { - let inner = self.inner.read(); - inner.last_flushed_wal_id - } - - pub(crate) fn status(&self) -> WalStatus { - self.inner.read().status(&self.table_store) - } - - /// Append row entries to the current WAL. Returns a watcher for durability notification. - pub(crate) fn append( - &self, - entries: &[RowEntry], - ) -> Result>, SlateDBError> { - // TODO: check if the wal buffer is in a fatal error state. - self.inner.write().append(entries) + )?; + Ok(Self { + inner, + stats, + table_store, + max_wal_bytes_size, + last_flush_requested_epoch: AtomicU64::new(0), + task_executor, + }) } + //TODO: do we still need durable watchers here? /// Check if we need to flush the wal with considering max_wal_size. the checking over `max_wal_size` /// is not very strict, we have to ensure a write batch into a single WAL file. /// /// It's the caller's duty to call `maybe_trigger_flush` after calling `append`. - pub(crate) fn maybe_trigger_flush( + fn maybe_trigger_flush( &self, - ) -> Result>, SlateDBError> { + ) -> Result>, WalError> { let (durable_watcher, need_flush, flush_epoch) = { let inner = self.inner.read(); // checks the size of the current wal @@ -214,58 +183,95 @@ impl WalBufferManager { } } - let status = self.status(); + let status = self.status()?; self.stats .estimated_bytes .set(status.estimated_bytes as i64); Ok(durable_watcher) } - pub(crate) fn observer(&self) -> WalObserver { - WalObserver { - inner: self.inner.clone(), - table_store: self.table_store.clone(), - flush_tx: self.flush_tx.clone(), - } - } - /// Send a flush request to the background flush worker. fn send_flush_request( &self, - result_tx: Option>>, - ) -> Result<(), SlateDBError> { + result_tx: Option>>, + ) -> Result<(), WalError> { self.stats.flush_requests.increment(1); - self.flush_tx.send(WalFlushWork::Flush { result_tx }) + self.inner + .read() + .send_flush_msg(WalFlushWork::Flush { result_tx }) } +} - pub(crate) fn flush( - &self, - ) -> Result>, SlateDBError> { +#[async_trait] +impl WalWriter for WalBufferManager { + fn status(&self) -> Result { + self.inner.read().status(&self.table_store) + } + + /// Append row entries to the current WAL. Returns a watcher for durability notification. + async fn append(&mut self, entries: &[RowEntry]) -> Result<(), WalError> { + self.inner.write().append(entries)?; + self.maybe_trigger_flush()?; + Ok(()) + } + + fn observer(&self) -> Box { + Box::new(WalObserver { + inner: self.inner.clone(), + table_store: self.table_store.clone(), + }) + } + + async fn flush(&mut self) -> Result { let (result_tx, result_rx) = oneshot::channel(); self.send_flush_request(Some(result_tx))?; - Ok(result_rx) + Ok(async { + result_rx + .await + .unwrap_or_else(|e| Err(WalError::InternalError(Arc::new(e)))) + } + .boxed()) } - #[allow(dead_code)] - pub(crate) async fn close(&self) -> Result<(), SlateDBError> { - let task_executor = self + async fn close(&mut self) -> Result<(), WalError> { + if let Some(result) = self .task_executor - .as_ref() - .expect("task executor should be initialized"); - task_executor.shutdown_task(WAL_BUFFER_TASK_NAME).await + .shutdown_or_deregister_task(WAL_BUFFER_TASK_NAME) + .await + { + return Ok(result?); + }; + self.inner + .write() + .drain_on_close(WalError::Closed, &self.table_store); + Ok(()) } } impl WalBufferManagerInner { - fn append( - &mut self, - entries: &[RowEntry], - ) -> Result>, SlateDBError> { + fn check_exited(&self) -> Result<(), WalError> { + match self.flush_task_exited_reason.as_ref() { + Some(err) => Err(err.clone()), + None => Ok(()), + } + } + + fn send_flush_msg(&self, msg: WalFlushWork) -> Result<(), WalError> { + self.check_exited()?; + // TODO: there is a small window here where the dispatcher closes `flush_rx` before + // calling cleanup. In this case we may have exited with a different error than + // a clean close. To fix this we'd need some pre-cleanup hook the dispatcher can + // call to propagate the error + self.flush_tx.send(msg).map_err(|_e| WalError::Closed) + } + + fn append(&mut self, entries: &[RowEntry]) -> Result<(), WalError> { // TODO: validate the seq number is always increasing. + self.check_exited()?; for entry in entries { self.current_wal.append(entry.clone()); } - Ok(self.current_wal.durable_watcher()) + Ok(()) } fn needs_flush(&self, table_store: &TableStore, max_wal_bytes_size: usize) -> (bool, u64) { @@ -303,7 +309,16 @@ impl WalBufferManagerInner { current_wal_size + imm_wal_size } - fn status(&self, table_store: &TableStore) -> WalStatus { + fn status(&self, table_store: &TableStore) -> Result { + let status = self.compute_status(table_store); + if status.closed_reason.is_none() { + Ok(status) + } else { + Err(status) + } + } + + fn compute_status(&self, table_store: &TableStore) -> WalStatus { let flushing_wal_entries_count = self .immutable_wals .iter() @@ -311,6 +326,7 @@ impl WalBufferManagerInner { .sum::(); let buffered_wal_entries_count = self.current_wal.len() + flushing_wal_entries_count; WalStatus { + closed_reason: self.flush_task_exited_reason.clone(), estimated_bytes: self.estimated_bytes(table_store), last_flushed_wal_id: self.last_flushed_wal_id, last_flushed_seq: self.last_flushed_seq, @@ -318,6 +334,19 @@ impl WalBufferManagerInner { } } + fn drain_on_close( + &mut self, + reason: WalError, + table_store: &TableStore, + ) -> (WalStatus, Vec<(u64, Arc)>) { + self.flush_task_exited_reason = Some(reason); + self.freeze_current_wal(); + let unflushed_wals = self.flushing_wals(); + self.immutable_wals.clear(); + let status = self.compute_status(table_store); + (status, unflushed_wals) + } + fn freeze_current_wal(&mut self) { if self.current_wal.is_empty() { return; @@ -434,10 +463,10 @@ impl WalBufferIterator { enum WalFlushWork { Flush { - result_tx: Option>>, + result_tx: Option>>, }, Subscribe { - listener: WalStatusListener, + listener: wal::WalStatusListener, }, } @@ -455,7 +484,7 @@ struct WalFlushHandler { inner: Arc>, table_store: Arc, stats: Arc, - listener: Option, + listener: Option, } impl WalFlushHandler { @@ -481,7 +510,7 @@ impl WalFlushHandler { let status = { let mut inner = self.inner.write(); inner.record_flushed_wal(wal_id, &wal); - inner.status(&self.table_store) + inner.compute_status(&self.table_store) }; // we notify the listener first since that updates the oracle, and then notify @@ -490,7 +519,7 @@ impl WalFlushHandler { // after notifying flushed before the wal memory is actually released. // TODO: once we change writes to block on the durable seq num from the oracle we // can simplify this and fully drop the wal before notifying listeners - self.notify_listener(WalEvent::WalFlushed(status)); + self.notify_listener(wal::WalEvent::WalFlushed(status)); wal.notify_durable(result.clone()); if Arc::strong_count(&wal) > 1 { warn!("outstanding references to wal id {} after flushing", wal_id); @@ -519,7 +548,7 @@ impl WalFlushHandler { Ok(()) } - fn notify_listener(&self, event: WalEvent) { + fn notify_listener(&self, event: wal::WalEvent) { if let Some(l) = self.listener.as_ref() { (*l)(event); } @@ -543,10 +572,10 @@ impl MessageHandler for WalFlushHandler { WalFlushWork::Flush { result_tx } => { if let Some(result_tx) = result_tx { let result = self.do_flush().await; - let _ = result_tx.send(result.clone()); - result + let _ = result_tx.send(result.clone().map_err(WalError::from)); + Ok(result?) } else { - self.do_flush().await + Ok(self.do_flush().await?) } } WalFlushWork::Subscribe { listener } => { @@ -563,7 +592,17 @@ impl MessageHandler for WalFlushHandler { mut messages: BoxStream<'async_trait, WalFlushWork>, result: Result<(), SlateDBError>, ) -> Result<(), SlateDBError> { - let error = result.err().unwrap_or(SlateDBError::Closed); + let error = result + .clone() + .err() + .map(WalError::from) + .unwrap_or(WalError::Closed); + + let (final_status, unflushed) = self + .inner + .write() + .drain_on_close(error.clone(), &self.table_store); + self.notify_listener(WalEvent::WalClosed(final_status.clone())); // drain remaining messages while let Some(msg) = messages.next().await { @@ -573,20 +612,17 @@ impl MessageHandler for WalFlushHandler { let _ = result_tx.send(Err(error.clone())); } } - WalFlushWork::Subscribe { listener: _ } => {} + WalFlushWork::Subscribe { listener } => { + (*listener)(WalEvent::WalClosed(final_status.clone())) + } } } // notify all the flushing wals to be finished with fatal error or shutdown // error. we need ensure all the wal tables finally get notified. freeze current // WAL to notify writers in the subsequent flushing_wals loop. - let flushing_wals = { - let mut inner = self.inner.write(); - inner.freeze_current_wal(); - inner.flushing_wals() - }; - for (_, wal) in flushing_wals.iter() { - wal.notify_durable(Err(error.clone())); + for (_, wal) in unflushed { + wal.notify_durable(Err(result.clone().err().unwrap_or(SlateDBError::Closed))); } Ok(()) } @@ -594,44 +630,21 @@ impl MessageHandler for WalFlushHandler { /// Interface for getting information about the current state of the Wal #[derive(Clone)] -pub(crate) struct WalObserver { +struct WalObserver { inner: Arc>, table_store: Arc, - flush_tx: SafeSender, -} - -/// Describes the current status of the WAL -#[derive(Debug, Clone)] -pub(crate) struct WalStatus { - /// The estimated in-memory bytes used by the WAL to buffer unflushed writes. - pub(crate) estimated_bytes: usize, - /// The id of the last WAL file that was durably flushed - pub(crate) last_flushed_wal_id: u64, - /// The last sequence number that was durably flushed - pub(crate) last_flushed_seq: Option, - /// The number of writes currently buffered - #[allow(dead_code)] - pub(crate) buffered_wal_entries_count: usize, -} - -/// An event emitted by [`WalBufferManager`] to subscribers. -#[derive(Debug, Clone)] -pub(crate) enum WalEvent { - /// Emitted when a WAL file is durably flushed to storage. On receipt of this event, SlateDB - /// notifies write tasks blocked on [`crate::config::WriteOptions::await_durable`] - WalFlushed(WalStatus), } -impl WalObserver { +impl wal::WalObserver for WalObserver { /// Gets information about the Wal buffer's current state - pub(crate) fn status(&self) -> WalStatus { + fn status(&self) -> Result { self.inner.read().status(self.table_store.as_ref()) } - pub(crate) fn subscribe(&self, listener: WalStatusListener) -> Result<(), SlateDBError> { - self.flush_tx - .send(WalFlushWork::Subscribe { listener }) - .map_err(|_err| SlateDBError::Closed) + fn subscribe(&self, listener: wal::WalStatusListener) -> Result<(), WalError> { + self.inner + .read() + .send_flush_msg(WalFlushWork::Subscribe { listener }) } } @@ -673,7 +686,7 @@ pub mod stats { mod tests { use super::*; use crate::block_cache_policy::BlockCachePolicy; - use crate::db_status::DbStatusManager; + use crate::db_status::{ClosedResultWriter, DbStatusManager}; use crate::format::sst::SsTableFormat; use crate::iter::RowEntryIterator; use crate::manifest::SsTableView; @@ -689,7 +702,6 @@ mod tests { lookup_metric, DefaultMetricsRecorder, MetricLevel, MetricsRecorderHelper, }; use slatedb_common::MockSystemClock; - use std::sync::atomic::AtomicBool; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -886,7 +898,7 @@ mod tests { async fn setup_wal_buffer_with_args( flush_interval: Duration, - listener: WalStatusListener, + listener: wal::WalStatusListener, ) -> ( WalBufferManager, Arc, @@ -904,31 +916,35 @@ mod tests { )); let test_clock = Arc::new(MockSystemClock::new()); let system_clock = Arc::new(DefaultSystemClock::new()); - let status_manager = DbStatusManager::new(0); + let status_manager = Arc::new(DbStatusManager::new(0)); let oracle = Arc::new(DbOracle::new(0, 0, 0, status_manager.clone())); let recorder = Arc::new(DefaultMetricsRecorder::new()); let helper = MetricsRecorderHelper::new(recorder.clone(), MetricLevel::default()); - let mut wal_buffer = WalBufferManager::new( + let task_executor = Arc::new(MessageHandlerExecutor::new( status_manager.clone(), + system_clock.clone(), + )); + let wal_buffer = WalBufferManager::start_new( + status_manager.result_reader(), &helper, 0, // recent_flushed_wal_id table_store.clone(), 1000, // max_wal_bytes_size Some(flush_interval), // max_flush_interval - ); + task_executor.clone(), + ) + .await + .unwrap(); let observer = wal_buffer.observer(); observer .subscribe(Arc::new(move |status| { (*listener)(status.clone()); - let WalEvent::WalFlushed(status) = status; + let wal::WalEvent::WalFlushed(status) = status else { + return; + }; oracle.advance_durable_seq(status.last_flushed_seq.unwrap_or(0)) })) .unwrap(); - let task_executor = Arc::new(MessageHandlerExecutor::new( - Arc::new(status_manager), - system_clock.clone(), - )); - wal_buffer.init(task_executor.clone()).await.unwrap(); task_executor .monitor_on(&Handle::current()) .expect("failed to monitor executor"); @@ -937,17 +953,23 @@ mod tests { #[tokio::test] async fn test_basic_append_and_flush_operations() { - let (wal_buffer, table_store, _, _) = setup_wal_buffer().await; + let (mut wal_buffer, table_store, _, _) = setup_wal_buffer().await; // Append some entries let entry1 = make_entry("key1", "value1", 1, None); let entry2 = make_entry("key2", "value2", 2, None); - wal_buffer.append(std::slice::from_ref(&entry1)).unwrap(); - wal_buffer.append(std::slice::from_ref(&entry2)).unwrap(); + wal_buffer + .append(std::slice::from_ref(&entry1)) + .await + .unwrap(); + wal_buffer + .append(std::slice::from_ref(&entry2)) + .await + .unwrap(); // Flush the buffer - wal_buffer.flush().unwrap().await.unwrap().unwrap(); + wal_buffer.flush().await.unwrap().await.unwrap(); // Verify entries were written to storage let sst_iter_options = SstIteratorOptions { @@ -979,39 +1001,39 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_size_based_flush_triggering() { - let (wal_buffer, _, _, _) = setup_wal_buffer_with_flush_interval(Duration::MAX).await; + let (mut wal_buffer, _, _, _) = setup_wal_buffer_with_flush_interval(Duration::MAX).await; // Append entries until we exceed the size threshold let mut seq = 1; - while wal_buffer.status().estimated_bytes < wal_buffer.max_wal_bytes_size { + while wal_buffer.status().unwrap().estimated_bytes < wal_buffer.max_wal_bytes_size { let entry = make_entry(&format!("key{}", seq), &format!("value{}", seq), seq, None); - wal_buffer.append(&[entry]).unwrap(); + wal_buffer.append(&[entry]).await.unwrap(); seq += 1; } let mut reader = wal_buffer.maybe_trigger_flush().unwrap(); reader.await_value().await.unwrap(); - assert_eq!(wal_buffer.last_flushed_wal_id(), 1); + assert_eq!(wal_buffer.status().unwrap().last_flushed_wal_id, 1); } #[tokio::test] async fn test_immutable_wal_reclaim() { - let (wal_buffer, _, _, _) = setup_wal_buffer().await; + let (mut wal_buffer, _, _, _) = setup_wal_buffer().await; // Append entries to create multiple WALs for i in 0..100 { let seq = i + 1; let entry = make_entry(&format!("key{}", i), &format!("value{}", i), seq, None); - wal_buffer.append(&[entry]).unwrap(); - wal_buffer.flush().unwrap().await.unwrap().unwrap(); + wal_buffer.append(&[entry]).await.unwrap(); + wal_buffer.flush().await.unwrap().await.unwrap(); } - assert_eq!(wal_buffer.last_flushed_wal_id(), 100); + assert_eq!(wal_buffer.status().unwrap().last_flushed_wal_id, 100); assert_eq!(wal_buffer.inner.read().immutable_wals.len(), 0); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_maybe_trigger_flush_spams_flush_requests() { - let (wal_buffer, _, _, recorder) = + let (mut wal_buffer, _, _, recorder) = setup_wal_buffer_with_flush_interval(Duration::MAX).await; // Simulate many writers each appending a small entry and calling @@ -1021,7 +1043,7 @@ mod tests { let num_writes: u64 = 100; for seq in 1..=num_writes { let entry = make_entry(&format!("key{}", seq), &format!("value{}", seq), seq, None); - wal_buffer.append(&[entry]).unwrap(); + wal_buffer.append(&[entry]).await.unwrap(); wal_buffer.maybe_trigger_flush().unwrap(); } @@ -1029,7 +1051,7 @@ mod tests { lookup_metric(&recorder, stats::WAL_BUFFER_FLUSH_REQUESTS).unwrap(); // Explicitly flush to drain everything, including any partial current WAL. - wal_buffer.flush().unwrap().await.unwrap().unwrap(); + wal_buffer.flush().await.unwrap().await.unwrap(); let actual_flushes = lookup_metric(&recorder, stats::WAL_BUFFER_FLUSHES).unwrap(); @@ -1049,7 +1071,7 @@ mod tests { ); } - fn recording_listener() -> (WalStatusListener, Arc>>) { + fn recording_listener() -> (wal::WalStatusListener, Arc>>) { let events = Arc::new(std::sync::Mutex::new(Vec::new())); let recorder = events.clone(); let listener = Arc::new(move |event| { @@ -1062,21 +1084,24 @@ mod tests { async fn test_listener_notified_when_flush_task_flushes_wal() { // given: let (listener, events) = recording_listener(); - let (wal_buffer, _, _, _) = setup_wal_buffer_with_args(Duration::MAX, listener).await; + let (mut wal_buffer, _, _, _) = setup_wal_buffer_with_args(Duration::MAX, listener).await; // when: Append an entry and explicitly flush it, driving the background flush task. wal_buffer .append(&[make_entry("key1", "value1", 1, None)]) + .await .unwrap(); - wal_buffer.flush().unwrap().await.unwrap().unwrap(); + wal_buffer.flush().await.unwrap().await.unwrap(); // then: the listener should have been notified that wal 1 was flushed. let recorded = events.lock().unwrap().clone(); let mut flushed: Vec<_> = recorded .iter() - .map(|e| { - let WalEvent::WalFlushed(status) = e; - status + .filter_map(|e| { + let wal::WalEvent::WalFlushed(status) = e else { + return None; + }; + Some(status) }) .collect(); assert_eq!(flushed.len(), 1); @@ -1084,54 +1109,4 @@ mod tests { assert_eq!(status.last_flushed_wal_id, 1); assert_eq!(status.last_flushed_seq, Some(1)); } - - #[tokio::test] - async fn test_listener_notified_before_table_waiters() { - // given: - let object_store: Arc = Arc::new(InMemory::new()); - let table_store = Arc::new(TableStore::new( - ObjectStores::new(object_store, None), - SsTableFormat::default(), - Path::from("/root"), - None, - TableStoreKind::Main, - BlockCachePolicy::default(), - )); - let system_clock = Arc::new(DefaultSystemClock::new()); - let status_manager = DbStatusManager::new(0); - let recorder = Arc::new(DefaultMetricsRecorder::new()); - let helper = MetricsRecorderHelper::new(recorder.clone(), MetricLevel::default()); - let mut wal_buffer = WalBufferManager::new( - status_manager.clone(), - &helper, - 0, - table_store.clone(), - 1000, - Some(Duration::MAX), - ); - let task_executor = Arc::new(MessageHandlerExecutor::new( - Arc::new(status_manager), - system_clock.clone(), - )); - wal_buffer.init(task_executor.clone()).await.unwrap(); - task_executor - .monitor_on(&Handle::current()) - .expect("failed to monitor executor"); - let waiter = wal_buffer - .append(&[make_entry("key1", "value1", 1, None)]) - .unwrap(); - let called = Arc::new(AtomicBool::new(false)); - let this_called = called.clone(); - let listener = move |event| { - this_called.store(true, Ordering::SeqCst); - assert!(matches!(event, WalEvent::WalFlushed(_))); - // verifies that the table is not yet notified - assert!(waiter.read().is_none()) - }; - wal_buffer.observer().subscribe(Arc::new(listener)).unwrap(); - - // when/then: - wal_buffer.flush().unwrap().await.unwrap().unwrap(); - assert!(called.load(Ordering::SeqCst)); - } } From 779c90557376cec137fe1821525d4ff155b6a37c Mon Sep 17 00:00:00 2001 From: Uttom Akash Date: Tue, 28 Jul 2026 23:00:22 +0600 Subject: [PATCH 44/63] Fix panic when probing a zero-bit prefix Bloom filter (#1969) --- slatedb/src/filter.rs | 56 +++++++++++++ slatedb/tests/prefix_filter.rs | 140 ++++++++++++++++++++++++++++++++- 2 files changed, 195 insertions(+), 1 deletion(-) diff --git a/slatedb/src/filter.rs b/slatedb/src/filter.rs index 85e25a7df..8239e2250 100644 --- a/slatedb/src/filter.rs +++ b/slatedb/src/filter.rs @@ -122,6 +122,11 @@ impl BloomFilter { } fn might_contain(&self, hash: u64) -> bool { + // A filter built with zero extracted hashes has zero bits: nothing can + // match, and probing it would divide by zero in probes_for_key. + if self.buffer.is_empty() { + return false; + } for p in probes_for_key(hash, self.num_probes, self.filter_bits()) { if !check_bit(p as usize, &self.buffer) { return false; @@ -442,4 +447,55 @@ mod tests { expected_size ); } + + /// Extracts a fixed 4-byte prefix; shorter targets yield no prefix. + struct GatedFixed4; + + impl PrefixExtractor for GatedFixed4 { + fn name(&self) -> &str { + "gated_fixed_4" + } + + fn prefix_len(&self, target: &PrefixTarget) -> Option { + let bytes = match target { + PrefixTarget::Point(k) => k.as_ref(), + PrefixTarget::Prefix(p) => p.as_ref(), + }; + (bytes.len() >= 4).then_some(4) + } + } + + #[test] + fn test_prefix_only_filter_with_no_extracted_prefixes() { + // Zero extracted prefixes + whole-key filtering off = zero-bit filter. + // Probing it must not panic, and a miss is safe: nothing was hashed in. + let mut builder = BloomFilterBuilder::new(10, false, Some(Arc::new(GatedFixed4))); + builder.add_key(&Bytes::from_static(b"a")); + builder.add_key(&Bytes::from_static(b"b")); + let filter = builder.build_filter(); + assert!(filter.buffer.is_empty()); + + assert!(!filter.might_match(&FilterQuery::prefix(Bytes::from_static(b"aaaa")))); + assert!(!filter.might_match(&FilterQuery::point(Bytes::from_static(b"aaaa_key")))); + + // Queries the extractor rejects never reach the filter and must keep + // reporting "might match", so the stored short keys stay reachable. + assert!(filter.might_match(&FilterQuery::prefix(Bytes::from_static(b"a")))); + assert!(filter.might_match(&FilterQuery::point(Bytes::from_static(b"a")))); + } + + #[test] + fn test_combined_filter_with_no_extracted_prefixes() { + // With whole-key filtering on, every key hashes into the filter even + // when the extractor yields nothing, so the empty-filter guard never fires. + let mut builder = BloomFilterBuilder::new(10, true, Some(Arc::new(GatedFixed4))); + builder.add_key(&Bytes::from_static(b"a")); + builder.add_key(&Bytes::from_static(b"b")); + let filter = builder.build_filter(); + assert!(!filter.buffer.is_empty()); + + // Point lookups take the whole-key path and find the stored keys. + assert!(filter.might_match(&FilterQuery::point(Bytes::from_static(b"a")))); + assert!(filter.might_match(&FilterQuery::point(Bytes::from_static(b"b")))); + } } diff --git a/slatedb/tests/prefix_filter.rs b/slatedb/tests/prefix_filter.rs index 23f49563e..ed7ec109d 100644 --- a/slatedb/tests/prefix_filter.rs +++ b/slatedb/tests/prefix_filter.rs @@ -1,6 +1,6 @@ //! Integration tests for the prefix bloom filter. //! -//! Three test modules: +//! Four test modules: //! * [`composite_filters`]: integration tests that configure two filter //! policies on a single DB (one full-key, one conditional prefix) and //! verify reads work after closing/reopening with policies in a different @@ -8,6 +8,9 @@ //! * [`subrange`]: integration tests for `scan_prefix` restricted to a //! subrange, asserting both correct results and actual SST pruning via //! the filter-negative counter. +//! * [`empty_prefix_filter`]: regression tests for #1966 — a persisted +//! zero-bit prefix filter must report a miss instead of panicking, on +//! fresh and reopened DBs. //! * [`prop_test`]: a property test asserting that `scan_prefix` (with and //! without a subrange) returns the same results with and without a prefix //! bloom filter configured. The filter must never introduce false @@ -381,6 +384,141 @@ mod subrange { } } +mod empty_prefix_filter { + use std::sync::Arc; + + use slatedb::config::{FlushOptions, FlushType, PutOptions, Settings, WriteOptions}; + use slatedb::db_stats::{ + FILTER_KIND_LABEL, FILTER_KIND_POINT, FILTER_KIND_PREFIX, SST_FILTER_NEGATIVE_COUNT, + }; + use slatedb::object_store::memory::InMemory; + use slatedb::object_store::ObjectStore; + use slatedb::{BloomFilterPolicy, Db, PrefixExtractor, PrefixTarget}; + use slatedb_common::metrics::{DefaultMetricsRecorder, MetricValue}; + + const PREFIX_LEN: usize = 4; + + /// Extracts a 4-byte prefix only from inputs of at least 4 bytes, so an + /// SST holding only shorter keys builds a zero-bit prefix filter. + struct GatedPrefixExtractor; + + impl PrefixExtractor for GatedPrefixExtractor { + fn name(&self) -> &str { + "gated4" + } + + fn prefix_len(&self, target: &PrefixTarget) -> Option { + let input = match target { + PrefixTarget::Point(k) => k.as_ref(), + PrefixTarget::Prefix(p) => p.as_ref(), + }; + (input.len() >= PREFIX_LEN).then_some(PREFIX_LEN) + } + } + + fn filter_negatives(recorder: &DefaultMetricsRecorder, kind: &'static str) -> u64 { + recorder + .snapshot() + .by_name_and_labels(SST_FILTER_NEGATIVE_COUNT, &[(FILTER_KIND_LABEL, kind)]) + .map(|m| match m.value { + MetricValue::Counter(v) => v, + ref other => panic!("expected counter, got {:?}", other), + }) + .unwrap_or(0) + } + + async fn open_db(store: Arc, recorder: Arc) -> Db { + Db::builder("/test/empty_prefix_filter", store) + .with_settings(Settings { + min_filter_keys: 0, + compactor_options: None, + ..Settings::default() + }) + .with_filter_policies(vec![Arc::new( + BloomFilterPolicy::new(10) + .with_whole_key_filtering(false) + .with_prefix_extractor(Arc::new(GatedPrefixExtractor)), + )]) + .with_metrics_recorder(recorder) + .build() + .await + .expect("failed to build db") + } + + async fn collect_keys(mut iter: slatedb::DbIterator) -> Vec> { + let mut keys = Vec::new(); + while let Some(kv) = iter.next().await.expect("iterator next failed") { + keys.push(kv.key.to_vec()); + } + keys + } + + async fn assert_empty_filter_reads(db: &Db, recorder: &DefaultMetricsRecorder) { + // Extractor-rejected queries bypass the filter; short keys stay reachable. + for key in [b"a".as_slice(), b"b".as_slice()] { + let got = db.get(key).await.expect("get failed"); + assert!(got.is_some(), "expected key {:?} to be present", key); + } + let iter = db.scan_prefix(b"a", ..).await.expect("scan_prefix failed"); + assert_eq!(collect_keys(iter).await, vec![b"a".to_vec()]); + + // Extractor-accepted queries probe the zero-bit filter: before the + // fix this panicked with a division by zero; now the SST is skipped. + let negatives_before = filter_negatives(recorder, FILTER_KIND_PREFIX); + let iter = db + .scan_prefix(b"aaaa", ..) + .await + .expect("scan_prefix failed"); + assert!(collect_keys(iter).await.is_empty()); + assert_eq!( + filter_negatives(recorder, FILTER_KIND_PREFIX) - negatives_before, + 1, + "expected the SST to be skipped on the empty prefix filter" + ); + + let negatives_before = filter_negatives(recorder, FILTER_KIND_POINT); + let got = db.get(b"aaaa").await.expect("get failed"); + assert!(got.is_none()); + assert_eq!( + filter_negatives(recorder, FILTER_KIND_POINT) - negatives_before, + 1, + "expected the SST to be skipped on the empty prefix filter" + ); + } + + #[tokio::test] + async fn empty_filter_reports_miss_after_write_and_reopen() { + let store: Arc = Arc::new(InMemory::new()); + let recorder = Arc::new(DefaultMetricsRecorder::new()); + let db = open_db(store.clone(), recorder.clone()).await; + + let put = PutOptions::default(); + let write = WriteOptions { + await_durable: false, + seqnum: 0, + }; + for key in [b"a".as_slice(), b"b".as_slice()] { + db.put_with_options(key, b"v", &put, &write) + .await + .expect("put failed"); + } + db.flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .expect("memtable flush failed"); + + assert_empty_filter_reads(&db, &recorder).await; + db.close().await.expect("close failed"); + + // Reopen so the zero-bit filter is decoded from the persisted SST. + let recorder = Arc::new(DefaultMetricsRecorder::new()); + let db = open_db(store, recorder.clone()).await; + assert_empty_filter_reads(&db, &recorder).await; + db.close().await.expect("close failed"); + } +} + mod prop_test { use std::sync::Arc; From 54b2f33e9bde3bce8dad49b419ba5d3d0dbe9265 Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 28 Jul 2026 11:02:26 -0700 Subject: [PATCH 45/63] fix flaky test_size_based_flush_triggering (#1979) --- slatedb/src/wal_buffer.rs | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/slatedb/src/wal_buffer.rs b/slatedb/src/wal_buffer.rs index 027af0031..a20020808 100644 --- a/slatedb/src/wal_buffer.rs +++ b/slatedb/src/wal_buffer.rs @@ -701,7 +701,6 @@ mod tests { use slatedb_common::metrics::{ lookup_metric, DefaultMetricsRecorder, MetricLevel, MetricsRecorderHelper, }; - use slatedb_common::MockSystemClock; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -879,7 +878,7 @@ mod tests { async fn setup_wal_buffer() -> ( WalBufferManager, Arc, - Arc, + Arc, Arc, ) { setup_wal_buffer_with_flush_interval(Duration::from_millis(10)).await @@ -890,7 +889,7 @@ mod tests { ) -> ( WalBufferManager, Arc, - Arc, + Arc, Arc, ) { setup_wal_buffer_with_args(flush_interval, Arc::new(|_status| {})).await @@ -902,7 +901,7 @@ mod tests { ) -> ( WalBufferManager, Arc, - Arc, + Arc, Arc, ) { let object_store: Arc = Arc::new(InMemory::new()); @@ -914,7 +913,6 @@ mod tests { TableStoreKind::Main, BlockCachePolicy::default(), )); - let test_clock = Arc::new(MockSystemClock::new()); let system_clock = Arc::new(DefaultSystemClock::new()); let status_manager = Arc::new(DbStatusManager::new(0)); let oracle = Arc::new(DbOracle::new(0, 0, 0, status_manager.clone())); @@ -948,7 +946,7 @@ mod tests { task_executor .monitor_on(&Handle::current()) .expect("failed to monitor executor"); - (wal_buffer, table_store, test_clock, recorder) + (wal_buffer, table_store, status_manager, recorder) } #[tokio::test] @@ -1001,17 +999,20 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_size_based_flush_triggering() { - let (mut wal_buffer, _, _, _) = setup_wal_buffer_with_flush_interval(Duration::MAX).await; - - // Append entries until we exceed the size threshold - let mut seq = 1; - while wal_buffer.status().unwrap().estimated_bytes < wal_buffer.max_wal_bytes_size { - let entry = make_entry(&format!("key{}", seq), &format!("value{}", seq), seq, None); - wal_buffer.append(&[entry]).await.unwrap(); - seq += 1; - } - let mut reader = wal_buffer.maybe_trigger_flush().unwrap(); - reader.await_value().await.unwrap(); + // Append an oversized entry to trigger a flush, then wait until its sequence is durable. + let (mut wal_buffer, _, status_manager, _) = + setup_wal_buffer_with_flush_interval(Duration::MAX).await; + let seq = 1; + let value = "v".repeat(wal_buffer.max_wal_bytes_size); + wal_buffer + .append(&[make_entry("key", &value, seq, None)]) + .await + .unwrap(); + status_manager + .subscribe() + .wait_for(|status| status.durable_seq >= seq) + .await + .unwrap(); assert_eq!(wal_buffer.status().unwrap().last_flushed_wal_id, 1); } From 71a4963aef65de62bc5ebbdba8ebc04d374ecefb Mon Sep 17 00:00:00 2001 From: Rui Fan <1996fanrui@gmail.com> Date: Tue, 28 Jul 2026 20:10:24 +0200 Subject: [PATCH 46/63] [1977] Fix flaky test_backpressure_waiter_exits_when_db_is_fenced (#1978) --- slatedb/src/db.rs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/slatedb/src/db.rs b/slatedb/src/db.rs index 13d43a4d9..31699c192 100644 --- a/slatedb/src/db.rs +++ b/slatedb/src/db.rs @@ -5231,14 +5231,7 @@ mod tests { db.put_with_options(b"key1", &large_value, &PutOptions::default(), &write_opts) .await .unwrap(); - assert_eq!( - db.inner - .wal_observer - .status() - .unwrap() - .buffered_wal_entries_count, - 1 - ); + assert!(!db.inner.state.read().state().imm_memtable.is_empty()); // Start backpressure on a cloned inner handle. This parks the task on // the same wait path used by writers before they enqueue a batch. @@ -5246,7 +5239,7 @@ mod tests { let mut backpressure_task = tokio::spawn(async move { inner.maybe_apply_backpressure().await }); - // Wait until the task has observed the buffered WAL bytes and incremented + // Wait until the task has observed the unflushed memtable and incremented // the backpressure counter, proving it is inside the wait path. tokio::time::timeout(Duration::from_secs(60), async { loop { From 42a9f151f7719d98125a37fb3567e2931bc43a4f Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 28 Jul 2026 12:35:08 -0700 Subject: [PATCH 47/63] Fix test_compactor_applies_output_cache_policy (#1981) --- slatedb/src/compactor.rs | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/slatedb/src/compactor.rs b/slatedb/src/compactor.rs index f5188772b..bc48229e6 100644 --- a/slatedb/src/compactor.rs +++ b/slatedb/src/compactor.rs @@ -1443,6 +1443,7 @@ mod tests { use ulid::Ulid; use super::*; + use crate::batch::WriteBatch; use crate::block_cache_policy::BlockCachePolicy; use crate::compaction_worker::WorkerMessage; use crate::compactions_store::{FenceableCompactions, StoredCompactions}; @@ -1791,19 +1792,21 @@ mod tests { .await .unwrap(); + // Keep all entries in one memtable so the explicit flush produces one + // L0 SST regardless of the configured memtable size threshold. + let mut batch = WriteBatch::new(); for key in [b"a", b"b", b"c", b"d"] { - db.put_with_options( - key, - b"value", - &PutOptions::default(), - &WriteOptions { - await_durable: false, - ..Default::default() - }, - ) - .await - .unwrap(); + batch.put(key, b"value"); } + db.write_with_options( + batch, + &WriteOptions { + await_durable: false, + ..Default::default() + }, + ) + .await + .unwrap(); db.flush_with_options(FlushOptions { flush_type: FlushType::MemTable, }) From 402a3cf94d6900732a0d2e61ff2f4dd8ed150e76 Mon Sep 17 00:00:00 2001 From: nomiero Date: Tue, 28 Jul 2026 15:26:04 -0700 Subject: [PATCH 48/63] convert block prefetch panic to error (#1983) --- slatedb/src/sst_iter.rs | 72 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/slatedb/src/sst_iter.rs b/slatedb/src/sst_iter.rs index 8f158b7c4..ff3855d20 100644 --- a/slatedb/src/sst_iter.rs +++ b/slatedb/src/sst_iter.rs @@ -1,5 +1,6 @@ use async_trait::async_trait; use bytes::Bytes; +use log::error; use slatedb_common::metrics::CounterFn; use std::cmp::min; use std::collections::VecDeque; @@ -10,7 +11,7 @@ use tokio::task::JoinHandle; use crate::block_iterator::DataBlockIterator; use crate::bytes_range::BytesRange; -use crate::db_state::SsTableView; +use crate::db_state::{SsTableId, SsTableView}; use crate::db_stats::DbStats; use crate::error::SlateDBError; use crate::filter_policy::{FilterContext, FilterQuery, NamedFilter}; @@ -22,6 +23,7 @@ use crate::{ partitioned_keyspace, tablestore::TableStore, types::RowEntry, + utils::panic_string, }; enum FetchTask { @@ -443,6 +445,7 @@ impl<'a> InternalSstIterator<'a> { return Ok(None); } let sst_version = self.view.table_as_ref().sst.format_version; + let sst_id = self.view.table_as_ref().sst.id; loop { if spawn_fetches { self.spawn_fetches(); @@ -450,7 +453,9 @@ impl<'a> InternalSstIterator<'a> { if let Some(fetch_task) = self.fetch_tasks.front_mut() { match fetch_task { FetchTask::InFlight(jh) => { - let blocks = jh.await.expect("join task failed")?; + let blocks = jh + .await + .map_err(|join_err| block_fetch_join_error(join_err, sst_id))??; *fetch_task = FetchTask::Finished(blocks); } FetchTask::Finished(blocks) => { @@ -1057,6 +1062,26 @@ impl RowEntryIterator for SstIterator<'_> { } } +/// Converts a failed join on a block fetch task into an error. +/// +/// A fetch task is cancelled when the runtime it was spawned on shuts down, +/// so the iterator reports the cancellation to its caller instead of panicking +/// the task that is awaiting the fetch. +fn block_fetch_join_error(join_err: tokio::task::JoinError, sst_id: SsTableId) -> SlateDBError { + let task_name = format!("sst_block_fetch[{:?}]", sst_id); + match join_err.try_into_panic() { + Ok(panic_err) => { + error!( + "sst block fetch task panicked unexpectedly. [task_name={}, panic={}]", + task_name, + panic_string(&panic_err), + ); + SlateDBError::BackgroundTaskPanic(task_name) + } + Err(_) => SlateDBError::BackgroundTaskCancelled(task_name), + } +} + #[cfg(test)] mod tests { use super::*; @@ -2806,4 +2831,47 @@ mod tests { let kv: KeyValue = entry.into(); assert_eq!(kv.key.as_ref(), b"key_040"); } + + #[tokio::test] + async fn test_next_iter_prefetch_task_cancelled() { + let object_store: Arc = Arc::new(InMemory::new()); + let table_store = Arc::new(TableStore::new( + ObjectStores::new(object_store, None), + SsTableFormat::default(), + Path::from(""), + None, + TableStoreKind::Main, + BlockCachePolicy::default(), + )); + let sst = build_single_block_sst(&table_store, &[b"key1", b"key2"]).await; + + // A runtime that is shut down before anything is spawned on it. + // `shutdown_background` rather than a plain drop, which would itself + // panic inside an async context. + let dead = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .build() + .unwrap(); + let dead_handle = dead.handle().clone(); + dead.shutdown_background(); + + // Initialization walks `advance_block` -> `next_iter(true)` -> + // `spawn_fetches`, so the first fetch is spawned onto - and cancelled + // by - the dead runtime while its context is entered. + let result = { + let _guard = dead_handle.enter(); + SstIterator::new_owned_initialized( + .., + sst, + table_store.clone(), + SstIteratorOptions::default(), + ) + .await + }; + + let Err(err) = result else { + panic!("a cancelled prefetch task must be reported as an error"); + }; + assert!(matches!(err, SlateDBError::BackgroundTaskCancelled(_))); + } } From 69892b3513e7bddf1ff6a2a0253986e310c598b2 Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 28 Jul 2026 19:40:06 -0700 Subject: [PATCH 49/63] Add simple trivial moves to compactor scheduler (#1982) --- slatedb-dst/src/utils.rs | 1 + slatedb/src/compactor.rs | 167 ++++++++++++++++-- slatedb/src/compactor_state.rs | 93 ++++++++++ slatedb/src/config.rs | 19 +- slatedb/src/test_utils.rs | 15 +- .../content/docs/docs/design/compaction.mdx | 11 ++ 6 files changed, 291 insertions(+), 15 deletions(-) diff --git a/slatedb-dst/src/utils.rs b/slatedb-dst/src/utils.rs index 930fde63b..69c36de83 100644 --- a/slatedb-dst/src/utils.rs +++ b/slatedb-dst/src/utils.rs @@ -108,6 +108,7 @@ pub fn build_settings_compactor(rng: &mut impl Rng) -> CompactorOptions { manifest_update_timeout: rng .random_range(Duration::from_millis(100)..Duration::from_secs(60)), max_concurrent_compactions: rng.random_range(1..=4), + enable_trivial_move: rng.random_bool(0.5), scheduler_options: SizeTieredCompactionSchedulerOptions { min_compaction_sources, max_compaction_sources, diff --git a/slatedb/src/compactor.rs b/slatedb/src/compactor.rs index bc48229e6..7fb86a66e 100644 --- a/slatedb/src/compactor.rs +++ b/slatedb/src/compactor.rs @@ -1176,13 +1176,13 @@ impl CompactorEventHandler { } /// Validates every `Submitted` compaction against the current manifest and - /// promotes the valid tiered specs to `Scheduled` — the coordinator's - /// "ready for a worker to claim" state. Drain specs short-circuit the - /// executor and are applied directly to the in-memory manifest (→ + /// promotes valid tiered specs to `Scheduled` — the coordinator's "ready + /// for a worker to claim" state. Drain specs and trivial moves short-circuit + /// the executor and are applied directly to the in-memory manifest (→ /// `Completed`). Invalid specs are marked `Failed`. State changes are /// persisted before any worker (including the local executor) can act on - /// them: when any submission was a drain the manifest and `.compactions` - /// are written together, otherwise `.compactions` alone. + /// them: when a submission changed the manifest, the manifest and + /// `.compactions` are written together, otherwise `.compactions` alone. /// /// Workers exclusively claim `Scheduled` entries; they never act on /// `Submitted`. Routing validation through this single chokepoint keeps @@ -1200,7 +1200,7 @@ impl CompactorEventHandler { return Ok(()); } - let any_drain = submitted_compactions.iter().any(|c| c.spec().is_drain()); + let mut manifest_changed = false; for compaction in &submitted_compactions { // Validate the candidate compaction; mark as failed if invalid. @@ -1215,12 +1215,22 @@ impl CompactorEventHandler { continue; } - // Drain specs apply the watermark advance and SR removal directly, - // marking the compaction Completed. They never enter Scheduled - // because no worker runs them. Tiered specs become Scheduled so a - // worker can claim them. + // Coordinator-local compactions never enter Scheduled because no + // worker runs them. Everything else becomes ready to claim. + let trivial_move_output = self + .options + .enable_trivial_move + .then(|| compaction.trivial_move_output(self.state().db_state())) + .flatten(); + if compaction.spec().is_drain() { self.state_mut().finish_drain_compaction(compaction.id()); + manifest_changed = true; + } else if let Some(output_sr) = trivial_move_output { + info!("trivially moving compaction [spec={}]", compaction.spec()); + self.state_mut() + .finish_compaction(compaction.id(), output_sr); + manifest_changed = true; } else { self.state_mut().update_compaction(&compaction.id(), |c| { c.clear_ctx(); @@ -1229,8 +1239,11 @@ impl CompactorEventHandler { } } - if any_drain { + if manifest_changed { self.state_writer.write_state_safely().await?; + self.stats + .last_compaction_ts + .set(self.system_clock.now().timestamp()); } else { self.state_writer.write_compactions_safely().await?; } @@ -1474,7 +1487,9 @@ mod tests { use crate::proptest_util::rng; use crate::sst_iter::{SstIterator, SstIteratorOptions}; use crate::tablestore::{TableStore, TableStoreKind}; - use crate::test_utils::{assert_iterator, FixedThreeBytePrefixExtractor, GatedObjectStore}; + use crate::test_utils::{ + assert_iterator, bounded_sst_view, FixedThreeBytePrefixExtractor, GatedObjectStore, + }; use crate::types::KeyValue; use crate::types::RowEntry; use bytes::Bytes; @@ -5139,6 +5154,134 @@ mod tests { ); } + #[tokio::test] + async fn test_maybe_validate_submitted_compactions_completes_trivial_move() { + let options = Arc::new(CompactorOptions { + enable_trivial_move: true, + ..compactor_options() + }); + let mut fixture = CompactorEventHandlerTestFixture::new_with_clock( + Arc::new(DefaultSystemClock::new()), + options, + ) + .await; + let l0 = bounded_sst_view(2, b"m", b"n"); + let sr_first = bounded_sst_view(1, b"a", b"b"); + let sr_last = bounded_sst_view(3, b"z", b"z"); + let core = &mut fixture + .handler + .state_writer + .state + .manifest_mut_for_test() + .value + .core; + Arc::make_mut(&mut core.tree).l0 = VecDeque::from([l0.clone()]); + Arc::make_mut(&mut core.tree).compacted = vec![SortedRun { + id: 1, + sst_views: vec![sr_first.clone(), sr_last.clone()], + }]; + + let compaction_id = Ulid::new(); + fixture + .handler + .state_mut() + .add_compaction(Compaction::new( + compaction_id, + CompactionSpec::new(vec![SourceId::SstView(l0.id), SourceId::SortedRun(1)], 2), + )) + .expect("failed to add compaction"); + + fixture + .handler + .maybe_validate_submitted_compactions() + .await + .unwrap(); + + let state = fixture.handler.state(); + assert_eq!( + state + .compactions() + .value + .get(&compaction_id) + .expect("missing compaction") + .status(), + CompactionStatus::Completed + ); + assert!(state.db_state().tree.l0.is_empty()); + assert_eq!(state.db_state().tree.compacted.len(), 1); + let output = &state.db_state().tree.compacted[0]; + assert_eq!(output.id, 2); + assert_eq!( + output + .sst_views + .iter() + .map(|view| view.id) + .collect::>(), + vec![sr_first.id, l0.id, sr_last.id] + ); + + let expected_output = output.clone(); + let stored_manifest = fixture.latest_db_state().await; + assert!(stored_manifest.tree.l0.is_empty()); + assert_eq!(stored_manifest.tree.compacted[0], expected_output); + } + + #[tokio::test] + async fn test_maybe_validate_submitted_compactions_schedules_when_trivial_move_disabled() { + let options = Arc::new(CompactorOptions { + enable_trivial_move: false, + ..compactor_options() + }); + let mut fixture = CompactorEventHandlerTestFixture::new_with_clock( + Arc::new(DefaultSystemClock::new()), + options, + ) + .await; + let l0 = bounded_sst_view(2, b"m", b"n"); + let sr_first = bounded_sst_view(1, b"a", b"b"); + let sr_last = bounded_sst_view(3, b"z", b"z"); + let core = &mut fixture + .handler + .state_writer + .state + .manifest_mut_for_test() + .value + .core; + Arc::make_mut(&mut core.tree).l0 = VecDeque::from([l0.clone()]); + Arc::make_mut(&mut core.tree).compacted = vec![SortedRun { + id: 1, + sst_views: vec![sr_first, sr_last], + }]; + let compaction_id = Ulid::new(); + fixture + .handler + .state_mut() + .add_compaction(Compaction::new( + compaction_id, + CompactionSpec::new(vec![SourceId::SstView(l0.id), SourceId::SortedRun(1)], 2), + )) + .unwrap(); + + fixture + .handler + .maybe_validate_submitted_compactions() + .await + .unwrap(); + + let state = fixture.handler.state(); + assert_eq!( + state + .compactions() + .value + .get(&compaction_id) + .unwrap() + .status(), + CompactionStatus::Scheduled + ); + assert_eq!(state.db_state().tree.l0.len(), 1); + assert_eq!(state.db_state().tree.compacted[0].id, 1); + } + #[tokio::test] async fn test_maybe_validate_submitted_compactions_marks_invalid_failed() { let mut fixture = CompactorEventHandlerTestFixture::new().await; diff --git a/slatedb/src/compactor_state.rs b/slatedb/src/compactor_state.rs index 0c47dce2c..ad51aa7b6 100644 --- a/slatedb/src/compactor_state.rs +++ b/slatedb/src/compactor_state.rs @@ -519,6 +519,35 @@ impl Compaction { .collect() } + /// Builds the output run when all input SST views have disjoint effective + /// key ranges. Reusing the views avoids reading or rewriting SST data. + pub(crate) fn trivial_move_output(&self, db_state: &ManifestCore) -> Option { + let destination = self.spec.destination()?; + let mut sst_views = self.get_l0_sst_views(db_state); + sst_views.extend( + self.get_sorted_runs(db_state) + .into_iter() + .flat_map(|sr| sr.sst_views), + ); + sst_views.sort_by(|left, right| { + left.compacted_effective_range() + .comparable_start_bound() + .cmp(&right.compacted_effective_range().comparable_start_bound()) + }); + + (!sst_views.is_empty() + && sst_views.windows(2).all(|pair| { + pair[0] + .compacted_effective_range() + .intersect(pair[1].compacted_effective_range()) + .is_none() + })) + .then_some(SortedRun { + id: destination, + sst_views, + }) + } + /// The stable id (ULID) used to track this compaction across messages and attempts. pub fn id(&self) -> Ulid { self.id @@ -1242,6 +1271,7 @@ mod tests { use crate::manifest::store::test_utils::new_dirty_manifest; use crate::manifest::store::{ManifestStore, StoredManifest}; use crate::manifest::{LsmTreeState, Segment}; + use crate::test_utils::bounded_sst_view; use crate::utils::IdGenerator; use bytes::Bytes; use object_store::memory::InMemory; @@ -1273,6 +1303,69 @@ mod tests { .with_ctx(Some(CompactionContext::new(subcompactions, Some(0)))) } + #[test] + fn test_trivial_move_output_builds_sorted_run_from_disjoint_inputs() { + let l0 = bounded_sst_view(2, b"m", b"n"); + let sr_first = bounded_sst_view(1, b"a", b"b"); + let sr_last = bounded_sst_view(3, b"z", b"z"); + let mut db_state = ManifestCore::new(); + Arc::make_mut(&mut db_state.tree).l0 = VecDeque::from([l0.clone()]); + Arc::make_mut(&mut db_state.tree).compacted = vec![SortedRun { + id: 1, + sst_views: vec![sr_first.clone(), sr_last.clone()], + }]; + let compaction = Compaction::new( + Ulid::new(), + CompactionSpec::new(vec![SstView(l0.id), SourceId::SortedRun(1)], 2), + ); + + let output = compaction + .trivial_move_output(&db_state) + .expect("disjoint inputs should be a trivial move"); + + assert_eq!(output.id, 2); + assert_eq!( + output + .sst_views + .iter() + .map(|view| view.id) + .collect::>(), + vec![sr_first.id, l0.id, sr_last.id] + ); + } + + #[test] + fn test_trivial_move_output_rejects_overlapping_inputs() { + let l0 = bounded_sst_view(1, b"a", b"m"); + let sr_view = bounded_sst_view(2, b"m", b"z"); + let mut db_state = ManifestCore::new(); + Arc::make_mut(&mut db_state.tree).l0 = VecDeque::from([l0.clone()]); + Arc::make_mut(&mut db_state.tree).compacted = vec![SortedRun { + id: 1, + sst_views: vec![sr_view], + }]; + let compaction = Compaction::new( + Ulid::new(), + CompactionSpec::new(vec![SstView(l0.id), SourceId::SortedRun(1)], 2), + ); + + assert!(compaction.trivial_move_output(&db_state).is_none()); + } + + #[test] + fn test_trivial_move_output_rejects_drain() { + let db_state = ManifestCore::new(); + let compaction = Compaction::new( + Ulid::new(), + CompactionSpec::drain_segment( + Bytes::from_static(b"segment/"), + vec![SourceId::SortedRun(1)], + ), + ); + + assert!(compaction.trivial_move_output(&db_state).is_none()); + } + fn set_test_subcompactions(compaction: &mut Compaction, subcompactions: Vec) { compaction.set_ctx(Some(CompactionContext::new(subcompactions, Some(0)))); } diff --git a/slatedb/src/config.rs b/slatedb/src/config.rs index 9e1ab4640..a1dcc46ce 100644 --- a/slatedb/src/config.rs +++ b/slatedb/src/config.rs @@ -59,6 +59,7 @@ //! [compactor_options] //! poll_interval = "5s" //! max_concurrent_compactions = 4 +//! enable_trivial_move = false //! //! [compactor_options.worker] //! max_sst_size = 1073741824 @@ -110,6 +111,7 @@ //! "compactor_options": { //! "poll_interval": "5s", //! "max_concurrent_compactions": 4, +//! "enable_trivial_move": false, //! "worker": { //! "max_sst_size": 1073741824 //! }, @@ -165,6 +167,7 @@ //! compactor_options: //! poll_interval: '5s' //! max_concurrent_compactions: 4 +//! enable_trivial_move: false //! worker: //! max_sst_size: 1073741824 //! scheduler_options: @@ -210,7 +213,7 @@ use crate::error::SlateDBError; use crate::garbage_collector::{DEFAULT_INTERVAL, DEFAULT_MIN_AGE}; -fn default_boundary_files_enabled() -> bool { +fn default_true() -> bool { true } @@ -1167,6 +1170,16 @@ pub struct CompactorOptions { /// The maximum number of concurrent compactions to execute at once pub max_concurrent_compactions: usize, + /// Whether the coordinator may complete compactions with non-overlapping + /// input SSTs by moving them directly into the destination sorted run, + /// without dispatching a worker job. Because a trivial move does not rewrite + /// rows, it does not remove tombstones, apply compaction filters, or process + /// merges during that compaction. It also preserves the input SST sizes, + /// which can increase manifest size and read amplification compared with + /// rewriting inputs into larger output SSTs. Defaults to false. + #[serde(default)] + pub enable_trivial_move: bool, + /// Scheduler-specific options expressed as string key/value pairs. #[serde(default)] pub scheduler_options: HashMap, @@ -1219,6 +1232,7 @@ impl Default for CompactorOptions { poll_interval: Duration::from_secs(5), manifest_update_timeout: Duration::from_secs(300), max_concurrent_compactions: 4, + enable_trivial_move: false, scheduler_options: HashMap::new(), worker: Some(CompactionWorkerOptions::default()), metric_level: None, @@ -1239,6 +1253,7 @@ impl std::fmt::Debug for CompactorOptions { "max_concurrent_compactions", &self.max_concurrent_compactions, ) + .field("enable_trivial_move", &self.enable_trivial_move) .field("scheduler_options", &self.scheduler_options) .field("worker", &self.worker) .field("metric_level", &self.metric_level) @@ -1509,7 +1524,7 @@ pub struct GarbageCollectorOptions { /// deleted metadata ID and incorrectly report its stale update as successful. Set `min_age` /// longer than the maximum lifetime of a stale process, and use the same setting for every /// garbage collector operating on the database. - #[serde(default = "default_boundary_files_enabled")] + #[serde(default = "default_true")] pub boundary_files_enabled: bool, /// Controls wrapper-level retries for this garbage collector's object-store diff --git a/slatedb/src/test_utils.rs b/slatedb/src/test_utils.rs index 42edf27c5..df22e7009 100644 --- a/slatedb/src/test_utils.rs +++ b/slatedb/src/test_utils.rs @@ -2,9 +2,10 @@ use crate::compactor::{CompactionScheduler, CompactionSchedulerSupplier}; use crate::compactor_state::{CompactionSpec, SourceId}; use crate::compactor_state_protocols::CompactorStateView; use crate::config::{CompactorOptions, PutOptions, WriteOptions}; -use crate::db_state::{SortedRun, SsTableHandle, SsTableId, SsTableView, SstType}; +use crate::db_state::{SortedRun, SsTableHandle, SsTableId, SsTableInfo, SsTableView, SstType}; use crate::error::{RetryReason, SlateDBError}; use crate::format::row::SstRowCodecV0; +use crate::format::sst::SST_FORMAT_VERSION_LATEST; use crate::iter::{IterationOrder, RowEntryIterator}; use crate::object_store_tag::ObjectStoreCallTag; use crate::tablestore::{TableStore, TableStoreKind}; @@ -35,6 +36,18 @@ use tracing_subscriber::fmt::format::FmtSpan; use tracing_subscriber::EnvFilter; use ulid::Ulid; +pub(crate) fn bounded_sst_view(id: u64, first: &'static [u8], last: &'static [u8]) -> SsTableView { + SsTableView::identity(SsTableHandle::new( + SsTableId::Compacted(Ulid::from_parts(id, 0)), + SST_FORMAT_VERSION_LATEST, + SsTableInfo { + first_entry: Some(Bytes::from_static(first)), + last_entry: Some(Bytes::from_static(last)), + ..SsTableInfo::default() + }, + )) +} + /// Asserts that the iterator returns the exact set of expected values in correct order. pub(crate) async fn assert_iterator(iterator: &mut T, entries: Vec) { iterator diff --git a/website/src/content/docs/docs/design/compaction.mdx b/website/src/content/docs/docs/design/compaction.mdx index 797d4b24a..10b1500db 100644 --- a/website/src/content/docs/docs/design/compaction.mdx +++ b/website/src/content/docs/docs/design/compaction.mdx @@ -29,6 +29,17 @@ sorted run. It implements the [`CompactionExecutor`] trait. Currently, the only is the [`TokioCompactionExecutor`](https://github.com/slatedb/slatedb/blob/main/slatedb/src/compactor_executor.rs), which runs compaction on a local tokio runtime. +## Trivial moves + +When every input SST has a non-overlapping effective key range, the compactor can reuse the +existing SSTs in the destination sorted run. The coordinator completes this +[trivial move](https://github.com/facebook/rocksdb/wiki/Compaction-Trivial-Move) itself, +without dispatching the job to an executor or reading and rewriting SST data. This reduces +compaction I/O while still producing a sorted run of non-overlapping SSTs. + +Trivial moves are disabled by default. Set `CompactorOptions::enable_trivial_move` to `true` +to enable them. + To split data with different read/write profiles into independent LSM trees — each with its own compaction policy — and to retire whole key ranges cheaply, see [Segmented Compaction](/docs/design/segmented-compaction). From 4a9a2e4346e3863d2c71e0e40158afa8b08990d8 Mon Sep 17 00:00:00 2001 From: JayJamieson <38236622+JayJamieson@users.noreply.github.com> Date: Thu, 30 Jul 2026 03:58:23 +1200 Subject: [PATCH 50/63] Add DbIterator.next_batch to the UniFFI binding (#1986) --- bindings/go/uniffi/slatedb.go | 110 +++++++++ bindings/go/uniffi/slatedb.h | 11 + bindings/go/uniffi/slatedb_test.go | 370 +++++++++++++++++++++++++++++ bindings/uniffi/src/iterator.rs | 120 ++++++++++ 4 files changed, 611 insertions(+) diff --git a/bindings/go/uniffi/slatedb.go b/bindings/go/uniffi/slatedb.go index 0683f3f83..afa3390e8 100644 --- a/bindings/go/uniffi/slatedb.go +++ b/bindings/go/uniffi/slatedb.go @@ -1398,6 +1398,15 @@ func uniffiCheckChecksums() { panic("slatedb: uniffi_slatedb_uniffi_checksum_method_dbiterator_next: UniFFI API checksum mismatch") } } + { + checksum := rustCall(func(_uniffiStatus *C.RustCallStatus) C.uint16_t { + return C.uniffi_slatedb_uniffi_checksum_method_dbiterator_next_batch() + }) + if checksum != 61234 { + // If this happens try cleaning and rebuilding your project + panic("slatedb: uniffi_slatedb_uniffi_checksum_method_dbiterator_next_batch: UniFFI API checksum mismatch") + } + } { checksum := rustCall(func(_uniffiStatus *C.RustCallStatus) C.uint16_t { return C.uniffi_slatedb_uniffi_checksum_method_dbiterator_seek() @@ -4578,6 +4587,16 @@ func (_ FfiDestroyerDbCache) Destroy(value *DbCache) { type DbIteratorInterface interface { // Returns the next key/value pair from the iterator. Next() (*KeyValue, error) + // Returns up to `max` key/value pairs from the iterator in one call. + // + // Locks the iterator once and pulls rows until it yields `max` items or the + // iterator is exhausted. A returned vector shorter than `max` (including an + // empty vector) means the iterator is exhausted. `max == 0` returns an empty + // vector without advancing. + // + // This exists so that callers crossing a foreign-function boundary can drain + // a scan with one call per batch instead of one call per row. + NextBatch(max uint32) ([]KeyValue, error) // Seeks the iterator to the first entry at or after `key`. Seek(key []byte) error } @@ -4623,6 +4642,50 @@ func (_self *DbIterator) Next() (*KeyValue, error) { return res, err } +// Returns up to `max` key/value pairs from the iterator in one call. +// +// Locks the iterator once and pulls rows until it yields `max` items or the +// iterator is exhausted. A returned vector shorter than `max` (including an +// empty vector) means the iterator is exhausted. `max == 0` returns an empty +// vector without advancing. +// +// This exists so that callers crossing a foreign-function boundary can drain +// a scan with one call per batch instead of one call per row. +func (_self *DbIterator) NextBatch(max uint32) ([]KeyValue, error) { + _pointer := _self.ffiObject.incrementPointer("*DbIterator") + defer _self.ffiObject.decrementPointer() + res, err := uniffiRustCallAsync[*Error]( + FfiConverterErrorINSTANCE, + // completeFn + func(handle C.uint64_t, status *C.RustCallStatus) RustBufferI { + res := C.ffi_slatedb_uniffi_rust_future_complete_rust_buffer(handle, status) + return GoRustBuffer{ + inner: res, + } + }, + // liftFn + func(ffi RustBufferI) []KeyValue { + return FfiConverterSequenceKeyValueINSTANCE.Lift(ffi) + }, + C.uniffi_slatedb_uniffi_fn_method_dbiterator_next_batch( + _pointer, FfiConverterUint32INSTANCE.Lower(max)), + // pollFn + func(handle C.uint64_t, continuation C.UniffiRustFutureContinuationCallback, data C.uint64_t) { + C.ffi_slatedb_uniffi_rust_future_poll_rust_buffer(handle, continuation, data) + }, + // freeFn + func(handle C.uint64_t) { + C.ffi_slatedb_uniffi_rust_future_free_rust_buffer(handle) + }, + ) + + if err == nil { + return res, nil + } + + return res, err +} + // Seeks the iterator to the first entry at or after `key`. func (_self *DbIterator) Seek(key []byte) error { _pointer := _self.ffiObject.incrementPointer("*DbIterator") @@ -14006,6 +14069,53 @@ func (FfiDestroyerSequenceExternalDb) Destroy(sequence []ExternalDb) { } } +type FfiConverterSequenceKeyValue struct{} + +var FfiConverterSequenceKeyValueINSTANCE = FfiConverterSequenceKeyValue{} + +func (c FfiConverterSequenceKeyValue) Lift(rb RustBufferI) []KeyValue { + return LiftFromRustBuffer[[]KeyValue](c, rb) +} + +func (c FfiConverterSequenceKeyValue) Read(reader io.Reader) []KeyValue { + length := readInt32(reader) + if length == 0 { + return nil + } + result := make([]KeyValue, 0, length) + for i := int32(0); i < length; i++ { + result = append(result, FfiConverterKeyValueINSTANCE.Read(reader)) + } + return result +} + +func (c FfiConverterSequenceKeyValue) Lower(value []KeyValue) C.RustBuffer { + return LowerIntoRustBuffer[[]KeyValue](c, value) +} + +func (c FfiConverterSequenceKeyValue) LowerExternal(value []KeyValue) ExternalCRustBuffer { + return RustBufferFromC(LowerIntoRustBuffer[[]KeyValue](c, value)) +} + +func (c FfiConverterSequenceKeyValue) Write(writer io.Writer, value []KeyValue) { + if len(value) > math.MaxInt32 { + panic("[]KeyValue is too large to fit into Int32") + } + + writeInt32(writer, int32(len(value))) + for _, item := range value { + FfiConverterKeyValueINSTANCE.Write(writer, item) + } +} + +type FfiDestroyerSequenceKeyValue struct{} + +func (FfiDestroyerSequenceKeyValue) Destroy(sequence []KeyValue) { + for _, value := range sequence { + FfiDestroyerKeyValue{}.Destroy(value) + } +} + type FfiConverterSequenceMetric struct{} var FfiConverterSequenceMetricINSTANCE = FfiConverterSequenceMetric{} diff --git a/bindings/go/uniffi/slatedb.h b/bindings/go/uniffi/slatedb.h index 27f6de14c..5d21f81ce 100644 --- a/bindings/go/uniffi/slatedb.h +++ b/bindings/go/uniffi/slatedb.h @@ -1349,6 +1349,11 @@ void uniffi_slatedb_uniffi_fn_free_dbiterator(uint64_t handle, RustCallStatus *o uint64_t uniffi_slatedb_uniffi_fn_method_dbiterator_next(uint64_t ptr ); #endif +#ifndef UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_FN_METHOD_DBITERATOR_NEXT_BATCH +#define UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_FN_METHOD_DBITERATOR_NEXT_BATCH +uint64_t uniffi_slatedb_uniffi_fn_method_dbiterator_next_batch(uint64_t ptr, uint32_t max +); +#endif #ifndef UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_FN_METHOD_DBITERATOR_SEEK #define UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_FN_METHOD_DBITERATOR_SEEK uint64_t uniffi_slatedb_uniffi_fn_method_dbiterator_seek(uint64_t ptr, RustBuffer key @@ -2689,6 +2694,12 @@ uint16_t uniffi_slatedb_uniffi_checksum_method_prefixextractor_prefix_len(void #define UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_CHECKSUM_METHOD_DBITERATOR_NEXT uint16_t uniffi_slatedb_uniffi_checksum_method_dbiterator_next(void +); +#endif +#ifndef UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_CHECKSUM_METHOD_DBITERATOR_NEXT_BATCH +#define UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_CHECKSUM_METHOD_DBITERATOR_NEXT_BATCH +uint16_t uniffi_slatedb_uniffi_checksum_method_dbiterator_next_batch(void + ); #endif #ifndef UNIFFI_FFIDEF_UNIFFI_SLATEDB_UNIFFI_CHECKSUM_METHOD_DBITERATOR_SEEK diff --git a/bindings/go/uniffi/slatedb_test.go b/bindings/go/uniffi/slatedb_test.go index 384ec5164..dfcc42dbe 100644 --- a/bindings/go/uniffi/slatedb_test.go +++ b/bindings/go/uniffi/slatedb_test.go @@ -3103,3 +3103,373 @@ func TestDbTtl(t *testing.T) { }) } } + +// batchSeedRow describes one row written by seedBatchRows. +type batchSeedRow struct { + key string + value string + ttl slatedb.Ttl +} + +// batchSeedRows mixes rows with and without a TTL so that both the Some and the +// None case of KeyValue.ExpireTs round trip through NextBatch. Six rows lets a +// batch size of 3 or 6 exercise the exact-multiple case, where the drain loop +// needs one extra call returning an empty slice to detect exhaustion. +var batchSeedRows = []batchSeedRow{ + {key: "batch:01", value: "one", ttl: slatedb.TtlNoExpiry{}}, + {key: "batch:02", value: "two", ttl: slatedb.TtlExpireAfterTicks{Field0: batchSeedTtlTicks}}, + {key: "batch:03", value: "three", ttl: slatedb.TtlNoExpiry{}}, + {key: "batch:04", value: "four", ttl: slatedb.TtlExpireAfterTicks{Field0: batchSeedTtlTicks}}, + {key: "batch:05", value: "five", ttl: slatedb.TtlNoExpiry{}}, + {key: "batch:06", value: "six", ttl: slatedb.TtlExpireAfterTicks{Field0: batchSeedTtlTicks}}, +} + +// batchSeedTtlTicks is far enough in the future that TTL'd seed rows never +// expire mid-test, while still producing a non-nil ExpireTs. +const batchSeedTtlTicks = 3_600_000 + +func seedBatchRows(t *testing.T, db *slatedb.Db) { + t.Helper() + + writeOptions := slatedb.WriteOptions{AwaitDurable: true} + for _, row := range batchSeedRows { + putOptions := slatedb.PutOptions{Ttl: row.ttl} + if _, err := db.PutWithOptions([]byte(row.key), []byte(row.value), putOptions, writeOptions); err != nil { + t.Fatalf("PutWithOptions(%q): %v", row.key, err) + } + } +} + +// scanBatchRows opens a full scan, which covers exactly the seed rows written +// by seedBatchRows. An unbounded range keeps Seek keys unambiguous: they are +// whole keys rather than suffixes relative to a prefix. +func scanBatchRows(t *testing.T, db *slatedb.Db) *slatedb.DbIterator { + t.Helper() + + iter, err := db.Scan(slatedb.KeyRange{}) + if err != nil { + t.Fatalf("Scan(): %v", err) + } + t.Cleanup(iter.Destroy) + return iter +} + +// drainBatch drains iter with NextBatch(max), applying the documented +// exhaustion rule: a batch shorter than max (including an empty one) ends the +// scan. When the row count is an exact multiple of max this performs one extra +// call that returns an empty slice. +func drainBatch(t *testing.T, iter *slatedb.DbIterator, max uint32) []slatedb.KeyValue { + t.Helper() + + if max == 0 { + t.Fatalf("drainBatch requires max > 0; NextBatch(0) never advances") + } + + var rows []slatedb.KeyValue + for { + batch, err := iter.NextBatch(max) + if err != nil { + t.Fatalf("NextBatch(%d): %v", max, err) + } + rows = append(rows, batch...) + if len(batch) < int(max) { + return rows + } + } +} + +func int64PtrString(value *int64) string { + if value == nil { + return "" + } + return fmt.Sprintf("%d", *value) +} + +// requireKeyValuesEqual asserts two row slices are identical across every +// KeyValue field, not just key and value. +func requireKeyValuesEqual(t *testing.T, context string, got []slatedb.KeyValue, want []slatedb.KeyValue) { + t.Helper() + + if len(got) != len(want) { + t.Fatalf("%s: got %d rows, want %d", context, len(got), len(want)) + } + + for i := range want { + gotRow, wantRow := got[i], want[i] + if !bytes.Equal(gotRow.Key, wantRow.Key) { + t.Fatalf("%s: row %d key: got %q, want %q", context, i, gotRow.Key, wantRow.Key) + } + if !bytes.Equal(gotRow.Value, wantRow.Value) { + t.Fatalf("%s: row %d value: got %q, want %q", context, i, gotRow.Value, wantRow.Value) + } + if gotRow.Seq != wantRow.Seq { + t.Fatalf("%s: row %d seq: got %d, want %d", context, i, gotRow.Seq, wantRow.Seq) + } + if gotRow.CreateTs != wantRow.CreateTs { + t.Fatalf("%s: row %d create_ts: got %d, want %d", context, i, gotRow.CreateTs, wantRow.CreateTs) + } + if (gotRow.ExpireTs == nil) != (wantRow.ExpireTs == nil) || + (gotRow.ExpireTs != nil && *gotRow.ExpireTs != *wantRow.ExpireTs) { + t.Fatalf("%s: row %d expire_ts: got %s, want %s", + context, i, int64PtrString(gotRow.ExpireTs), int64PtrString(wantRow.ExpireTs)) + } + } +} + +func TestDbIteratorNextBatchMatchesNext(t *testing.T) { + store := newMemoryStore(t) + handle := openTestDB(t, store, nil) + seedBatchRows(t, handle.db) + + // Row-by-row Next() is the oracle every batch size is compared against. + want := drainIterator(t, scanBatchRows(t, handle.db)) + if len(want) != len(batchSeedRows) { + t.Fatalf("oracle drain: got %d rows, want %d", len(want), len(batchSeedRows)) + } + + // Guard against the differential assertion going vacuous on expire_ts: the + // seed set must actually produce both Some and None. + var withTTL, withoutTTL int + for _, row := range want { + if row.ExpireTs != nil { + withTTL++ + } else { + withoutTTL++ + } + } + if withTTL == 0 || withoutTTL == 0 { + t.Fatalf("seed rows must cover both expire_ts states: with=%d without=%d", withTTL, withoutTTL) + } + + // 3 and 6 divide the row count exactly; 4, 5 and 7 leave a short final + // batch; 1 must behave exactly like repeated Next(); 1000 exceeds the row + // count entirely. + for _, max := range []uint32{1, 2, 3, 4, 5, 6, 7, 1000} { + t.Run(fmt.Sprintf("max=%d", max), func(t *testing.T) { + got := drainBatch(t, scanBatchRows(t, handle.db), max) + requireKeyValuesEqual(t, fmt.Sprintf("NextBatch(%d) drain", max), got, want) + }) + } +} + +func TestDbIteratorNextBatchLargerThanRowCount(t *testing.T) { + store := newMemoryStore(t) + handle := openTestDB(t, store, nil) + seedBatchRows(t, handle.db) + + want := drainIterator(t, scanBatchRows(t, handle.db)) + + iter := scanBatchRows(t, handle.db) + first, err := iter.NextBatch(1000) + if err != nil { + t.Fatalf("NextBatch(1000): %v", err) + } + requireKeyValuesEqual(t, "single oversized NextBatch", first, want) + + second, err := iter.NextBatch(1000) + if err != nil { + t.Fatalf("NextBatch(1000) after exhaustion: %v", err) + } + if len(second) != 0 { + t.Fatalf("NextBatch(1000) after exhaustion: got %d rows, want 0", len(second)) + } +} + +func TestDbIteratorNextBatchEmptyRange(t *testing.T) { + store := newMemoryStore(t) + handle := openTestDB(t, store, nil) + seedBatchRows(t, handle.db) + + // A range that starts past every seeded key. + iter, err := handle.db.Scan(slatedb.KeyRange{ + Start: bytesPtr([]byte("zzz:")), + StartInclusive: true, + }) + if err != nil { + t.Fatalf("Scan(empty range): %v", err) + } + t.Cleanup(iter.Destroy) + + batch, err := iter.NextBatch(16) + if err != nil { + t.Fatalf("NextBatch(16) on empty range: %v", err) + } + if len(batch) != 0 { + t.Fatalf("NextBatch(16) on empty range: got %d rows, want 0", len(batch)) + } +} + +func TestDbIteratorNextBatchZeroMaxDoesNotAdvance(t *testing.T) { + store := newMemoryStore(t) + handle := openTestDB(t, store, nil) + seedBatchRows(t, handle.db) + + want := drainIterator(t, scanBatchRows(t, handle.db)) + + iter := scanBatchRows(t, handle.db) + for i := 0; i < 3; i++ { + batch, err := iter.NextBatch(0) + if err != nil { + t.Fatalf("NextBatch(0) call %d: %v", i, err) + } + if len(batch) != 0 { + t.Fatalf("NextBatch(0) call %d: got %d rows, want 0", i, len(batch)) + } + } + + // The zero-max calls must not have consumed anything. + got, err := iter.NextBatch(1000) + if err != nil { + t.Fatalf("NextBatch(1000) after NextBatch(0): %v", err) + } + requireKeyValuesEqual(t, "NextBatch(1000) after NextBatch(0)", got, want) +} + +func TestDbIteratorNextBatchAfterSeek(t *testing.T) { + store := newMemoryStore(t) + handle := openTestDB(t, store, nil) + seedBatchRows(t, handle.db) + + seekKey := []byte("batch:04") + + oracle := scanBatchRows(t, handle.db) + if err := oracle.Seek(seekKey); err != nil { + t.Fatalf("Seek(%q) on oracle iterator: %v", seekKey, err) + } + want := drainIterator(t, oracle) + if len(want) != 3 { + t.Fatalf("oracle drain after seek: got %d rows, want 3", len(want)) + } + + t.Run("seek then batch drain", func(t *testing.T) { + iter := scanBatchRows(t, handle.db) + if err := iter.Seek(seekKey); err != nil { + t.Fatalf("Seek(%q): %v", seekKey, err) + } + requireKeyValuesEqual(t, "NextBatch(2) after seek", drainBatch(t, iter, 2), want) + }) + + t.Run("seek mid batch drain", func(t *testing.T) { + iter := scanBatchRows(t, handle.db) + if _, err := iter.NextBatch(2); err != nil { + t.Fatalf("NextBatch(2) before seek: %v", err) + } + if err := iter.Seek(seekKey); err != nil { + t.Fatalf("Seek(%q) mid-drain: %v", seekKey, err) + } + requireKeyValuesEqual(t, "NextBatch(2) after mid-drain seek", drainBatch(t, iter, 2), want) + }) +} + +// benchScanRows is the number of rows each scan benchmark drains. +const benchScanRows = 1000 + +func openBenchDB(b *testing.B) *slatedb.Db { + b.Helper() + + store, err := slatedb.ObjectStoreResolve("memory:///") + if err != nil { + b.Fatalf("ObjectStoreResolve(memory:///): %v", err) + } + b.Cleanup(store.Destroy) + + builder := slatedb.NewDbBuilder(testDBPath, store) + defer builder.Destroy() + + db, err := builder.Build() + if err != nil { + b.Fatalf("Build(): %v", err) + } + b.Cleanup(func() { + if err := db.Shutdown(); err != nil { + b.Errorf("Shutdown(): %v", err) + } + db.Destroy() + }) + + writeOptions := slatedb.WriteOptions{AwaitDurable: false} + putOptions := slatedb.PutOptions{Ttl: slatedb.TtlDefault{}} + for i := 0; i < benchScanRows; i++ { + key := []byte(fmt.Sprintf("bench:%06d", i)) + value := []byte(fmt.Sprintf("value-%06d", i)) + if _, err := db.PutWithOptions(key, value, putOptions, writeOptions); err != nil { + b.Fatalf("PutWithOptions(%q): %v", key, err) + } + } + if err := db.Flush(); err != nil { + b.Fatalf("Flush(): %v", err) + } + + return db +} + +func benchScanIterator(b *testing.B, db *slatedb.Db) *slatedb.DbIterator { + b.Helper() + + iter, err := db.Scan(slatedb.KeyRange{}) + if err != nil { + b.Fatalf("Scan(): %v", err) + } + return iter +} + +// BenchmarkScanNext measures the row-at-a-time drain: one async FFI call, and +// the RustBuffer decode that comes with it, per row. +func BenchmarkScanNext(b *testing.B) { + db := openBenchDB(b) + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + iter := benchScanIterator(b, db) + rows := 0 + for { + row, err := iter.Next() + if err != nil { + b.Fatalf("Next(): %v", err) + } + if row == nil { + break + } + rows++ + } + iter.Destroy() + if rows != benchScanRows { + b.Fatalf("drained %d rows, want %d", rows, benchScanRows) + } + } +} + +// BenchmarkScanNextBatch measures the same drain amortized over batches, which +// is the point of NextBatch: the per-call cost is paid once per batch instead +// of once per row. +func BenchmarkScanNextBatch(b *testing.B) { + db := openBenchDB(b) + + for _, max := range []uint32{16, 64, 256, 1024} { + b.Run(fmt.Sprintf("max=%d", max), func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + iter := benchScanIterator(b, db) + rows := 0 + for { + batch, err := iter.NextBatch(max) + if err != nil { + b.Fatalf("NextBatch(%d): %v", max, err) + } + rows += len(batch) + if len(batch) < int(max) { + break + } + } + iter.Destroy() + if rows != benchScanRows { + b.Fatalf("drained %d rows, want %d", rows, benchScanRows) + } + } + }) + } +} diff --git a/bindings/uniffi/src/iterator.rs b/bindings/uniffi/src/iterator.rs index 9eaeaa47b..3ec61dd1d 100644 --- a/bindings/uniffi/src/iterator.rs +++ b/bindings/uniffi/src/iterator.rs @@ -4,6 +4,10 @@ use crate::error::Error; use crate::types::KeyValue; use crate::validation::validate_key; +/// Upper bound on the number of rows `next_batch` preallocates room for, so a +/// caller passing a very large `max` cannot force a large allocation up front. +const MAX_BATCH_PREALLOC: u32 = 1024; + /// Async iterator returned by scan APIs. #[derive(uniffi::Object)] pub struct DbIterator { @@ -26,6 +30,27 @@ impl DbIterator { Ok(guard.next().await?.map(KeyValue::from)) } + /// Returns up to `max` key/value pairs from the iterator in one call. + /// + /// Locks the iterator once and pulls rows until it yields `max` items or the + /// iterator is exhausted. A returned vector shorter than `max` (including an + /// empty vector) means the iterator is exhausted. `max == 0` returns an empty + /// vector without advancing. + /// + /// This exists so that callers crossing a foreign-function boundary can drain + /// a scan with one call per batch instead of one call per row. + pub async fn next_batch(&self, max: u32) -> Result, Error> { + let mut guard = self.inner.lock().await; + let mut out = Vec::with_capacity(max.min(MAX_BATCH_PREALLOC) as usize); + for _ in 0..max { + match guard.next().await? { + Some(kv) => out.push(KeyValue::from(kv)), + None => break, + } + } + Ok(out) + } + /// Seeks the iterator to the first entry at or after `key`. pub async fn seek(&self, key: Vec) -> Result<(), Error> { validate_key(&key)?; @@ -33,3 +58,98 @@ impl DbIterator { guard.seek(key).await.map_err(Into::into) } } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use slatedb::object_store::memory::InMemory; + + use super::DbIterator; + + const ROWS: u32 = 6; + + async fn seeded_db() -> slatedb::Db { + let db = slatedb::Db::builder("test", Arc::new(InMemory::new())) + .build() + .await + .expect("failed to open db"); + for i in 0..ROWS { + db.put(format!("key{i:02}"), format!("value{i:02}")) + .await + .expect("failed to put row"); + } + db + } + + async fn scan_all(db: &slatedb::Db) -> DbIterator { + DbIterator::new(db.scan(..).await.expect("failed to scan")) + } + + /// Drains an iterator one row at a time; the oracle for the batch results. + async fn drain_rows(iter: &DbIterator) -> Vec { + let mut rows = Vec::new(); + while let Some(row) = iter.next().await.expect("next() failed") { + rows.push(row); + } + rows + } + + #[tokio::test] + async fn next_batch_matches_next() { + let db = seeded_db().await; + let want = drain_rows(&scan_all(&db).await).await; + assert_eq!(want.len(), ROWS as usize); + + // 3 and 6 divide the row count exactly; 4 and 7 leave a short final + // batch; 1 must behave like repeated next(). + for max in [1u32, 3, 4, 6, 7, 1000] { + let iter = scan_all(&db).await; + let mut got = Vec::new(); + loop { + let batch = iter.next_batch(max).await.expect("next_batch() failed"); + let exhausted = batch.len() < max as usize; + got.extend(batch); + if exhausted { + break; + } + } + assert_eq!(got, want, "next_batch({max}) disagreed with next()"); + } + } + + #[tokio::test] + async fn next_batch_zero_max_does_not_advance() { + let db = seeded_db().await; + let iter = scan_all(&db).await; + + assert!(iter + .next_batch(0) + .await + .expect("next_batch(0) failed") + .is_empty()); + + let rows = iter + .next_batch(1000) + .await + .expect("next_batch(1000) failed"); + assert_eq!(rows.len(), ROWS as usize); + } + + #[tokio::test] + async fn next_batch_returns_empty_once_exhausted() { + let db = seeded_db().await; + let iter = scan_all(&db).await; + + let rows = iter + .next_batch(1000) + .await + .expect("next_batch(1000) failed"); + assert_eq!(rows.len(), ROWS as usize); + assert!(iter + .next_batch(1000) + .await + .expect("next_batch(1000) after exhaustion failed") + .is_empty()); + } +} From 81acc3cca2f4de62a54ad82f3ecaa8263412a4d1 Mon Sep 17 00:00:00 2001 From: Chris Date: Wed, 29 Jul 2026 09:28:27 -0700 Subject: [PATCH 51/63] Probe latest sequenced objects before listing (#1968) --- .../0032-cached-probing-sequenced-metadata.md | 556 ++++++++++++++++++ slatedb-txn-obj/src/lib.rs | 8 +- slatedb-txn-obj/src/object_store.rs | 376 +++++++++++- 3 files changed, 915 insertions(+), 25 deletions(-) create mode 100644 rfcs/0032-cached-probing-sequenced-metadata.md diff --git a/rfcs/0032-cached-probing-sequenced-metadata.md b/rfcs/0032-cached-probing-sequenced-metadata.md new file mode 100644 index 000000000..e672d19b1 --- /dev/null +++ b/rfcs/0032-cached-probing-sequenced-metadata.md @@ -0,0 +1,556 @@ +# Cached Probing for Sequenced Metadata + +Table of Contents: + + + +- [Summary](#summary) +- [Motivation](#motivation) +- [Goals](#goals) +- [Non-Goals](#non-goals) +- [Design](#design) + - [Latest Object Cache](#latest-object-cache) + - [Cold Reads](#cold-reads) + - [Warm Reads](#warm-reads) + - [Write-Through Updates](#write-through-updates) + - [Boundary Checks and Garbage Collection](#boundary-checks-and-garbage-collection) + - [Concurrency and Sharing](#concurrency-and-sharing) + - [Failure Handling](#failure-handling) +- [Impact Analysis](#impact-analysis) +- [Operations](#operations) + - [Performance and Cost](#performance-and-cost) + - [Observability](#observability) + - [Compatibility](#compatibility) +- [Testing](#testing) +- [Rollout](#rollout) +- [Alternatives](#alternatives) + - [LIST on Every Latest Read](#list-on-every-latest-read) + - [Cache Only the Latest ID](#cache-only-the-latest-id) + - [Unbounded Linear Probing](#unbounded-linear-probing) + - [Exponential and Binary Probing](#exponential-and-binary-probing) + - [Parallel Window Probing](#parallel-window-probing) + - [HEAD Probing](#head-probing) + - [Adaptive Probe Limit](#adaptive-probe-limit) + - [CURRENT Pointer](#current-pointer) + - [Authoritative CURRENT Pointer](#authoritative-current-pointer) +- [Open Questions](#open-questions) +- [References](#references) + + + +Status: Draft + +Authors: + +* [Chris Riccomini](https://github.com/criccomini) + +## Summary + +SlateDB finds the latest sequenced metadata object (`.manifest` or +`.compactions`) by listing every object in the metadata directory, selecting +the highest ID, and reading that object. This RFC replaces LIST on the common +path with a process-local cache and consecutive GET probes. + +Each sequenced metadata store (`ManifestStore` and `CompactionsStore`) caches +the highest object ID and its encoded bytes. A latest read starts at the next +ID and issues up to four GETs. A 404 means the last successful GET or cached +object is the latest version. Four consecutive hits fall back to the existing +LIST path. A store with an empty cache also uses LIST to establish its starting +point. + +Successful writes update the same cache. Boundary checks from RFC-0026 remain +in place and invalidate cached objects rejected by the GC boundary. + +## Motivation + +SlateDB stores manifests and compaction state as immutable, consecutively +numbered objects: + +```text +manifest/00000000000000000012.manifest +manifest/00000000000000000013.manifest + +compactions/00000000000000000007.compactions +compactions/00000000000000000008.compactions +``` + +`ObjectStoreSequencedStorageProtocol::try_read_latest` currently performs these +operations: + +1. LIST the namespace. +2. Sort the returned metadata by object ID. +3. GET the object with the highest ID. +4. Check the object ID against the GC boundary. + +The LIST cost repeats on refreshes even when no writer has added a new object. +LIST also returns metadata for the full retained history, although the caller +needs one object. + +This pattern is expensive and wasteful for idle processes. A database that's +idle for five minutes incurs 560 LIST requests with default configuration. +This comes out to $24.19 per-month in AWS S3 us-east-1 pricing. + +Sequenced metadata already supplies a cheaper lookup mechanism. Once a process +has seen ID `N`, it can GET `N+1`. A 404 means `N` is still latest. If `N+1` +exists, the process continues with `N+2`. The protocol's consecutive-ID +invariant makes the first 404 a valid stopping condition. Probing stops after +four successful GETs so a reader that is far behind catches up with LIST instead +of downloading every intervening object. + +## Goals + +- Idle databases should cost less than $5 per month. +- Protocol should be friendly to lagging readers. +- Remove LIST when a warm reader is close to the latest version. +- Keep the existing object layout, write protocol, and GC boundary semantics. +- Avoid rereading an unchanged latest object. +- Preserve the existing LIST fallback for cold stores, lagging readers, and GC + races. + +## Non-Goals + +- Remove LIST from APIs that enumerate historical versions. +- Add a durable pointer object or change the metadata commit point. +- Share cache state across processes. +- Change manifest or compactions serialization. +- Replace the GC boundary protocol from RFC-0026. + +## Design + +### Latest Object Cache + +Each `ObjectStoreSequencedStorageProtocol` stores one cache entry: + +```rust +Mutex> +``` + +The cache contains encoded bytes rather than `T`. This avoids adding a +`T: Clone` bound and lets the write path reuse the bytes it already encoded. + +Cache updates are monotonic. An update replaces the entry only when its ID is +higher than the cached ID: + +```rust +fn maybe_cache_latest(id, bytes): + lock latest + if latest is empty or id > latest.id: + latest = (id, bytes) +``` + +The cache holds one object body per protocol instance. A manifest can be +several MiB, so this changes the steady-state memory footprint by the size of +the latest encoded manifest and compactions object for each live store +instance. + +### Cold Reads + +A protocol instance starts with an empty cache. Its first latest read keeps the +existing behavior: + +```text +LIST namespace + | + +-- empty --------------------> return None + | + +-- select highest ID N + | + +-- GET N succeeds ---> cache (N, bytes), return N + | + +-- GET N is 404 ------> repeat LIST +``` + +The LIST retry covers the race where GC deletes the listed object before the +GET completes. Other object-store errors and codec errors return to the caller. + +### Warm Reads + +A warm read snapshots the cache, releases the mutex, and starts probing at the +next ID: + +```text +cached (N, bytes_N) + | + +-- GET N+1 is 404 -----------> decode bytes_N, return N + | + +-- GET N+1 succeeds + | + +-- cache (N+1, bytes_N+1) + +-- GET N+2 + | + +-- continue until the first 404 or fourth hit + | + +-- fourth hit ---> fall back to LIST +``` + +The implementation never holds the cache mutex across an object-store request. +Concurrent readers may issue duplicate probes. They cannot move the cache +backward. + +The first missing successor terminates the probe. It does not trigger LIST. +The cached bytes let the reader return the latest object without rereading it. +Four consecutive successful probes fall through to the cold LIST path. + +### Write-Through Updates + +The write path encodes a new value once: + +1. Compute the next consecutive ID. +2. Encode the value. +3. PUT the object with create-if-absent. +4. If the PUT succeeds, cache the ID and encoded bytes when the ID is higher + than the current cache entry. +5. Run the existing boundary check. + +The generic checked write still decides whether the caller observes success. +`write_unchecked` caches the object after its physical PUT because unchecked +reads expose physically present objects. + +A write performed through the same protocol instance moves the read watermark +without a LIST or a successful GET. The next latest read probes the following +ID and returns the cached bytes on 404. + +### Boundary Checks and Garbage Collection + +RFC-0026 remains part of the protocol. The cache does not make deleted IDs safe +to reuse and does not replace the durable GC boundary. + +A cached object can become stale after another process advances the boundary +and GC deletes an old prefix. Checked latest reads already validate their +result against the boundary: + +1. The probing path returns cached ID `N`. +2. The boundary rejects `N`. +3. The protocol invalidates the cache entry for `N`. +4. The generic latest-read loop retries. +5. The empty cache sends the retry through LIST. + +An object above `N` may appear during the first probe. In that case, probing +advances the cache and can find an object above the boundary without LIST. + +Deleting a cached object through the same protocol invalidates the matching +entry after the object-store DELETE succeeds. If another process deletes +objects covered by a newer GC boundary, checked reads reject and invalidate +any cached object covered by that boundary before returning it. + +### Concurrency and Sharing + +The cache belongs to `ObjectStoreSequencedStorageProtocol`, not the underlying +`ObjectStore`. Components share it only when they share the same protocol +instance, normally through an `Arc` or +`Arc`. + +Database components constructed together should reuse those store instances +where their lifetimes allow it. A separately constructed GC process, admin +client, or external compactor starts with an empty cache and pays one cold +LIST. There is no process-wide registry keyed by object-store path. + +The cache update rule handles concurrent reads and writes: + +- A lower ID never replaces a higher cached ID. +- Locks cover only cloning or replacing the cache entry. +- Object bytes are immutable after create-if-absent succeeds. +- A boundary rejection clears only the matching cached ID. It does not discard + a higher entry installed by another operation. + +### Failure Handling + +The probing path distinguishes a missing successor from other failures: + +- 404 for `N+1`: return cached `N`. +- Successful GET for `N+1`: cache it and continue. +- Four consecutive successful GETs: fall back to LIST. +- Other GET error: return the error. +- Decode error: return the codec error. + +## Impact Analysis + +SlateDB features and components that this RFC interacts with. + +### Core API & Query Semantics + +- [ ] Basic KV API (`get`/`put`/`delete`) +- [ ] Range queries, iterators, seek semantics +- [ ] Range deletions +- [ ] Error model, API errors + +### Consistency, Isolation, and Multi-Versioning + +- [ ] Transactions +- [ ] Snapshots +- [ ] Sequence numbers + +### Time, Retention, and Derived State + +- [ ] Time to live (TTL) +- [ ] Compaction filters +- [ ] Merge operator +- [ ] Change Data Capture (CDC) + +### Metadata, Coordination, and Lifecycles + +- [x] Manifest format +- [x] Checkpoints +- [ ] Clones +- [x] Garbage collection +- [ ] Database splitting and merging +- [ ] Multi-writer + +The manifest and compactions payload formats do not change. This RFC changes +how callers locate the latest encoded object. + +### Compaction + +- [x] Compaction state persistence +- [ ] Compaction filters +- [ ] Compaction strategies +- [ ] Distributed compaction +- [ ] Compactions format + +### Storage Engine Internals + +- [ ] Write-ahead log (WAL) +- [ ] Block cache +- [ ] Object store cache +- [ ] Indexing (bloom filters, metadata) +- [ ] SST format or block format + +### Ecosystem & Operations + +- [ ] CLI tools +- [ ] Language bindings (Go/Python/etc) +- [ ] Observability (metrics/logging/tracing) + +## Operations + +### Performance and Cost + +The proposal moves object-store work from LIST to GET without adding a write +request. + +| Operation | Existing LIST path | Cached probing | +|---|---:|---:| +| Successful write | 1 object PUT + boundary GET | Same | +| Cold latest read | 1 LIST + object GET + boundary GET | Same | +| Warm unchanged read | 1 LIST + object GET + boundary GET | 1 missing-successor GET + boundary GET | +| Warm read fewer than 4 versions behind | 1 LIST + object GET + boundary GET | `k` GET hits + 1 GET miss + boundary GET | +| Warm read at least 4 versions behind | 1 LIST + object GET + boundary GET | 4 GET hits + 1 LIST + object GET + boundary GET | + +Object stores tend to bill LIST with PUT-class requests and GETs at a lower +request rate. Exact prices depend on the provider and region. Probing should +cost less when protocol instances live long enough to amortize their cold LIST +and usually lag by a small number of versions. + +A process that falls far behind issues at most four probe GETs before using +LIST. The ratio below captures that workload: + +```text +probe_gets / latest_reads +``` + +A ratio near `1` means most reads issue one 404 probe and return cached bytes. +A ratio near `4` means readers often hit the probe limit and fall back to LIST. + +The proposal adds one encoded object body per live protocol instance. It does +not add objects to storage or increase write amplification. + +### Observability + +Add counters for: + +- latest cache hits and cold misses; +- successful and missing probe GETs; +- cache advances from reads and writes; and +- boundary-driven cache invalidations. + +Log probe-limit LIST fallbacks at debug level with the object directory, last +seen ID, and probe count. + +Record a histogram of probes per latest read. Existing object-store metrics +continue to report request latency and failures. + +No configuration is required. + +### Compatibility + +The object layout and payload formats do not change. Old and new SlateDB +versions can read objects written by either implementation. + +Rolling upgrades are safe. Old processes continue to use LIST. New processes +build their cache from LIST and then probe. Each process retains RFC-0026 +boundary checks, so mixed versions do not change GC fencing. + +No public Rust API or language binding changes. + +## Testing + +- Unit test to verify reads fall back to LIST after four consecutive GET hits. +- Run benchmarks to verify an idle SlateDB with default configuration stays + below $5 per month in AWS S3 us-east-1 pricing. + +## Rollout + +The change is internal and does not require a feature flag. + +## Alternatives + +### LIST on Every Latest Read + +Keep listing the namespace and reading the highest object on every refresh. +This has no process-local state and catches up in one LIST plus one GET, +regardless of lag. + +The retained history makes LIST more expensive than the information needed by +the caller. Repeating it when no version changed also adds avoidable request +cost and latency. + +### Unbounded Linear Probing + +Continue reading `N+1`, `N+2`, and so on until the first 404. A reader that is +`k` versions behind uses `k+1` GETs and avoids LIST. Each successful GET also +supplies the bytes needed if that object is latest. + +The request count and latency grow linearly with lag. A process resuming after +a long pause could download hundreds of obsolete objects before returning. +The four-GET limit gives the common case the same behavior while bounding the +catch-up cost. + +### Exponential and Binary Probing + +`TableStore::last_seen_wal_id` uses exponential probing followed by binary +search to find the latest WAL SST. Starting at `N`, it probes offsets +`1, 2, 4, 8, ...` in parallel groups of eight until one is missing, then +binary searches between the highest hit and the first miss. Contiguous IDs make +the existence test monotonic. A pure binary search would not work because the +reader has no upper bound before the exponential phase. + +This finds a frontier `k` versions away with `O(log k)` existence checks. The +reader must then GET the latest object because the search only establishes its +ID. For small `k`, linear GETs use fewer requests and already have the latest +bytes. Exponential probing is a better fit when large gaps are common enough to +justify the extra code and concurrent request fan-out. + +### Parallel Window Probing + +Issue a fixed window of successor reads concurrently. If the window contains a +404, the object before the first missing ID is latest. If every request +succeeds, issue another window or fall back to LIST. A window of four can cross +four versions in one round trip. + +Issuing the full window on every read turns an unchanged warm read from one +request into four. A hybrid can probe `N+1` first and issue a window only after +that request succeeds, but requests beyond the first missing ID do no useful +work. This option trades request count for catch-up latency. + +### HEAD Probing + +Use HEAD requests to locate the frontier, then GET only the latest object. +This avoids downloading intermediate object bodies when a reader is behind. +An unchanged reader still needs one missing-successor request and can return +its cached bytes. + +For small gaps, HEAD probing adds a final GET that linear GET probing avoids. +Many object stores bill HEAD and GET at the same request rate, so this option +reduces transferred bytes without reducing request charges. + +### Adaptive Probe Limit + +Change the probe limit based on recent reads. A store could raise the limit +after repeated LIST fallbacks and lower it after 404s. This may help workloads +where writers publish bursts that often exceed four versions. + +The store would need more cache state and a tuning policy. A fixed limit has a +predictable request bound and keeps behavior consistent across protocol +instances. + +### CURRENT Pointer + +Store a small mutable `CURRENT` object containing the latest sequence ID. +Readers GET `CURRENT` and then GET the referenced metadata object. With a local +bytes cache, an unchanged pointer can return the cached object after one GET. + +A pointer can support racing writes without making the pointer the commit +point. Starting from object `N-1`, a writer races: + +1. create object `N` with create-if-absent; and +2. update `CURRENT` from `N-1` to `N`. + +The two operations can complete in either order: + +| Result | Recovery | +|---|---| +| Both succeed | `N` is current. | +| Object succeeds, pointer fails | Retry `CURRENT=N`. | +| Pointer succeeds, object fails | Readers return `N-1`; writers retry object `N`. | +| Object already exists | Another writer won `N`; refresh and return a write conflict. | + +A writer that observes `CURRENT=N` with object `N` missing must retry `N`. +Advancing to `N+1` would create a gap and make reader fallback ambiguous. +Once object `N` exists, it remains part of the sequence after `CURRENT` +advances. + +This version of `CURRENT` is an advisory cursor. Object creation still commits +a version, so RFC-0026 boundary files remain necessary. A stale writer could +otherwise recreate an ID deleted by GC. + +`CURRENT` removes cold LISTs and the four-probe catch-up fallback. It also adds +a mutable pointer update to every write. Its steady-state request profile is: + +| Operation | Cached probing | `CURRENT` cursor | +|---|---:|---:| +| Successful write | 1 object PUT + boundary GET | 1 object PUT + 1 pointer PUT + boundary GET | +| Warm unchanged read | 1 successor GET + boundary GET | 1 pointer GET + boundary GET | +| Cold latest read | 1 LIST + object GET + boundary GET | 1 pointer GET + object GET + boundary GET | + +Both designs issue one lookup GET on an unchanged warm read. The pointer pays +an extra PUT on every write to avoid cold LISTs and catch-up probes. Long-lived +SlateDB processes with shared store instances should pay less with probing. +Short-lived readers or processes that lag many versions may favor a pointer. +A pointer that leads a failed object write adds a 404 GET and a fallback GET +until some writer fills the reserved ID. + +Useful break-even inputs are: + +```text +latest_reads +cold_list_fallbacks +probe_gets +successful_writes +``` + +The pointer costs less when the LIST and probe requests it avoids cost more +than one extra pointer PUT per successful write, including retries during write +contention. + +### Authoritative CURRENT Pointer + +`CURRENT` could become the commit point instead of a cursor. A stale writer +would write a candidate object and then conditionally update `CURRENT`. If the +conditional update failed, readers would ignore the candidate. This could +replace the GC boundary because recreating a deleted object would not publish +it. + +That design changes the meaning of physical metadata files. A file could exist +without ever becoming a committed historical version. A crash after writing +deterministic object `N+1` but before updating `CURRENT` would also block later +create-if-absent attempts for `N+1`. + +Writers could use unique candidate keys or skip occupied IDs, but both choices +change the current layout and historical listing semantics. Racing publication +before the candidate is durable can leave `CURRENT` pointing to a missing +object. Reader fallback can handle the missing object, but the pointer then +needs reservation and recovery rules. + +Cached probing keeps object creation as the commit point and preserves +contiguous IDs. Existing tools can continue to treat physical sequenced +objects as history. + +## Open Questions + +None. + +## References + +- [RFC-0001: Manifest](0001-manifest.md) +- [RFC-0026: Garbage Collector Boundary Files for Sequenced Metadata](0026-garbage-collector-boundary.md) +- [slatedb/slatedb#1215: listed file missing before read](https://github.com/slatedb/slatedb/issues/1215) diff --git a/slatedb-txn-obj/src/lib.rs b/slatedb-txn-obj/src/lib.rs index 9d1f6b172..4a3d26c9e 100644 --- a/slatedb-txn-obj/src/lib.rs +++ b/slatedb-txn-obj/src/lib.rs @@ -604,8 +604,9 @@ pub trait BoundaryObject: Send + Sync { /// latest-version reads retry [`SequencedStorageProtocol::try_read_latest_unchecked`] until the /// returned ID is above the durable boundary; and [`SequencedStorageProtocol::delete`] only deletes /// versions at or below the boundary. Methods with `_unchecked` in their names, along with -/// [`SequencedStorageProtocol::list`], expose physically present versions without boundary -/// filtering. +/// [`SequencedStorageProtocol::list`], do not filter against the durable boundary. +/// [`SequencedStorageProtocol::try_read_latest_unchecked`] may return a process-local cached version +/// after another process has deleted it from storage. #[async_trait] pub trait SequencedStorageProtocol: TransactionalStorageProtocol + BoundaryObject @@ -622,6 +623,9 @@ pub trait SequencedStorageProtocol: /// Read the latest version without checking it against the durable boundary. /// + /// Implementations may serve a process-local cached version that is no longer physically + /// present in storage. + /// /// Implementations provide this storage primitive and should rely on the generic /// [`TransactionalStorageProtocol::try_read_latest`] implementation for normal checked reads. async fn try_read_latest_unchecked( diff --git a/slatedb-txn-obj/src/object_store.rs b/slatedb-txn-obj/src/object_store.rs index d0abda30e..caf667621 100644 --- a/slatedb-txn-obj/src/object_store.rs +++ b/slatedb-txn-obj/src/object_store.rs @@ -3,6 +3,7 @@ use crate::{ BoundaryObject, MonotonicId, ObjectCodec, SequencedStorageProtocol, TransactionalObjectError, }; use async_trait::async_trait; +use bytes::Bytes; use futures::StreamExt; use log::{debug, error, warn}; use object_store::path::Path; @@ -17,6 +18,8 @@ use std::collections::Bound::Unbounded; use std::ops::RangeBounds; use std::sync::Arc; +const MAX_PROBES: usize = 4; + /// Implements `SequencedStorageProtocol` on object storage. /// /// ## File layout and naming @@ -33,6 +36,7 @@ pub struct ObjectStoreSequencedStorageProtocol { codec: Box>, file_suffix: &'static str, boundary: Arc, + latest: Mutex>, } impl ObjectStoreSequencedStorageProtocol { @@ -72,6 +76,7 @@ impl ObjectStoreSequencedStorageProtocol { codec, file_suffix, boundary, + latest: Mutex::new(None), } } @@ -95,6 +100,47 @@ impl ObjectStoreSequencedStorageProtocol { _ => Err(TransactionalObjectError::InvalidObjectState), } } + + fn maybe_cache_latest(&self, id: MonotonicId, bytes: Bytes) { + let mut latest = self.latest.lock(); + if latest + .as_ref() + .map(|(cached_id, _)| id > *cached_id) + .unwrap_or(true) + { + *latest = Some((id, bytes)); + } + } + + fn invalidate_cached(&self, id: MonotonicId) { + let mut latest = self.latest.lock(); + if matches!(latest.as_ref(), Some((cached_id, _)) if *cached_id == id) { + *latest = None; + } + } + + fn invalidate_cached_through(&self, boundary: MonotonicId) { + let mut latest = self.latest.lock(); + if matches!(latest.as_ref(), Some((cached_id, _)) if *cached_id <= boundary) { + *latest = None; + } + } + + async fn try_read_bytes_unchecked( + &self, + id: MonotonicId, + ) -> Result, TransactionalObjectError> { + let path = self.path_for(id); + match self.object_store.get(&path).await { + Ok(obj) => obj + .bytes() + .await + .map(Some) + .map_err(TransactionalObjectError::from), + Err(Error::NotFound { .. }) => Ok(None), + Err(e) => Err(TransactionalObjectError::from(e)), + } + } } /// Implements [`BoundaryObject`] on object storage. @@ -304,11 +350,17 @@ impl BoundaryObject for ObjectStoreBoundaryObject { #[async_trait] impl BoundaryObject for ObjectStoreSequencedStorageProtocol { async fn check(&self, id: MonotonicId) -> Result<(), TransactionalObjectError> { - self.boundary.check(id).await + let result = self.boundary.check(id).await; + if matches!(&result, Err(TransactionalObjectError::ObjectVersionExists)) { + self.invalidate_cached(id); + } + result } async fn advance(&self, boundary: MonotonicId) -> Result<(), TransactionalObjectError> { - self.boundary.advance(boundary).await + self.boundary.advance(boundary).await?; + self.invalidate_cached_through(boundary); + Ok(()) } } @@ -323,10 +375,11 @@ impl SequencedStorageProtocol for ObjectStoreSequencedStorage .map(|id| id.next()) .unwrap_or(MonotonicId::initial()); let path = self.path_for(id); + let bytes = self.codec.encode(new_value); self.object_store .put_opts( &path, - PutPayload::from_bytes(self.codec.encode(new_value)), + PutPayload::from_bytes(bytes.clone()), PutOptions::from(PutMode::Create), ) .await @@ -337,32 +390,71 @@ impl SequencedStorageProtocol for ObjectStoreSequencedStorage TransactionalObjectError::from(err) } })?; + self.maybe_cache_latest(id, bytes); Ok(id) } async fn try_read_latest_unchecked( &self, ) -> Result, TransactionalObjectError> { + let cached = self.latest.lock().clone(); + if let Some((mut id, mut bytes)) = cached { + for _ in 0..MAX_PROBES { + let next_id = id.next(); + match self.try_read_bytes_unchecked(next_id).await? { + Some(next_bytes) => { + id = next_id; + bytes = next_bytes; + self.maybe_cache_latest(id, bytes.clone()); + } + // IDs are consecutive, so a missing successor means the + // cached object is the latest version. + None => { + return self + .codec + .decode(&bytes) + .map(|value| Some((id, value))) + .map_err(CallbackError); + } + } + } + debug!( + "latest read probe limit reached, falling back to list [directory={}, last_seen_id={}, probes={}]", + self.dir_path, + id.id(), + MAX_PROBES, + ); + } + loop { let files = self.list(Unbounded, Unbounded).await?; + let cached = self.latest.lock().clone(); if let Some(file) = files.last() { - let result = self - .try_read_unchecked(file.id) - .await - .map(|opt| opt.map(|v| (file.id, v))); - match result { + // Reuse cached bytes when LIST selects the cached ID instead of issuing another GET. + let bytes = match cached { + Some((cached_id, bytes)) if cached_id == file.id => Some(bytes), + _ => self.try_read_bytes_unchecked(file.id).await?, + }; + match bytes { // File listed but not found. Probably deleted by GC. Retry list/read. // See https://github.com/slatedb/slatedb/issues/1215 for more details. - Ok(None) => { + None => { warn!( "listed file missing on read, retrying [location={}]", file.metadata.location, ); } - _ => return result, + Some(bytes) => { + let value = self.codec.decode(&bytes).map_err(CallbackError)?; + self.maybe_cache_latest(file.id, bytes); + return Ok(Some((file.id, value))); + } } } else { - // No files found, so return None + // Clear the observed entry so a later read cannot resurrect it after an empty LIST. + if let Some((cached_id, _)) = cached { + self.invalidate_cached(cached_id); + } break; } } @@ -373,16 +465,9 @@ impl SequencedStorageProtocol for ObjectStoreSequencedStorage &self, id: MonotonicId, ) -> Result, TransactionalObjectError> { - let path = self.path_for(id); - match self.object_store.get(&path).await { - Ok(obj) => match obj.bytes().await { - Ok(bytes) => self.codec.decode(&bytes).map(Some).map_err(CallbackError), - Err(e) => Err(TransactionalObjectError::from(e)), - }, - Err(e) => match e { - Error::NotFound { .. } => Ok(None), - _ => Err(TransactionalObjectError::from(e)), - }, + match self.try_read_bytes_unchecked(id).await? { + Some(bytes) => self.codec.decode(&bytes).map(Some).map_err(CallbackError), + None => Ok(None), } } @@ -421,7 +506,9 @@ impl SequencedStorageProtocol for ObjectStoreSequencedStorage self.object_store .delete(&path) .await - .map_err(TransactionalObjectError::from) + .map_err(TransactionalObjectError::from)?; + self.invalidate_cached(id); + Ok(()) } } @@ -447,7 +534,7 @@ mod tests { }; use std::collections::Bound::{Excluded, Included, Unbounded}; use std::fmt; - use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex as StdMutex}; use tokio::sync::Notify; @@ -574,6 +661,8 @@ mod tests { inner: InMemory, get_opts_calls: AtomicUsize, if_none_match_gets: AtomicUsize, + list_calls: AtomicUsize, + list_empty: AtomicBool, blocking_not_found: StdMutex>, } @@ -583,6 +672,8 @@ mod tests { inner: InMemory::new(), get_opts_calls: AtomicUsize::new(0), if_none_match_gets: AtomicUsize::new(0), + list_calls: AtomicUsize::new(0), + list_empty: AtomicBool::new(false), blocking_not_found: StdMutex::new(None), } } @@ -655,6 +746,10 @@ mod tests { } fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, ObjectStoreResult> { + self.list_calls.fetch_add(1, Ordering::SeqCst); + if self.list_empty.load(Ordering::SeqCst) { + return stream::empty().boxed(); + } self.inner.list(prefix) } @@ -675,6 +770,30 @@ mod tests { } } + fn new_counting_protocol() -> ( + Arc, + Arc>, + ) { + let counting_store = Arc::new(CountingGetStore::new()); + let object_store: Arc = counting_store.clone(); + let protocol = Arc::new(ObjectStoreSequencedStorageProtocol::new( + &Path::from("/root"), + object_store, + "test", + "val", + Box::new(TestValCodec), + )); + (counting_store, protocol) + } + + async fn put_test_value(store: &CountingGetStore, id: u64, value: &TestVal) { + let path = Path::from(format!("/root/test/{:020}.val", id)); + store + .put(&path, PutPayload::from_bytes(TestValCodec.encode(value))) + .await + .unwrap(); + } + #[tokio::test] async fn test_boundary_check_allows_missing_boundary() { let object_store: Arc = Arc::new(InMemory::new()); @@ -928,6 +1047,216 @@ mod tests { assert!(missing.is_none()); } + #[tokio::test] + async fn test_latest_read_caches_bytes_after_list() { + let (object_store, store) = new_counting_protocol(); + let expected = TestVal { + epoch: 1, + payload: 10, + }; + put_test_value(&object_store, 1, &expected).await; + + let first = store.try_read_latest_unchecked().await.unwrap().unwrap(); + assert_eq!((MonotonicId::new(1), expected.clone()), first); + assert_eq!(1, object_store.list_calls.load(Ordering::SeqCst)); + + let gets = object_store.get_opts_calls.load(Ordering::SeqCst); + let second = store.try_read_latest_unchecked().await.unwrap().unwrap(); + assert_eq!((MonotonicId::new(1), expected), second); + assert_eq!(1, object_store.list_calls.load(Ordering::SeqCst)); + assert_eq!( + gets + 1, + object_store.get_opts_calls.load(Ordering::SeqCst), + "the warm read should only probe id 2" + ); + } + + #[tokio::test] + async fn test_latest_read_linearly_probes_from_cached_id() { + let (object_store, store) = new_counting_protocol(); + let first = TestVal { + epoch: 1, + payload: 10, + }; + let second = TestVal { + epoch: 1, + payload: 20, + }; + let third = TestVal { + epoch: 1, + payload: 30, + }; + put_test_value(&object_store, 1, &first).await; + store.try_read_latest_unchecked().await.unwrap().unwrap(); + + put_test_value(&object_store, 2, &second).await; + put_test_value(&object_store, 3, &third).await; + + let latest = store.try_read_latest_unchecked().await.unwrap().unwrap(); + assert_eq!((MonotonicId::new(3), third), latest); + assert_eq!(1, object_store.list_calls.load(Ordering::SeqCst)); + assert_eq!( + Some(MonotonicId::new(3)), + store.latest.lock().as_ref().map(|(id, _)| *id) + ); + } + + #[tokio::test] + async fn test_latest_read_falls_back_to_list_after_four_probes() { + let (object_store, store) = new_counting_protocol(); + let first = TestVal { + epoch: 1, + payload: 1, + }; + put_test_value(&object_store, 1, &first).await; + store.try_read_latest_unchecked().await.unwrap().unwrap(); + assert_eq!(1, object_store.list_calls.load(Ordering::SeqCst)); + + for id in 2..=8 { + let value = TestVal { + epoch: 1, + payload: id, + }; + put_test_value(&object_store, id, &value).await; + } + + let gets = object_store.get_opts_calls.load(Ordering::SeqCst); + let latest = store.try_read_latest_unchecked().await.unwrap().unwrap(); + + assert_eq!(MonotonicId::new(8), latest.0); + assert_eq!(8, latest.1.payload); + assert_eq!(2, object_store.list_calls.load(Ordering::SeqCst)); + assert_eq!( + gets + 5, + object_store.get_opts_calls.load(Ordering::SeqCst), + "the warm read should issue four probes and GET the object selected by LIST" + ); + } + + #[tokio::test] + async fn test_latest_read_reuses_cached_bytes_after_four_probes() { + let (object_store, store) = new_counting_protocol(); + let first = TestVal { + epoch: 1, + payload: 1, + }; + put_test_value(&object_store, 1, &first).await; + store.try_read_latest_unchecked().await.unwrap().unwrap(); + + for id in 2..=5 { + let value = TestVal { + epoch: 1, + payload: id, + }; + put_test_value(&object_store, id, &value).await; + } + + let gets = object_store.get_opts_calls.load(Ordering::SeqCst); + let latest = store.try_read_latest_unchecked().await.unwrap().unwrap(); + + assert_eq!(MonotonicId::new(5), latest.0); + assert_eq!(5, latest.1.payload); + assert_eq!(2, object_store.list_calls.load(Ordering::SeqCst)); + assert_eq!( + gets + 4, + object_store.get_opts_calls.load(Ordering::SeqCst), + "the LIST fallback should reuse the bytes from the fourth probe" + ); + } + + #[tokio::test] + async fn test_latest_read_invalidates_cache_when_list_is_empty() { + let (object_store, store) = new_counting_protocol(); + let first = TestVal { + epoch: 1, + payload: 1, + }; + put_test_value(&object_store, 1, &first).await; + store.try_read_latest_unchecked().await.unwrap().unwrap(); + + for id in 2..=5 { + let value = TestVal { + epoch: 1, + payload: id, + }; + put_test_value(&object_store, id, &value).await; + } + object_store.list_empty.store(true, Ordering::SeqCst); + + let latest = store.try_read_latest_unchecked().await.unwrap(); + + assert!(latest.is_none()); + assert!(store.latest.lock().is_none()); + } + + #[tokio::test] + async fn test_successful_write_populates_latest_cache() { + let (object_store, store) = new_counting_protocol(); + let expected = TestVal { + epoch: 1, + payload: 10, + }; + + let id = store.write(None, &expected).await.unwrap(); + let latest = store.try_read_latest().await.unwrap().unwrap(); + + assert_eq!((id, expected), latest); + assert_eq!(0, object_store.list_calls.load(Ordering::SeqCst)); + } + + #[tokio::test] + async fn test_boundary_rejection_invalidates_latest_cache() { + let (_, store) = new_counting_protocol(); + let value = TestVal { + epoch: 1, + payload: 10, + }; + let id = store.write(None, &value).await.unwrap(); + store.advance(id).await.unwrap(); + + let error = store.check(id).await.unwrap_err(); + + assert!(matches!( + error, + TransactionalObjectError::ObjectVersionExists + )); + assert!(store.latest.lock().is_none()); + } + + #[tokio::test] + async fn test_boundary_advance_invalidates_cached_entries_through_boundary() { + let (_, store) = new_counting_protocol(); + let first_id = store + .write( + None, + &TestVal { + epoch: 1, + payload: 10, + }, + ) + .await + .unwrap(); + let second_id = store + .write( + Some(first_id), + &TestVal { + epoch: 1, + payload: 20, + }, + ) + .await + .unwrap(); + + store.advance(first_id).await.unwrap(); + assert_eq!( + Some(second_id), + store.latest.lock().as_ref().map(|(id, _)| *id) + ); + + store.advance(second_id).await.unwrap(); + assert!(store.latest.lock().is_none()); + } + /// Validate that try_read_latest retries when a listed file is missing on read. #[tokio::test] async fn test_try_read_latest_retries_missing_listed_file() { @@ -961,6 +1290,7 @@ mod tests { flaky_store.clone(), "test", )), + latest: parking_lot::Mutex::new(None), }; let latest = store.try_read_latest().await.unwrap().unwrap(); From 7db4911082c8af96beb4be3ec2e4f8cbf0b142c8 Mon Sep 17 00:00:00 2001 From: criccomini Date: Wed, 29 Jul 2026 16:52:29 +0000 Subject: [PATCH 52/63] Bump version to 0.15.0 --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 8 ++++---- bindings/java/gradle.properties | 2 +- bindings/node/package.json | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 609dcfc66..53af1e213 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -914,7 +914,7 @@ dependencies = [ [[package]] name = "examples" -version = "0.14.1" +version = "0.15.0" dependencies = [ "anyhow", "object_store", @@ -3211,7 +3211,7 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "slatedb" -version = "0.14.1" +version = "0.15.0" dependencies = [ "async-channel", "async-trait", @@ -3271,7 +3271,7 @@ dependencies = [ [[package]] name = "slatedb-bencher" -version = "0.14.1" +version = "0.15.0" dependencies = [ "bytes", "chrono", @@ -3289,7 +3289,7 @@ dependencies = [ [[package]] name = "slatedb-cli" -version = "0.14.1" +version = "0.15.0" dependencies = [ "chrono", "clap", @@ -3309,7 +3309,7 @@ dependencies = [ [[package]] name = "slatedb-common" -version = "0.14.1" +version = "0.15.0" dependencies = [ "chrono", "log", @@ -3323,7 +3323,7 @@ dependencies = [ [[package]] name = "slatedb-dst" -version = "0.14.1" +version = "0.15.0" dependencies = [ "async-trait", "bytes", @@ -3348,7 +3348,7 @@ dependencies = [ [[package]] name = "slatedb-txn-obj" -version = "0.14.1" +version = "0.15.0" dependencies = [ "async-trait", "bytes", @@ -3365,7 +3365,7 @@ dependencies = [ [[package]] name = "slatedb-uniffi" -version = "0.14.1" +version = "0.15.0" dependencies = [ "chrono", "figment", diff --git a/Cargo.toml b/Cargo.toml index c0bb1d446..7953fb6ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ members = [ ] [workspace.package] -version = "0.14.1" +version = "0.15.0" edition = "2021" repository = "https://github.com/slatedb/slatedb" license = "Apache-2.0" @@ -69,9 +69,9 @@ serde = "1.0" serde_json = "1.0.142" siphasher = "1" smallvec = "1.15.1" -slatedb = { path = "slatedb", version = "0.14.1" } -slatedb-common = { path = "slatedb-common", version = "0.14.1" } -slatedb-txn-obj = { path = "slatedb-txn-obj", version = "0.14.1" } +slatedb = { path = "slatedb", version = "0.15.0" } +slatedb-common = { path = "slatedb-common", version = "0.15.0" } +slatedb-txn-obj = { path = "slatedb-txn-obj", version = "0.15.0" } snap = "1.1.1" sysinfo = "0.35.2" thiserror = "1.0.63" diff --git a/bindings/java/gradle.properties b/bindings/java/gradle.properties index 072faeb2e..dfb204fc3 100644 --- a/bindings/java/gradle.properties +++ b/bindings/java/gradle.properties @@ -1 +1 @@ -version=0.14.1-SNAPSHOT +version=0.15.0-SNAPSHOT diff --git a/bindings/node/package.json b/bindings/node/package.json index 4b7a9d4fe..91bc2b891 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -1,6 +1,6 @@ { "name": "@slatedb/uniffi", - "version": "0.14.1", + "version": "0.15.0", "description": "Node.js bindings for SlateDB generated from UniFFI and packaged with native libraries.", "license": "Apache-2.0", "type": "module", From 8742c847d89965d73a899e338ef3e5c15795499f Mon Sep 17 00:00:00 2001 From: xav-db Date: Thu, 16 Jul 2026 14:16:25 +0100 Subject: [PATCH 53/63] Add request-scoped query storage metrics --- .../src/cached_object_store/object_store.rs | 16 +- slatedb/src/db_cache/mod.rs | 19 ++- slatedb/src/instrumented_object_store.rs | 6 + slatedb/src/lib.rs | 5 + slatedb/src/query_metrics.rs | 151 ++++++++++++++++++ 5 files changed, 184 insertions(+), 13 deletions(-) create mode 100644 slatedb/src/query_metrics.rs diff --git a/slatedb/src/cached_object_store/object_store.rs b/slatedb/src/cached_object_store/object_store.rs index 216a408a3..110f964b8 100644 --- a/slatedb/src/cached_object_store/object_store.rs +++ b/slatedb/src/cached_object_store/object_store.rs @@ -7,6 +7,7 @@ use crate::cached_object_store::storage_fs::FsCacheStorage; use crate::cached_object_store::LocalCacheEntry; use crate::config::ObjectStoreCacheOptions; use crate::object_store_tag::ObjectStoreCallTag; +use crate::query_metrics::{self, QueryCacheKind}; use bytes::{Bytes, BytesMut}; use futures::{future::BoxFuture, stream, stream::BoxStream, StreamExt}; use object_store::{path::Path, GetOptions, GetResult, ObjectMeta, ObjectStore, ObjectStoreExt}; @@ -310,12 +311,17 @@ impl CachedObjectStore { let location = location.clone(); async move { this.stats.object_store_cache_part_access.increment(1); - let (bytes, part_source) = this + let result = this .read_part(&location, part_id, range_in_part, force_refresh) - .await?; - if head_source == ReadResultSource::Disk - && part_source == ReadResultSource::Disk - { + .await; + let Ok((bytes, part_source)) = result else { + query_metrics::record_cache_access(QueryCacheKind::Object, false); + return result.map(|(bytes, _)| bytes); + }; + let hit = head_source == ReadResultSource::Disk + && part_source == ReadResultSource::Disk; + query_metrics::record_cache_access(QueryCacheKind::Object, hit); + if hit { this.stats.object_store_cache_part_hits.increment(1); } Ok::(bytes) diff --git a/slatedb/src/db_cache/mod.rs b/slatedb/src/db_cache/mod.rs index 1831702c8..6c35744f2 100644 --- a/slatedb/src/db_cache/mod.rs +++ b/slatedb/src/db_cache/mod.rs @@ -27,6 +27,7 @@ use crate::db_state::SsTableId; use crate::filter_policy::NamedFilter; use crate::flatbuffer_types::SsTableIndexOwned; use crate::format::block::Block; +use crate::query_metrics::{self, QueryCacheKind}; use crate::sst_stats::SstStats; use slatedb_common::clock::SystemClock; use slatedb_common::metrics::MetricsRecorderHelper; @@ -683,6 +684,7 @@ impl DbCacheWrapper { } fn record_hit(&self, block_type: &str) { + query_metrics::record_cache_access(QueryCacheKind::Block, true); match block_type { "block" => self.stats.data_block_hit.increment(1), "index" => self.stats.index_hit.increment(1), @@ -693,6 +695,7 @@ impl DbCacheWrapper { } fn record_miss(&self, block_type: &str) { + query_metrics::record_cache_access(QueryCacheKind::Block, false); match block_type { "block" => self.stats.data_block_miss.increment(1), "index" => self.stats.index_miss.increment(1), @@ -744,9 +747,9 @@ impl DbCache for DbCacheWrapper { } }; if entry.is_some() { - self.stats.data_block_hit.increment(1); + self.record_hit("block"); } else { - self.stats.data_block_miss.increment(1); + self.record_miss("block"); } Ok(entry) } @@ -761,9 +764,9 @@ impl DbCache for DbCacheWrapper { } }; if entry.is_some() { - self.stats.index_hit.increment(1); + self.record_hit("index"); } else { - self.stats.index_miss.increment(1); + self.record_miss("index"); } Ok(entry) } @@ -778,9 +781,9 @@ impl DbCache for DbCacheWrapper { } }; if entry.is_some() { - self.stats.filter_hit.increment(1); + self.record_hit("filter"); } else { - self.stats.filter_miss.increment(1); + self.record_miss("filter"); } Ok(entry) } @@ -795,9 +798,9 @@ impl DbCache for DbCacheWrapper { } }; if entry.is_some() { - self.stats.stats_hit.increment(1); + self.record_hit("stats"); } else { - self.stats.stats_miss.increment(1); + self.record_miss("stats"); } Ok(entry) } diff --git a/slatedb/src/instrumented_object_store.rs b/slatedb/src/instrumented_object_store.rs index 822111e47..e87d632b3 100644 --- a/slatedb/src/instrumented_object_store.rs +++ b/slatedb/src/instrumented_object_store.rs @@ -42,6 +42,7 @@ use object_store::{ use slatedb_common::metrics::MetricsRecorderHelper; use crate::object_stores::ObjectStoreType; +use crate::query_metrics; /// Which SlateDB component is issuing object store requests. /// @@ -155,6 +156,7 @@ impl ObjectStore for InstrumentedObjectStore { location: &Path, options: GetOptions, ) -> object_store::Result { + query_metrics::record_object_storage_read(); let metric = if options.head { &self.stats.head } else if options.range.is_some() { @@ -173,6 +175,7 @@ impl ObjectStore for InstrumentedObjectStore { location: &Path, ranges: &[Range], ) -> object_store::Result> { + query_metrics::record_object_storage_read(); let start = Instant::now(); let result = self.inner.get_ranges(location, ranges).await; self.stats @@ -233,6 +236,7 @@ impl ObjectStore for InstrumentedObjectStore { } fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { + query_metrics::record_object_storage_read(); self.stats.list.increment(1); self.inner.list(prefix) } @@ -242,11 +246,13 @@ impl ObjectStore for InstrumentedObjectStore { prefix: Option<&Path>, offset: &Path, ) -> BoxStream<'static, object_store::Result> { + query_metrics::record_object_storage_read(); self.stats.list_with_offset.increment(1); self.inner.list_with_offset(prefix, offset) } async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { + query_metrics::record_object_storage_read(); let start = Instant::now(); let result = self.inner.list_with_delimiter(prefix).await; self.stats diff --git a/slatedb/src/lib.rs b/slatedb/src/lib.rs index f9d05befe..0ba56acc8 100644 --- a/slatedb/src/lib.rs +++ b/slatedb/src/lib.rs @@ -69,6 +69,10 @@ pub use merge_operator::{MergeOperator, MergeOperatorError}; pub use ops::{DbCacheManagerOps, DbMetadataOps, DbReadOps, DbTransactionOps, DbWriteOps}; pub use paths::PathResolver; pub use prefix_extractor::{PrefixExtractor, PrefixTarget}; +pub use query_metrics::{ + scope_query_metrics, QueryCacheKind, QueryCacheStatistics, QueryMetricsObserver, + QueryMetricsSnapshot, +}; pub use slatedb_common::{DbRand, IdentifiedObjectMetadata, ObjectMetadata}; #[cfg(test)] pub use sst_builder::BlockFormat; @@ -93,6 +97,7 @@ pub mod db_stats; pub mod manifest; pub mod object_store_tag; pub mod prefix_extractor; +pub mod query_metrics; pub mod seq_tracker; pub mod size_tiered_compaction; pub mod wal; diff --git a/slatedb/src/query_metrics.rs b/slatedb/src/query_metrics.rs new file mode 100644 index 000000000..8ab2d64f0 --- /dev/null +++ b/slatedb/src/query_metrics.rs @@ -0,0 +1,151 @@ +//! Request-scoped storage metrics for foreground query execution. +//! +//! The observer is installed with [`scope_query_metrics`]. Storage paths record +//! against the current task scope without reading or subtracting process-wide +//! metrics, so concurrent queries cannot contaminate each other's counters. + +use std::future::Future; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + +tokio::task_local! { + static QUERY_METRICS: QueryMetricsObserver; +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum QueryCacheKind { + Block, + Object, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct QueryCacheStatistics { + pub hits: u64, + pub misses: u64, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct QueryMetricsSnapshot { + pub block_cache: QueryCacheStatistics, + pub object_cache: QueryCacheStatistics, + pub object_storage_reads: u64, +} + +#[derive(Clone, Debug, Default)] +pub struct QueryMetricsObserver { + inner: Arc, +} + +#[derive(Debug, Default)] +struct QueryMetricsInner { + block_cache_hits: AtomicU64, + block_cache_misses: AtomicU64, + object_cache_hits: AtomicU64, + object_cache_misses: AtomicU64, + object_storage_reads: AtomicU64, +} + +impl QueryMetricsObserver { + #[must_use] + pub fn snapshot(&self) -> QueryMetricsSnapshot { + QueryMetricsSnapshot { + block_cache: QueryCacheStatistics { + hits: self.inner.block_cache_hits.load(Ordering::Relaxed), + misses: self.inner.block_cache_misses.load(Ordering::Relaxed), + }, + object_cache: QueryCacheStatistics { + hits: self.inner.object_cache_hits.load(Ordering::Relaxed), + misses: self.inner.object_cache_misses.load(Ordering::Relaxed), + }, + object_storage_reads: self.inner.object_storage_reads.load(Ordering::Relaxed), + } + } + + fn record_cache(&self, kind: QueryCacheKind, hit: bool) { + let counter = match (kind, hit) { + (QueryCacheKind::Block, true) => &self.inner.block_cache_hits, + (QueryCacheKind::Block, false) => &self.inner.block_cache_misses, + (QueryCacheKind::Object, true) => &self.inner.object_cache_hits, + (QueryCacheKind::Object, false) => &self.inner.object_cache_misses, + }; + counter.fetch_add(1, Ordering::Relaxed); + } + + fn record_object_storage_read(&self) { + self.inner + .object_storage_reads + .fetch_add(1, Ordering::Relaxed); + } +} + +/// Runs one foreground query with an isolated storage observer. +/// +/// ``` +/// # tokio_test::block_on(async { +/// use slatedb::{QueryMetricsObserver, scope_query_metrics}; +/// +/// let observer = QueryMetricsObserver::default(); +/// scope_query_metrics(observer.clone(), async {}).await; +/// assert_eq!(observer.snapshot().object_storage_reads, 0); +/// # }); +/// ``` +pub async fn scope_query_metrics(observer: QueryMetricsObserver, future: F) -> F::Output +where + F: Future, +{ + QUERY_METRICS.scope(observer, future).await +} + +pub(crate) fn record_cache_access(kind: QueryCacheKind, hit: bool) { + let _ = QUERY_METRICS.try_with(|observer| observer.record_cache(kind, hit)); +} + +pub(crate) fn record_object_storage_read() { + let _ = QUERY_METRICS.try_with(QueryMetricsObserver::record_object_storage_read); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn concurrent_scopes_do_not_share_counters() { + let first = QueryMetricsObserver::default(); + let second = QueryMetricsObserver::default(); + + tokio::join!( + scope_query_metrics(first.clone(), async { + record_cache_access(QueryCacheKind::Block, true); + tokio::task::yield_now().await; + record_object_storage_read(); + }), + scope_query_metrics(second.clone(), async { + record_cache_access(QueryCacheKind::Object, false); + tokio::task::yield_now().await; + record_cache_access(QueryCacheKind::Object, false); + }), + ); + + assert_eq!( + first.snapshot(), + QueryMetricsSnapshot { + block_cache: QueryCacheStatistics { hits: 1, misses: 0 }, + object_storage_reads: 1, + ..QueryMetricsSnapshot::default() + } + ); + assert_eq!( + second.snapshot(), + QueryMetricsSnapshot { + object_cache: QueryCacheStatistics { hits: 0, misses: 2 }, + ..QueryMetricsSnapshot::default() + } + ); + } + + #[tokio::test] + async fn records_outside_scope_are_ignored() { + record_cache_access(QueryCacheKind::Block, false); + record_object_storage_read(); + } +} From 7869213976e27279083d74d4b0453cc6d1f9d7a6 Mon Sep 17 00:00:00 2001 From: Matthew Sanetra <41018997+matthewsanetra@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:55:41 +0100 Subject: [PATCH 54/63] Add snapshot isolation to `DbReader` --- slatedb-dst/src/actors/bank/auditor.rs | 107 +- slatedb-dst/tests/bank.rs | 12 +- slatedb/Cargo.toml | 10 + slatedb/benches/db_reader_memory_scaling.rs | 501 ++++++ slatedb/benches/db_reader_scaling.rs | 1250 +++++++++++++ slatedb/src/db_cache/mod.rs | 68 +- slatedb/src/db_iter.rs | 45 + slatedb/src/db_reader.rs | 1735 +++++++++++++++++-- slatedb/src/db_snapshot.rs | 119 +- slatedb/src/db_state.rs | 6 +- slatedb/src/db_stats.rs | 26 + slatedb/src/error.rs | 12 + slatedb/src/manifest/store.rs | 252 ++- slatedb/src/reader.rs | 12 +- 14 files changed, 3935 insertions(+), 220 deletions(-) create mode 100644 slatedb/benches/db_reader_memory_scaling.rs create mode 100644 slatedb/benches/db_reader_scaling.rs diff --git a/slatedb-dst/src/actors/bank/auditor.rs b/slatedb-dst/src/actors/bank/auditor.rs index cdb3b9349..99d7e14ad 100644 --- a/slatedb-dst/src/actors/bank/auditor.rs +++ b/slatedb-dst/src/actors/bank/auditor.rs @@ -1,8 +1,11 @@ +use std::collections::{HashMap, VecDeque}; +use std::sync::{Arc, Mutex}; use std::time::Duration; use async_trait::async_trait; use rand::RngCore; use slatedb::config::DbReaderOptions; +use slatedb::db_cache::{CachedEntry, CachedKey, DbCache}; use slatedb::{DbReadOps, DbReader, Error}; use tracing::{info, instrument}; @@ -19,6 +22,8 @@ pub enum BankAuditView { Snapshot, /// Audit a long-lived read-only `DbReader`. Reader { options: DbReaderOptions }, + /// Audit an O(1) snapshot captured from a long-lived read-only `DbReader`. + ReaderSnapshot { options: DbReaderOptions }, } impl BankAuditView { @@ -27,6 +32,7 @@ impl BankAuditView { Self::Regular => "db", Self::Snapshot => "db_snapshot", Self::Reader { .. } => "db_reader", + Self::ReaderSnapshot { .. } => "db_reader_snapshot", } } } @@ -89,6 +95,21 @@ impl Actor for AuditorActor { .expect("bank reader auditor should have opened a reader"); audit_bank_view(reader, &self.bank, self.step).await?; } + BankAuditView::ReaderSnapshot { options } => { + if self.reader.is_none() { + self.reader = Some(open_bank_reader(ctx, options).await?); + } + let reader = self + .reader + .as_ref() + .expect("bank reader snapshot auditor should have opened a reader"); + let snapshot = reader.snapshot().await?; + // Make the capture-to-read window explicit so transfer, flush, + // compaction, GC, and fencing actors can advance after this + // snapshot has fixed its manifest generation and sequence. + tokio::task::yield_now().await; + audit_bank_view(snapshot.as_ref(), &self.bank, self.step).await?; + } }; self.step += 1; @@ -121,7 +142,11 @@ async fn open_bank_reader(ctx: &ActorCtx, options: DbReaderOptions) -> Result Result, +} + +#[derive(Default)] +struct DeterministicDbCacheInner { + entries: HashMap, + insertion_order: VecDeque, + size: usize, +} + +impl DeterministicDbCache { + fn new(capacity: usize) -> Self { + Self { + capacity, + inner: Mutex::new(DeterministicDbCacheInner::default()), + } + } + + fn get(&self, key: &CachedKey) -> Option { + self.inner.lock().unwrap().entries.get(key).cloned() + } +} + +#[async_trait] +impl DbCache for DeterministicDbCache { + async fn get_block(&self, key: &CachedKey) -> Result, Error> { + Ok(self.get(key)) + } + + async fn get_index(&self, key: &CachedKey) -> Result, Error> { + Ok(self.get(key)) + } + + async fn get_filter(&self, key: &CachedKey) -> Result, Error> { + Ok(self.get(key)) + } + + async fn get_stats(&self, key: &CachedKey) -> Result, Error> { + Ok(self.get(key)) + } + + async fn insert(&self, key: CachedKey, value: CachedEntry) { + let value_size = value.size(); + if value_size > self.capacity { + return; + } + + let mut inner = self.inner.lock().unwrap(); + if let Some(previous) = inner.entries.insert(key.clone(), value) { + inner.size = inner.size.saturating_sub(previous.size()) + value_size; + } else { + inner.size += value_size; + inner.insertion_order.push_back(key); + } + + while inner.size > self.capacity { + let Some(oldest) = inner.insertion_order.pop_front() else { + break; + }; + if let Some(evicted) = inner.entries.remove(&oldest) { + inner.size = inner.size.saturating_sub(evicted.size()); + } + } + } + + async fn remove(&self, key: &CachedKey) { + let mut inner = self.inner.lock().unwrap(); + if let Some(removed) = inner.entries.remove(key) { + inner.size = inner.size.saturating_sub(removed.size()); + } + inner.insertion_order.retain(|queued| queued != key); + } + + fn entry_count(&self) -> u64 { + self.inner.lock().unwrap().entries.len() as u64 + } +} + async fn audit_bank_view(reader: &R, bank: &BankAccounts, step: u64) -> Result<(), Error> where R: DbReadOps + Sync, diff --git a/slatedb-dst/tests/bank.rs b/slatedb-dst/tests/bank.rs index 5462536da..03c22e6b4 100644 --- a/slatedb-dst/tests/bank.rs +++ b/slatedb-dst/tests/bank.rs @@ -152,9 +152,19 @@ fn run_bank(seed: u64, shutdown_at_ms: i64) -> Result<(), Box *mut u8 { + // SAFETY: Delegates the exact layout to the system allocator. + let pointer = unsafe { System.alloc(layout) }; + if !pointer.is_null() { + record_allocation(layout.size()); + } + pointer + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + // SAFETY: Delegates the exact layout to the system allocator. + let pointer = unsafe { System.alloc_zeroed(layout) }; + if !pointer.is_null() { + record_allocation(layout.size()); + } + pointer + } + + unsafe fn dealloc(&self, pointer: *mut u8, layout: Layout) { + LIVE_BYTES.fetch_sub(layout.size(), Ordering::Relaxed); + // SAFETY: `pointer` was allocated with this allocator and `layout` is unchanged. + unsafe { System.dealloc(pointer, layout) }; + } + + unsafe fn realloc(&self, pointer: *mut u8, old: Layout, new_size: usize) -> *mut u8 { + // SAFETY: Delegates the original pointer/layout and requested size unchanged. + let new_pointer = unsafe { System.realloc(pointer, old, new_size) }; + if !new_pointer.is_null() { + ALLOCATED_BYTES.fetch_add(new_size, Ordering::Relaxed); + ALLOCATION_CALLS.fetch_add(1, Ordering::Relaxed); + let live = if new_size >= old.size() { + LIVE_BYTES.fetch_add(new_size - old.size(), Ordering::Relaxed) + new_size + - old.size() + } else { + LIVE_BYTES.fetch_sub(old.size() - new_size, Ordering::Relaxed) + - (old.size() - new_size) + }; + PEAK_LIVE_BYTES.fetch_max(live, Ordering::Relaxed); + } + new_pointer + } +} + +#[global_allocator] +static GLOBAL_ALLOCATOR: TrackingAllocator = TrackingAllocator; + +#[derive(Clone, Copy, Debug)] +struct MemoryWindow { + live_before: usize, + allocated_before: usize, + calls_before: usize, +} + +#[derive(Clone, Copy, Debug)] +struct MemorySample { + retained_bytes: i64, + peak_extra_bytes: usize, + allocated_bytes: usize, + allocation_calls: usize, +} + +impl MemoryWindow { + fn start() -> Self { + let live_before = LIVE_BYTES.load(Ordering::SeqCst); + PEAK_LIVE_BYTES.store(live_before, Ordering::SeqCst); + Self { + live_before, + allocated_before: ALLOCATED_BYTES.load(Ordering::SeqCst), + calls_before: ALLOCATION_CALLS.load(Ordering::SeqCst), + } + } + + fn finish(self) -> MemorySample { + let live_after = LIVE_BYTES.load(Ordering::SeqCst); + MemorySample { + retained_bytes: live_after as i64 - self.live_before as i64, + peak_extra_bytes: PEAK_LIVE_BYTES + .load(Ordering::SeqCst) + .saturating_sub(self.live_before), + allocated_bytes: ALLOCATED_BYTES + .load(Ordering::SeqCst) + .saturating_sub(self.allocated_before), + allocation_calls: ALLOCATION_CALLS + .load(Ordering::SeqCst) + .saturating_sub(self.calls_before), + } + } +} + +#[derive(Clone, Copy)] +enum Segmentation { + None, + Fixed4, +} + +impl Segmentation { + fn label(self) -> &'static str { + match self { + Self::None => "unsegmented", + Self::Fixed4 => "segmented", + } + } +} + +struct Fixed4Extractor; + +impl PrefixExtractor for Fixed4Extractor { + fn name(&self) -> &str { + "reader-memory-bench-fixed4" + } + + fn prefix_len(&self, target: &PrefixTarget) -> Option { + let len = match target { + PrefixTarget::Point(key) | PrefixTarget::Prefix(key) => key.len(), + }; + (len >= 4).then_some(4) + } +} + +fn writer_settings() -> Settings { + Settings { + flush_interval: None, + compactor_options: None, + garbage_collector_options: None, + l0_sst_size_bytes: 256 * 1024 * 1024, + l0_max_ssts: 16_384, + l0_max_ssts_per_key: 16_384, + ..Settings::default() + } +} + +fn reader_options(max_memtable_bytes: u64) -> DbReaderOptions { + DbReaderOptions { + manifest_poll_interval: POLL_INTERVAL, + checkpoint_lifetime: Duration::from_secs(60), + max_memtable_bytes, + ..DbReaderOptions::default() + } +} + +fn key_for(index: usize, segmentation: Segmentation) -> Bytes { + match segmentation { + Segmentation::None => Bytes::from(format!("key-{index:08}")), + Segmentation::Fixed4 => Bytes::from(format!("{index:04}-key-{index:08}")), + } +} + +async fn open_db( + path: &str, + store: Arc, + clock: Arc, + segmentation: Segmentation, +) -> Db { + let mut builder = Db::builder(path, store) + .with_settings(writer_settings()) + .with_system_clock(clock); + if matches!(segmentation, Segmentation::Fixed4) { + builder = builder.with_segment_extractor(Arc::new(Fixed4Extractor)); + } + builder.build().await.expect("DB open failed") +} + +async fn open_reader( + path: &str, + store: Arc, + clock: Arc, + segmentation: Segmentation, + recorder: Arc, +) -> DbReader { + let mut builder = DbReader::builder(path, store) + .with_options(reader_options(1)) + .with_system_clock(clock) + .with_metrics_recorder(recorder.clone()) + .with_db_cache_disabled(); + if matches!(segmentation, Segmentation::Fixed4) { + builder = builder.with_segment_extractor(Arc::new(Fixed4Extractor)); + } + let reader = builder.build().await.expect("reader open failed"); + wait_for_counter(|| scalar(&recorder, MANIFEST_POLLS), 1).await; + reader +} + +async fn write_wal(db: &Db, index: usize, segmentation: Segmentation) { + db.put_with_options( + &key_for(index, segmentation), + b"value-value-value-value-value", + &PutOptions::default(), + &WriteOptions { + await_durable: false, + ..WriteOptions::default() + }, + ) + .await + .expect("put failed"); + db.flush_with_options(FlushOptions { + flush_type: FlushType::Wal, + }) + .await + .expect("WAL flush failed"); +} + +fn scalar(recorder: &DefaultMetricsRecorder, name: &str) -> u64 { + lookup_metric(recorder, name).unwrap_or(0) as u64 +} + +async fn wait_for_counter(mut current: F, target: u64) +where + F: FnMut() -> u64, +{ + for _ in 0..100_000 { + if current() >= target { + return; + } + tokio::task::yield_now().await; + } + panic!( + "timed out waiting for reader poll: current={}, target={target}", + current() + ); +} + +async fn settle() { + for _ in 0..100 { + tokio::task::yield_now().await; + } +} + +struct ReaderFixture { + db: Db, + reader: Arc, + recorder: Arc, + clock: Arc, + segmentation: Segmentation, + next_key: usize, +} + +impl ReaderFixture { + async fn new(history: usize, segmentation: Segmentation) -> Self { + let path = format!("bench/db-reader-memory/{}", Uuid::new_v4()); + let store = Arc::new(InMemory::new()); + let clock = Arc::new(MockSystemClock::new()); + let db = open_db(&path, Arc::clone(&store), Arc::clone(&clock), segmentation).await; + for index in 0..history { + write_wal(&db, index, segmentation).await; + } + let recorder = Arc::new(DefaultMetricsRecorder::new()); + let reader = Arc::new( + open_reader( + &path, + store, + Arc::clone(&clock), + segmentation, + Arc::clone(&recorder), + ) + .await, + ); + Self { + db, + reader, + recorder, + clock, + segmentation, + next_key: history, + } + } + + async fn append_wals(&mut self, count: usize) { + for _ in 0..count { + write_wal(&self.db, self.next_key, self.segmentation).await; + self.next_key += 1; + } + } + + async fn poll(&self, expected_new_wals: u64) { + let poll_target = scalar(&self.recorder, MANIFEST_POLLS) + 1; + let replay_target = scalar(&self.recorder, REPLAY_SSTS) + expected_new_wals; + self.clock.advance(POLL_INTERVAL).await; + wait_for_counter(|| scalar(&self.recorder, MANIFEST_POLLS), poll_target).await; + if expected_new_wals > 0 { + wait_for_counter(|| scalar(&self.recorder, REPLAY_SSTS), replay_target).await; + } + } + + async fn close(self) { + self.reader.close().await.expect("reader close failed"); + self.db.close().await.expect("DB close failed"); + } +} + +fn percentile_i64(samples: &[MemorySample], field: impl Fn(&MemorySample) -> i64) -> i64 { + let mut values = samples.iter().map(field).collect::>(); + values.sort_unstable(); + values[(values.len() - 1) / 2] +} + +fn percentile_usize(samples: &[MemorySample], field: impl Fn(&MemorySample) -> usize) -> usize { + let mut values = samples.iter().map(field).collect::>(); + values.sort_unstable(); + values[(values.len() - 1) / 2] +} + +fn print_samples( + suite: &str, + case: &str, + history: usize, + delta: usize, + samples: &[MemorySample], + notes: &str, +) { + println!( + "MEMORY\t{suite}\t{case}\t{history}\t{delta}\t{}\t{}\t{}\t{}\t{}\t{notes}", + samples.len(), + percentile_i64(samples, |sample| sample.retained_bytes), + percentile_usize(samples, |sample| sample.peak_extra_bytes), + percentile_usize(samples, |sample| sample.allocated_bytes), + percentile_usize(samples, |sample| sample.allocation_calls), + ); +} + +async fn benchmark_incremental_replay_memory(full: bool) { + println!("SECTION\tincremental_replay_memory"); + let histories = if full { + vec![0, 32, 128, 512] + } else { + vec![0, 128] + }; + let deltas = if full { vec![1, 8, 32] } else { vec![1, 8] }; + let reps = if full { 9 } else { 5 }; + for segmentation in [Segmentation::None, Segmentation::Fixed4] { + for &history in &histories { + for &delta in &deltas { + let mut samples = Vec::with_capacity(reps); + for _ in 0..reps { + let mut fixture = ReaderFixture::new(history, segmentation).await; + fixture.append_wals(delta).await; + settle().await; + let window = MemoryWindow::start(); + fixture.poll(delta as u64).await; + settle().await; + samples.push(window.finish()); + fixture.close().await; + } + print_samples( + "replay", + "poll_delta", + history, + delta, + &samples, + segmentation.label(), + ); + } + } + } +} + +async fn benchmark_snapshot_memory(full: bool) { + println!("SECTION\tsnapshot_memory"); + let histories = if full { + vec![0, 128, 512] + } else { + vec![0, 128] + }; + let snapshot_count = if full { 10_000 } else { 2_000 }; + for history in histories { + let fixture = ReaderFixture::new(history, Segmentation::None).await; + let mut snapshots: Vec> = Vec::with_capacity(snapshot_count); + settle().await; + let window = MemoryWindow::start(); + for _ in 0..snapshot_count { + snapshots.push( + fixture + .reader + .snapshot() + .await + .expect("snapshot creation failed"), + ); + } + black_box(&snapshots); + settle().await; + let held = window.finish(); + let live_before_drop = LIVE_BYTES.load(Ordering::SeqCst); + snapshots.clear(); + settle().await; + let released = live_before_drop.saturating_sub(LIVE_BYTES.load(Ordering::SeqCst)); + println!( + "SNAPSHOT\thistory={history}\tcount={snapshot_count}\tretained_bytes={}\tbytes_per_snapshot={:.2}\talloc_calls_per_snapshot={:.2}\treleased_bytes={released}", + held.retained_bytes, + held.retained_bytes as f64 / snapshot_count as f64, + held.allocation_calls as f64 / snapshot_count as f64, + ); + fixture.close().await; + } +} + +async fn benchmark_old_generation_release(full: bool) { + println!("SECTION\told_generation_release"); + let histories = if full { + vec![1, 8, 32, 128, 512] + } else { + vec![8, 128] + }; + let reps = if full { 5 } else { 3 }; + for history in histories { + let mut released_samples = Vec::with_capacity(reps); + for _ in 0..reps { + let fixture = ReaderFixture::new(history, Segmentation::None).await; + let old_snapshot = fixture.reader.snapshot().await.expect("snapshot failed"); + fixture + .db + .flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .expect("L0 flush failed"); + fixture.poll(0).await; + settle().await; + let before = LIVE_BYTES.load(Ordering::SeqCst); + drop(old_snapshot); + settle().await; + let after = LIVE_BYTES.load(Ordering::SeqCst); + released_samples.push(before.saturating_sub(after)); + fixture.close().await; + } + released_samples.sort_unstable(); + println!( + "RELEASE\thistory={history}\treps={reps}\treleased_bytes_p50={}", + released_samples[(released_samples.len() - 1) / 2] + ); + } +} + +async fn run() { + let full = std::env::var_os("SLATEDB_READER_BENCH_FULL").is_some(); + println!( + "CONFIG\tprofile={}\tfull={full}\tallocator=system-counting", + if cfg!(debug_assertions) { + "debug" + } else { + "release" + } + ); + println!("HEADER\tsuite\tcase\thistory\tdelta\treps\tretained_p50_bytes\tpeak_extra_p50_bytes\tallocated_p50_bytes\tallocation_calls_p50\tnotes"); + benchmark_incremental_replay_memory(full).await; + benchmark_snapshot_memory(full).await; + benchmark_old_generation_release(full).await; +} + +fn main() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("failed to build Tokio runtime"); + runtime.block_on(run()); +} diff --git a/slatedb/benches/db_reader_scaling.rs b/slatedb/benches/db_reader_scaling.rs new file mode 100644 index 000000000..278cb4ea0 --- /dev/null +++ b/slatedb/benches/db_reader_scaling.rs @@ -0,0 +1,1250 @@ +//! End-to-end scaling benchmarks for reader-backed snapshots and incremental WAL replay. +//! +//! This target deliberately uses fixed repetitions instead of Criterion's adaptive +//! iteration count. Replay, checkpoint rollover, and generation cleanup mutate state, +//! and their untimed fixture setup is substantially more expensive than the operation +//! under test. Fixed repetitions keep those costs out of the samples without causing +//! Criterion to request millions of fixture rebuilds. +//! +//! Run the standard matrix: +//! cargo bench -p slatedb --bench db_reader_scaling --features test-util +//! +//! Run the extended edge-case matrix: +//! SLATEDB_READER_BENCH_FULL=1 cargo bench -p slatedb --bench db_reader_scaling --features test-util + +use std::future::Future; +use std::hint::black_box; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use bytes::Bytes; +use object_store::memory::InMemory; +use object_store::path::Path; +use object_store::{ObjectStore, ObjectStoreExt}; +use slatedb::config::{ + CheckpointOptions, CheckpointScope, DbReaderOptions, FlushOptions, FlushType, PutOptions, + Settings, WriteOptions, +}; +use slatedb::instrumented_object_store_stats; +use slatedb::{Db, DbReader, DbSnapshot, PrefixExtractor, PrefixTarget}; +use slatedb_common::metrics::{lookup_metric, lookup_metric_with_labels, DefaultMetricsRecorder}; +use slatedb_common::{MockSystemClock, SystemClock}; +use tokio::sync::Barrier; +use uuid::Uuid; + +const POLL_INTERVAL: Duration = Duration::from_millis(10); +const REPLAY_SSTS: &str = "slatedb.db.reader_wal_replay_ssts"; +const MANIFEST_POLLS: &str = "slatedb.db.reader_manifest_polls"; + +#[derive(Clone, Copy)] +enum Segmentation { + None, + Fixed4, +} + +impl Segmentation { + fn label(self) -> &'static str { + match self { + Self::None => "unsegmented", + Self::Fixed4 => "segmented", + } + } +} + +struct Fixed4Extractor; + +impl PrefixExtractor for Fixed4Extractor { + fn name(&self) -> &str { + "reader-bench-fixed4" + } + + fn prefix_len(&self, target: &PrefixTarget) -> Option { + let len = match target { + PrefixTarget::Point(key) | PrefixTarget::Prefix(key) => key.len(), + }; + (len >= 4).then_some(4) + } +} + +#[derive(Debug)] +struct Summary { + reps: usize, + min_us: f64, + p50_us: f64, + p95_us: f64, + mean_us: f64, + max_us: f64, +} + +impl Summary { + fn from_durations(samples: Vec) -> Self { + assert!(!samples.is_empty()); + let mut micros = samples + .into_iter() + .map(|sample| sample.as_secs_f64() * 1_000_000.0) + .collect::>(); + micros.sort_by(f64::total_cmp); + let reps = micros.len(); + let percentile = |p: f64| { + let index = ((reps - 1) as f64 * p).round() as usize; + micros[index] + }; + Self { + reps, + min_us: micros[0], + p50_us: percentile(0.50), + p95_us: percentile(0.95), + mean_us: micros.iter().sum::() / reps as f64, + max_us: micros[reps - 1], + } + } +} + +fn print_summary(suite: &str, case: &str, scale: usize, summary: &Summary, notes: &str) { + println!( + "RESULT\t{suite}\t{case}\t{scale}\t{}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{notes}", + summary.reps, + summary.min_us, + summary.p50_us, + summary.p95_us, + summary.mean_us, + summary.max_us, + ); +} + +async fn measure_async(warmup: usize, reps: usize, mut operation: F) -> Summary +where + F: FnMut() -> Fut, + Fut: Future, +{ + for _ in 0..warmup { + operation().await; + } + let mut samples = Vec::with_capacity(reps); + for _ in 0..reps { + let start = Instant::now(); + operation().await; + samples.push(start.elapsed()); + } + Summary::from_durations(samples) +} + +fn writer_settings() -> Settings { + Settings { + flush_interval: None, + compactor_options: None, + garbage_collector_options: None, + l0_sst_size_bytes: 256 * 1024 * 1024, + l0_max_ssts: 16_384, + l0_max_ssts_per_key: 16_384, + ..Settings::default() + } +} + +fn quiet_reader_options(max_memtable_bytes: u64) -> DbReaderOptions { + DbReaderOptions { + manifest_poll_interval: Duration::from_secs(60 * 60), + checkpoint_lifetime: Duration::from_secs(3 * 60 * 60), + max_memtable_bytes, + ..DbReaderOptions::default() + } +} + +fn polling_reader_options( + max_memtable_bytes: u64, + checkpoint_lifetime: Duration, +) -> DbReaderOptions { + DbReaderOptions { + manifest_poll_interval: POLL_INTERVAL, + checkpoint_lifetime, + max_memtable_bytes, + ..DbReaderOptions::default() + } +} + +fn key_for(index: usize, segmentation: Segmentation) -> Bytes { + match segmentation { + Segmentation::None => Bytes::from(format!("key-{index:08}")), + Segmentation::Fixed4 => Bytes::from(format!("{index:04}-key-{index:08}")), + } +} + +async fn write_wal(db: &Db, index: usize, segmentation: Segmentation) -> Bytes { + let key = key_for(index, segmentation); + db.put_with_options( + &key, + b"value-value-value-value-value", + &PutOptions::default(), + &WriteOptions { + await_durable: false, + ..WriteOptions::default() + }, + ) + .await + .expect("put failed"); + db.flush_with_options(FlushOptions { + flush_type: FlushType::Wal, + }) + .await + .expect("WAL flush failed"); + key +} + +async fn open_db( + path: &str, + store: Arc, + clock: Option>, + segmentation: Segmentation, +) -> Db { + let mut builder = Db::builder(path, store).with_settings(writer_settings()); + if let Some(clock) = clock { + builder = builder.with_system_clock(clock); + } + if matches!(segmentation, Segmentation::Fixed4) { + builder = builder.with_segment_extractor(Arc::new(Fixed4Extractor)); + } + builder.build().await.expect("DB open failed") +} + +async fn open_reader( + path: &str, + store: Arc, + clock: Option>, + segmentation: Segmentation, + options: DbReaderOptions, + recorder: Arc, +) -> DbReader { + let mut builder = DbReader::builder(path, store) + .with_options(options) + .with_metrics_recorder(recorder.clone()) + .with_db_cache_disabled(); + if let Some(clock) = clock { + builder = builder.with_system_clock(clock); + } + if matches!(segmentation, Segmentation::Fixed4) { + builder = builder.with_segment_extractor(Arc::new(Fixed4Extractor)); + } + let reader = builder.build().await.expect("reader open failed"); + // The dispatcher's first ticker fires immediately. Wait for the complete + // startup poll, not merely its first object-store request, so the first + // timed sample cannot race startup work. + wait_for_counter(|| scalar(&recorder, MANIFEST_POLLS), 1).await; + reader +} + +struct ReaderFixture { + path: String, + store: Arc, + db: Db, + reader: Arc, + recorder: Arc, + clock: Option>, + segmentation: Segmentation, + next_key: usize, + latest_key: Option, +} + +impl ReaderFixture { + async fn wal_history( + wal_count: usize, + segmentation: Segmentation, + polling: bool, + checkpoint_lifetime: Duration, + ) -> Self { + let path = format!("bench/db-reader-scaling/{}", Uuid::new_v4()); + let store = Arc::new(InMemory::new()); + let clock = polling.then(|| Arc::new(MockSystemClock::new())); + let db = open_db(&path, Arc::clone(&store), clock.clone(), segmentation).await; + let mut latest_key = None; + for index in 0..wal_count { + latest_key = Some(write_wal(&db, index, segmentation).await); + } + let recorder = Arc::new(DefaultMetricsRecorder::new()); + let options = if polling { + polling_reader_options(1, checkpoint_lifetime) + } else { + quiet_reader_options(1) + }; + let reader = Arc::new( + open_reader( + &path, + Arc::clone(&store), + clock.clone(), + segmentation, + options, + Arc::clone(&recorder), + ) + .await, + ); + Self { + path, + store, + db, + reader, + recorder, + clock, + segmentation, + next_key: wal_count, + latest_key, + } + } + + async fn append_wals(&mut self, count: usize) { + for _ in 0..count { + self.latest_key = Some(write_wal(&self.db, self.next_key, self.segmentation).await); + self.next_key += 1; + } + } + + async fn close(self) { + self.reader.close().await.expect("reader close failed"); + self.db.close().await.expect("DB close failed"); + } +} + +fn scalar(recorder: &DefaultMetricsRecorder, name: &str) -> u64 { + lookup_metric(recorder, name).unwrap_or(0) as u64 +} + +fn object_store_requests(recorder: &DefaultMetricsRecorder, api: &'static str) -> u64 { + lookup_metric_with_labels( + recorder, + instrumented_object_store_stats::REQUEST_COUNT, + &[ + ("component", "reader"), + ("store_type", "main"), + ("op", if api == "put" { "put" } else { "get" }), + ("api", api), + ], + ) + .unwrap_or(0) as u64 +} + +#[derive(Clone, Copy, Default)] +struct ReaderCounters { + lists: u64, + heads: u64, + gets: u64, + get_ranges: u64, + puts: u64, + replay_ssts: u64, +} + +impl ReaderCounters { + fn capture(recorder: &DefaultMetricsRecorder) -> Self { + Self { + lists: object_store_requests(recorder, "list"), + heads: object_store_requests(recorder, "head"), + gets: object_store_requests(recorder, "get"), + get_ranges: object_store_requests(recorder, "get_range"), + puts: object_store_requests(recorder, "put"), + replay_ssts: scalar(recorder, REPLAY_SSTS), + } + } + + fn delta(self, before: Self) -> Self { + Self { + lists: self.lists - before.lists, + heads: self.heads - before.heads, + gets: self.gets - before.gets, + get_ranges: self.get_ranges - before.get_ranges, + puts: self.puts - before.puts, + replay_ssts: self.replay_ssts - before.replay_ssts, + } + } + + fn note(self, reps: usize) -> String { + let reps = reps as f64; + format!( + "list/op={:.2};head/op={:.2};get/op={:.2};range/op={:.2};put/op={:.2};replay_sst/op={:.2}", + self.lists as f64 / reps, + self.heads as f64 / reps, + self.gets as f64 / reps, + self.get_ranges as f64 / reps, + self.puts as f64 / reps, + self.replay_ssts as f64 / reps, + ) + } +} + +async fn wait_for_counter(mut current: F, target: u64) +where + F: FnMut() -> u64, +{ + for _ in 0..100_000 { + if current() >= target { + return; + } + tokio::task::yield_now().await; + } + panic!( + "timed out waiting for reader poll completion: current={}, target={target}", + current() + ); +} + +async fn trigger_poll(fixture: &ReaderFixture, expected_new_wals: u64) { + advance_and_wait_poll(fixture, POLL_INTERVAL, expected_new_wals).await; +} + +async fn advance_and_wait_poll(fixture: &ReaderFixture, advance: Duration, expected_new_wals: u64) { + let clock = fixture + .clock + .as_ref() + .expect("polling fixture has no clock"); + let completed_poll_target = scalar(&fixture.recorder, MANIFEST_POLLS) + 1; + let replay_target = scalar(&fixture.recorder, REPLAY_SSTS) + expected_new_wals; + clock.advance(advance).await; + wait_for_counter( + || scalar(&fixture.recorder, MANIFEST_POLLS), + completed_poll_target, + ) + .await; + if expected_new_wals > 0 { + wait_for_counter(|| scalar(&fixture.recorder, REPLAY_SSTS), replay_target).await; + } +} + +async fn benchmark_usual_paths(scales: &[usize], full: bool) { + println!("SECTION\tusual_paths"); + for &wal_count in scales { + let fixture = ReaderFixture::wal_history( + wal_count, + Segmentation::None, + false, + Duration::from_secs(60), + ) + .await; + let snapshot = fixture.reader.snapshot().await.expect("snapshot failed"); + let latest = fixture + .latest_key + .clone() + .unwrap_or_else(|| Bytes::from_static(b"missing")); + let snapshot_reps = if full { 20_000 } else { 5_000 }; + let read_reps = if full { 2_000 } else { 500 }; + let scan_reps = if full { 250 } else { 75 }; + + let summary = measure_async(200, snapshot_reps, || async { + let snapshot = fixture.reader.snapshot().await.expect("snapshot failed"); + black_box(snapshot.seq()); + }) + .await; + print_summary("usual", "snapshot_create_drop", wal_count, &summary, ""); + + let summary = measure_async(30, read_reps, || async { + black_box(fixture.reader.get(&latest).await.expect("get failed")); + }) + .await; + print_summary("usual", "reader_get_latest", wal_count, &summary, ""); + + let summary = measure_async(30, read_reps, || async { + black_box( + fixture + .reader + .get(b"definitely-missing") + .await + .expect("missing get failed"), + ); + }) + .await; + print_summary("usual", "reader_get_missing", wal_count, &summary, ""); + + let summary = measure_async(30, read_reps, || async { + black_box(snapshot.get(&latest).await.expect("snapshot get failed")); + }) + .await; + print_summary("usual", "snapshot_get_latest", wal_count, &summary, ""); + + let summary = measure_async(10, scan_reps, || async { + let mut iter = fixture.reader.scan(..).await.expect("scan failed"); + black_box(iter.next().await.expect("scan next failed")); + }) + .await; + print_summary("usual", "reader_scan_first", wal_count, &summary, ""); + + let summary = measure_async(10, scan_reps, || async { + let mut iter = snapshot.scan(..).await.expect("snapshot scan failed"); + black_box(iter.next().await.expect("snapshot scan next failed")); + }) + .await; + print_summary("usual", "snapshot_scan_first", wal_count, &summary, ""); + + let full_scan_reps = if wal_count <= 32 { 50 } else { 15 }; + let summary = measure_async(3, full_scan_reps, || async { + let mut iter = fixture.reader.scan(..).await.expect("scan failed"); + let mut count = 0usize; + while iter.next().await.expect("scan next failed").is_some() { + count += 1; + } + black_box(count); + }) + .await; + print_summary("usual", "reader_scan_full", wal_count, &summary, ""); + + drop(snapshot); + fixture.close().await; + } +} + +async fn benchmark_bulk_iterator(full: bool) { + println!("SECTION\tbulk_iterator"); + let row_count = if full { 20_000 } else { 5_000 }; + let path = format!("bench/db-reader-bulk/{}", Uuid::new_v4()); + let store = Arc::new(InMemory::new()); + let db = open_db(&path, Arc::clone(&store), None, Segmentation::None).await; + for index in 0..row_count { + let key = key_for(index, Segmentation::None); + db.put_with_options( + &key, + b"value", + &PutOptions::default(), + &WriteOptions { + await_durable: false, + ..WriteOptions::default() + }, + ) + .await + .expect("put failed"); + } + db.flush_with_options(FlushOptions { + flush_type: FlushType::Wal, + }) + .await + .expect("WAL flush failed"); + let recorder = Arc::new(DefaultMetricsRecorder::new()); + let reader = open_reader( + &path, + Arc::clone(&store), + None, + Segmentation::None, + quiet_reader_options(u64::MAX), + recorder, + ) + .await; + + let summary = measure_async(2, if full { 12 } else { 5 }, || async { + let mut iter = db.scan(..).await.expect("DB scan failed"); + let mut count = 0usize; + while iter.next().await.expect("DB scan next failed").is_some() { + count += 1; + } + assert_eq!(count, row_count); + }) + .await; + print_summary("bulk", "db_scan_one_memtable", row_count, &summary, ""); + + let summary = measure_async(2, if full { 12 } else { 5 }, || async { + let mut iter = reader.scan(..).await.expect("reader scan failed"); + let mut count = 0usize; + while iter + .next() + .await + .expect("reader scan next failed") + .is_some() + { + count += 1; + } + assert_eq!(count, row_count); + }) + .await; + print_summary( + "bulk", + "reader_scan_one_replay_memtable", + row_count, + &summary, + "", + ); + + let snapshot = reader.snapshot().await.expect("snapshot failed"); + let summary = measure_async(2, if full { 12 } else { 5 }, || async { + let mut iter = snapshot.scan(..).await.expect("snapshot scan failed"); + let mut count = 0usize; + while iter + .next() + .await + .expect("snapshot scan next failed") + .is_some() + { + count += 1; + } + assert_eq!(count, row_count); + }) + .await; + print_summary( + "bulk", + "snapshot_scan_one_replay_memtable", + row_count, + &summary, + "", + ); + drop(snapshot); + reader.close().await.expect("reader close failed"); + db.close().await.expect("DB close failed"); +} + +async fn benchmark_iterator_contention(full: bool) { + println!("SECTION\titerator_contention"); + let row_count = if full { 5_000 } else { 2_000 }; + let path = format!("bench/db-reader-iterator-contention/{}", Uuid::new_v4()); + let store = Arc::new(InMemory::new()); + let db = Arc::new(open_db(&path, Arc::clone(&store), None, Segmentation::None).await); + for index in 0..row_count { + db.put_with_options( + &key_for(index, Segmentation::None), + b"value", + &PutOptions::default(), + &WriteOptions { + await_durable: false, + ..WriteOptions::default() + }, + ) + .await + .expect("put failed"); + } + db.flush_with_options(FlushOptions { + flush_type: FlushType::Wal, + }) + .await + .expect("WAL flush failed"); + let reader = Arc::new( + open_reader( + &path, + Arc::clone(&store), + None, + Segmentation::None, + quiet_reader_options(u64::MAX), + Arc::new(DefaultMetricsRecorder::new()), + ) + .await, + ); + + // Populate both read paths' caches before comparing their concurrent scans. + let mut db_warmup = db.scan(..).await.expect("DB warmup scan failed"); + while db_warmup + .next() + .await + .expect("DB warmup next failed") + .is_some() + {} + let mut reader_warmup = reader.scan(..).await.expect("reader warmup scan failed"); + while reader_warmup + .next() + .await + .expect("reader warmup next failed") + .is_some() + {} + + let task_counts = if full { + vec![1, 2, 4, 8, 16, 32] + } else { + vec![1, 4, 16] + }; + for task_count in task_counts { + let barrier = Arc::new(Barrier::new(task_count + 1)); + let mut tasks = Vec::with_capacity(task_count); + for _ in 0..task_count { + let db = Arc::clone(&db); + let barrier = Arc::clone(&barrier); + tasks.push(tokio::spawn(async move { + barrier.wait().await; + let mut iter = db.scan(..).await.expect("DB scan failed"); + let mut count = 0usize; + while iter.next().await.expect("DB scan next failed").is_some() { + count += 1; + } + count + })); + } + barrier.wait().await; + let start = Instant::now(); + for task in tasks { + assert_eq!(task.await.expect("DB scan task failed"), row_count); + } + let elapsed = start.elapsed(); + let total_rows = task_count * row_count; + let summary = Summary::from_durations(vec![elapsed.div_f64(total_rows as f64)]); + print_summary( + "contention", + "db_scan_per_row", + task_count, + &summary, + &format!( + "throughput_rows_s={:.0};total_rows={total_rows}", + total_rows as f64 / elapsed.as_secs_f64() + ), + ); + + let barrier = Arc::new(Barrier::new(task_count + 1)); + let mut tasks = Vec::with_capacity(task_count); + for _ in 0..task_count { + let reader = Arc::clone(&reader); + let barrier = Arc::clone(&barrier); + tasks.push(tokio::spawn(async move { + barrier.wait().await; + let mut iter = reader.scan(..).await.expect("reader scan failed"); + let mut count = 0usize; + while iter + .next() + .await + .expect("reader scan next failed") + .is_some() + { + count += 1; + } + count + })); + } + barrier.wait().await; + let start = Instant::now(); + for task in tasks { + assert_eq!(task.await.expect("reader scan task failed"), row_count); + } + let elapsed = start.elapsed(); + let total_rows = task_count * row_count; + let summary = Summary::from_durations(vec![elapsed.div_f64(total_rows as f64)]); + print_summary( + "contention", + "reader_scan_per_row", + task_count, + &summary, + &format!( + "throughput_rows_s={:.0};total_rows={total_rows}", + total_rows as f64 / elapsed.as_secs_f64() + ), + ); + } + + reader.close().await.expect("reader close failed"); + db.close().await.expect("DB close failed"); +} + +async fn benchmark_incremental_replay(histories: &[usize], deltas: &[usize], full: bool) { + println!("SECTION\tincremental_replay"); + for segmentation in [Segmentation::None, Segmentation::Fixed4] { + for &history in histories { + for &delta in deltas { + let reps = if full { 12 } else { 5 }; + let mut samples = Vec::with_capacity(reps); + let mut counter_total = ReaderCounters::default(); + for _ in 0..reps { + // Rebuild outside the timed interval so every sample starts + // from exactly `history`, rather than accumulating each + // preceding sample's delta into its advertised baseline. + let mut fixture = ReaderFixture::wal_history( + history, + segmentation, + true, + Duration::from_secs(60), + ) + .await; + fixture.append_wals(delta).await; + let before = ReaderCounters::capture(&fixture.recorder); + let start = Instant::now(); + trigger_poll(&fixture, delta as u64).await; + samples.push(start.elapsed()); + let counters = ReaderCounters::capture(&fixture.recorder).delta(before); + counter_total.lists += counters.lists; + counter_total.heads += counters.heads; + counter_total.gets += counters.gets; + counter_total.get_ranges += counters.get_ranges; + counter_total.puts += counters.puts; + counter_total.replay_ssts += counters.replay_ssts; + fixture.close().await; + } + let summary = Summary::from_durations(samples); + let notes = format!("mode={};{}", segmentation.label(), counter_total.note(reps)); + print_summary( + "replay", + "poll_delta", + history * 10_000 + delta, + &summary, + ¬es, + ); + } + } + } +} + +async fn benchmark_generation_rollover(scales: &[usize], full: bool) { + println!("SECTION\tgeneration_rollover"); + for &history in scales.iter().filter(|&&value| value > 0) { + let reps = if full && history <= 32 { 5 } else { 2 }; + let mut samples = Vec::with_capacity(reps); + let mut counter_total = ReaderCounters { + lists: 0, + heads: 0, + gets: 0, + get_ranges: 0, + puts: 0, + replay_ssts: 0, + }; + for _ in 0..reps { + let fixture = ReaderFixture::wal_history( + history, + Segmentation::None, + true, + Duration::from_secs(60), + ) + .await; + let before_manifest = fixture.reader.manifest().id(); + fixture + .db + .flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .expect("L0 flush failed"); + let before = ReaderCounters::capture(&fixture.recorder); + let start = Instant::now(); + advance_and_wait_poll(&fixture, POLL_INTERVAL, 0).await; + assert!(fixture.reader.manifest().id() > before_manifest); + samples.push(start.elapsed()); + let delta = ReaderCounters::capture(&fixture.recorder).delta(before); + counter_total.lists += delta.lists; + counter_total.heads += delta.heads; + counter_total.gets += delta.gets; + counter_total.get_ranges += delta.get_ranges; + counter_total.puts += delta.puts; + counter_total.replay_ssts += delta.replay_ssts; + fixture.close().await; + } + let summary = Summary::from_durations(samples); + print_summary( + "rollover", + "flush_all_replayed_wals_to_l0", + history, + &summary, + &counter_total.note(reps), + ); + } +} + +async fn benchmark_final_old_snapshot_drop(scales: &[usize], full: bool) { + println!("SECTION\tfinal_old_snapshot_drop"); + for &history in scales.iter().filter(|&&value| value > 0) { + let reps = if full && history <= 32 { 8 } else { 3 }; + let mut samples = Vec::with_capacity(reps); + for _ in 0..reps { + let fixture = ReaderFixture::wal_history( + history, + Segmentation::None, + true, + Duration::from_secs(60), + ) + .await; + let old_snapshot = fixture.reader.snapshot().await.expect("snapshot failed"); + let before_manifest = fixture.reader.manifest().id(); + fixture + .db + .flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .expect("L0 flush failed"); + advance_and_wait_poll(&fixture, POLL_INTERVAL, 0).await; + assert!(fixture.reader.manifest().id() > before_manifest); + + // The reader has moved to an L0-backed generation, so this is the + // last strong owner of the old persistent replay-memtable chain. + let start = Instant::now(); + drop(old_snapshot); + samples.push(start.elapsed()); + fixture.close().await; + } + let summary = Summary::from_durations(samples); + print_summary( + "cleanup", + "drop_final_old_snapshot", + history, + &summary, + "old_generation_replay_memtables", + ); + } +} + +async fn benchmark_snapshot_contention(full: bool) { + println!("SECTION\tsnapshot_contention"); + let fixture = + ReaderFixture::wal_history(128, Segmentation::None, false, Duration::from_secs(60)).await; + let per_task = if full { 25_000 } else { 5_000 }; + let task_counts = if full { + vec![1, 2, 4, 8, 16, 32] + } else { + vec![1, 4, 16] + }; + for task_count in task_counts { + let barrier = Arc::new(Barrier::new(task_count + 1)); + let mut tasks = Vec::with_capacity(task_count); + for _ in 0..task_count { + let reader = Arc::clone(&fixture.reader); + let barrier = Arc::clone(&barrier); + tasks.push(tokio::spawn(async move { + barrier.wait().await; + for _ in 0..per_task { + let snapshot = reader.snapshot().await.expect("snapshot failed"); + black_box(snapshot.seq()); + } + })); + } + barrier.wait().await; + let start = Instant::now(); + for task in tasks { + task.await.expect("snapshot task failed"); + } + let elapsed = start.elapsed(); + let operations = task_count * per_task; + let per_operation = elapsed.div_f64(operations as f64); + let summary = Summary::from_durations(vec![per_operation]); + let throughput = operations as f64 / elapsed.as_secs_f64(); + print_summary( + "contention", + "snapshot_create_drop", + task_count, + &summary, + &format!("throughput_ops_s={throughput:.0};total_ops={operations}"), + ); + } + fixture.close().await; +} + +async fn benchmark_manifest_file_listing(scales: &[usize], full: bool) { + println!("SECTION\tmanifest_file_listing"); + for &file_count in scales { + let fixture = + ReaderFixture::wal_history(0, Segmentation::None, true, Duration::from_secs(60)).await; + let source_id = fixture.reader.manifest().id(); + // Opening a managed reader creates a checkpoint and therefore advances + // the manifest ID. Small requested scales can be below that fixture + // baseline, so report and populate the actual retained-file count. + let target_file_count = file_count.max(source_id as usize); + let source = Path::from(format!( + "{}/manifest/{source_id:020}.manifest", + fixture.path + )); + for id in (source_id + 1)..=target_file_count as u64 { + let destination = Path::from(format!("{}/manifest/{id:020}.manifest", fixture.path)); + fixture + .store + .copy(&source, &destination) + .await + .expect("manifest copy failed"); + } + let reps = if full { 25 } else { 8 }; + for _ in 0..2 { + trigger_poll(&fixture, 0).await; + } + let before = ReaderCounters::capture(&fixture.recorder); + let summary = measure_async(0, reps, || async { + trigger_poll(&fixture, 0).await; + }) + .await; + let counters = ReaderCounters::capture(&fixture.recorder).delta(before); + print_summary( + "manifest", + "no_change_poll_by_retained_files", + target_file_count, + &summary, + &counters.note(reps), + ); + fixture.close().await; + } +} + +async fn benchmark_checkpoint_heavy_manifest(scales: &[usize], full: bool) { + println!("SECTION\tcheckpoint_heavy_manifest"); + for &checkpoint_count in scales { + let path = format!("bench/db-reader-checkpoints/{}", Uuid::new_v4()); + let store = Arc::new(InMemory::new()); + let clock = Arc::new(MockSystemClock::new()); + let db = open_db( + &path, + Arc::clone(&store), + Some(Arc::clone(&clock)), + Segmentation::None, + ) + .await; + for _ in 0..checkpoint_count { + db.create_checkpoint(CheckpointScope::Durable, &CheckpointOptions::default()) + .await + .expect("user checkpoint creation failed"); + } + let recorder = Arc::new(DefaultMetricsRecorder::new()); + let reader = open_reader( + &path, + Arc::clone(&store), + Some(Arc::clone(&clock)), + Segmentation::None, + polling_reader_options(1, Duration::from_secs(1)), + Arc::clone(&recorder), + ) + .await; + let fixture = ReaderFixture { + path, + store, + db, + reader: Arc::new(reader), + recorder, + clock: Some(clock), + segmentation: Segmentation::None, + next_key: 0, + latest_key: None, + }; + + let reps = if full { 15 } else { 5 }; + for _ in 0..2 { + trigger_poll(&fixture, 0).await; + } + let before = ReaderCounters::capture(&fixture.recorder); + let summary = measure_async(0, reps, || async { + trigger_poll(&fixture, 0).await; + }) + .await; + let counters = ReaderCounters::capture(&fixture.recorder).delta(before); + print_summary( + "manifest", + "no_change_poll_by_checkpoint_count", + checkpoint_count, + &summary, + &counters.note(reps), + ); + + let refresh_reps = if full { 7 } else { 3 }; + let before = ReaderCounters::capture(&fixture.recorder); + let mut samples = Vec::with_capacity(refresh_reps); + for _ in 0..refresh_reps { + let start = Instant::now(); + advance_and_wait_poll(&fixture, Duration::from_millis(501), 0).await; + samples.push(start.elapsed()); + } + let counters = ReaderCounters::capture(&fixture.recorder).delta(before); + let summary = Summary::from_durations(samples); + print_summary( + "manifest", + "refresh_one_managed_checkpoint", + checkpoint_count, + &summary, + &counters.note(refresh_reps), + ); + fixture.close().await; + } +} + +async fn build_pinned_generations( + generation_count: usize, +) -> (ReaderFixture, Vec>) { + let fixture = + ReaderFixture::wal_history(0, Segmentation::None, true, Duration::from_secs(60)).await; + let mut snapshots = vec![fixture.reader.snapshot().await.expect("snapshot failed")]; + for index in 1..generation_count { + let key = key_for(index, Segmentation::None); + fixture + .db + .put_with_options( + &key, + b"value", + &PutOptions::default(), + &WriteOptions { + await_durable: false, + ..WriteOptions::default() + }, + ) + .await + .expect("put failed"); + fixture + .db + .flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .expect("L0 flush failed"); + let old_manifest = fixture.reader.manifest().id(); + advance_and_wait_poll(&fixture, POLL_INTERVAL, 0).await; + assert!(fixture.reader.manifest().id() > old_manifest); + snapshots.push(fixture.reader.snapshot().await.expect("snapshot failed")); + } + (fixture, snapshots) +} + +async fn benchmark_pinned_generations(scales: &[usize], full: bool) { + println!("SECTION\tpinned_generations"); + for &generation_count in scales { + let (fixture, snapshots) = build_pinned_generations(generation_count).await; + let poll_reps = if full { 15 } else { 5 }; + for _ in 0..2 { + trigger_poll(&fixture, 0).await; + } + let before = ReaderCounters::capture(&fixture.recorder); + let summary = measure_async(0, poll_reps, || async { + trigger_poll(&fixture, 0).await; + }) + .await; + let counters = ReaderCounters::capture(&fixture.recorder).delta(before); + print_summary( + "generation", + "no_change_poll_while_pinned", + generation_count, + &summary, + &counters.note(poll_reps), + ); + + let refresh_reps = if full { 5 } else { 2 }; + let before = ReaderCounters::capture(&fixture.recorder); + let mut refresh_samples = Vec::with_capacity(refresh_reps); + for _ in 0..refresh_reps { + let start = Instant::now(); + advance_and_wait_poll(&fixture, Duration::from_secs(31), 0).await; + refresh_samples.push(start.elapsed()); + } + let counters = ReaderCounters::capture(&fixture.recorder).delta(before); + let summary = Summary::from_durations(refresh_samples); + print_summary( + "generation", + "refresh_all_pinned", + generation_count, + &summary, + &counters.note(refresh_reps), + ); + + drop(snapshots); + let before = ReaderCounters::capture(&fixture.recorder); + let start = Instant::now(); + trigger_poll(&fixture, 0).await; + let summary = Summary::from_durations(vec![start.elapsed()]); + let counters = ReaderCounters::capture(&fixture.recorder).delta(before); + print_summary( + "generation", + "delete_released", + generation_count, + &summary, + &counters.note(1), + ); + fixture.close().await; + } +} + +async fn benchmark_fixed_reader_open(scales: &[usize], full: bool) { + println!("SECTION\tfixed_reader_open"); + for &wal_count in scales { + let path = format!("bench/db-reader-fixed/{}", Uuid::new_v4()); + let store = Arc::new(InMemory::new()); + let db = open_db(&path, Arc::clone(&store), None, Segmentation::None).await; + for index in 0..wal_count { + write_wal(&db, index, Segmentation::None).await; + } + let checkpoint = db + .create_checkpoint(CheckpointScope::Durable, &CheckpointOptions::default()) + .await + .expect("checkpoint creation failed"); + let reps = if full { 10 } else { 3 }; + let mut samples = Vec::with_capacity(reps); + let mut counter_total = ReaderCounters { + lists: 0, + heads: 0, + gets: 0, + get_ranges: 0, + puts: 0, + replay_ssts: 0, + }; + for _ in 0..reps { + let recorder = Arc::new(DefaultMetricsRecorder::new()); + let before = ReaderCounters::capture(&recorder); + let object_store: Arc = store.clone(); + let start = Instant::now(); + let reader = DbReader::builder(path.as_str(), object_store) + .with_checkpoint_id(checkpoint.id) + .with_options(quiet_reader_options(1)) + .with_metrics_recorder(recorder.clone()) + .with_db_cache_disabled() + .build() + .await + .expect("fixed reader open failed"); + samples.push(start.elapsed()); + let delta = ReaderCounters::capture(&recorder).delta(before); + counter_total.lists += delta.lists; + counter_total.heads += delta.heads; + counter_total.gets += delta.gets; + counter_total.get_ranges += delta.get_ranges; + counter_total.puts += delta.puts; + counter_total.replay_ssts += delta.replay_ssts; + reader.close().await.expect("fixed reader close failed"); + } + let summary = Summary::from_durations(samples); + print_summary( + "open", + "fixed_checkpoint_reader", + wal_count, + &summary, + &counter_total.note(reps), + ); + db.close().await.expect("DB close failed"); + } +} + +async fn run() { + let full = std::env::var_os("SLATEDB_READER_BENCH_FULL").is_some(); + println!( + "CONFIG\tprofile={}\tfull={full}", + if cfg!(debug_assertions) { + "debug" + } else { + "release" + } + ); + println!("HEADER\tsuite\tcase\tscale\treps\tmin_us\tp50_us\tp95_us\tmean_us\tmax_us\tnotes"); + + let usual_scales = if full { + vec![0, 1, 8, 32, 128, 512] + } else { + vec![0, 8, 128] + }; + benchmark_usual_paths(&usual_scales, full).await; + benchmark_bulk_iterator(full).await; + benchmark_iterator_contention(full).await; + + let replay_histories = if full { + vec![0, 32, 128, 512] + } else { + vec![0, 128] + }; + let replay_deltas = if full { vec![1, 8, 32] } else { vec![1, 8] }; + benchmark_incremental_replay(&replay_histories, &replay_deltas, full).await; + benchmark_generation_rollover(&usual_scales, full).await; + benchmark_final_old_snapshot_drop(&usual_scales, full).await; + benchmark_snapshot_contention(full).await; + + let manifest_files = if full { + vec![2, 16, 64, 256, 1_024, 4_096] + } else { + vec![2, 64, 1_024] + }; + benchmark_manifest_file_listing(&manifest_files, full).await; + + let checkpoint_counts = if full { + vec![0, 8, 32, 128] + } else { + vec![0, 32] + }; + benchmark_checkpoint_heavy_manifest(&checkpoint_counts, full).await; + + let generation_counts = if full { + vec![1, 8, 32, 128] + } else { + vec![1, 8] + }; + benchmark_pinned_generations(&generation_counts, full).await; + + let fixed_open_scales = if full { + vec![0, 1, 8, 32, 128] + } else { + vec![0, 8, 32] + }; + benchmark_fixed_reader_open(&fixed_open_scales, full).await; +} + +fn main() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("failed to build Tokio runtime"); + runtime.block_on(run()); +} diff --git a/slatedb/src/db_cache/mod.rs b/slatedb/src/db_cache/mod.rs index 6c35744f2..82fdaf538 100644 --- a/slatedb/src/db_cache/mod.rs +++ b/slatedb/src/db_cache/mod.rs @@ -46,7 +46,17 @@ pub const DEFAULT_BLOCK_CACHE_CAPACITY: u64 = 512 * 1024 * 1024; pub const DEFAULT_META_CACHE_CAPACITY: u64 = 128 * 1024 * 1024; /// Atomic counter to generate unique scope IDs for `DbCacheWrapper` instances. -static NEXT_CACHE_SCOPE_ID: AtomicU64 = AtomicU64::new(0); +/// Scope `0` belongs exclusively to cache keys serialized before scoping was +/// introduced, so live wrappers start at `1` and can never alias those entries. +static NEXT_CACHE_SCOPE_ID: AtomicU64 = AtomicU64::new(1); + +fn next_cache_scope_id() -> u64 { + NEXT_CACHE_SCOPE_ID + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + current.checked_add(1) + }) + .expect("cache scope IDs exhausted") +} /// A `FnOnce` returning a future that produces a [`CachedEntry`] on cache miss. /// @@ -654,7 +664,7 @@ impl DbCacheWrapper { Self { stats: DbCacheStats::new(recorder), cache, - scope_id: NEXT_CACHE_SCOPE_ID.fetch_add(1, Ordering::Relaxed), + scope_id: next_cache_scope_id(), last_err_log_time: Mutex::new(None), system_clock, } @@ -1054,6 +1064,7 @@ pub(crate) mod test_utils { use crate::db_cache::{CachedEntry, CachedKey, DbCache}; use async_trait::async_trait; use std::collections::HashMap; + use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; /// A cache that always returns an error from get operations. @@ -1094,43 +1105,72 @@ pub(crate) mod test_utils { pub(crate) struct TestCache { items: Mutex>, + hits: AtomicU64, + misses: AtomicU64, + inserts: AtomicU64, } impl TestCache { pub(crate) fn new() -> Self { Self { items: Mutex::new(HashMap::new()), + hits: AtomicU64::new(0), + misses: AtomicU64::new(0), + inserts: AtomicU64::new(0), } } pub(crate) fn keys(&self) -> Vec { self.items.lock().unwrap().keys().cloned().collect() } + + fn get(&self, key: &CachedKey) -> Option { + let entry = self.items.lock().unwrap().get(key).cloned(); + if entry.is_some() { + self.hits.fetch_add(1, Ordering::Relaxed); + } else { + self.misses.fetch_add(1, Ordering::Relaxed); + } + entry + } + + pub(crate) fn hits(&self) -> u64 { + self.hits.load(Ordering::Relaxed) + } + + pub(crate) fn misses(&self) -> u64 { + self.misses.load(Ordering::Relaxed) + } + + pub(crate) fn inserts(&self) -> u64 { + self.inserts.load(Ordering::Relaxed) + } + + pub(crate) fn clear(&self) { + self.items.lock().unwrap().clear(); + } } #[async_trait] impl DbCache for TestCache { async fn get_block(&self, key: &CachedKey) -> Result, crate::Error> { - let guard = self.items.lock().unwrap(); - Ok(guard.get(key).cloned()) + Ok(self.get(key)) } async fn get_index(&self, key: &CachedKey) -> Result, crate::Error> { - let guard = self.items.lock().unwrap(); - Ok(guard.get(key).cloned()) + Ok(self.get(key)) } async fn get_filter(&self, key: &CachedKey) -> Result, crate::Error> { - let guard = self.items.lock().unwrap(); - Ok(guard.get(key).cloned()) + Ok(self.get(key)) } async fn get_stats(&self, key: &CachedKey) -> Result, crate::Error> { - let guard = self.items.lock().unwrap(); - Ok(guard.get(key).cloned()) + Ok(self.get(key)) } async fn insert(&self, key: CachedKey, value: CachedEntry) { + self.inserts.fetch_add(1, Ordering::Relaxed); let mut guard = self.items.lock().unwrap(); guard.insert(key, value); } @@ -1509,6 +1549,14 @@ mod tests { let shared_cache: Arc = Arc::new(TestCache::new()); let cache_a = DbCacheWrapper::new(shared_cache.clone(), &recorder_a, system_clock.clone()); let cache_b = DbCacheWrapper::new(shared_cache.clone(), &recorder_b, system_clock); + assert_ne!( + cache_a.scope_id, 0, + "live wrappers must not use legacy scope 0" + ); + assert_ne!( + cache_b.scope_id, 0, + "live wrappers must not use legacy scope 0" + ); assert_ne!(cache_a.scope_id, cache_b.scope_id); let policy = BloomFilterPolicy::new(1); diff --git a/slatedb/src/db_iter.rs b/slatedb/src/db_iter.rs index 2886b3e65..5c4e5ac66 100644 --- a/slatedb/src/db_iter.rs +++ b/slatedb/src/db_iter.rs @@ -15,6 +15,29 @@ use async_trait::async_trait; use bytes::Bytes; use std::collections::VecDeque; use std::ops::RangeBounds; +use std::sync::Arc; + +pub(crate) trait DbIteratorGuard: Send + Sync { + fn enter(&self) -> Result<(), SlateDBError>; + fn exit(&self); +} + +struct DbIteratorGuardPermit { + guard: Arc, +} + +impl DbIteratorGuardPermit { + fn acquire(guard: Arc) -> Result { + guard.enter()?; + Ok(Self { guard }) + } +} + +impl Drop for DbIteratorGuardPermit { + fn drop(&mut self) { + self.guard.exit(); + } +} /// [`DbIteratorRangeTracker`] records the *requested* scan range of a /// [`DbIterator`] so that the transaction manager can detect read-write @@ -183,6 +206,10 @@ pub struct DbIterator { iter: Box, invalidated_error: Option, last_key: Option, + /// Keeps any reader-generation state needed by the iterator alive. The + /// concrete type is intentionally opaque so the common iterator does not + /// depend on `DbReader` internals. + iteration_guard: Option>, } impl DbIterator { @@ -253,9 +280,25 @@ impl DbIterator { iter, invalidated_error: None, last_key: None, + iteration_guard: None, }) } + pub(crate) fn with_iteration_guard(mut self, guard: Arc) -> Self + where + T: DbIteratorGuard + 'static, + { + self.iteration_guard = Some(guard); + self + } + + fn acquire_iteration_guard(&self) -> Result, SlateDBError> { + self.iteration_guard + .as_ref() + .map(|guard| DbIteratorGuardPermit::acquire(Arc::clone(guard))) + .transpose() + } + /// Get the next key-value pair. /// /// This method filters out tombstones and returns the user-facing [`KeyValue`] struct, @@ -280,6 +323,7 @@ impl DbIterator { } pub(crate) async fn next_entry(&mut self) -> Result, SlateDBError> { + let _permit = self.acquire_iteration_guard()?; if let Some(error) = self.invalidated_error.clone() { Err(error) } else { @@ -328,6 +372,7 @@ impl DbIterator { /// /// Returns [`Error`] if the iterator has been invalidated in order to reclaim resources. pub async fn seek>(&mut self, next_key: K) -> Result<(), crate::Error> { + let _permit = self.acquire_iteration_guard().map_err(crate::Error::from)?; let next_key = next_key.as_ref(); if let Some(error) = self.invalidated_error.clone() { Err(error.into()) diff --git a/slatedb/src/db_reader.rs b/slatedb/src/db_reader.rs index 5f0aed5ce..bae55e551 100644 --- a/slatedb/src/db_reader.rs +++ b/slatedb/src/db_reader.rs @@ -5,6 +5,7 @@ use crate::config::{CheckpointOptions, DbReaderOptions, ReadOptions, ScanOptions use crate::db_cache::CacheTarget; use crate::db_cache_manager; use crate::db_common::extract_segment_prefix; +use crate::db_iter::DbIteratorGuard; use crate::db_state::{collect_touched_segments, SsTableId}; use crate::db_stats::DbStats; use crate::db_status::{ClosedResultWriter, DbStatus, DbStatusManager}; @@ -24,7 +25,7 @@ use crate::tablestore::TableStore; use crate::types::KeyValue; use crate::utils::IdGenerator; use crate::wal_replay::{WalReplayIterator, WalReplayOptions}; -use crate::{Checkpoint, DbIterator}; +use crate::{Checkpoint, DbIterator, DbSnapshot}; use crate::{DbCacheManagerOps, DbMetadataOps, DbReadOps}; use async_trait::async_trait; use bytes::Bytes; @@ -35,11 +36,13 @@ use object_store::ObjectStore; use parking_lot::RwLock; use slatedb_common::clock::SystemClock; use slatedb_common::DbRand; -use std::collections::{BTreeSet, VecDeque}; +use std::collections::{BTreeSet, HashMap}; use std::ops::Sub; -use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::LazyLock; +use std::sync::{Arc, Weak}; use tokio::runtime::Handle; +use tokio::sync::Notify; use uuid::Uuid; pub(crate) const DB_READER_TASK_NAME: &str = "manifest_poller"; @@ -70,12 +73,17 @@ pub enum DbReaderMode { /// Read-only interface for accessing a database from either /// the latest persistent state or from an arbitrary checkpoint. +/// +/// A reader that follows the latest state is read-only with respect to user +/// data, but it appends manifest versions to create, refresh, and release GC +/// checkpoints. Opening from an explicit checkpoint does not start that +/// manifest-writing poller. pub struct DbReader { inner: Arc, task_executor: MessageHandlerExecutor, } -struct DbReaderInner { +pub(crate) struct DbReaderInner { manifest_store: Arc, table_store: Arc, options: DbReaderOptions, @@ -84,6 +92,7 @@ struct DbReaderInner { system_clock: Arc, oracle: Arc, reader: Reader, + db_stats: DbStats, status_manager: DbStatusManager, segment_extractor: Option>, rand: Arc, @@ -100,15 +109,210 @@ enum DbReaderMessage { } #[derive(Clone)] -struct ReaderState { - manifest_id: u64, - checkpoint: Option, - manifest: Manifest, - imm_memtable: VecDeque>, +pub(crate) struct ReaderState { + generation: Arc, + imm_memtable: ReplayMemtables, last_wal_id: u64, last_remote_persisted_seq: u64, } +struct ReaderGeneration { + manifest_id: u64, + checkpoint: Option>, + manifest: Manifest, + operation_state: AtomicUsize, + operation_drained: Notify, +} + +const GENERATION_INVALID: usize = 1 << (usize::BITS - 1); +const GENERATION_OPERATION_COUNT: usize = !GENERATION_INVALID; + +struct ReaderGenerationPermit { + generation: Arc, +} + +impl Drop for ReaderGenerationPermit { + fn drop(&mut self) { + self.generation.exit_operation(); + } +} + +impl ReaderGeneration { + fn new( + manifest_id: u64, + checkpoint: Option, + manifest: Manifest, + ) -> Arc { + Arc::new(Self { + manifest_id, + checkpoint: checkpoint.map(RwLock::new), + manifest, + operation_state: AtomicUsize::new(0), + operation_drained: Notify::new(), + }) + } + + fn checkpoint(&self) -> Option { + self.checkpoint.as_ref().map(|checkpoint| checkpoint.read().clone()) + } + + fn managed_checkpoint(&self) -> &RwLock { + self.checkpoint + .as_ref() + .expect("managed reader generation must have a checkpoint") + } + + fn invalidate(&self) { + self.operation_state + .fetch_or(GENERATION_INVALID, Ordering::AcqRel); + } + + fn acquire(self: &Arc) -> Result { + self.enter_operation()?; + Ok(ReaderGenerationPermit { + generation: Arc::clone(self), + }) + } + + fn enter_operation(&self) -> Result<(), SlateDBError> { + let mut state = self.operation_state.load(Ordering::Acquire); + loop { + if state & GENERATION_INVALID != 0 { + let checkpoint_id = self + .checkpoint() + .expect("only checkpoint-backed generations can be invalidated") + .id; + return Err(SlateDBError::CheckpointLeaseLost(checkpoint_id)); + } + assert_ne!( + state & GENERATION_OPERATION_COUNT, + GENERATION_OPERATION_COUNT, + "reader generation in-flight operation count overflow" + ); + match self.operation_state.compare_exchange_weak( + state, + state + 1, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return Ok(()), + Err(current) => state = current, + } + } + } + + fn exit_operation(&self) { + let previous = self.operation_state.fetch_sub(1, Ordering::AcqRel); + assert!( + previous & GENERATION_OPERATION_COUNT > 0, + "reader generation in-flight operation count underflow" + ); + if previous & GENERATION_OPERATION_COUNT == 1 { + self.operation_drained.notify_waiters(); + } + } + + async fn drain(&self) { + loop { + let notified = self.operation_drained.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if self.operation_state.load(Ordering::Acquire) & GENERATION_OPERATION_COUNT == 0 { + return; + } + notified.await; + } + } +} + +impl DbIteratorGuard for ReaderGeneration { + fn enter(&self) -> Result<(), SlateDBError> { + self.enter_operation() + } + + fn exit(&self) { + self.exit_operation(); + } +} + +/// Persistent newest-first collection of WAL-replayed immutable memtables. +/// +/// A new replay poll only prepends newly decoded tables and shares the +/// existing tail with snapshots and in-flight reads. This makes applying a WAL +/// delta proportional to the delta rather than to the complete replay history. +#[derive(Clone, Default)] +struct ReplayMemtables { + head: Option>, + len: usize, +} + +struct ReplayMemtableNode { + table: Arc, + older: Option>, +} + +type ReplayPublisher<'a> = &'a mut (dyn FnMut(&ReplayMemtables, u64, u64) + Send); + +impl ReaderState { + pub(crate) fn applied_seq(&self) -> u64 { + self.last_remote_persisted_seq + } +} + +impl ReplayMemtables { + fn prepend(&mut self, table: Arc) { + self.head = Some(Arc::new(ReplayMemtableNode { + table, + older: self.head.take(), + })); + self.len += 1; + } + + fn front(&self) -> Option<&Arc> { + self.head.as_ref().map(|node| &node.table) + } + + fn iter(&self) -> ReplayMemtablesIter<'_> { + ReplayMemtablesIter { + next: self.head.as_deref(), + } + } + + fn len(&self) -> usize { + self.len + } + + #[cfg(test)] + fn is_empty(&self) -> bool { + self.len == 0 + } +} + +impl FromIterator> for ReplayMemtables { + fn from_iter>>(iter: T) -> Self { + let tables = iter.into_iter().collect::>(); + let mut result = Self::default(); + for table in tables.into_iter().rev() { + result.prepend(table); + } + result + } +} + +struct ReplayMemtablesIter<'a> { + next: Option<&'a ReplayMemtableNode>, +} + +impl Iterator for ReplayMemtablesIter<'_> { + type Item = Arc; + + fn next(&mut self) -> Option { + let node = self.next?; + self.next = node.older.as_deref(); + Some(Arc::clone(&node.table)) + } +} + static EMPTY_TABLE: LazyLock> = LazyLock::new(|| Arc::new(KVTable::new())); impl DbStateReader for ReaderState { @@ -116,18 +320,21 @@ impl DbStateReader for ReaderState { Arc::clone(&EMPTY_TABLE) } - fn imm_memtable(&self) -> &VecDeque> { - &self.imm_memtable + fn imm_memtables(&self) -> Box> + '_> { + Box::new(self.imm_memtable.iter()) } fn core(&self) -> &ManifestCore { - &self.manifest.core + &self.generation.manifest.core } } impl From<&ReaderState> for VersionedManifest { fn from(state: &ReaderState) -> Self { - Self::from_manifest(state.manifest_id, state.manifest.clone()) + Self::from_manifest( + state.generation.manifest_id, + state.generation.manifest.clone(), + ) } } @@ -156,16 +363,19 @@ impl DbReaderInner { }; let replay_new_wals = !matches!(mode, DbReaderMode::Checkpoint(_)) && !options.skip_wal_replay; + let db_stats = DbStats::new(&recorder); let initial_state = Arc::new( Self::build_reader_state( checkpoint, manifest_id, initial_manifest, - VecDeque::new(), + ReplayMemtables::default(), replay_new_wals, Arc::clone(&table_store), &options, segment_extractor.as_ref(), + None, + &db_stats, ) .await?, ); @@ -190,12 +400,10 @@ impl DbReaderInner { status_manager.clone(), )); - let db_stats = DbStats::new(&recorder); - let state = RwLock::new(initial_state); let reader = Reader::new( Arc::clone(&table_store), - db_stats, + db_stats.clone(), Arc::clone(&mono_clock), oracle.clone(), merge_operator, @@ -210,6 +418,7 @@ impl DbReaderInner { system_clock, oracle, reader, + db_stats, status_manager, segment_extractor, rand, @@ -265,11 +474,26 @@ impl DbReaderInner { ) -> Result, SlateDBError> { self.check_closed()?; let db_state = Arc::clone(&self.state.read()); + let _permit = db_state.generation.acquire()?; self.reader .get_key_value_with_options(key, options, db_state.as_ref(), None, None) .await } + pub(crate) async fn snapshot_get_key_value_with_options + Send>( + &self, + state: Arc, + max_seq: u64, + key: K, + options: &ReadOptions, + ) -> Result, SlateDBError> { + self.check_closed()?; + let _permit = state.generation.acquire()?; + self.reader + .get_key_value_with_options(key, options, state.as_ref(), None, Some(max_seq)) + .await + } + async fn scan_with_options( &self, range: BytesRange, @@ -278,7 +502,9 @@ impl DbReaderInner { ) -> Result { self.check_closed()?; let db_state = Arc::clone(&self.state.read()); - self.reader + let _permit = db_state.generation.acquire()?; + let iter = self + .reader .scan_with_options( range, options, @@ -289,7 +515,34 @@ impl DbReaderInner { prefix, }, ) - .await + .await?; + Ok(iter.with_iteration_guard(Arc::clone(&db_state.generation))) + } + + pub(crate) async fn snapshot_scan_with_options( + &self, + state: Arc, + max_seq: u64, + range: BytesRange, + options: &ScanOptions, + prefix: Option, + ) -> Result { + self.check_closed()?; + let _permit = state.generation.acquire()?; + let iter = self + .reader + .scan_with_options( + range, + options, + ScanContext { + db_state: state.as_ref(), + write_batch_iter: None, + max_seq: Some(max_seq), + prefix, + }, + ) + .await?; + Ok(iter.with_iteration_guard(Arc::clone(&state.generation))) } fn should_reestablish_checkpoint(&self, latest: &ManifestCore) -> bool { @@ -306,24 +559,17 @@ impl DbReaderInner { || latest.segments != current_state.segments } - async fn replace_checkpoint( + async fn create_checkpoint( &self, stored_manifest: &mut StoredManifest, ) -> Result { - let current_checkpoint_id = self - .state - .read() - .checkpoint - .as_ref() - .expect("managed reader must have a checkpoint") - .id; let options = CheckpointOptions { lifetime: Some(self.options.checkpoint_lifetime), ..CheckpointOptions::default() }; let new_checkpoint_id = self.rand.rng().gen_uuid(); stored_manifest - .replace_checkpoint(current_checkpoint_id, new_checkpoint_id, &options) + .write_checkpoint(new_checkpoint_id, &options) .await } @@ -349,39 +595,42 @@ impl DbReaderInner { if self.options.skip_wal_replay { return Ok(()); } - let last_replayed_wal_id = self.state.read().last_wal_id; - let last_seen_wal_id = self - .table_store - .last_seen_wal_id(last_replayed_wal_id) - .await?; - if last_seen_wal_id > last_replayed_wal_id { - let current_state = Arc::clone(&self.state.read()); - let mut imm_memtable = current_state.imm_memtable().clone(); - - let (last_wal_id, last_committed_seq) = Self::replay_wal_into( - Arc::clone(&self.table_store), - &self.options, - current_state.core(), - &mut imm_memtable, - true, - self.segment_extractor.as_ref(), - ) - .await?; + let current_state = Arc::clone(&self.state.read()); + let mut imm_memtable = current_state.imm_memtable.clone(); + let generation = Arc::clone(¤t_state.generation); + let mut publish = + |imm_memtable: &ReplayMemtables, last_wal_id: u64, last_committed_seq: u64| { + self.oracle.advance_durable_seq(last_committed_seq); + self.db_stats + .reader_replay_memtables + .set(imm_memtable.len() as i64); + let mut write_guard = self.state.write(); + *write_guard = Arc::new(ReaderState { + generation: Arc::clone(&generation), + imm_memtable: imm_memtable.clone(), + last_wal_id, + last_remote_persisted_seq: last_committed_seq, + }); + drop(write_guard); + self.status_manager + .report_memtable_segments(collect_touched_segments(self.state.read().as_ref())); + }; - self.oracle.advance_durable_seq(last_committed_seq); - let mut write_guard = self.state.write(); - *write_guard = Arc::new(ReaderState { - manifest_id: current_state.manifest_id, - checkpoint: current_state.checkpoint.clone(), - manifest: current_state.manifest.clone(), - imm_memtable, - last_wal_id, - last_remote_persisted_seq: last_committed_seq, - }); - drop(write_guard); - self.status_manager - .report_memtable_segments(collect_touched_segments(self.state.read().as_ref())); - } + Self::replay_wal_into( + Arc::clone(&self.table_store), + &self.options, + current_state.core(), + &mut imm_memtable, + Some(( + current_state.last_wal_id, + current_state.last_remote_persisted_seq, + )), + true, + self.segment_extractor.as_ref(), + Some(&mut publish), + Some(&self.db_stats), + ) + .await?; Ok(()) } @@ -402,7 +651,13 @@ impl DbReaderInner { manifest: Manifest, ) -> Result { let prior = self.state.read().clone(); - let mut imm_memtable = VecDeque::new(); + let replay_cursor = Some(( + prior.last_wal_id.max(manifest.core.replay_after_wal_id), + prior + .last_remote_persisted_seq + .max(manifest.core.last_l0_seq), + )); + let mut retained_memtables = Vec::new(); for table in prior.imm_memtable.iter() { let table_meta = table.table().metadata(); @@ -411,7 +666,7 @@ impl DbReaderInner { continue; } else if table_meta.first_seq > manifest.core.last_l0_seq { // Keep the entire table since all rows are newer than L0+. - imm_memtable.push_back(Arc::clone(table)); + retained_memtables.push(table); } else { // The table has some rows that are newer than L0+ and some that are older. This // happens when the table spans multiple WAL files. Some of those WAL files can @@ -424,10 +679,11 @@ impl DbReaderInner { )?; // Push to the back because we are iterating prior from newest to oldest, and we // want the imm memtables in checkpoint state to be ordered the same way. - imm_memtable.push_back(Arc::new(filtered_table)); + retained_memtables.push(Arc::new(filtered_table)); } } + let imm_memtable = retained_memtables.into_iter().collect(); Self::build_reader_state( checkpoint, manifest_id, @@ -437,6 +693,8 @@ impl DbReaderInner { Arc::clone(&self.table_store), &self.options, self.segment_extractor.as_ref(), + replay_cursor, + &self.db_stats, ) .await } @@ -445,26 +703,33 @@ impl DbReaderInner { checkpoint: Option, manifest_id: u64, manifest: Manifest, - mut imm_memtable: VecDeque>, + mut imm_memtable: ReplayMemtables, replay_new_wals: bool, table_store: Arc, options: &DbReaderOptions, segment_extractor: Option<&Arc>, + replay_cursor: Option<(u64, u64)>, + db_stats: &DbStats, ) -> Result { let (last_wal_id, last_committed_seq) = Self::replay_wal_into( Arc::clone(&table_store), options, &manifest.core, &mut imm_memtable, + replay_cursor, replay_new_wals, segment_extractor, + None, + Some(db_stats), ) .await?; + db_stats + .reader_replay_memtables + .set(imm_memtable.len() as i64); + Ok(ReaderState { - manifest_id, - checkpoint, - manifest, + generation: ReaderGeneration::new(manifest_id, checkpoint, manifest), imm_memtable, last_wal_id, last_remote_persisted_seq: last_committed_seq, @@ -481,7 +746,7 @@ impl DbReaderInner { latest_manifest: VersionedManifest, ) -> Result<(), SlateDBError> { let manifest_id = latest_manifest.id; - if manifest_id <= self.state.read().manifest_id { + if manifest_id <= self.state.read().generation.manifest_id { return self.maybe_replay_new_wals().await; } @@ -493,15 +758,14 @@ impl DbReaderInner { Ok(()) } + #[cfg(test)] async fn maybe_refresh_checkpoint( &self, stored_manifest: &mut StoredManifest, ) -> Result<(), SlateDBError> { - let checkpoint = self - .state - .read() - .checkpoint - .clone() + let generation = Arc::clone(&self.state.read().generation); + let checkpoint = generation + .checkpoint() .expect("managed reader must have a checkpoint"); let half_lifetime = self .options @@ -524,7 +788,7 @@ impl DbReaderInner { // GC reaped it. Re-establish a fresh checkpoint against the latest // manifest instead of failing the reader permanently. warn!("reader checkpoint missing, re-establishing [checkpoint_id={id}]"); - let checkpoint = self.replace_checkpoint(stored_manifest).await?; + let checkpoint = self.create_checkpoint(stored_manifest).await?; self.reestablish_checkpoint(checkpoint).await?; return Ok(()); } @@ -534,20 +798,11 @@ impl DbReaderInner { // Update our local checkpoint copy so we know the latest expiration time // and can calculate future refresh deadlines correctly. { - let mut write_guard = self.state.write(); - let current_state = write_guard.as_ref(); - // Defensively, only update checkpoint if the id and expiry still match. - if current_state - .checkpoint - .as_ref() - .is_some_and(|current_checkpoint| { - current_checkpoint.id == checkpoint.id - && current_checkpoint.expire_time == checkpoint.expire_time - }) + let mut current_checkpoint = generation.managed_checkpoint().write(); + if current_checkpoint.id == checkpoint.id + && current_checkpoint.expire_time == checkpoint.expire_time { - let mut updated_state = current_state.clone(); - updated_state.checkpoint = Some(refreshed_checkpoint.clone()); - *write_guard = Arc::new(updated_state); + *current_checkpoint = refreshed_checkpoint.clone(); } } @@ -563,9 +818,7 @@ impl DbReaderInner { self: &Arc, task_executor: &MessageHandlerExecutor, ) -> Result<(), SlateDBError> { - let poller = ManifestPoller { - inner: Arc::clone(self), - }; + let poller = ManifestPoller::new(Arc::clone(self)); let (_tx, rx) = async_channel::unbounded(); let result = task_executor.add_handler( DB_READER_TASK_NAME.to_string(), @@ -581,9 +834,12 @@ impl DbReaderInner { table_store: Arc, reader_options: &DbReaderOptions, core: &ManifestCore, - into_tables: &mut VecDeque>, + into_tables: &mut ReplayMemtables, + replay_cursor: Option<(u64, u64)>, replay_new_wals: bool, segment_extractor: Option<&Arc>, + mut publish: Option>, + db_stats: Option<&DbStats>, ) -> Result<(u64, u64), SlateDBError> { let sst_iter_options = SstIteratorOptions { max_fetch_tasks: 1, @@ -597,14 +853,16 @@ impl DbReaderInner { }; let (mut replay_after_wal_id, mut last_committed_seq) = - if let Some(latest_replayed_table) = into_tables.front() { - ( - latest_replayed_table.recent_flushed_wal_id(), - latest_replayed_table.table().last_seq().unwrap_or(0), - ) - } else { - (core.replay_after_wal_id, core.last_l0_seq) - }; + replay_cursor.unwrap_or_else(|| { + if let Some(latest_replayed_table) = into_tables.front() { + ( + latest_replayed_table.recent_flushed_wal_id(), + latest_replayed_table.table().last_seq().unwrap_or(0), + ) + } else { + (core.replay_after_wal_id, core.last_l0_seq) + } + }); let wal_id_end = if replay_new_wals { table_store.last_seen_wal_id(replay_after_wal_id).await? + 1 } else { @@ -634,7 +892,16 @@ impl DbReaderInner { Err(err) => return Err(err), } { assert!(replayed_table.last_wal_id > replay_after_wal_id); + let replayed_ssts = replayed_table.last_wal_id - replay_after_wal_id; replay_after_wal_id = replayed_table.last_wal_id; + if let Some(db_stats) = db_stats { + let metadata = replayed_table.table.metadata(); + db_stats.reader_wal_replay_ssts.increment(replayed_ssts); + db_stats + .reader_wal_replay_bytes + .increment(metadata.entries_size_in_bytes as u64); + db_stats.reader_wal_replay_batches.increment(1); + } if !replayed_table.table.is_empty() && replayed_table.last_seq > last_committed_seq { let first_seq = replayed_table .table @@ -653,7 +920,10 @@ impl DbReaderInner { } let imm_memtable = ImmutableMemtable::new(replayed_table.table, replayed_table.last_wal_id); - into_tables.push_front(Arc::new(imm_memtable)); + into_tables.prepend(Arc::new(imm_memtable)); + } + if let Some(publish) = publish.as_mut() { + publish(into_tables, replay_after_wal_id, last_committed_seq); } } @@ -708,6 +978,149 @@ impl DbReaderInner { struct ManifestPoller { inner: Arc, + generations: HashMap>, +} + +impl ManifestPoller { + fn new(inner: Arc) -> Self { + let mut generations = HashMap::new(); + if inner.mode == DbReaderMode::ManagedCheckpoint { + let generation = Arc::clone(&inner.state.read().generation); + let checkpoint_id = generation + .checkpoint() + .expect("managed reader must have a checkpoint") + .id; + generations.insert(checkpoint_id, Arc::downgrade(&generation)); + } + let poller = Self { + inner, + generations, + }; + poller.report_active_checkpoints(); + poller + } + + fn report_active_checkpoints(&self) { + self.inner + .db_stats + .reader_active_checkpoints + .set(self.generations.len() as i64); + } + + fn register_current_generation(&mut self) { + let generation = Arc::clone(&self.inner.state.read().generation); + let checkpoint_id = generation + .checkpoint() + .expect("managed reader must have a checkpoint") + .id; + self.generations + .insert(checkpoint_id, Arc::downgrade(&generation)); + self.report_active_checkpoints(); + } + + async fn delete_released_checkpoints( + &mut self, + manifest: &mut StoredManifest, + ) -> Result<(), SlateDBError> { + let released = self + .generations + .iter() + .filter_map(|(id, generation)| generation.upgrade().is_none().then_some(*id)) + .collect::>(); + if released.is_empty() { + return Ok(()); + } + manifest.delete_checkpoints(&released).await?; + for id in released { + self.generations.remove(&id); + } + self.report_active_checkpoints(); + Ok(()) + } + + async fn refresh_live_checkpoints( + &mut self, + manifest: &mut StoredManifest, + ) -> Result<(), SlateDBError> { + let half_lifetime = self + .inner + .options + .checkpoint_lifetime + .checked_div(2) + .expect("checkpoint lifetime division failed"); + loop { + let live = self + .generations + .iter() + .filter_map(|(id, generation)| { + generation.upgrade().map(|generation| (*id, generation)) + }) + .collect::>(); + let now = self.inner.system_clock.now(); + let refresh_due = live.iter().any(|(_, generation)| { + generation + .checkpoint() + .expect("managed reader generation must have a checkpoint") + .expire_time + .is_some_and(|expiry| now > expiry.sub(half_lifetime)) + }); + if !refresh_due { + return Ok(()); + } + + let ids = live.iter().map(|(id, _)| *id).collect::>(); + match manifest + .refresh_checkpoints(&ids, self.inner.options.checkpoint_lifetime) + .await + { + Ok(refreshed) => { + for checkpoint in refreshed { + if let Some(generation) = + self.generations.get(&checkpoint.id).and_then(Weak::upgrade) + { + *generation.managed_checkpoint().write() = checkpoint; + } + } + return Ok(()); + } + Err(SlateDBError::CheckpointMissing(id)) => { + warn!("reader checkpoint lease lost [checkpoint_id={id}]"); + if let Some(generation) = self.generations.remove(&id).and_then(|g| g.upgrade()) + { + generation.invalidate(); + generation.drain().await; + } + self.report_active_checkpoints(); + + let current_id = self + .inner + .state + .read() + .generation + .checkpoint() + .expect("managed reader must have a checkpoint") + .id; + if current_id == id { + let checkpoint = self.inner.create_checkpoint(manifest).await?; + self.inner.reestablish_checkpoint(checkpoint).await?; + self.register_current_generation(); + } + // Other live generations may be due at the same time. Retry + // the batch immediately after removing the missing lease so + // one lost checkpoint cannot make their refresh a poll late. + } + Err(err) => return Err(err), + } + } + } +} + +impl Drop for ManifestPoller { + fn drop(&mut self) { + // The gauge tracks only GC checkpoints actively managed by this + // poller. Reset it even if startup, cleanup, or the poller task fails. + self.inner.db_stats.reader_active_checkpoints.set(0); + } } #[async_trait] @@ -728,24 +1141,30 @@ impl MessageHandler for ManifestPoller { self.inner.system_clock.clone(), ) .await?; + self.delete_released_checkpoints(&mut manifest).await?; let latest_manifest = manifest.manifest(); if self .inner .should_reestablish_checkpoint(&latest_manifest.core) { - let checkpoint = self.inner.replace_checkpoint(&mut manifest).await?; + let checkpoint = self.inner.create_checkpoint(&mut manifest).await?; self.inner.reestablish_checkpoint(checkpoint).await?; + self.register_current_generation(); } else { self.inner.maybe_replay_new_wals().await?; } - self.inner.maybe_refresh_checkpoint(&mut manifest).await + self.refresh_live_checkpoints(&mut manifest).await?; + self.inner.db_stats.reader_manifest_polls.increment(1); + Ok(()) } DbReaderMode::FollowLatest => { let result = self.inner.refresh_latest_manifest().await; if let Err(error) = result { warn!("failed to refresh reader to latest manifest [error={error:?}]"); + } else { + self.inner.db_stats.reader_manifest_polls.increment(1); } Ok(()) } @@ -767,19 +1186,29 @@ impl MessageHandler for ManifestPoller { self.inner.system_clock.clone(), ) .await?; - let checkpoint_id = self - .inner - .state - .read() - .checkpoint - .as_ref() - .expect("managed reader must have a checkpoint") - .id; - info!( - "deleting reader established checkpoint for shutdown [checkpoint_id={}]", - checkpoint_id - ); - manifest.delete_checkpoint(checkpoint_id).await?; + let checkpoint_ids = self.generations.keys().copied().collect::>(); + if !checkpoint_ids.is_empty() { + let live_generations = self + .generations + .values() + .filter_map(Weak::upgrade) + .collect::>(); + // Invalidate every generation before waiting on any one of them, + // otherwise operations could continue entering a later gate while + // shutdown is draining an earlier one. + for generation in &live_generations { + generation.invalidate(); + } + for generation in live_generations { + generation.drain().await; + } + info!( + "deleting reader established checkpoints for shutdown [checkpoint_ids={:?}]", + checkpoint_ids + ); + manifest.delete_checkpoints(&checkpoint_ids).await?; + } + self.inner.db_stats.reader_active_checkpoints.set(0); Ok(()) } } @@ -814,11 +1243,11 @@ impl DbReader { path: object_store::path::Path, ) -> Result<(), SlateDBError> { let state = Arc::clone(&self.inner.state.read()); - let external_ssts = state.manifest.external_ssts(); + let external_ssts = state.generation.manifest.external_ssts(); let path_resolver = PathResolver::new_with_external_ssts(path, external_ssts); let cache_opts = &self.inner.options.object_store_cache_options; crate::utils::preload_cache_from_manifest( - &state.manifest.core, + &state.generation.manifest.core, cached_obj_store, &path_resolver, cache_opts.preload_disk_cache_on_startup, @@ -828,9 +1257,10 @@ impl DbReader { } /// Creates a database reader that can read the contents of a database (but cannot write any - /// data). [`DbReaderMode`] controls whether the reader manages a checkpoint, remains pinned to - /// a supplied checkpoint, or follows the latest manifest without garbage-collection - /// protection. + /// user data). [`DbReaderMode`] controls whether the reader manages GC checkpoints, remains + /// pinned to a supplied checkpoint, or follows the latest manifest without GC protection. + /// Managed readers retain each generation's checkpoint until all snapshots, iterators, and + /// in-flight reads using that generation are gone. pub async fn open>( path: P, object_store: Arc, @@ -845,6 +1275,44 @@ impl DbReader { .await } + /// Captures the reader's latest fully applied state as a read-only + /// snapshot-isolation transaction. + /// + /// This is an O(1) local operation: WAL discovery and replay remain the + /// responsibility of the long-lived reader poller. Creating a snapshot + /// never performs object-store I/O or forces a flush. Snapshots created + /// from the same manifest generation share one GC checkpoint. + pub async fn snapshot(&self) -> Result, crate::Error> { + if self.inner.mode == DbReaderMode::FollowLatest { + return Err(SlateDBError::DbReaderSnapshotUnsupportedInFollowLatest.into()); + } + loop { + self.inner.check_closed()?; + let state = Arc::clone(&self.inner.state.read()); + match state.generation.acquire() { + Ok(_permit) => { + // Keep the generation gate held until the snapshot has + // captured the state. Lease-loss recovery cannot + // invalidate this generation between validation and + // construction. + return Ok(DbSnapshot::new_reader(Arc::clone(&self.inner), state)); + } + Err(err @ SlateDBError::CheckpointLeaseLost(_)) => { + // Recovery may have replaced the invalid generation while + // we were waiting for its gate. Retry only in that case; + // otherwise report the lease loss instead of returning a + // snapshot whose every read is guaranteed to fail. + let current = Arc::clone(&self.inner.state.read()); + if !Arc::ptr_eq(&state, ¤t) { + continue; + } + return Err(err.into()); + } + Err(err) => return Err(err.into()), + } + } + } + /// Creates a new builder for a database reader at the given path. /// /// # Arguments @@ -1260,6 +1728,11 @@ impl DbReader { /// ``` /// pub async fn close(&self) -> Result<(), crate::Error> { + // Fixed-checkpoint readers do not have a manifest poller to publish + // their clean shutdown, so close the shared status explicitly for + // both reader modes before shutting down any managed task. + self.inner.status_manager.write_result(Ok(())); + self.task_executor .shutdown_task(DB_READER_TASK_NAME) .await @@ -1386,13 +1859,17 @@ fn has_not_found_object_store_error(err: &(dyn std::error::Error + 'static)) -> #[cfg(test)] mod tests { - use super::{DbReaderMessage, ManifestPoller, ReaderState}; + use super::{ + DbReaderMessage, ManifestPoller, ReaderGeneration, ReaderState, ReplayMemtables, + }; use crate::block_cache_policy::BlockCachePolicy; use crate::clock::MonotonicClock; use crate::config::{ CheckpointOptions, CheckpointScope, FlushOptions, FlushType, MergeOptions, PutOptions, Settings, WriteOptions, }; + use crate::db_cache::test_utils::TestCache; + use crate::db_cache::DbCache; use crate::db_reader::{DbReader, DbReaderInner, DbReaderMode, DbReaderOptions}; use crate::db_state::SsTableId; use crate::db_stats::DbStats; @@ -1427,6 +1904,28 @@ mod tests { use std::time::Duration; use uuid::Uuid; + async fn wait_for_reader_generation_change(reader: &DbReader, previous: Uuid) { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if reader + .inner + .state + .read() + .generation + .checkpoint() + .unwrap() + .id + != previous + { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("reader did not install a new manifest generation"); + } + #[tokio::test] async fn should_get_value_from_db() { let object_store: Arc = Arc::new(InMemory::new()); @@ -1821,9 +2320,17 @@ mod tests { assert!(latest_manifest.id > initial_manifest_id); assert!(latest_manifest.manifest.core.checkpoints.is_empty()); - let mut poller = ManifestPoller { - inner: Arc::clone(&reader.inner), + let snapshot_error = match reader.snapshot().await { + Ok(_) => panic!("FollowLatest must not create an unprotected snapshot"), + Err(error) => error, }; + assert_eq!(snapshot_error.kind(), crate::ErrorKind::Invalid); + assert!(snapshot_error + .to_string() + .contains("snapshots are unsupported in FollowLatest mode")); + assert!(recording_store.write_kinds().is_empty()); + + let mut poller = ManifestPoller::new(Arc::clone(&reader.inner)); poller.handle(DbReaderMessage::PollManifest).await.unwrap(); assert!(reader.manifest().id() >= latest_manifest.id); @@ -1893,9 +2400,7 @@ mod tests { saved_manifests.push((location, bytes)); } - let mut poller = ManifestPoller { - inner: Arc::clone(&reader.inner), - }; + let mut poller = ManifestPoller::new(Arc::clone(&reader.inner)); poller.handle(DbReaderMessage::PollManifest).await.unwrap(); assert_eq!(reader.manifest().id(), manifest_id); @@ -2169,7 +2674,13 @@ mod tests { ) .await .unwrap(); - let reader_checkpoint_id = inner.state.read().checkpoint.as_ref().unwrap().id; + let reader_checkpoint_id = inner + .state + .read() + .generation + .checkpoint() + .unwrap() + .id; // Simulate the writer's GC reaping the expired checkpoint. let mut stored_manifest = StoredManifest::load(Arc::clone(&manifest_store), clock.clone()) @@ -2191,7 +2702,13 @@ mod tests { .unwrap(); // The reader should have replaced the reaped checkpoint with a new one. - let new_checkpoint_id = inner.state.read().checkpoint.as_ref().unwrap().id; + let new_checkpoint_id = inner + .state + .read() + .generation + .checkpoint() + .unwrap() + .id; assert_ne!(reader_checkpoint_id, new_checkpoint_id); let latest_manifest = manifest_store.read_latest_manifest().await.unwrap(); let checkpoints = &latest_manifest.manifest.core.checkpoints; @@ -2229,12 +2746,559 @@ mod tests { } #[tokio::test] - async fn replay_wal_into_should_use_latest_existing_table_and_keep_newest_first_order() { + async fn reader_snapshot_should_remain_stable_while_wal_replay_advances() { let object_store: Arc = Arc::new(InMemory::new()); - let path = Path::from("/tmp/test_db_reader_replay_order"); + let path = Path::from("/tmp/test_db_reader_snapshot_wal_isolation"); let test_provider = TestProvider::new(path, Arc::clone(&object_store)); - let table_store = test_provider.table_store(); - + let db = test_provider + .new_db(Settings { + flush_interval: None, + compactor_options: None, + garbage_collector_options: None, + ..Settings::default() + }) + .await + .unwrap(); + + let write_options = WriteOptions { + await_durable: false, + ..WriteOptions::default() + }; + db.put_with_options(b"key", b"v1", &PutOptions::default(), &write_options) + .await + .unwrap(); + db.flush_with_options(FlushOptions { + flush_type: FlushType::Wal, + }) + .await + .unwrap(); + + let reader = test_provider + .new_db_reader( + DbReaderOptions { + manifest_poll_interval: Duration::from_millis(10), + ..DbReaderOptions::default() + }, + None, + None, + ) + .await + .unwrap(); + let snapshot = reader.snapshot().await.unwrap(); + + db.put_with_options(b"key", b"v2", &PutOptions::default(), &write_options) + .await + .unwrap(); + db.flush_with_options(FlushOptions { + flush_type: FlushType::Wal, + }) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(20)).await; + + assert_eq!(reader.get(b"key").await.unwrap(), Some(Bytes::from("v2"))); + assert_eq!(snapshot.get(b"key").await.unwrap(), Some(Bytes::from("v1"))); + assert_eq!( + reader.snapshot().await.unwrap().get(b"key").await.unwrap(), + Some(Bytes::from("v2")) + ); + } + + #[tokio::test] + async fn reader_snapshot_isolation_should_survive_db_cache_hits_and_eviction() { + let object_store: Arc = Arc::new(InMemory::new()); + let path = Path::from("/tmp/test_db_reader_snapshot_db_cache_isolation"); + let db = Db::builder(path.clone(), Arc::clone(&object_store)) + .with_settings(Settings { + flush_interval: None, + compactor_options: None, + garbage_collector_options: None, + ..Settings::default() + }) + .build() + .await + .unwrap(); + let write_options = WriteOptions { + await_durable: false, + ..WriteOptions::default() + }; + db.put_with_options(b"key", b"v1", &PutOptions::default(), &write_options) + .await + .unwrap(); + db.flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .unwrap(); + + let cache = Arc::new(TestCache::new()); + let reader = DbReader::builder(path, Arc::clone(&object_store)) + .with_options(DbReaderOptions { + manifest_poll_interval: Duration::from_millis(10), + ..DbReaderOptions::default() + }) + .with_db_cache(cache.clone()) + .build() + .await + .unwrap(); + let old_snapshot = reader.snapshot().await.unwrap(); + let old_generation = reader + .inner + .state + .read() + .generation + .checkpoint() + .unwrap() + .id; + + assert_eq!( + old_snapshot.get(b"key").await.unwrap(), + Some(Bytes::from_static(b"v1")) + ); + assert!( + cache.inserts() > 0, + "the cold read should populate the cache" + ); + let hits_after_cold_read = cache.hits(); + assert_eq!( + old_snapshot.get(b"key").await.unwrap(), + Some(Bytes::from_static(b"v1")) + ); + assert!( + cache.hits() > hits_after_cold_read, + "the second old-generation read should hit the cache" + ); + + db.put_with_options(b"key", b"v2", &PutOptions::default(), &write_options) + .await + .unwrap(); + db.flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .unwrap(); + wait_for_reader_generation_change(&reader, old_generation).await; + + assert_eq!(reader.get(b"key").await.unwrap(), Some(Bytes::from("v2"))); + let hits_before_old_generation_read = cache.hits(); + assert_eq!( + old_snapshot.get(b"key").await.unwrap(), + Some(Bytes::from_static(b"v1")) + ); + assert!( + cache.hits() > hits_before_old_generation_read, + "warming the new SST must not displace or alias the old SST's cache key" + ); + + cache.clear(); + assert_eq!(cache.entry_count(), 0); + let misses_before_reload = cache.misses(); + let inserts_before_reload = cache.inserts(); + assert_eq!( + old_snapshot.get(b"key").await.unwrap(), + Some(Bytes::from_static(b"v1")) + ); + assert!(cache.misses() > misses_before_reload); + assert!( + cache.inserts() > inserts_before_reload, + "an evicted old-generation block should reload from its checkpoint-pinned SST" + ); + assert_eq!( + reader.snapshot().await.unwrap().get(b"key").await.unwrap(), + Some(Bytes::from_static(b"v2")) + ); + + drop(old_snapshot); + reader.close().await.unwrap(); + db.close().await.unwrap(); + } + + #[tokio::test] + async fn reader_snapshot_isolation_should_survive_warm_object_store_cache_hits() { + use crate::cached_object_store::stats::PART_HIT_COUNT; + use slatedb_common::metrics::{lookup_metric, DefaultMetricsRecorder}; + + let object_store: Arc = Arc::new(InMemory::new()); + let path = Path::from("/tmp/test_db_reader_snapshot_object_cache_isolation"); + let db = Db::builder(path.clone(), Arc::clone(&object_store)) + .with_settings(Settings { + flush_interval: None, + compactor_options: None, + garbage_collector_options: None, + ..Settings::default() + }) + .build() + .await + .unwrap(); + let write_options = WriteOptions { + await_durable: false, + ..WriteOptions::default() + }; + db.put_with_options(b"key", b"v1", &PutOptions::default(), &write_options) + .await + .unwrap(); + db.flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .unwrap(); + + let cache_dir = tempfile::Builder::new() + .prefix("dbreader_snapshot_object_cache_") + .tempdir() + .unwrap(); + let mut options = DbReaderOptions { + manifest_poll_interval: Duration::from_millis(10), + ..DbReaderOptions::default() + }; + options.object_store_cache_options.root_folder = Some(cache_dir.path().to_path_buf()); + options.object_store_cache_options.part_size_bytes = 1024; + options.object_store_cache_options.scan_interval = None; + let metrics = Arc::new(DefaultMetricsRecorder::new()); + let reader = DbReader::builder(path, Arc::clone(&object_store)) + .with_options(options) + // Force SST reads through the object-store cache instead of allowing + // the in-memory block cache to satisfy the second read first. + .with_db_cache_disabled() + .with_metrics_recorder(metrics.clone()) + .build() + .await + .unwrap(); + let old_snapshot = reader.snapshot().await.unwrap(); + let old_generation = reader + .inner + .state + .read() + .generation + .checkpoint() + .unwrap() + .id; + + assert_eq!( + old_snapshot.get(b"key").await.unwrap(), + Some(Bytes::from_static(b"v1")) + ); + let hits_after_cold_read = lookup_metric(&metrics, PART_HIT_COUNT).unwrap_or(0); + assert_eq!( + old_snapshot.get(b"key").await.unwrap(), + Some(Bytes::from_static(b"v1")) + ); + assert!( + lookup_metric(&metrics, PART_HIT_COUNT).unwrap_or(0) > hits_after_cold_read, + "the second read should be served from the local object-store cache" + ); + + db.put_with_options(b"key", b"v2", &PutOptions::default(), &write_options) + .await + .unwrap(); + db.flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .unwrap(); + wait_for_reader_generation_change(&reader, old_generation).await; + assert_eq!(reader.get(b"key").await.unwrap(), Some(Bytes::from("v2"))); + + let hits_before_old_generation_read = lookup_metric(&metrics, PART_HIT_COUNT).unwrap_or(0); + assert_eq!( + old_snapshot.get(b"key").await.unwrap(), + Some(Bytes::from_static(b"v1")) + ); + assert!( + lookup_metric(&metrics, PART_HIT_COUNT).unwrap_or(0) > hits_before_old_generation_read, + "the old snapshot should still use the cached bytes for its own immutable SST" + ); + + drop(old_snapshot); + reader.close().await.unwrap(); + db.close().await.unwrap(); + } + + #[tokio::test] + async fn reader_snapshot_creation_should_do_no_io_and_share_generation_checkpoint() { + let recording = Arc::new(test_utils::RecordingObjectStore::new(Arc::new( + InMemory::new(), + ))); + let object_store: Arc = recording.clone(); + let path = Path::from("/tmp/test_db_reader_snapshot_no_io"); + let test_provider = TestProvider::new(path, Arc::clone(&object_store)); + let db = test_provider.new_db(Settings::default()).await.unwrap(); + db.put(b"key", b"value").await.unwrap(); + db.flush().await.unwrap(); + let checkpoint = db + .create_checkpoint(CheckpointScope::All, &CheckpointOptions::default()) + .await + .unwrap(); + let reader = test_provider + .new_db_reader(DbReaderOptions::default(), Some(checkpoint.id), None) + .await + .unwrap(); + recording.clear(); + + let snapshots = futures::future::try_join_all((0..100).map(|_| reader.snapshot())) + .await + .unwrap(); + + assert_eq!(100, snapshots.len()); + assert!(recording.get_kinds(false).is_empty()); + assert!(recording.get_kinds(true).is_empty()); + assert!(recording.write_kinds().is_empty()); + let manifest = test_provider + .manifest_store() + .read_latest_manifest() + .await + .unwrap(); + assert_eq!(1, manifest.manifest.core.checkpoints.len()); + } + + #[tokio::test] + async fn reader_snapshot_should_reject_an_invalid_current_generation() { + let object_store: Arc = Arc::new(InMemory::new()); + let path = Path::from("/tmp/test_db_reader_snapshot_invalid_generation"); + let test_provider = TestProvider::new(path, Arc::clone(&object_store)); + let db = test_provider.new_db(Settings::default()).await.unwrap(); + let checkpoint = db + .create_checkpoint(CheckpointScope::All, &CheckpointOptions::default()) + .await + .unwrap(); + let reader = test_provider + .new_db_reader(DbReaderOptions::default(), Some(checkpoint.id), None) + .await + .unwrap(); + reader.inner.state.read().generation.invalidate(); + + let result = reader.snapshot().await; + + let err = match result { + Ok(_) => panic!("an invalid generation must not produce a snapshot"), + Err(err) => err, + }; + assert!(err.to_string().contains("reader checkpoint lease lost")); + } + + #[tokio::test] + async fn snapshot_should_keep_old_generation_checkpoint_until_drop() { + let object_store: Arc = Arc::new(InMemory::new()); + let path = Path::from("/tmp/test_db_reader_snapshot_checkpoint_retention"); + let test_provider = TestProvider::new(path, Arc::clone(&object_store)); + let db = test_provider + .new_db(Settings { + flush_interval: None, + compactor_options: None, + garbage_collector_options: None, + ..Settings::default() + }) + .await + .unwrap(); + let write_options = WriteOptions { + await_durable: false, + ..WriteOptions::default() + }; + db.put_with_options(b"key", b"v1", &PutOptions::default(), &write_options) + .await + .unwrap(); + db.flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .unwrap(); + + let reader = test_provider + .new_db_reader( + DbReaderOptions { + manifest_poll_interval: Duration::from_millis(10), + ..DbReaderOptions::default() + }, + None, + None, + ) + .await + .unwrap(); + let snapshot = reader.snapshot().await.unwrap(); + let old_checkpoint_id = reader + .inner + .state + .read() + .generation + .checkpoint() + .unwrap() + .id; + + db.put_with_options(b"key", b"v2", &PutOptions::default(), &write_options) + .await + .unwrap(); + db.flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(20)).await; + + let new_checkpoint_id = reader + .inner + .state + .read() + .generation + .checkpoint() + .unwrap() + .id; + assert_ne!(old_checkpoint_id, new_checkpoint_id); + let manifest = test_provider + .manifest_store() + .read_latest_manifest() + .await + .unwrap(); + assert!(manifest + .manifest + .core + .find_checkpoint(old_checkpoint_id) + .is_some()); + assert!(manifest + .manifest + .core + .find_checkpoint(new_checkpoint_id) + .is_some()); + assert_eq!(snapshot.get(b"key").await.unwrap(), Some(Bytes::from("v1"))); + + drop(snapshot); + tokio::time::sleep(Duration::from_millis(20)).await; + + let manifest = test_provider + .manifest_store() + .read_latest_manifest() + .await + .unwrap(); + assert!(manifest + .manifest + .core + .find_checkpoint(old_checkpoint_id) + .is_none()); + assert!(manifest + .manifest + .core + .find_checkpoint(new_checkpoint_id) + .is_some()); + } + + #[tokio::test] + async fn snapshot_iterator_should_retain_checkpoint_after_snapshot_drop() { + let object_store: Arc = Arc::new(InMemory::new()); + let path = Path::from("/tmp/test_db_reader_snapshot_iterator_retention"); + let test_provider = TestProvider::new(path, Arc::clone(&object_store)); + let db = test_provider + .new_db(Settings { + flush_interval: None, + compactor_options: None, + garbage_collector_options: None, + ..Settings::default() + }) + .await + .unwrap(); + let write_options = WriteOptions { + await_durable: false, + ..WriteOptions::default() + }; + db.put_with_options(b"a", b"old", &PutOptions::default(), &write_options) + .await + .unwrap(); + db.flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .unwrap(); + + let reader = test_provider + .new_db_reader( + DbReaderOptions { + manifest_poll_interval: Duration::from_millis(10), + ..DbReaderOptions::default() + }, + None, + None, + ) + .await + .unwrap(); + let snapshot = reader.snapshot().await.unwrap(); + let mut iter = snapshot.scan(..).await.unwrap(); + let old_checkpoint_id = reader + .inner + .state + .read() + .generation + .checkpoint() + .unwrap() + .id; + drop(snapshot); + + db.put_with_options(b"b", b"new", &PutOptions::default(), &write_options) + .await + .unwrap(); + db.flush_with_options(FlushOptions { + flush_type: FlushType::MemTable, + }) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(20)).await; + + let manifest = test_provider + .manifest_store() + .read_latest_manifest() + .await + .unwrap(); + assert!(manifest + .manifest + .core + .find_checkpoint(old_checkpoint_id) + .is_some()); + let row = iter.next().await.unwrap().unwrap(); + assert_eq!(row.key, Bytes::from("a")); + assert_eq!(row.value, Bytes::from("old")); + assert!(iter.next().await.unwrap().is_none()); + + drop(iter); + tokio::time::sleep(Duration::from_millis(20)).await; + let manifest = test_provider + .manifest_store() + .read_latest_manifest() + .await + .unwrap(); + assert!(manifest + .manifest + .core + .find_checkpoint(old_checkpoint_id) + .is_none()); + } + + #[tokio::test] + async fn generation_drain_should_wait_for_in_flight_operations_and_reject_new_ones() { + let clock: Arc = Arc::new(DefaultSystemClock::new()); + let generation = ReaderGeneration::new( + 1, + Some(test_checkpoint(1, clock)), + Manifest::initial(ManifestCore::new()), + ); + let permit = generation.acquire().unwrap(); + generation.invalidate(); + let draining_generation = Arc::clone(&generation); + let drain = tokio::spawn(async move { draining_generation.drain().await }); + tokio::task::yield_now().await; + assert!(!drain.is_finished()); + + drop(permit); + drain.await.unwrap(); + assert!(matches!( + generation.acquire(), + Err(SlateDBError::CheckpointLeaseLost(_)) + )); + } + + #[tokio::test] + async fn replay_wal_into_should_use_latest_existing_table_and_keep_newest_first_order() { + let object_store: Arc = Arc::new(InMemory::new()); + let path = Path::from("/tmp/test_db_reader_replay_order"); + let test_provider = TestProvider::new(path, Arc::clone(&object_store)); + let table_store = test_provider.table_store(); + write_wal_sst( Arc::clone(&table_store), 3, @@ -2250,15 +3314,18 @@ mod tests { .await .unwrap(); - let mut into_tables = VecDeque::new(); - into_tables.push_front(immutable_memtable( - 3, - vec![RowEntry::new_value(b"stale_key", b"stale_value", 3)], - )); - into_tables.push_back(immutable_memtable( - 2, - vec![RowEntry::new_value(b"older_key", b"older_value", 2)], - )); + let mut into_tables: ReplayMemtables = [ + immutable_memtable( + 3, + vec![RowEntry::new_value(b"stale_key", b"stale_value", 3)], + ), + immutable_memtable( + 2, + vec![RowEntry::new_value(b"older_key", b"older_value", 2)], + ), + ] + .into_iter() + .collect(); let mut core = ManifestCore::new(); core.next_wal_sst_id = 5; @@ -2268,8 +3335,11 @@ mod tests { &DbReaderOptions::default(), &core, &mut into_tables, + None, false, None, + None, + None, ) .await .unwrap(); @@ -2289,6 +3359,73 @@ mod tests { .await; } + #[test] + fn replay_memtable_prepend_should_share_the_existing_tail() { + let mut original = ReplayMemtables::default(); + original.prepend(immutable_memtable( + 1, + vec![RowEntry::new_value(b"old", b"value", 1)], + )); + let original_head = Arc::clone(original.head.as_ref().unwrap()); + + let mut extended = original.clone(); + extended.prepend(immutable_memtable( + 2, + vec![RowEntry::new_value(b"new", b"value", 2)], + )); + + let shared_tail = extended.head.as_ref().unwrap().older.as_ref().unwrap(); + assert!(Arc::ptr_eq(shared_tail, &original_head)); + assert_eq!(1, original.len()); + assert_eq!(2, extended.len()); + } + + #[tokio::test] + async fn replay_wal_into_should_publish_each_bounded_batch() { + let object_store: Arc = Arc::new(InMemory::new()); + let path = Path::from("/tmp/test_db_reader_incremental_replay_publication"); + let test_provider = TestProvider::new(path, Arc::clone(&object_store)); + let table_store = test_provider.table_store(); + let rows = (1..=3) + .map(|seq| RowEntry::new_value(format!("key-{seq}").as_bytes(), &[b'x'; 128], seq)) + .collect::>(); + for (wal_id, row) in rows.iter().cloned().enumerate() { + write_wal_sst(Arc::clone(&table_store), wal_id as u64 + 1, vec![row]) + .await + .unwrap(); + } + let max_memtable_bytes = + table_store.estimate_encoded_size_compacted(1, rows[0].estimated_size()) as u64; + let options = DbReaderOptions { + max_memtable_bytes, + ..DbReaderOptions::default() + }; + let mut core = ManifestCore::new(); + core.next_wal_sst_id = 4; + let mut into_tables = ReplayMemtables::default(); + let mut publications = Vec::new(); + let mut publish = |tables: &ReplayMemtables, wal_id, seq| { + publications.push((wal_id, seq, tables.len())); + }; + + let result = DbReaderInner::replay_wal_into( + Arc::clone(&table_store), + &options, + &core, + &mut into_tables, + None, + false, + None, + Some(&mut publish), + None, + ) + .await + .unwrap(); + + assert_eq!((3, 3), result); + assert_eq!(vec![(1, 1, 1), (2, 2, 2), (3, 3, 3)], publications); + } + #[test] fn has_not_found_object_store_error_should_walk_nested_error_sources() { let err = crate::Error::from(SlateDBError::from(object_store::Error::NotFound { @@ -2324,7 +3461,7 @@ mod tests { .await .unwrap(); - let mut into_tables = VecDeque::new(); + let mut into_tables = ReplayMemtables::default(); let mut core = ManifestCore::new(); core.next_wal_sst_id = 3; @@ -2333,8 +3470,11 @@ mod tests { &DbReaderOptions::default(), &core, &mut into_tables, + None, false, None, + None, + None, ) .await .unwrap(); @@ -2371,7 +3511,7 @@ mod tests { .await .unwrap(); - let mut into_tables = VecDeque::new(); + let mut into_tables = ReplayMemtables::default(); let mut core = ManifestCore::new(); // Force the reader to attempt to read up to 4 even though 3 and 4 don't exist. core.next_wal_sst_id = 4; @@ -2385,8 +3525,11 @@ mod tests { &reader_options, &core, &mut into_tables, + None, false, None, + None, + None, ) .await .unwrap(); @@ -2413,7 +3556,7 @@ mod tests { let test_provider = TestProvider::new(path, Arc::clone(&object_store)); let table_store = test_provider.table_store(); - let mut into_tables = VecDeque::new(); + let mut into_tables = ReplayMemtables::default(); let core = ManifestCore::new(); let (last_wal_id, last_committed_seq) = DbReaderInner::replay_wal_into( @@ -2421,8 +3564,11 @@ mod tests { &DbReaderOptions::default(), &core, &mut into_tables, + None, true, None, + None, + None, ) .await .unwrap(); @@ -2444,7 +3590,7 @@ mod tests { .await .unwrap(); - let mut into_tables = VecDeque::new(); + let mut into_tables = ReplayMemtables::default(); let core = ManifestCore::new(); let (last_wal_id, last_committed_seq) = DbReaderInner::replay_wal_into( @@ -2452,8 +3598,11 @@ mod tests { &DbReaderOptions::default(), &core, &mut into_tables, + None, true, None, + None, + None, ) .await .unwrap(); @@ -2480,8 +3629,8 @@ mod tests { .await .unwrap(); - let mut into_tables = VecDeque::new(); - into_tables.push_front(immutable_memtable( + let mut into_tables = ReplayMemtables::default(); + into_tables.prepend(immutable_memtable( 5, vec![ RowEntry::new_value(b"existing_key_1", b"existing_value_1", 9), @@ -2498,18 +3647,47 @@ mod tests { &DbReaderOptions::default(), &core, &mut into_tables, + None, + true, + None, + None, + None, + ) + .await + .unwrap(); + + assert_eq!(last_wal_id, 6); + assert_eq!(last_committed_seq, 10); + + let head_after_first_replay = Arc::clone(into_tables.head.as_ref().unwrap()); + let (last_wal_id, last_committed_seq) = DbReaderInner::replay_wal_into( + Arc::clone(&table_store), + &DbReaderOptions::default(), + &core, + &mut into_tables, + Some((last_wal_id, last_committed_seq)), true, None, + None, + None, ) .await .unwrap(); assert_eq!(last_wal_id, 6); assert_eq!(last_committed_seq, 10); + assert!(Arc::ptr_eq( + into_tables.head.as_ref().unwrap(), + &head_after_first_replay + )); } #[tokio::test(start_paused = true)] async fn should_fail_new_reads_if_manifest_poller_crashes() { + use slatedb_common::metrics::{ + lookup_metric, DefaultMetricsRecorder, MetricLevel, MetricsRecorderHelper, + }; + let object_store: Arc = Arc::new(InMemory::new()); let path = Path::from("/tmp/test_kv_store"); let test_provider = TestProvider::new(path.clone(), Arc::clone(&object_store)); @@ -2519,10 +3697,27 @@ mod tests { manifest_poll_interval: Duration::from_millis(500), ..DbReaderOptions::default() }; - let reader = test_provider - .new_db_reader(reader_options, None, None) - .await - .unwrap(); + let metrics_recorder = Arc::new(DefaultMetricsRecorder::new()); + let reader = DbReader::open_internal( + test_provider.manifest_store(), + test_provider.table_store(), + DbReaderMode::ManagedCheckpoint, + None, + None, + reader_options, + Arc::clone(&test_provider.system_clock), + Arc::clone(&test_provider.rand), + MetricsRecorderHelper::new(metrics_recorder.clone(), MetricLevel::default()), + ) + .await + .unwrap(); + assert_eq!( + Some(1), + lookup_metric( + &metrics_recorder, + crate::db_stats::READER_ACTIVE_CHECKPOINTS + ) + ); fail_parallel::cfg( Arc::clone(&test_provider.fp_registry), @@ -2532,8 +3727,14 @@ mod tests { .unwrap(); tokio::time::sleep(Duration::from_millis(20)).await; let result = reader.get(b"key").await.unwrap_err(); - dbg!(&result); assert_eq!(result.to_string(), "Unavailable error: io error (oops)"); + assert_eq!( + Some(0), + lookup_metric( + &metrics_recorder, + crate::db_stats::READER_ACTIVE_CHECKPOINTS + ) + ); } #[tokio::test] @@ -2917,12 +4118,14 @@ mod tests { // Seed the prior checkpoint state with IMMs. let input_tables: Vec<_> = case.tables.iter().map(InputMemtable::build).collect(); let prior_state = ReaderState { - manifest_id: stored_manifest.id(), - checkpoint: Some(test_checkpoint( + generation: ReaderGeneration::new( stored_manifest.id(), - test_provider.system_clock.clone(), - )), - manifest: stored_manifest.manifest().clone(), + Some(test_checkpoint( + stored_manifest.id(), + test_provider.system_clock.clone(), + )), + stored_manifest.manifest().clone(), + ), imm_memtable: input_tables.iter().cloned().collect(), last_wal_id: 0, last_remote_persisted_seq: 0, @@ -2948,9 +4151,10 @@ mod tests { // directly. skip_wal_replay keeps the test scoped to the IMM retention logic. let oracle = Arc::new(DbReaderOracle::new(0, DbStatusManager::new(0))); let recorder = slatedb_common::metrics::MetricsRecorderHelper::noop(); + let db_stats = DbStats::new(&recorder); let reader = Reader::new( Arc::clone(&table_store), - DbStats::new(&recorder), + db_stats.clone(), Arc::new(MonotonicClock::new( test_provider.system_clock.clone(), i64::MIN, @@ -2970,6 +4174,7 @@ mod tests { system_clock: test_provider.system_clock.clone(), oracle, reader, + db_stats, status_manager: DbStatusManager::new(0), segment_extractor: None, rand: test_provider.rand.clone(), @@ -2985,7 +4190,10 @@ mod tests { .unwrap(); // The rebuilt checkpoint should reflect the new manifest. - assert_eq!(rebuilt_state.manifest.core.last_l0_seq, case.last_l0_seq); + assert_eq!( + rebuilt_state.generation.manifest.core.last_l0_seq, + case.last_l0_seq + ); assert_eq!(rebuilt_state.imm_memtable.len(), case.expected.len()); for (rebuilt_table, expected_table) in @@ -3021,22 +4229,27 @@ mod tests { let table_store = test_provider.table_store(); let prior_state = ReaderState { - manifest_id: 1, - checkpoint: Some(test_checkpoint(1, test_provider.system_clock.clone())), - manifest: Manifest::initial(current_core.clone()), - imm_memtable: VecDeque::from([immutable_memtable( + generation: ReaderGeneration::new( + 1, + Some(test_checkpoint(1, test_provider.system_clock.clone())), + Manifest::initial(current_core.clone()), + ), + imm_memtable: [immutable_memtable( 1, vec![RowEntry::new_value(b"key", b"value", 10)], - )]), + )] + .into_iter() + .collect(), last_wal_id: 1, last_remote_persisted_seq: 10, }; let oracle = Arc::new(DbReaderOracle::new(0, DbStatusManager::new(0))); let recorder = slatedb_common::metrics::MetricsRecorderHelper::noop(); + let db_stats = DbStats::new(&recorder); let reader = Reader::new( Arc::clone(&table_store), - DbStats::new(&recorder), + db_stats.clone(), Arc::new(MonotonicClock::new( test_provider.system_clock.clone(), i64::MIN, @@ -3053,6 +4266,7 @@ mod tests { system_clock: test_provider.system_clock.clone(), oracle, reader, + db_stats, status_manager: DbStatusManager::new(0), segment_extractor: None, rand: test_provider.rand.clone(), @@ -3239,7 +4453,9 @@ mod tests { #[tokio::test] async fn should_record_metrics_with_recorder() { - use slatedb_common::metrics::{lookup_metric_with_labels, DefaultMetricsRecorder}; + use slatedb_common::metrics::{ + lookup_metric, lookup_metric_with_labels, DefaultMetricsRecorder, + }; let object_store: Arc = Arc::new(InMemory::new()); let path = Path::from("/tmp/test_db_reader_metrics"); @@ -3273,6 +4489,180 @@ mod tests { ), Some(1) ); + for _ in 0..1_000 { + if lookup_metric(&metrics_recorder, crate::db_stats::READER_MANIFEST_POLLS) == Some(1) { + break; + } + tokio::task::yield_now().await; + } + assert_eq!( + lookup_metric(&metrics_recorder, crate::db_stats::READER_MANIFEST_POLLS,), + Some(1) + ); + } + + #[tokio::test] + async fn should_record_incremental_wal_replay_metrics() { + use slatedb_common::metrics::{lookup_metric, DefaultMetricsRecorder}; + + let object_store: Arc = Arc::new(InMemory::new()); + let path = Path::from("/tmp/test_db_reader_wal_replay_metrics"); + let db = Db::builder(path.clone(), Arc::clone(&object_store)) + .with_settings(Settings { + flush_interval: None, + compactor_options: None, + garbage_collector_options: None, + ..Settings::default() + }) + .build() + .await + .unwrap(); + db.put_with_options( + b"key", + b"value", + &PutOptions::default(), + &WriteOptions { + await_durable: false, + ..WriteOptions::default() + }, + ) + .await + .unwrap(); + db.flush_with_options(FlushOptions { + flush_type: FlushType::Wal, + }) + .await + .unwrap(); + + let metrics_recorder = Arc::new(DefaultMetricsRecorder::new()); + let reader = DbReader::builder(path, object_store) + .with_metrics_recorder(metrics_recorder.clone()) + .build() + .await + .unwrap(); + + assert!( + lookup_metric(&metrics_recorder, crate::db_stats::READER_WAL_REPLAY_SSTS) + .is_some_and(|value| value > 0) + ); + assert!( + lookup_metric(&metrics_recorder, crate::db_stats::READER_WAL_REPLAY_BYTES) + .is_some_and(|value| value > 0) + ); + assert!(lookup_metric( + &metrics_recorder, + crate::db_stats::READER_WAL_REPLAY_BATCHES + ) + .is_some_and(|value| value > 0)); + assert_eq!( + Some(1), + lookup_metric(&metrics_recorder, crate::db_stats::READER_REPLAY_MEMTABLES) + ); + assert_eq!( + Some(1), + lookup_metric( + &metrics_recorder, + crate::db_stats::READER_ACTIVE_CHECKPOINTS + ) + ); + + reader.close().await.unwrap(); + assert_eq!( + Some(0), + lookup_metric( + &metrics_recorder, + crate::db_stats::READER_ACTIVE_CHECKPOINTS + ) + ); + db.close().await.unwrap(); + } + + #[tokio::test] + async fn fixed_checkpoint_reader_should_not_manage_or_delete_its_checkpoint() { + use slatedb_common::metrics::{lookup_metric, DefaultMetricsRecorder}; + + let object_store: Arc = Arc::new(InMemory::new()); + let path = Path::from("/tmp/test_fixed_db_reader_checkpoint_metric"); + let db = Db::builder(path.clone(), Arc::clone(&object_store)) + .with_settings(Settings::default()) + .build() + .await + .unwrap(); + db.put(b"key", b"value").await.unwrap(); + let checkpoint = db + .create_checkpoint(CheckpointScope::All, &CheckpointOptions::default()) + .await + .unwrap(); + db.close().await.unwrap(); + + let metrics_recorder = Arc::new(DefaultMetricsRecorder::new()); + let reader = DbReader::builder(path.clone(), Arc::clone(&object_store)) + .with_reader_mode(DbReaderMode::Checkpoint(checkpoint.id)) + .with_metrics_recorder(metrics_recorder.clone()) + .build() + .await + .unwrap(); + + assert_eq!( + Some(0), + lookup_metric( + &metrics_recorder, + crate::db_stats::READER_ACTIVE_CHECKPOINTS + ) + ); + assert_eq!( + reader.get(b"key").await.unwrap(), + Some(Bytes::from("value")) + ); + + reader.close().await.unwrap(); + assert_eq!(reader.status().close_reason, Some(CloseReason::Clean)); + assert!(reader.get(b"key").await.is_err()); + assert_eq!( + Some(0), + lookup_metric( + &metrics_recorder, + crate::db_stats::READER_ACTIVE_CHECKPOINTS + ) + ); + + let manifest_store = ManifestStore::new(&path, Arc::clone(&object_store)); + let manifest = manifest_store.read_latest_manifest().await.unwrap(); + assert!(manifest + .manifest + .core + .find_checkpoint(checkpoint.id) + .is_some()); + + let drop_metrics_recorder = Arc::new(DefaultMetricsRecorder::new()); + let dropped_reader = DbReader::builder(path, Arc::clone(&object_store)) + .with_reader_mode(DbReaderMode::Checkpoint(checkpoint.id)) + .with_metrics_recorder(drop_metrics_recorder.clone()) + .build() + .await + .unwrap(); + assert_eq!( + Some(0), + lookup_metric( + &drop_metrics_recorder, + crate::db_stats::READER_ACTIVE_CHECKPOINTS + ) + ); + drop(dropped_reader); + assert_eq!( + Some(0), + lookup_metric( + &drop_metrics_recorder, + crate::db_stats::READER_ACTIVE_CHECKPOINTS + ) + ); + + let manifest = manifest_store.read_latest_manifest().await.unwrap(); + assert!(manifest + .manifest + .core + .find_checkpoint(checkpoint.id) + .is_some()); } impl TestProvider { @@ -3428,7 +4818,18 @@ mod tests { let timeout = Duration::from_secs(30); let start = tokio::time::Instant::now(); loop { - if reader.inner.state.read().manifest.core.tree.l0.len() == 1 { + if reader + .inner + .state + .read() + .generation + .manifest + .core + .tree + .l0 + .len() + == 1 + { break; } // The reader poller may observe the pre-flush manifest on one tick and diff --git a/slatedb/src/db_snapshot.rs b/slatedb/src/db_snapshot.rs index 90834c5a4..e99b1b24b 100644 --- a/slatedb/src/db_snapshot.rs +++ b/slatedb/src/db_snapshot.rs @@ -8,13 +8,30 @@ use crate::db_iter::DbIterator; use crate::types::KeyValue; use crate::db::DbInner; +use crate::db_reader::{DbReaderInner, ReaderState}; use crate::reader::ScanContext; use crate::DbReadOps; +/// An immutable snapshot-isolation view created by either [`crate::Db`] or +/// [`crate::DbReader`]. +/// +/// Reader-backed snapshots pin the reader's already-replayed local state. They +/// do not replay WALs or create a checkpoint per snapshot; snapshots from the +/// same manifest generation share the reader's GC checkpoint. pub struct DbSnapshot { - snapshot_id: Uuid, started_seq: u64, - db_inner: Arc, + backend: DbSnapshotBackend, +} + +enum DbSnapshotBackend { + Db { + snapshot_id: Uuid, + inner: Arc, + }, + Reader { + inner: Arc, + state: Arc, + }, } impl DbSnapshot { @@ -22,9 +39,18 @@ impl DbSnapshot { let (snapshot_id, started_seq) = db_inner.snapshot_manager.new_snapshot(seq); Arc::new(Self { - snapshot_id, started_seq, - db_inner, + backend: DbSnapshotBackend::Db { + snapshot_id, + inner: db_inner, + }, + }) + } + + pub(crate) fn new_reader(inner: Arc, state: Arc) -> Arc { + Arc::new(Self { + started_seq: state.applied_seq(), + backend: DbSnapshotBackend::Reader { inner, state }, }) } @@ -78,15 +104,32 @@ impl DbSnapshot { key: K, options: &ReadOptions, ) -> Result, crate::Error> { - self.db_inner.check_closed()?; - let db_state = self.db_inner.state.read().view(); - let kv = self - .db_inner - .reader - .get_key_value_with_options(key, options, &db_state, None, Some(self.started_seq)) - .await - .map_err(crate::Error::from)?; - Ok(kv) + match &self.backend { + DbSnapshotBackend::Db { inner, .. } => { + inner.check_closed()?; + let db_state = inner.state.read().view(); + inner + .reader + .get_key_value_with_options( + key, + options, + &db_state, + None, + Some(self.started_seq), + ) + .await + .map_err(crate::Error::from) + } + DbSnapshotBackend::Reader { inner, state } => inner + .snapshot_get_key_value_with_options( + Arc::clone(state), + self.started_seq, + key, + options, + ) + .await + .map_err(crate::Error::from), + } } /// Scan a range of keys using the default scan options. @@ -186,22 +229,36 @@ impl DbSnapshot { options: &ScanOptions, prefix: Option, ) -> Result { - self.db_inner.check_closed()?; - let db_state = self.db_inner.state.read().view(); - self.db_inner - .reader - .scan_with_options( - range, - options, - ScanContext { - db_state: &db_state, - write_batch_iter: None, - max_seq: Some(self.started_seq), + match &self.backend { + DbSnapshotBackend::Db { inner, .. } => { + inner.check_closed()?; + let db_state = inner.state.read().view(); + inner + .reader + .scan_with_options( + range, + options, + ScanContext { + db_state: &db_state, + write_batch_iter: None, + max_seq: Some(self.started_seq), + prefix, + }, + ) + .await + .map_err(Into::into) + } + DbSnapshotBackend::Reader { inner, state } => inner + .snapshot_scan_with_options( + Arc::clone(state), + self.started_seq, + range, + options, prefix, - }, - ) - .await - .map_err(Into::into) + ) + .await + .map_err(Into::into), + } } } @@ -250,9 +307,9 @@ impl DbReadOps for DbSnapshot { impl Drop for DbSnapshot { fn drop(&mut self) { - self.db_inner - .snapshot_manager - .drop_snapshot(&self.snapshot_id); + if let DbSnapshotBackend::Db { snapshot_id, inner } = &self.backend { + inner.snapshot_manager.drop_snapshot(snapshot_id); + } } } diff --git a/slatedb/src/db_state.rs b/slatedb/src/db_state.rs index dffa54812..7287da5ad 100644 --- a/slatedb/src/db_state.rs +++ b/slatedb/src/db_state.rs @@ -686,8 +686,8 @@ impl DbStateReader for DbStateView { Arc::clone(&self.memtable) } - fn imm_memtable(&self) -> &VecDeque> { - &self.state.imm_memtable + fn imm_memtables(&self) -> Box> + '_> { + Box::new(self.state.imm_memtable.iter().cloned()) } fn core(&self) -> &ManifestCore { @@ -717,7 +717,7 @@ pub(crate) fn collect_touched_segments( return std::collections::BTreeSet::new(); } let mut set = reader.memtable().touched_segments(); - for imm in reader.imm_memtable() { + for imm in reader.imm_memtables() { set.extend(imm.table().touched_segments()); } set diff --git a/slatedb/src/db_stats.rs b/slatedb/src/db_stats.rs index 89a40c7c6..d615044c6 100644 --- a/slatedb/src/db_stats.rs +++ b/slatedb/src/db_stats.rs @@ -40,6 +40,12 @@ pub const SST_FILTER_NEGATIVE_COUNT: &str = db_stat_name!("sst_filter_negative_c /// write_amp = (`WAL_FLUSH_BYTES` + `L0_FLUSH_BYTES` + `compactor::stats::BYTES_COMPACTED`) /// / `MEMTABLE_WRITE_BYTES` pub const MEMTABLE_WRITE_BYTES: &str = db_stat_name!("memtable_write_bytes"); +pub const READER_WAL_REPLAY_SSTS: &str = db_stat_name!("reader_wal_replay_ssts"); +pub const READER_WAL_REPLAY_BYTES: &str = db_stat_name!("reader_wal_replay_bytes"); +pub const READER_WAL_REPLAY_BATCHES: &str = db_stat_name!("reader_wal_replay_batches"); +pub const READER_REPLAY_MEMTABLES: &str = db_stat_name!("reader_replay_memtables"); +pub const READER_ACTIVE_CHECKPOINTS: &str = db_stat_name!("reader_active_checkpoints"); +pub const READER_MANIFEST_POLLS: &str = db_stat_name!("reader_manifest_polls"); /// Label key distinguishing filter metrics for point lookups from those for /// prefix scans. Value is one of [`FILTER_KIND_POINT`] or @@ -75,6 +81,12 @@ pub(crate) struct DbStatsInner { pub(crate) merge_operator_read_operands: Arc, pub(crate) merge_operator_flush_operands: Arc, pub(crate) memtable_write_bytes: Arc, + pub(crate) reader_wal_replay_ssts: Arc, + pub(crate) reader_wal_replay_bytes: Arc, + pub(crate) reader_wal_replay_batches: Arc, + pub(crate) reader_replay_memtables: Arc, + pub(crate) reader_active_checkpoints: Arc, + pub(crate) reader_manifest_polls: Arc, } #[derive(Clone)] @@ -161,6 +173,20 @@ impl DbStats { .description(MERGE_OPERATOR_OPERANDS_DESCRIPTION) .register(), memtable_write_bytes: recorder.counter(MEMTABLE_WRITE_BYTES).register(), + reader_wal_replay_ssts: recorder.counter(READER_WAL_REPLAY_SSTS).register(), + reader_wal_replay_bytes: recorder.counter(READER_WAL_REPLAY_BYTES).register(), + reader_wal_replay_batches: recorder.counter(READER_WAL_REPLAY_BATCHES).register(), + reader_replay_memtables: recorder.gauge(READER_REPLAY_MEMTABLES).register(), + reader_active_checkpoints: recorder + .gauge(READER_ACTIVE_CHECKPOINTS) + .description( + "Number of GC checkpoints currently managed by this DbReader's manifest poller", + ) + .register(), + reader_manifest_polls: recorder + .counter(READER_MANIFEST_POLLS) + .description("Number of successful DbReader manifest polls completed") + .register(), }; DbStats { inner: Arc::new(inner), diff --git a/slatedb/src/error.rs b/slatedb/src/error.rs index ea5802b35..4327435bd 100644 --- a/slatedb/src/error.rs +++ b/slatedb/src/error.rs @@ -191,6 +191,15 @@ pub(crate) enum SlateDBError { #[error("checkpoint missing. checkpoint_id=`{0}`")] CheckpointMissing(Uuid), + #[error("checkpoint already exists. checkpoint_id=`{0}`")] + CheckpointAlreadyExists(Uuid), + + #[error("reader checkpoint lease lost. checkpoint_id=`{0}`")] + CheckpointLeaseLost(Uuid), + + #[error("reader snapshots are unsupported in FollowLatest mode")] + DbReaderSnapshotUnsupportedInFollowLatest, + #[error( "unsupported {format_name} format version. supported_versions=`{supported_versions:?}`, actual_version=`{actual_version}`" )] @@ -653,6 +662,7 @@ impl From for Error { SlateDBError::FoyerError(err) => Error::unavailable(msg).with_source(Box::new(err)), SlateDBError::TransactionalObjectTimeout { .. } => Error::unavailable(msg), SlateDBError::WalUnavailable(src) => Error::unavailable(msg).with_source(Box::new(src)), + SlateDBError::CheckpointLeaseLost(_) => Error::unavailable(msg), // Invalid errors SlateDBError::InvalidCachePartSize => Error::invalid(msg), @@ -671,6 +681,7 @@ impl From for Error { SlateDBError::InvalidCheckpointLifetime(_) => Error::invalid(msg), SlateDBError::InvalidManifestPollInterval(_) => Error::invalid(msg), SlateDBError::CheckpointLifetimeTooShort { .. } => Error::invalid(msg), + SlateDBError::DbReaderSnapshotUnsupportedInFollowLatest => Error::invalid(msg), SlateDBError::SeekKeyOutOfRange { .. } => Error::invalid(msg), SlateDBError::SeekKeyLessThanLastReturnedKey => Error::invalid(msg), SlateDBError::IdenticalClonePaths { .. } => Error::invalid(msg), @@ -712,6 +723,7 @@ impl From for Error { SlateDBError::BlockTransformError => Error::data(msg), SlateDBError::InvalidRowFlags { .. } => Error::data(msg), SlateDBError::CheckpointMissing(_) => Error::data(msg), + SlateDBError::CheckpointAlreadyExists(_) => Error::data(msg), SlateDBError::InvalidVersion { .. } => Error::data(msg), SlateDBError::ManifestMissing(_) => Error::data(msg), SlateDBError::LatestTransactionalObjectVersionMissing => Error::data(msg), diff --git a/slatedb/src/manifest/store.rs b/slatedb/src/manifest/store.rs index 8707dc04b..016b97f37 100644 --- a/slatedb/src/manifest/store.rs +++ b/slatedb/src/manifest/store.rs @@ -2,7 +2,8 @@ use crate::checkpoint::Checkpoint; use crate::config::CheckpointOptions; use crate::error::SlateDBError; use crate::error::SlateDBError::{ - CheckpointMissing, InvalidDBState, LatestTransactionalObjectVersionMissing, ManifestMissing, + CheckpointAlreadyExists, CheckpointMissing, InvalidDBState, + LatestTransactionalObjectVersionMissing, ManifestMissing, }; use crate::flatbuffer_types::FlatBufferManifestCodec; use crate::manifest::{Manifest, ManifestCore, VersionedManifest}; @@ -16,7 +17,7 @@ use slatedb_txn_obj::{ DirtyObject, FenceableTransactionalObject, MonotonicId, SequencedStorageProtocol, SimpleTransactionalObject, TransactionalObject, TransactionalStorageProtocol, }; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::ops::RangeBounds; use std::sync::Arc; use std::time::Duration; @@ -301,6 +302,14 @@ impl StoredManifest { self.inner .maybe_apply_update(|sr| { let mut new_val = sr.object().clone(); + if new_val + .core + .checkpoints + .iter() + .any(|checkpoint| checkpoint.id == checkpoint_id) + { + return Err(CheckpointAlreadyExists(checkpoint_id)); + } let checkpoint = Self::new_checkpoint( &new_val, sr.id().into(), @@ -346,9 +355,42 @@ impl StoredManifest { .await?) } + /// Deletes several checkpoints in one manifest append. Missing checkpoint + /// IDs are treated as already deleted so cleanup remains idempotent. + pub(crate) async fn delete_checkpoints( + &mut self, + checkpoint_ids: &[Uuid], + ) -> Result<(), SlateDBError> { + if checkpoint_ids.is_empty() { + return Ok(()); + } + let checkpoint_ids = checkpoint_ids.iter().copied().collect::>(); + Ok(self + .inner + .maybe_apply_update(|sr| { + let mut new_val = sr.object().clone(); + let before = new_val.core.checkpoints.len(); + new_val + .core + .checkpoints + .retain(|cp| !checkpoint_ids.contains(&cp.id)); + let result: Result>, SlateDBError> = + if new_val.core.checkpoints.len() == before { + Ok(None) + } else { + let mut dirty = sr.prepare_dirty()?; + dirty.value = new_val; + Ok(Some(dirty)) + }; + result + }) + .await?) + } + /// Replace an existing checkpoint with a new checkpoint. If the old checkpoint /// is missing, the new checkpoint will still be added. This helps avoid /// issuing two manifest updates when creating a new checkpoint. + #[cfg(test)] pub(crate) async fn replace_checkpoint( &mut self, old_checkpoint_id: Uuid, @@ -386,6 +428,7 @@ impl StoredManifest { Ok(new_checkpoint) } + #[cfg(test)] pub(crate) async fn refresh_checkpoint( &mut self, checkpoint_id: Uuid, @@ -417,6 +460,47 @@ impl StoredManifest { Ok(checkpoint) } + /// Refreshes several checkpoint leases in one manifest append. + pub(crate) async fn refresh_checkpoints( + &mut self, + checkpoint_ids: &[Uuid], + new_lifetime: Duration, + ) -> Result, SlateDBError> { + if checkpoint_ids.is_empty() { + return Ok(Vec::new()); + } + let checkpoint_ids = checkpoint_ids.iter().copied().collect::>(); + let clock = Arc::clone(&self.clock); + self.inner + .maybe_apply_update(|sr| { + let mut new_val = sr.object().clone(); + for checkpoint_id in &checkpoint_ids { + if !new_val + .core + .checkpoints + .iter() + .any(|checkpoint| checkpoint.id == *checkpoint_id) + { + return Err(CheckpointMissing(*checkpoint_id)); + } + } + let expire_time = clock.now() + new_lifetime; + for checkpoint in &mut new_val.core.checkpoints { + if checkpoint_ids.contains(&checkpoint.id) { + checkpoint.expire_time = Some(expire_time); + } + } + let mut dirty = sr.prepare_dirty()?; + dirty.value = new_val; + Ok(Some(dirty)) + }) + .await?; + Ok(checkpoint_ids + .iter() + .filter_map(|id| self.db_state().find_checkpoint(*id).cloned()) + .collect()) + } + pub(crate) async fn update( &mut self, dirty: DirtyObject, @@ -1299,6 +1383,73 @@ mod tests { ); } + #[tokio::test] + async fn should_refresh_checkpoints_atomically_in_one_manifest() { + let ms = new_memory_manifest_store(); + let mut sm = StoredManifest::create_new_db( + ms, + ManifestCore::new(), + Arc::new(DefaultSystemClock::new()), + ) + .await + .unwrap(); + let options = CheckpointOptions { + lifetime: Some(Duration::from_secs(100)), + ..CheckpointOptions::default() + }; + let first = sm + .write_checkpoint(uuid::Uuid::new_v4(), &options) + .await + .unwrap(); + let second = sm + .write_checkpoint(uuid::Uuid::new_v4(), &options) + .await + .unwrap(); + let manifest_id = sm.id(); + + let refreshed = sm + .refresh_checkpoints(&[first.id, second.id], Duration::from_secs(500)) + .await + .unwrap(); + + assert_eq!(manifest_id + 1, sm.id()); + assert_eq!(2, refreshed.len()); + assert_eq!(refreshed[0].expire_time, refreshed[1].expire_time); + assert!(refreshed[0].expire_time > first.expire_time); + } + + #[tokio::test] + async fn should_not_partially_refresh_when_any_checkpoint_is_missing() { + let ms = new_memory_manifest_store(); + let mut sm = StoredManifest::create_new_db( + ms, + ManifestCore::new(), + Arc::new(DefaultSystemClock::new()), + ) + .await + .unwrap(); + let checkpoint = sm + .write_checkpoint(uuid::Uuid::new_v4(), &CheckpointOptions::default()) + .await + .unwrap(); + let manifest_id = sm.id(); + let missing = uuid::Uuid::new_v4(); + + let result = sm + .refresh_checkpoints(&[checkpoint.id, missing], Duration::from_secs(500)) + .await; + + assert!(matches!(result, Err(SlateDBError::CheckpointMissing(id)) if id == missing)); + assert_eq!(manifest_id, sm.id()); + assert_eq!( + checkpoint.expire_time, + sm.db_state() + .find_checkpoint(checkpoint.id) + .unwrap() + .expire_time + ); + } + #[tokio::test] async fn should_fail_refresh_if_checkpoint_missing() { let ms = new_memory_manifest_store(); @@ -1405,6 +1556,103 @@ mod tests { assert_eq!(None, sm.manifest().core.find_checkpoint(checkpoint.id)); } + #[tokio::test] + async fn should_delete_checkpoints_idempotently_in_one_manifest() { + let ms = new_memory_manifest_store(); + let mut sm = StoredManifest::create_new_db( + ms, + ManifestCore::new(), + Arc::new(DefaultSystemClock::new()), + ) + .await + .unwrap(); + let first = sm + .write_checkpoint(uuid::Uuid::new_v4(), &CheckpointOptions::default()) + .await + .unwrap(); + let second = sm + .write_checkpoint(uuid::Uuid::new_v4(), &CheckpointOptions::default()) + .await + .unwrap(); + let manifest_id = sm.id(); + + sm.delete_checkpoints(&[first.id, second.id]).await.unwrap(); + + assert_eq!(manifest_id + 1, sm.id()); + assert!(sm.db_state().checkpoints.is_empty()); + let manifest_id = sm.id(); + sm.delete_checkpoints(&[first.id, second.id]).await.unwrap(); + assert_eq!(manifest_id, sm.id()); + } + + #[tokio::test] + async fn should_reject_duplicate_checkpoint_ids_without_appending() { + let ms = new_memory_manifest_store(); + let mut sm = StoredManifest::create_new_db( + ms, + ManifestCore::new(), + Arc::new(DefaultSystemClock::new()), + ) + .await + .unwrap(); + let checkpoint_id = uuid::Uuid::new_v4(); + sm.write_checkpoint(checkpoint_id, &CheckpointOptions::default()) + .await + .unwrap(); + let manifest_id = sm.id(); + + let result = sm + .write_checkpoint(checkpoint_id, &CheckpointOptions::default()) + .await; + + assert!( + matches!(result, Err(SlateDBError::CheckpointAlreadyExists(id)) if id == checkpoint_id) + ); + assert_eq!(manifest_id, sm.id()); + assert_eq!(1, sm.db_state().checkpoints.len()); + } + + #[tokio::test] + async fn checkpoint_append_should_merge_with_a_concurrent_manifest_writer() { + let ms = new_memory_manifest_store(); + StoredManifest::create_new_db( + Arc::clone(&ms), + ManifestCore::new(), + Arc::new(DefaultSystemClock::new()), + ) + .await + .unwrap(); + let mut reader_manifest = + StoredManifest::load(Arc::clone(&ms), Arc::new(DefaultSystemClock::new())) + .await + .unwrap(); + let mut writer_manifest = + StoredManifest::load(Arc::clone(&ms), Arc::new(DefaultSystemClock::new())) + .await + .unwrap(); + let checkpoint_id = uuid::Uuid::new_v4(); + let checkpoint_options = CheckpointOptions::default(); + + let (checkpoint_result, writer_result) = tokio::join!( + reader_manifest.write_checkpoint(checkpoint_id, &checkpoint_options), + writer_manifest.maybe_apply_update(|manifest| { + let mut dirty = manifest.prepare_dirty()?; + dirty.value.core.last_l0_seq = 42; + Ok(Some(dirty)) + }) + ); + + checkpoint_result.unwrap(); + writer_result.unwrap(); + let latest = ms.read_latest_manifest().await.unwrap(); + assert_eq!(42, latest.manifest.core.last_l0_seq); + assert!(latest + .manifest + .core + .find_checkpoint(checkpoint_id) + .is_some()); + } + #[tokio::test] async fn should_ignore_missing_checkpoint_if_deleting() { let ms = new_memory_manifest_store(); diff --git a/slatedb/src/reader.rs b/slatedb/src/reader.rs index ad4b95ac0..b1faf711f 100644 --- a/slatedb/src/reader.rs +++ b/slatedb/src/reader.rs @@ -22,7 +22,9 @@ use std::sync::Arc; pub(crate) trait DbStateReader { fn memtable(&self) -> Arc; - fn imm_memtable(&self) -> &VecDeque>; + /// Returns immutable memtables newest-first. The iterator form permits + /// read-only replicas to use a structurally shared persistent chain. + fn imm_memtables(&self) -> Box> + '_>; fn core(&self) -> &ManifestCore; } @@ -141,7 +143,7 @@ impl Reader { ) -> Result { let mut memtables = VecDeque::new(); memtables.push_back(db_state.memtable()); - for memtable in db_state.imm_memtable() { + for memtable in db_state.imm_memtables() { memtables.push_back(memtable.table()); } let mem_iters = memtables @@ -381,7 +383,7 @@ impl Reader { .memtable() .range(range.clone(), sst_iter_options.order), )); - for memtable in db_state.imm_memtable() { + for memtable in db_state.imm_memtables() { all_iters.push(Box::new( memtable .table() @@ -613,8 +615,8 @@ mod tests { self.memtable.clone() } - fn imm_memtable(&self) -> &VecDeque> { - &self.imm_memtable + fn imm_memtables(&self) -> Box> + '_> { + Box::new(self.imm_memtable.iter().cloned()) } fn core(&self) -> &ManifestCore { From b313ac2c0929d64355c9877953c841e9c76e459e Mon Sep 17 00:00:00 2001 From: Matthew Sanetra <41018997+matthewsanetra@users.noreply.github.com> Date: Thu, 16 Jul 2026 18:13:09 +0100 Subject: [PATCH 55/63] Harden reader snapshot WAL replay test --- slatedb/src/db_reader.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/slatedb/src/db_reader.rs b/slatedb/src/db_reader.rs index bae55e551..773be0f5a 100644 --- a/slatedb/src/db_reader.rs +++ b/slatedb/src/db_reader.rs @@ -2794,7 +2794,16 @@ mod tests { }) .await .unwrap(); - tokio::time::sleep(Duration::from_millis(20)).await; + tokio::time::timeout(Duration::from_secs(10), async { + loop { + if reader.get(b"key").await.unwrap() == Some(Bytes::from_static(b"v2")) { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("live reader should replay the second WAL generation"); assert_eq!(reader.get(b"key").await.unwrap(), Some(Bytes::from("v2"))); assert_eq!(snapshot.get(b"key").await.unwrap(), Some(Bytes::from("v1"))); From d05328d352f991f28158ea2fe9ecf09c27bd5c78 Mon Sep 17 00:00:00 2001 From: xav-db Date: Mon, 6 Jul 2026 13:59:02 +0100 Subject: [PATCH 56/63] Add multi-get functionality across database components - Introduced `multi_get` and `multi_get_with_options` methods in `Db`, `DbReader`, `DbSnapshot`, and `DbTransaction` to retrieve multiple values efficiently. - Implemented a new `WriteBatchLookup` enum to facilitate lookup operations in `WriteBatch`. - Enhanced the `lookup_latest_for_key` method to support the new multi-get functionality. - Added tests to ensure correctness of multi-get operations, preserving input order and handling duplicates. This update improves the performance and usability of batch retrieval operations in the database. --- slatedb/src/batch.rs | 16 + slatedb/src/db.rs | 94 ++++++ slatedb/src/db_reader.rs | 105 ++++++- slatedb/src/db_snapshot.rs | 71 +++++ slatedb/src/db_state.rs | 2 +- slatedb/src/db_transaction.rs | 312 ++++++++++++++++++- slatedb/src/ops.rs | 48 +++ slatedb/src/reader.rs | 557 +++++++++++++++++++++++++++++++++- 8 files changed, 1198 insertions(+), 7 deletions(-) diff --git a/slatedb/src/batch.rs b/slatedb/src/batch.rs index 3b2e03dcd..c54586c66 100644 --- a/slatedb/src/batch.rs +++ b/slatedb/src/batch.rs @@ -69,6 +69,13 @@ pub(crate) enum WriteOp { Merge(Bytes, MergeOptions), } +pub(crate) enum WriteBatchLookup { + Put(Bytes), + Delete, + Merge, + NotPresent, +} + impl std::fmt::Debug for WriteOp { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn trunc(bytes: &Bytes) -> String { @@ -278,6 +285,15 @@ impl WriteBatch { self.ops.is_empty() } + pub(crate) fn lookup_latest_for_key(&self, key: &[u8]) -> WriteBatchLookup { + match self.ops.get(key).and_then(|ops| ops.last()) { + Some(WriteOp::Put(value, _)) => WriteBatchLookup::Put(value.clone()), + Some(WriteOp::Delete) => WriteBatchLookup::Delete, + Some(WriteOp::Merge(_, _)) => WriteBatchLookup::Merge, + None => WriteBatchLookup::NotPresent, + } + } + pub(crate) fn has_merge_ops(&self) -> bool { self.has_merge_ops } diff --git a/slatedb/src/db.rs b/slatedb/src/db.rs index 31699c192..825c5d0f3 100644 --- a/slatedb/src/db.rs +++ b/slatedb/src/db.rs @@ -232,6 +232,32 @@ impl DbInner { .await } + pub(crate) async fn multi_get_with_options>( + &self, + keys: &[K], + options: &ReadOptions, + ) -> Result>, SlateDBError> { + self.multi_get_key_value_with_options(keys, options) + .await + .map(|values| values.into_iter().map(|kv| kv.map(|kv| kv.value)).collect()) + } + + pub(crate) async fn multi_get_key_value_with_options>( + &self, + keys: &[K], + options: &ReadOptions, + ) -> Result>, SlateDBError> { + self.check_closed()?; + let db_state = self.state.read().view(); + let keys = keys + .iter() + .map(|key| Bytes::copy_from_slice(key.as_ref())) + .collect::>(); + self.reader + .multi_get_key_value_with_options(&keys, options, &db_state, None) + .await + } + /// Shared scan path for plain range scans and prefix scans. When /// `prefix` is set, every key in `range` starts with it and prefix /// bloom filters are consulted to skip non-matching SSTs. @@ -967,6 +993,31 @@ impl Db { Ok(kv) } + /// Get multiple values from the database with default read options. + /// + /// The returned vector preserves input order and duplicates. + pub async fn multi_get + Send + Sync>( + &self, + keys: &[K], + ) -> Result>, crate::Error> { + self.multi_get_with_options(keys, &ReadOptions::default()) + .await + } + + /// Get multiple values from the database with custom read options. + /// + /// The returned vector preserves input order and duplicates. + pub async fn multi_get_with_options + Send + Sync>( + &self, + keys: &[K], + options: &ReadOptions, + ) -> Result>, crate::Error> { + self.inner + .multi_get_with_options(keys, options) + .await + .map_err(crate::Error::from) + } + /// Scan a range of keys using the default scan options. /// /// returns a `DbIterator` @@ -1912,6 +1963,17 @@ impl DbReadOps for Db { Db::get_key_value_with_options(self, key, options).await } + async fn multi_get_with_options( + &self, + keys: &[K], + options: &ReadOptions, + ) -> Result>, crate::Error> + where + K: AsRef<[u8]> + Send + Sync, + { + Db::multi_get_with_options(self, keys, options).await + } + async fn scan_with_options( &self, range: T, @@ -2315,6 +2377,38 @@ mod tests { kv_store.close().await.unwrap(); } + #[tokio::test] + async fn test_multi_get_matches_repeated_get_with_duplicates() { + let object_store: Arc = Arc::new(InMemory::new()); + let kv_store = Db::builder("/tmp/test_multi_get", object_store) + .with_settings(test_db_options(0, 1024, None)) + .build() + .await + .unwrap(); + + kv_store.put(b"k1", b"v1").await.unwrap(); + kv_store.put(b"k2", b"v2").await.unwrap(); + kv_store.put(b"k3", b"v3").await.unwrap(); + kv_store.flush().await.unwrap(); + + let keys: [&[u8]; 6] = [b"k1", b"missing", b"k2", b"k1", b"k3", b"missing"]; + let multi = kv_store.multi_get(&keys).await.unwrap(); + let mut repeated = Vec::with_capacity(keys.len()); + for key in keys { + repeated.push(kv_store.get(key).await.unwrap()); + } + + assert_eq!(multi, repeated); + assert_eq!(multi[0], Some(Bytes::from_static(b"v1"))); + assert_eq!(multi[1], None); + assert_eq!(multi[2], Some(Bytes::from_static(b"v2"))); + assert_eq!(multi[3], Some(Bytes::from_static(b"v1"))); + assert_eq!(multi[4], Some(Bytes::from_static(b"v3"))); + assert_eq!(multi[5], None); + + kv_store.close().await.unwrap(); + } + #[tokio::test] async fn test_manifest_returns_current_versioned_manifest() { let object_store: Arc = Arc::new(InMemory::new()); diff --git a/slatedb/src/db_reader.rs b/slatedb/src/db_reader.rs index 773be0f5a..a03bee957 100644 --- a/slatedb/src/db_reader.rs +++ b/slatedb/src/db_reader.rs @@ -320,7 +320,7 @@ impl DbStateReader for ReaderState { Arc::clone(&EMPTY_TABLE) } - fn imm_memtables(&self) -> Box> + '_> { + fn imm_memtables(&self) -> Box> + Send + '_> { Box::new(self.imm_memtable.iter()) } @@ -480,6 +480,33 @@ impl DbReaderInner { .await } + async fn multi_get_with_options + Send + Sync>( + &self, + keys: &[K], + options: &ReadOptions, + ) -> Result>, SlateDBError> { + self.multi_get_key_value_with_options(keys, options) + .await + .map(|values| values.into_iter().map(|kv| kv.map(|kv| kv.value)).collect()) + } + + async fn multi_get_key_value_with_options + Send + Sync>( + &self, + keys: &[K], + options: &ReadOptions, + ) -> Result>, SlateDBError> { + self.check_closed()?; + let db_state = Arc::clone(&self.state.read()); + let _permit = db_state.generation.acquire()?; + let keys = keys + .iter() + .map(|key| Bytes::copy_from_slice(key.as_ref())) + .collect::>(); + self.reader + .multi_get_key_value_with_options(&keys, options, db_state.as_ref(), None) + .await + } + pub(crate) async fn snapshot_get_key_value_with_options + Send>( &self, state: Arc, @@ -494,6 +521,26 @@ impl DbReaderInner { .await } + pub(crate) async fn snapshot_multi_get_key_value_with_options< + K: AsRef<[u8]> + Send + Sync, + >( + &self, + state: Arc, + max_seq: u64, + keys: &[K], + options: &ReadOptions, + ) -> Result>, SlateDBError> { + self.check_closed()?; + let _permit = state.generation.acquire()?; + let keys = keys + .iter() + .map(|key| Bytes::copy_from_slice(key.as_ref())) + .collect::>(); + self.reader + .multi_get_key_value_with_options(&keys, options, state.as_ref(), Some(max_seq)) + .await + } + async fn scan_with_options( &self, range: BytesRange, @@ -1530,6 +1577,31 @@ impl DbReader { Ok(kv) } + /// Get multiple values from the reader with default read options. + /// + /// The returned vector preserves input order and duplicates. + pub async fn multi_get + Send + Sync>( + &self, + keys: &[K], + ) -> Result>, crate::Error> { + self.multi_get_with_options(keys, &ReadOptions::default()) + .await + } + + /// Get multiple values from the reader with custom read options. + /// + /// The returned vector preserves input order and duplicates. + pub async fn multi_get_with_options + Send + Sync>( + &self, + keys: &[K], + options: &ReadOptions, + ) -> Result>, crate::Error> { + self.inner + .multi_get_with_options(keys, options) + .await + .map_err(Into::into) + } + /// Scan a range of keys using the default scan options. /// /// returns a `DbIterator` @@ -1764,6 +1836,17 @@ impl DbReadOps for DbReader { DbReader::get_key_value_with_options(self, key, options).await } + async fn multi_get_with_options( + &self, + keys: &[K], + options: &ReadOptions, + ) -> Result>, crate::Error> + where + K: AsRef<[u8]> + Send + Sync, + { + DbReader::multi_get_with_options(self, keys, options).await + } + async fn scan_with_options( &self, range: T, @@ -2789,6 +2872,9 @@ mod tests { db.put_with_options(b"key", b"v2", &PutOptions::default(), &write_options) .await .unwrap(); + db.put_with_options(b"new", b"v3", &PutOptions::default(), &write_options) + .await + .unwrap(); db.flush_with_options(FlushOptions { flush_type: FlushType::Wal, }) @@ -2807,6 +2893,23 @@ mod tests { assert_eq!(reader.get(b"key").await.unwrap(), Some(Bytes::from("v2"))); assert_eq!(snapshot.get(b"key").await.unwrap(), Some(Bytes::from("v1"))); + let keys = [&b"key"[..], &b"key"[..], &b"new"[..]]; + assert_eq!( + snapshot.multi_get(&keys).await.unwrap(), + vec![ + Some(Bytes::from_static(b"v1")), + Some(Bytes::from_static(b"v1")), + None, + ] + ); + assert_eq!( + reader.multi_get(&keys).await.unwrap(), + vec![ + Some(Bytes::from_static(b"v2")), + Some(Bytes::from_static(b"v2")), + Some(Bytes::from_static(b"v3")), + ] + ); assert_eq!( reader.snapshot().await.unwrap().get(b"key").await.unwrap(), Some(Bytes::from("v2")) diff --git a/slatedb/src/db_snapshot.rs b/slatedb/src/db_snapshot.rs index e99b1b24b..c8a6b3b49 100644 --- a/slatedb/src/db_snapshot.rs +++ b/slatedb/src/db_snapshot.rs @@ -132,6 +132,66 @@ impl DbSnapshot { } } + /// Get multiple values from the snapshot with default read options. + /// + /// The returned vector preserves input order and duplicates. + pub async fn multi_get + Send + Sync>( + &self, + keys: &[K], + ) -> Result>, crate::Error> { + self.multi_get_with_options(keys, &ReadOptions::default()) + .await + } + + /// Get multiple values from the snapshot with custom read options. + /// + /// The returned vector preserves input order and duplicates. + pub async fn multi_get_with_options + Send + Sync>( + &self, + keys: &[K], + options: &ReadOptions, + ) -> Result>, crate::Error> { + self.multi_get_key_value_with_options(keys, options) + .await + .map(|values| values.into_iter().map(|kv| kv.map(|kv| kv.value)).collect()) + } + + async fn multi_get_key_value_with_options + Send + Sync>( + &self, + keys: &[K], + options: &ReadOptions, + ) -> Result>, crate::Error> { + match &self.backend { + DbSnapshotBackend::Db { inner, .. } => { + inner.check_closed()?; + let db_state = inner.state.read().view(); + let keys = keys + .iter() + .map(|key| Bytes::copy_from_slice(key.as_ref())) + .collect::>(); + inner + .reader + .multi_get_key_value_with_options( + &keys, + options, + &db_state, + Some(self.started_seq), + ) + .await + .map_err(crate::Error::from) + } + DbSnapshotBackend::Reader { inner, state } => inner + .snapshot_multi_get_key_value_with_options( + Arc::clone(state), + self.started_seq, + keys, + options, + ) + .await + .map_err(crate::Error::from), + } + } + /// Scan a range of keys using the default scan options. /// /// ## Arguments @@ -280,6 +340,17 @@ impl DbReadOps for DbSnapshot { DbSnapshot::get_key_value_with_options(self, key, options).await } + async fn multi_get_with_options( + &self, + keys: &[K], + options: &ReadOptions, + ) -> Result>, crate::Error> + where + K: AsRef<[u8]> + Send + Sync, + { + DbSnapshot::multi_get_with_options(self, keys, options).await + } + async fn scan_with_options( &self, range: T, diff --git a/slatedb/src/db_state.rs b/slatedb/src/db_state.rs index 7287da5ad..f4070243c 100644 --- a/slatedb/src/db_state.rs +++ b/slatedb/src/db_state.rs @@ -686,7 +686,7 @@ impl DbStateReader for DbStateView { Arc::clone(&self.memtable) } - fn imm_memtables(&self) -> Box> + '_> { + fn imm_memtables(&self) -> Box> + Send + '_> { Box::new(self.state.imm_memtable.iter().cloned()) } diff --git a/slatedb/src/db_transaction.rs b/slatedb/src/db_transaction.rs index ff5d49492..004150e18 100644 --- a/slatedb/src/db_transaction.rs +++ b/slatedb/src/db_transaction.rs @@ -4,7 +4,7 @@ use std::collections::HashSet; use std::sync::Arc; use uuid::Uuid; -use crate::batch::{WriteBatch, WriteBatchIterator}; +use crate::batch::{WriteBatch, WriteBatchIterator, WriteBatchLookup}; use crate::bytes_range::{ByteRangeBounds, BytesRange}; use crate::config::{MergeOptions, PutOptions, ReadOptions, ScanOptions, WriteOptions}; use crate::db::DbInner; @@ -184,6 +184,148 @@ impl DbTransaction { Ok(kv) } + /// Get multiple values from the transaction with default read options. + /// This operation tracks all read keys for conflict detection in SSI mode. + pub async fn multi_get + Send + Sync>( + &self, + keys: &[K], + ) -> Result>, crate::Error> { + self.multi_get_with_options(keys, &ReadOptions::default()) + .await + } + + /// Get multiple values from the transaction with custom read options. + /// This operation tracks all read keys for conflict detection in SSI mode. + pub async fn multi_get_with_options + Send + Sync>( + &self, + keys: &[K], + options: &ReadOptions, + ) -> Result>, crate::Error> { + self.multi_get_key_value_with_options(keys, options) + .await + .map(|values| values.into_iter().map(|kv| kv.map(|kv| kv.value)).collect()) + } + + async fn multi_get_key_value_with_options + Send + Sync>( + &self, + keys: &[K], + options: &ReadOptions, + ) -> Result>, crate::Error> { + self.db_inner.check_closed()?; + + if self.isolation_level == IsolationLevel::SerializableSnapshot { + let read_keys = keys + .iter() + .map(|key| Bytes::copy_from_slice(key.as_ref())) + .collect::>(); + self.txn_manager.track_read_keys(&self.txn_id, read_keys); + } + + let db_state = self.db_inner.state.read().view(); + + let mut key_to_idx = std::collections::HashMap::::with_capacity(keys.len()); + let mut unique_keys = Vec::::with_capacity(keys.len()); + let mut output_positions = Vec::>::with_capacity(keys.len()); + + for (output_idx, key) in keys.iter().enumerate() { + let key = Bytes::copy_from_slice(key.as_ref()); + if let Some(existing_idx) = key_to_idx.get(&key).copied() { + output_positions[existing_idx].push(output_idx); + continue; + } + + let key_idx = unique_keys.len(); + key_to_idx.insert(key.clone(), key_idx); + unique_keys.push(key); + output_positions.push(vec![output_idx]); + } + + let mut resolved = vec![false; unique_keys.len()]; + let mut values = vec![None; unique_keys.len()]; + let mut fallback_to_point_get = vec![false; unique_keys.len()]; + + { + let write_batch = self.write_batch.read(); + for (key_idx, key) in unique_keys.iter().enumerate() { + match write_batch.lookup_latest_for_key(key.as_ref()) { + WriteBatchLookup::Put(value) => { + resolved[key_idx] = true; + values[key_idx] = Some(KeyValue { + key: key.clone(), + value, + seq: u64::MAX, + create_ts: 0, + expire_ts: None, + }); + } + WriteBatchLookup::Delete => { + resolved[key_idx] = true; + } + WriteBatchLookup::Merge => { + fallback_to_point_get[key_idx] = true; + } + WriteBatchLookup::NotPresent => {} + } + } + } + + let reader_key_indices = unique_keys + .iter() + .enumerate() + .filter_map(|(idx, _)| (!resolved[idx] && !fallback_to_point_get[idx]).then_some(idx)) + .collect::>(); + if !reader_key_indices.is_empty() { + let reader_keys = reader_key_indices + .iter() + .map(|idx| unique_keys[*idx].clone()) + .collect::>(); + let reader_values = self + .db_inner + .reader + .multi_get_key_value_with_options( + &reader_keys, + options, + &db_state, + Some(self.started_seq), + ) + .await + .map_err(crate::Error::from)?; + for (key_idx, value) in reader_key_indices + .into_iter() + .zip(reader_values.into_iter()) + { + resolved[key_idx] = true; + values[key_idx] = value; + } + } + + for key_idx in fallback_to_point_get + .iter() + .enumerate() + .filter_map(|(idx, should_fallback)| should_fallback.then_some(idx)) + { + values[key_idx] = self + .get_key_value_with_options(unique_keys[key_idx].as_ref(), options) + .await?; + resolved[key_idx] = true; + } + + let mut result = vec![None; keys.len()]; + for (key_idx, positions) in output_positions.into_iter().enumerate() { + let value = if resolved[key_idx] { + values[key_idx].clone() + } else { + None + }; + + for position in positions { + result[position] = value.clone(); + } + } + + Ok(result) + } + /// Scan a range of keys using the default scan options. /// This operation will track the read range for conflict detection in SSI mode. /// @@ -375,6 +517,25 @@ impl DbTransaction { Ok(()) } + /// Put an owned key-value pair into the transaction without copying the + /// value bytes into the transaction write batch. + pub fn put_bytes(&self, key: Bytes, value: Bytes) -> Result<(), crate::Error> { + self.put_bytes_with_options(key, value, &PutOptions::default()) + } + + /// Put an owned key-value pair into the transaction with custom options. + pub fn put_bytes_with_options( + &self, + key: Bytes, + value: Bytes, + options: &PutOptions, + ) -> Result<(), crate::Error> { + self.write_batch + .write() + .put_bytes_with_options(key, value, options); + Ok(()) + } + /// Mark keys as read for conflict detection. /// /// This method explicitly tracks read operations for conflict detection. When keys are @@ -642,6 +803,17 @@ impl DbReadOps for DbTransaction { DbTransaction::get_key_value_with_options(self, key, options).await } + async fn multi_get_with_options( + &self, + keys: &[K], + options: &ReadOptions, + ) -> Result>, crate::Error> + where + K: AsRef<[u8]> + Send + Sync, + { + DbTransaction::multi_get_with_options(self, keys, options).await + } + async fn scan_with_options( &self, range: T, @@ -682,6 +854,15 @@ impl DbTransactionOps for DbTransaction { DbTransaction::put_with_options(self, key, value, options) } + fn put_bytes_with_options( + &self, + key: Bytes, + value: Bytes, + options: &PutOptions, + ) -> Result<(), crate::Error> { + DbTransaction::put_bytes_with_options(self, key, value, options) + } + fn delete>(&self, key: K) -> Result<(), crate::Error> { DbTransaction::delete(self, key) } @@ -839,6 +1020,135 @@ mod tests { txn.commit().await.unwrap(); } + #[tokio::test] + async fn test_txn_put_bytes_read_your_writes() { + let object_store: Arc = Arc::new(InMemory::new()); + let db = crate::Db::open("test_txn_put_bytes", object_store) + .await + .unwrap(); + + let txn = db.begin(IsolationLevel::Snapshot).await.unwrap(); + txn.put_bytes( + Bytes::from_static(b"bytes_key"), + Bytes::from_static(b"bytes_value"), + ) + .unwrap(); + + assert_eq!( + txn.get(b"bytes_key").await.unwrap(), + Some(Bytes::from_static(b"bytes_value")) + ); + txn.commit().await.unwrap(); + + assert_eq!( + db.get(b"bytes_key").await.unwrap(), + Some(Bytes::from_static(b"bytes_value")) + ); + } + + #[tokio::test] + async fn test_txn_multi_get_read_your_writes() { + let object_store: Arc = Arc::new(InMemory::new()); + let db = crate::Db::open("test_txn_multi_get_read_your_writes", object_store) + .await + .unwrap(); + + db.put(b"k1", b"db_v1").await.unwrap(); + + let txn = db + .begin(IsolationLevel::SerializableSnapshot) + .await + .unwrap(); + txn.put(b"k1", b"txn_v1").unwrap(); + txn.put_bytes(Bytes::from_static(b"k2"), Bytes::from_static(b"txn_v2")) + .unwrap(); + + let keys: [&[u8]; 4] = [b"k1", b"k2", b"missing", b"k1"]; + let values = txn.multi_get(&keys).await.unwrap(); + + assert_eq!(values[0], Some(Bytes::from_static(b"txn_v1"))); + assert_eq!(values[1], Some(Bytes::from_static(b"txn_v2"))); + assert_eq!(values[2], None); + assert_eq!(values[3], Some(Bytes::from_static(b"txn_v1"))); + + txn.commit().await.unwrap(); + } + + #[tokio::test] + async fn test_txn_multi_get_matches_get_for_pending_put_delete_and_isolation() { + let object_store: Arc = Arc::new(InMemory::new()); + let db = crate::Db::open("test_txn_multi_get_pending_overlay", object_store) + .await + .unwrap(); + + db.put(b"k1", b"db_v1").await.unwrap(); + db.put(b"k2", b"db_v2").await.unwrap(); + db.put(b"k3", b"db_v3").await.unwrap(); + + let txn = db.begin(IsolationLevel::Snapshot).await.unwrap(); + txn.put(b"k1", b"txn_v1").unwrap(); + txn.delete(b"k2").unwrap(); + txn.put(b"k4", b"txn_v4").unwrap(); + + let keys: [&[u8]; 7] = [b"k1", b"k2", b"k3", b"k4", b"missing", b"k1", b"k2"]; + let multi = txn.multi_get(&keys).await.unwrap(); + let mut single = Vec::with_capacity(keys.len()); + for key in keys { + single.push(txn.get(key).await.unwrap()); + } + + assert_eq!(multi, single); + assert_eq!(multi[0], Some(Bytes::from_static(b"txn_v1"))); + assert_eq!(multi[1], None); + assert_eq!(multi[2], Some(Bytes::from_static(b"db_v3"))); + assert_eq!(multi[3], Some(Bytes::from_static(b"txn_v4"))); + assert_eq!(multi[4], None); + assert_eq!(multi[5], Some(Bytes::from_static(b"txn_v1"))); + assert_eq!(multi[6], None); + + let concurrent = db.begin(IsolationLevel::Snapshot).await.unwrap(); + let concurrent_values = concurrent + .multi_get(&[b"k1".as_ref(), b"k2".as_ref(), b"k4".as_ref()]) + .await + .unwrap(); + assert_eq!(concurrent_values[0], Some(Bytes::from_static(b"db_v1"))); + assert_eq!(concurrent_values[1], Some(Bytes::from_static(b"db_v2"))); + assert_eq!(concurrent_values[2], None); + concurrent.commit().await.unwrap(); + + txn.commit().await.unwrap(); + } + + #[tokio::test] + async fn test_txn_multi_get_tracks_reads_for_serializable_conflicts() { + let object_store: Arc = Arc::new(InMemory::new()); + let db = crate::Db::open("test_txn_multi_get_conflicts", object_store) + .await + .unwrap(); + + db.put(b"k1", b"db_v1").await.unwrap(); + + let txn1 = db + .begin(IsolationLevel::SerializableSnapshot) + .await + .unwrap(); + let keys: [&[u8]; 3] = [b"k1", b"k1", b"missing"]; + let values = txn1.multi_get(&keys).await.unwrap(); + assert_eq!(values[0], Some(Bytes::from_static(b"db_v1"))); + assert_eq!(values[1], Some(Bytes::from_static(b"db_v1"))); + assert_eq!(values[2], None); + + let txn2 = db + .begin(IsolationLevel::SerializableSnapshot) + .await + .unwrap(); + txn2.put(b"k1", b"db_v2").unwrap(); + txn2.commit().await.unwrap(); + + txn1.put(b"k2", b"txn1_write").unwrap(); + assert!(txn1.commit().await.is_err()); + } + #[tokio::test] async fn test_txn_si_commit_conflict() { // Setup database with initial data diff --git a/slatedb/src/ops.rs b/slatedb/src/ops.rs index dcdd1cd40..40dfc6b39 100644 --- a/slatedb/src/ops.rs +++ b/slatedb/src/ops.rs @@ -67,6 +67,38 @@ pub trait DbReadOps { options: &ReadOptions, ) -> Result, crate::Error>; + /// Get multiple values from the database with default read options. + /// + /// The returned vector preserves the same order as the input keys, + /// including duplicates. + async fn multi_get(&self, keys: &[K]) -> Result>, crate::Error> + where + K: AsRef<[u8]> + Send + Sync, + { + self.multi_get_with_options(keys, &ReadOptions::default()) + .await + } + + /// Get multiple values from the database with custom read options. + /// + /// The default implementation is correctness-oriented and delegates to + /// repeated point reads. Concrete database handles override this with a + /// batched implementation. + async fn multi_get_with_options( + &self, + keys: &[K], + options: &ReadOptions, + ) -> Result>, crate::Error> + where + K: AsRef<[u8]> + Send + Sync, + { + let mut values = Vec::with_capacity(keys.len()); + for key in keys { + values.push(self.get_with_options(key, options).await?); + } + Ok(values) + } + /// Get a key-value pair from the database with default read options. /// /// Returns the key along with its value and metadata (sequence number, @@ -425,6 +457,22 @@ pub trait DbTransactionOps: DbReadOps { K: AsRef<[u8]>, V: AsRef<[u8]>; + /// Put an owned key-value pair into the transaction with default + /// `PutOptions`, avoiding the copies that [`Self::put`] performs when the + /// caller already has owned [`Bytes`]. + fn put_bytes(&self, key: Bytes, value: Bytes) -> Result<(), crate::Error> { + self.put_bytes_with_options(key, value, &PutOptions::default()) + } + + /// Put an owned key-value pair into the transaction with custom + /// `PutOptions`. + fn put_bytes_with_options( + &self, + key: Bytes, + value: Bytes, + options: &PutOptions, + ) -> Result<(), crate::Error>; + /// Delete a key from the transaction. The delete is buffered in the /// transaction's write batch until commit. fn delete>(&self, key: K) -> Result<(), crate::Error>; diff --git a/slatedb/src/reader.rs b/slatedb/src/reader.rs index b1faf711f..b9e436a98 100644 --- a/slatedb/src/reader.rs +++ b/slatedb/src/reader.rs @@ -1,30 +1,34 @@ use crate::batch::WriteBatchIterator; +use crate::block_iterator::DataBlockIterator; use crate::bytes_range::BytesRange; use crate::clock::MonotonicClock; use crate::config::{DurabilityLevel, ReadOptions, ScanOptions}; use crate::db_iter::{apply_filters, DbRecencyIterator}; use crate::db_stats::DbStats; +use crate::filter_policy::FilterQuery; +use crate::format::block::Block; use crate::iter::RowEntryIterator; use crate::manifest::{ManifestCore, Segment}; use crate::mem_table::{ImmutableMemtable, KVTable}; use crate::merge_operator::{instrument_merge_operator, MergeOperatorType}; use crate::oracle::Oracle; +use crate::partitioned_keyspace; use crate::segment_iterator::{build_segment_iter, SegmentScanContext}; use crate::sorted_run_iterator::SortedRunIterator; use crate::sst_iter::{SstIterator, SstIteratorOptions}; use crate::tablestore::TableStore; -use crate::types::KeyValue; +use crate::types::{KeyValue, RowEntry, ValueDeletable}; use crate::{error::SlateDBError, DbIterator}; use bytes::Bytes; -use std::collections::VecDeque; +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use std::sync::Arc; pub(crate) trait DbStateReader { fn memtable(&self) -> Arc; /// Returns immutable memtables newest-first. The iterator form permits /// read-only replicas to use a structurally shared persistent chain. - fn imm_memtables(&self) -> Box> + '_>; + fn imm_memtables(&self) -> Box> + Send + '_>; fn core(&self) -> &ManifestCore; } @@ -261,6 +265,549 @@ impl Reader { .transpose() } + pub(crate) async fn multi_get_key_value_with_options( + &self, + keys: &[Bytes], + options: &ReadOptions, + db_state: &(dyn DbStateReader + Sync + Send), + max_seq: Option, + ) -> Result>, SlateDBError> { + self.db_stats.get_requests.increment(1); + if keys.is_empty() { + return Ok(Vec::new()); + } + + let prepared_max_seq = + self.prepare_max_seq(max_seq, options.durability_filter, options.dirty); + let merge_operator_enabled = self.read_merge_operator.is_some(); + + let mut key_to_idx = HashMap::::with_capacity(keys.len()); + let mut unique_keys = Vec::::with_capacity(keys.len()); + let mut output_positions = Vec::>::with_capacity(keys.len()); + + for (output_idx, key) in keys.iter().enumerate() { + if let Some(existing_idx) = key_to_idx.get(key).copied() { + output_positions[existing_idx].push(output_idx); + continue; + } + + let key_idx = unique_keys.len(); + key_to_idx.insert(key.clone(), key_idx); + unique_keys.push(key.clone()); + output_positions.push(vec![output_idx]); + } + + let mut resolved = vec![false; unique_keys.len()]; + let mut values = vec![None; unique_keys.len()]; + let mut fallback_to_point_get = vec![false; unique_keys.len()]; + + self.resolve_rows_from_memtable_for_keys( + db_state.memtable(), + &unique_keys, + prepared_max_seq, + merge_operator_enabled, + &mut resolved, + &mut values, + &mut fallback_to_point_get, + ) + .await?; + + for imm in db_state.imm_memtables() { + if Self::all_done(&resolved, &fallback_to_point_get) { + break; + } + + self.resolve_rows_from_memtable_for_keys( + imm.table(), + &unique_keys, + prepared_max_seq, + merge_operator_enabled, + &mut resolved, + &mut values, + &mut fallback_to_point_get, + ) + .await?; + } + + self.resolve_rows_from_segments_for_keys( + db_state.core(), + &unique_keys, + prepared_max_seq, + options, + merge_operator_enabled, + &mut resolved, + &mut values, + &mut fallback_to_point_get, + ) + .await?; + + for key_idx in fallback_to_point_get + .iter() + .enumerate() + .filter_map(|(idx, should_fallback)| should_fallback.then_some(idx)) + { + values[key_idx] = self + .get_key_value_with_options( + unique_keys[key_idx].as_ref(), + options, + db_state, + None, + max_seq, + ) + .await?; + resolved[key_idx] = true; + } + + let mut result = vec![None; keys.len()]; + for (key_idx, positions) in output_positions.into_iter().enumerate() { + let value = if resolved[key_idx] { + values[key_idx].clone() + } else { + None + }; + + for position in positions { + result[position] = value.clone(); + } + } + + Ok(result) + } + + fn all_done(resolved: &[bool], fallback_to_point_get: &[bool]) -> bool { + resolved + .iter() + .zip(fallback_to_point_get) + .all(|(resolved, fallback)| *resolved || *fallback) + } + + async fn resolve_rows_from_memtable_for_keys( + &self, + table: Arc, + unique_keys: &[Bytes], + max_seq: Option, + merge_operator_enabled: bool, + resolved: &mut [bool], + values: &mut [Option], + fallback_to_point_get: &mut [bool], + ) -> Result<(), SlateDBError> { + if table.is_empty() { + return Ok(()); + } + + for (key_idx, key) in unique_keys.iter().enumerate() { + if resolved[key_idx] || fallback_to_point_get[key_idx] { + continue; + } + + let mut iter = table.range_ascending(key.clone()..=key.clone()); + while let Some(row) = iter.next().await? { + if !Self::row_visible(&row, max_seq) { + continue; + } + + Self::apply_source_row_result( + key_idx, + row, + merge_operator_enabled, + resolved, + values, + fallback_to_point_get, + )?; + break; + } + } + + Ok(()) + } + + async fn resolve_rows_from_segments_for_keys( + &self, + core: &ManifestCore, + unique_keys: &[Bytes], + max_seq: Option, + options: &ReadOptions, + merge_operator_enabled: bool, + resolved: &mut [bool], + values: &mut [Option], + fallback_to_point_get: &mut [bool], + ) -> Result<(), SlateDBError> { + if Self::all_done(resolved, fallback_to_point_get) { + return Ok(()); + } + + let mut groups = BTreeMap::)>::new(); + for (key_idx, key) in unique_keys.iter().enumerate() { + if resolved[key_idx] || fallback_to_point_get[key_idx] { + continue; + } + + let range = BytesRange::from_slice(key.as_ref()..=key.as_ref()); + let segment = match core.select_segments(&range) { + None => core.default_segment(), + Some(segments) => match segments.first() { + Some(segment) => segment.clone(), + None => continue, + }, + }; + + groups + .entry(segment.prefix.clone()) + .or_insert_with(|| (segment, Vec::new())) + .1 + .push(key_idx); + } + + for (_, (segment, key_indices)) in groups { + self.resolve_rows_from_lsm_tree_for_keys( + &segment, + &key_indices, + unique_keys, + max_seq, + options, + merge_operator_enabled, + resolved, + values, + fallback_to_point_get, + ) + .await?; + } + + Ok(()) + } + + async fn resolve_rows_from_lsm_tree_for_keys( + &self, + segment: &Segment, + key_indices: &[usize], + unique_keys: &[Bytes], + max_seq: Option, + options: &ReadOptions, + merge_operator_enabled: bool, + resolved: &mut [bool], + values: &mut [Option], + fallback_to_point_get: &mut [bool], + ) -> Result<(), SlateDBError> { + let mut unresolved = key_indices + .iter() + .copied() + .filter(|key_idx| !resolved[*key_idx] && !fallback_to_point_get[*key_idx]) + .collect::>(); + + for sst in segment.tree.l0.iter() { + if unresolved.is_empty() { + return Ok(()); + } + + self.resolve_rows_from_sst_for_keys( + sst, + &unresolved, + unique_keys, + max_seq, + options, + merge_operator_enabled, + resolved, + values, + fallback_to_point_get, + ) + .await?; + unresolved.retain(|idx| !resolved[*idx] && !fallback_to_point_get[*idx]); + } + + for sr in segment.tree.compacted.iter() { + if unresolved.is_empty() { + return Ok(()); + } + + self.resolve_rows_from_sorted_run_for_keys( + sr, + &unresolved, + unique_keys, + max_seq, + options, + merge_operator_enabled, + resolved, + values, + fallback_to_point_get, + ) + .await?; + unresolved.retain(|idx| !resolved[*idx] && !fallback_to_point_get[*idx]); + } + + Ok(()) + } + + async fn resolve_rows_from_sorted_run_for_keys( + &self, + sorted_run: &crate::db_state::SortedRun, + key_indices: &[usize], + unique_keys: &[Bytes], + max_seq: Option, + options: &ReadOptions, + merge_operator_enabled: bool, + resolved: &mut [bool], + values: &mut [Option], + fallback_to_point_get: &mut [bool], + ) -> Result<(), SlateDBError> { + let mut groups = BTreeMap::)>::new(); + for key_idx in key_indices.iter().copied() { + if resolved[key_idx] || fallback_to_point_get[key_idx] { + continue; + } + + for sst in sorted_run.tables_covering_point_key(unique_keys[key_idx].as_ref()) { + groups + .entry(sst.id) + .or_insert_with(|| (sst.clone(), Vec::new())) + .1 + .push(key_idx); + } + } + + for (_, (sst, group_key_indices)) in groups { + self.resolve_rows_from_sst_for_keys( + &sst, + &group_key_indices, + unique_keys, + max_seq, + options, + merge_operator_enabled, + resolved, + values, + fallback_to_point_get, + ) + .await?; + } + + Ok(()) + } + + async fn resolve_rows_from_sst_for_keys( + &self, + sst: &crate::db_state::SsTableView, + key_indices: &[usize], + unique_keys: &[Bytes], + max_seq: Option, + options: &ReadOptions, + merge_operator_enabled: bool, + resolved: &mut [bool], + values: &mut [Option], + fallback_to_point_get: &mut [bool], + ) -> Result<(), SlateDBError> { + let mut candidate_key_indices = Vec::with_capacity(key_indices.len()); + let filters = self.table_store.read_filters(&sst.sst, true).await?; + + for &key_idx in key_indices { + if resolved[key_idx] || fallback_to_point_get[key_idx] { + continue; + } + + let key = unique_keys[key_idx].as_ref(); + if sst + .calculate_view_range(BytesRange::from_slice(key..=key)) + .is_none() + { + continue; + } + + if filters.is_empty() { + candidate_key_indices.push(key_idx); + continue; + } + + let query = FilterQuery::point(unique_keys[key_idx].clone()) + .with_context(options.filter_context.clone()); + if filters.iter().all(|named| named.filter.might_match(&query)) { + self.db_stats.sst_filter_point_positives.increment(1); + candidate_key_indices.push(key_idx); + } else { + self.db_stats.sst_filter_point_negatives.increment(1); + } + } + + if candidate_key_indices.is_empty() { + return Ok(()); + } + + let index = self.table_store.read_index(&sst.sst, true).await?; + let block_to_key_indices = { + let index_ref = index.borrow(); + let mut block_to_key_indices = BTreeMap::>::new(); + for &key_idx in &candidate_key_indices { + let key = unique_keys[key_idx].as_ref(); + let start_block = + partitioned_keyspace::first_partition_including_or_after_key(&index_ref, key); + let end_block_exclusive = + partitioned_keyspace::last_partition_including_key(&index_ref, key) + .map(|last| last + 1) + .unwrap_or(start_block); + + if start_block >= end_block_exclusive { + continue; + } + + for block_idx in start_block..end_block_exclusive { + block_to_key_indices + .entry(block_idx) + .or_default() + .push(key_idx); + } + } + block_to_key_indices + }; + + if block_to_key_indices.is_empty() { + return Ok(()); + } + + let mut block_ranges = Vec::new(); + let mut range_start = None; + let mut previous_block = 0; + for &block_idx in block_to_key_indices.keys() { + if let Some(start) = range_start { + if block_idx == previous_block + 1 { + previous_block = block_idx; + } else { + block_ranges.push(start..(previous_block + 1)); + range_start = Some(block_idx); + previous_block = block_idx; + } + } else { + range_start = Some(block_idx); + previous_block = block_idx; + } + } + if let Some(start) = range_start { + block_ranges.push(start..(previous_block + 1)); + } + + let mut found_in_sst = HashSet::::new(); + for block_range in block_ranges { + let blocks = self + .table_store + .read_blocks_using_index( + &sst.sst, + index.clone(), + block_range.clone(), + options.cache_blocks, + ) + .await?; + + for (offset, block) in blocks.into_iter().enumerate() { + let block_idx = block_range.start + offset; + let Some(key_idxs) = block_to_key_indices.get(&block_idx) else { + continue; + }; + + self.resolve_rows_from_block_for_keys( + block, + sst.sst.format_version, + key_idxs, + unique_keys, + max_seq, + merge_operator_enabled, + resolved, + values, + fallback_to_point_get, + &mut found_in_sst, + ) + .await?; + } + } + + if !filters.is_empty() { + let false_positives = candidate_key_indices + .iter() + .filter(|key_idx| !found_in_sst.contains(key_idx)) + .count() as u64; + if false_positives > 0 { + self.db_stats + .sst_filter_point_false_positives + .increment(false_positives); + } + } + + Ok(()) + } + + async fn resolve_rows_from_block_for_keys( + &self, + block: Arc, + sst_version: u16, + key_indices: &[usize], + unique_keys: &[Bytes], + max_seq: Option, + merge_operator_enabled: bool, + resolved: &mut [bool], + values: &mut [Option], + fallback_to_point_get: &mut [bool], + found_in_sst: &mut HashSet, + ) -> Result<(), SlateDBError> { + let key_lookup = key_indices + .iter() + .copied() + .map(|key_idx| (unique_keys[key_idx].clone(), key_idx)) + .collect::>(); + + let mut iter = + DataBlockIterator::new(block, sst_version, crate::iter::IterationOrder::Ascending)?; + while let Some(row) = iter.next().await? { + let Some(&key_idx) = key_lookup.get(&row.key) else { + continue; + }; + if found_in_sst.contains(&key_idx) || !Self::row_visible(&row, max_seq) { + continue; + } + + found_in_sst.insert(key_idx); + Self::apply_source_row_result( + key_idx, + row, + merge_operator_enabled, + resolved, + values, + fallback_to_point_get, + )?; + } + + Ok(()) + } + + fn row_visible(row: &RowEntry, max_seq: Option) -> bool { + match max_seq { + Some(max_seq) => row.seq <= max_seq, + None => true, + } + } + + fn apply_source_row_result( + key_idx: usize, + row: RowEntry, + merge_operator_enabled: bool, + resolved: &mut [bool], + values: &mut [Option], + fallback_to_point_get: &mut [bool], + ) -> Result<(), SlateDBError> { + match &row.value { + ValueDeletable::Value(_) => { + resolved[key_idx] = true; + values[key_idx] = Some(KeyValue::from(row)); + } + ValueDeletable::Tombstone => { + resolved[key_idx] = true; + values[key_idx] = None; + } + ValueDeletable::Merge(_) => { + if !merge_operator_enabled { + return Err(SlateDBError::MergeOperatorMissing); + } + fallback_to_point_get[key_idx] = true; + } + } + + Ok(()) + } + /// Create an iterator over a key range. /// /// Produces a merged iterator over the provided `write_batch` (if any), @@ -615,7 +1162,9 @@ mod tests { self.memtable.clone() } - fn imm_memtables(&self) -> Box> + '_> { + fn imm_memtables( + &self, + ) -> Box> + Send + '_> { Box::new(self.imm_memtable.iter().cloned()) } From 4e3c5df0b529d9198837769ef7bda6bb0e65daf6 Mon Sep 17 00:00:00 2001 From: Matthew Sanetra <41018997+matthewsanetra@users.noreply.github.com> Date: Thu, 16 Jul 2026 21:54:31 +0100 Subject: [PATCH 57/63] ci: drop Windows test runner --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 90f1a5f35..7c0961558 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -121,7 +121,7 @@ jobs: strategy: fail-fast: false matrix: - os: [macos-latest, windows-latest] + os: [macos-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 From d439a32e2c30f5e2658390c5ff02d9da9653bc20 Mon Sep 17 00:00:00 2001 From: xav-db Date: Wed, 29 Jul 2026 13:25:17 +0100 Subject: [PATCH 58/63] Add SlateDB cache usage snapshots --- Cargo.toml | 1 + slatedb/Cargo.toml | 3 +- .../src/cached_object_store/object_store.rs | 5 + slatedb/src/cached_object_store/storage.rs | 6 + slatedb/src/cached_object_store/storage_fs.rs | 281 ++++++++++++++--- slatedb/src/db.rs | 25 ++ slatedb/src/db/builder.rs | 8 +- slatedb/src/db_cache/foyer.rs | 15 +- slatedb/src/db_cache/foyer_hybrid.rs | 287 +++++++++++++++++- slatedb/src/db_cache/mod.rs | 170 ++++++++++- slatedb/src/db_reader.rs | 26 ++ 11 files changed, 772 insertions(+), 55 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7953fb6ef..6b902ba0c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,6 +57,7 @@ log = "0.4.27" lru = "0.18" lz4_flex = "0.11.5" moka = "0.12.8" +mixtrics = "0.2" object_store = "0.14.0" ouroboros = "0.18" parking_lot = "0.12.4" diff --git a/slatedb/Cargo.toml b/slatedb/Cargo.toml index 217787355..d28d8a557 100644 --- a/slatedb/Cargo.toml +++ b/slatedb/Cargo.toml @@ -34,6 +34,7 @@ log = { workspace = true } lru = { workspace = true } lz4_flex = { workspace = true, optional = true } moka = { workspace = true, features = ["future"], optional = true } +mixtrics = { workspace = true, optional = true } object_store = { workspace = true } ouroboros = { workspace = true } parking_lot = { workspace = true } @@ -101,7 +102,7 @@ lz4 = ["dep:lz4_flex"] zstd = ["dep:zstd"] wal_disable = [] moka = ["dep:moka"] -foyer = ["dep:foyer"] +foyer = ["dep:foyer", "dep:mixtrics"] bench-internal = [] test-util = [ "tokio/test-util", diff --git a/slatedb/src/cached_object_store/object_store.rs b/slatedb/src/cached_object_store/object_store.rs index 110f964b8..cb860f64f 100644 --- a/slatedb/src/cached_object_store/object_store.rs +++ b/slatedb/src/cached_object_store/object_store.rs @@ -23,6 +23,7 @@ use std::{ops::Range, sync::Arc}; use crate::single_flight::SingleFlight; use crate::cached_object_store::storage::{LocalCacheStorage, PartID}; +use crate::db_cache::CacheUsageSnapshot; use crate::error::SlateDBError; use crate::utils::build_concurrent; use log::warn; @@ -73,6 +74,10 @@ pub struct CachedObjectStore { } impl CachedObjectStore { + pub(crate) fn usage_snapshot(&self) -> CacheUsageSnapshot { + self.cache_storage.usage_snapshot() + } + pub(crate) fn new( object_store: Arc, cache_storage: Arc, diff --git a/slatedb/src/cached_object_store/storage.rs b/slatedb/src/cached_object_store/storage.rs index 5ca2af16c..a487bd436 100644 --- a/slatedb/src/cached_object_store/storage.rs +++ b/slatedb/src/cached_object_store/storage.rs @@ -4,6 +4,8 @@ use object_store::{path::Path, Attribute, Attributes, ObjectMeta}; use serde::{Deserialize, Serialize}; use std::{collections::HashMap, fmt::Display, ops::Range}; +use crate::db_cache::CacheUsageSnapshot; + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct LocalCacheHead { pub location: String, @@ -74,6 +76,10 @@ pub trait LocalCacheStorage: Send + Sync + std::fmt::Debug + Display + 'static { fn entry(&self, location: &Path, part_size: usize) -> Box; async fn start_evictor(&self); + + fn usage_snapshot(&self) -> CacheUsageSnapshot { + CacheUsageSnapshot::Unavailable + } } #[async_trait] diff --git a/slatedb/src/cached_object_store/storage_fs.rs b/slatedb/src/cached_object_store/storage_fs.rs index 8a1500e53..b872c38a5 100644 --- a/slatedb/src/cached_object_store/storage_fs.rs +++ b/slatedb/src/cached_object_store/storage_fs.rs @@ -16,10 +16,11 @@ use std::ops::Range; use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering}; use std::sync::Arc; use std::time::Duration; -use tokio::sync::{Mutex, OnceCell}; +use tokio::sync::{Mutex, Notify, OnceCell}; use walkdir::WalkDir; use crate::cached_object_store::storage::{LocalCacheEntry, LocalCacheHead, LocalCacheStorage}; +use crate::db_cache::CacheUsageSnapshot; use crate::utils::format_bytes_si; /// A cached file handle node. Callers that obtain an `Arc` @@ -150,7 +151,7 @@ fn read_exact_at_offset(file: &std::fs::File, buf: &mut [u8], offset: u64) -> st #[derive(Debug)] pub struct FsCacheStorage { root_folder: std::path::PathBuf, - evictor: Option>, + evictor: Arc, rand: Arc, file_handle_cache: FileHandleCache, } @@ -166,17 +167,15 @@ impl FsCacheStorage { max_open_file_handles: usize, ) -> Self { let file_handle_cache = FileHandleCache::new(max_open_file_handles); - let evictor = max_cache_size_bytes.map(|max_cache_size_bytes| { - Arc::new(FsCacheEvictor::new( - root_folder.clone(), - max_cache_size_bytes, - scan_interval, - stats, - system_clock, - rand.clone(), - file_handle_cache.clone(), - )) - }); + let evictor = Arc::new(FsCacheEvictor::new( + root_folder.clone(), + max_cache_size_bytes, + scan_interval, + stats, + system_clock, + rand.clone(), + file_handle_cache.clone(), + )); Self { root_folder, @@ -202,7 +201,7 @@ impl LocalCacheStorage for FsCacheStorage { Box::new(FsCacheEntry { root_folder: self.root_folder.clone(), location: location.clone(), - evictor: self.evictor.clone(), + evictor: Some(self.evictor.clone()), part_size, rand: self.rand.clone(), file_handle_cache: self.file_handle_cache.clone(), @@ -210,9 +209,11 @@ impl LocalCacheStorage for FsCacheStorage { } async fn start_evictor(&self) { - if let Some(evictor) = &self.evictor { - evictor.start().await - } + self.evictor.start().await + } + + fn usage_snapshot(&self) -> CacheUsageSnapshot { + self.evictor.usage_snapshot() } } @@ -553,7 +554,7 @@ const QUEUE_FULL_LOG_INTERVAL_MS: i64 = 30_000; #[derive(Debug)] struct FsCacheEvictor { root_folder: std::path::PathBuf, - max_cache_size_bytes: usize, + max_cache_size_bytes: Option, scan_interval: Option, tx: tokio::sync::mpsc::Sender, rx: Mutex>>, @@ -566,12 +567,35 @@ struct FsCacheEvictor { system_clock: Arc, rand: Arc, file_handle_cache: FileHandleCache, + usage: Arc, + reconcile_notify: Arc, +} + +#[derive(Debug)] +struct FsCacheUsage { + initialized: AtomicBool, + used_bytes: AtomicU64, + capacity_bytes: Option, +} + +impl FsCacheUsage { + fn snapshot(&self) -> CacheUsageSnapshot { + if !self.initialized.load(Ordering::Acquire) { + return CacheUsageSnapshot::Initializing { + capacity_bytes: self.capacity_bytes, + }; + } + CacheUsageSnapshot::Ready { + used_bytes: self.used_bytes.load(Ordering::Acquire), + capacity_bytes: self.capacity_bytes, + } + } } impl FsCacheEvictor { fn new( root_folder: std::path::PathBuf, - max_cache_size_bytes: usize, + max_cache_size_bytes: Option, scan_interval: Option, stats: Arc, system_clock: Arc, @@ -594,16 +618,23 @@ impl FsCacheEvictor { system_clock, rand, file_handle_cache, + usage: Arc::new(FsCacheUsage { + initialized: AtomicBool::new(false), + used_bytes: AtomicU64::new(0), + capacity_bytes: max_cache_size_bytes.map(|bytes| bytes as u64), + }), + reconcile_notify: Arc::new(Notify::new()), } } async fn start(&self) { - let inner = Arc::new(FsCacheEvictorInner::new( + let inner = Arc::new(FsCacheEvictorInner::new_with_usage( self.root_folder.clone(), self.max_cache_size_bytes, self.stats.clone(), self.rand.clone(), self.file_handle_cache.clone(), + self.usage.clone(), )); let guard = self.rx.lock(); @@ -618,6 +649,7 @@ impl FsCacheEvictor { inner.clone(), self.scan_interval, self.system_clock.clone(), + self.reconcile_notify.clone(), ))) .ok(); @@ -635,6 +667,10 @@ impl FsCacheEvictor { self.started.load(Ordering::Acquire) } + fn usage_snapshot(&self) -> CacheUsageSnapshot { + self.usage.snapshot() + } + async fn background_evict( inner: Arc, mut rx: tokio::sync::mpsc::Receiver, @@ -666,14 +702,21 @@ impl FsCacheEvictor { inner: Arc, scan_interval: Option, system_clock: Arc, + reconcile_notify: Arc, ) { inner.clone().scan_entries(true).await; - if let Some(scan_interval) = scan_interval { - loop { - system_clock.clone().sleep(scan_interval).await; - inner.clone().scan_entries(true).await; + loop { + if let Some(scan_interval) = scan_interval { + let clock = system_clock.clone(); + tokio::select! { + () = clock.sleep(scan_interval) => {} + () = reconcile_notify.notified() => {} + } + } else { + reconcile_notify.notified().await; } + inner.clone().scan_entries(true).await; } } @@ -689,6 +732,8 @@ impl FsCacheEvictor { match self.tx.try_send((path, access)) { Ok(()) => true, Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => { + self.usage.initialized.store(false, Ordering::Release); + self.reconcile_notify.notify_one(); self.queue_full_count.fetch_add(1, Ordering::AcqRel); let now_ms = self.system_clock.now().timestamp_millis(); let last_log_ms = self.last_queue_full_log_ms.load(Ordering::Acquire); @@ -706,7 +751,11 @@ impl FsCacheEvictor { } false } - Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => false, + Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => { + self.usage.initialized.store(false, Ordering::Release); + self.reconcile_notify.notify_one(); + false + } } } } @@ -750,29 +799,52 @@ impl CacheState { #[derive(Debug)] struct FsCacheEvictorInner { root_folder: std::path::PathBuf, - max_cache_size_bytes: usize, + max_cache_size_bytes: Option, track_lock: Mutex<()>, cache_state: Mutex, - cache_size_bytes: AtomicU64, + usage: Arc, stats: Arc, rand: Arc, file_handle_cache: FileHandleCache, } impl FsCacheEvictorInner { + #[cfg(test)] fn new( root_folder: std::path::PathBuf, max_cache_size_bytes: usize, stats: Arc, rand: Arc, file_handle_cache: FileHandleCache, + ) -> Self { + Self::new_with_usage( + root_folder, + Some(max_cache_size_bytes), + stats, + rand, + file_handle_cache, + Arc::new(FsCacheUsage { + initialized: AtomicBool::new(false), + used_bytes: AtomicU64::new(0), + capacity_bytes: Some(max_cache_size_bytes as u64), + }), + ) + } + + fn new_with_usage( + root_folder: std::path::PathBuf, + max_cache_size_bytes: Option, + stats: Arc, + rand: Arc, + file_handle_cache: FileHandleCache, + usage: Arc, ) -> Self { Self { root_folder, max_cache_size_bytes, track_lock: Mutex::new(()), cache_state: Mutex::new(CacheState::default()), - cache_size_bytes: AtomicU64::new(0_u64), + usage, stats, rand, file_handle_cache, @@ -797,6 +869,7 @@ impl FsCacheEvictorInner { .await .unwrap_or_default(); + let scanned_paths: HashSet<_> = paths.iter().cloned().collect(); for path in paths { let metadata = match tokio::fs::metadata(&path).await { Ok(metadata) => metadata, @@ -820,6 +893,34 @@ impl FsCacheEvictorInner { self.track_entry_accessed(path, bytes, atime, evict).await; } + + let tracked_paths = self.cache_state.lock().await.keys.clone(); + let mut missing_paths = Vec::new(); + for path in tracked_paths { + if !scanned_paths.contains(&path) + && !tokio::fs::try_exists(&path).await.unwrap_or(false) + { + missing_paths.push(path); + } + } + if !missing_paths.is_empty() { + let _track_guard = self.track_lock.lock().await; + let mut cache_state = self.cache_state.lock().await; + for path in missing_paths { + if let Some(removed) = cache_state.remove_entry(&path) { + self.usage + .used_bytes + .fetch_sub(removed.size_bytes as u64, Ordering::AcqRel); + } + } + self.stats + .object_store_cache_keys + .set(cache_state.entries.len() as i64); + self.stats + .object_store_cache_bytes + .set(self.usage.used_bytes.load(Ordering::Relaxed) as i64); + } + self.usage.initialized.store(true, Ordering::Release); } /// track the cache entry access, and evict the cache files when the cache size exceeds the limit if evict is true, @@ -840,6 +941,18 @@ impl FsCacheEvictorInner { match cache_state.entries.get_mut(&path) { Some(entry) => { entry.access_time = accessed_time; + if entry.size_bytes != bytes { + if bytes > entry.size_bytes { + self.usage + .used_bytes + .fetch_add((bytes - entry.size_bytes) as u64, Ordering::AcqRel); + } else { + self.usage + .used_bytes + .fetch_sub((entry.size_bytes - bytes) as u64, Ordering::AcqRel); + } + entry.size_bytes = bytes; + } } None => { let key_index = cache_state.keys.len(); @@ -852,7 +965,8 @@ impl FsCacheEvictorInner { key_index, }, ); - self.cache_size_bytes + self.usage + .used_bytes .fetch_add(bytes as u64, Ordering::SeqCst); } } @@ -862,10 +976,13 @@ impl FsCacheEvictorInner { self.stats.object_store_cache_keys.set(entry_count as i64); self.stats .object_store_cache_bytes - .set(self.cache_size_bytes.load(Ordering::Relaxed) as i64); + .set(self.usage.used_bytes.load(Ordering::Relaxed) as i64); + let Some(max_cache_size_bytes) = self.max_cache_size_bytes else { + return 0; + }; // if the cache size is still below the limit, do nothing - if self.cache_size_bytes.load(Ordering::Relaxed) <= self.max_cache_size_bytes as u64 { + if self.usage.used_bytes.load(Ordering::Relaxed) <= max_cache_size_bytes as u64 { return 0; } // TODO: check the disk space ratio here, if the disk space is low, also triggers evict. @@ -877,10 +994,10 @@ impl FsCacheEvictorInner { // It's ok to call evict after inserting the new entry, because we will evict entries with eailer `accessed_time`. // This ensures that the newly added entry will not be evicted immediately. let evicted_bytes: usize = if evict - && self.cache_size_bytes.load(Ordering::Relaxed) > self.max_cache_size_bytes as u64 + && self.usage.used_bytes.load(Ordering::Relaxed) > max_cache_size_bytes as u64 { // We sacrifice floating-point precision error to prevent possible overflow(i.e. self.max_cache_size_bytes * 9 / 10). - let target_size = ((self.max_cache_size_bytes as f64) * 0.9) as u64; + let target_size = ((max_cache_size_bytes as f64) * 0.9) as u64; self.evict_to_target_size(target_size).await } else { 0 @@ -896,11 +1013,11 @@ impl FsCacheEvictorInner { let picked_targets = self.pick_evict_targets(target_size).await; if picked_targets.is_empty() { - if self.cache_size_bytes.load(Ordering::Relaxed) > target_size { + if self.usage.used_bytes.load(Ordering::Relaxed) > target_size { warn!( "cache_size_bytes still exceeds max_cache_size_bytes but no more entries can be evicted(cache_size_bytes={}, max_cache_size_bytes={})", - format_bytes_si(self.cache_size_bytes.load(Ordering::Relaxed)), - format_bytes_si(self.max_cache_size_bytes as u64) + format_bytes_si(self.usage.used_bytes.load(Ordering::Relaxed)), + format_bytes_si(self.max_cache_size_bytes.unwrap_or_default() as u64) ); } return 0; @@ -944,7 +1061,8 @@ impl FsCacheEvictorInner { for (target, target_bytes) in deleted_targets.iter() { if cache_state.remove_entry(target).is_some() { - self.cache_size_bytes + self.usage + .used_bytes .fetch_sub(*target_bytes as u64, Ordering::SeqCst); total_bytes += target_bytes; } @@ -963,7 +1081,7 @@ impl FsCacheEvictorInner { self.stats.object_store_cache_keys.set(entry_count as i64); self.stats .object_store_cache_bytes - .set(self.cache_size_bytes.load(Ordering::Relaxed) as i64); + .set(self.usage.used_bytes.load(Ordering::Relaxed) as i64); total_evicted_bytes } @@ -981,7 +1099,8 @@ impl FsCacheEvictorInner { let mut cache_state = self.cache_state.lock().await; for deleted_entry in deleted_entries { if let Some(removed) = cache_state.remove_entry(&deleted_entry) { - self.cache_size_bytes + self.usage + .used_bytes .fetch_sub(removed.size_bytes as u64, Ordering::SeqCst); } } @@ -992,7 +1111,7 @@ impl FsCacheEvictorInner { self.stats.object_store_cache_keys.set(entry_count as i64); self.stats .object_store_cache_bytes - .set(self.cache_size_bytes.load(Ordering::Relaxed) as i64); + .set(self.usage.used_bytes.load(Ordering::Relaxed) as i64); } /// Pick multiple eviction targets in a single pass using pick-of-2 strategy, which is an approximation @@ -1009,7 +1128,7 @@ impl FsCacheEvictorInner { let mut targets = Vec::new(); // Track the simulated cache size during eviction but do not modify the actual cache size until // after files are deleted. - let mut simulated_size = self.cache_size_bytes.load(Ordering::Relaxed); + let mut simulated_size = self.usage.used_bytes.load(Ordering::Relaxed); // Track which indices have been selected for eviction let mut picked_indices: HashSet = HashSet::new(); @@ -1242,7 +1361,7 @@ mod tests { let evictor = FsCacheEvictor::new( temp_dir.path().to_path_buf(), - 1024, + Some(1024), None, Arc::new(CachedObjectStoreStats::new(&recorder)), Arc::new(DefaultSystemClock::new()), @@ -1321,13 +1440,87 @@ mod tests { // rescan two times, the cache size should be 2049 unchanged evictor.clone().scan_entries(false).await; - let cache_size_bytes = evictor.cache_size_bytes.load(Ordering::SeqCst); + let cache_size_bytes = evictor.usage.used_bytes.load(Ordering::SeqCst); assert_eq!(cache_size_bytes, 2049); evictor.clone().scan_entries(false).await; - let cache_size_bytes = evictor.cache_size_bytes.load(Ordering::SeqCst); + let cache_size_bytes = evictor.usage.used_bytes.load(Ordering::SeqCst); assert_eq!(cache_size_bytes, 2049); } + #[tokio::test] + async fn usage_snapshot_reconciles_bounded_and_unbounded_cache_files() { + let temp_dir = tempfile::Builder::new() + .prefix("objstore_cache_usage_snapshot_") + .tempdir() + .unwrap(); + let recorder = slatedb_common::metrics::MetricsRecorderHelper::noop(); + let usage = Arc::new(FsCacheUsage { + initialized: AtomicBool::new(false), + used_bytes: AtomicU64::new(0), + capacity_bytes: None, + }); + let evictor = Arc::new(FsCacheEvictorInner::new_with_usage( + temp_dir.path().to_path_buf(), + None, + Arc::new(CachedObjectStoreStats::new(&recorder)), + Arc::new(DbRand::default()), + FileHandleCache::new(1000), + usage.clone(), + )); + assert_eq!( + usage.snapshot(), + CacheUsageSnapshot::Initializing { + capacity_bytes: None, + } + ); + + let path = gen_rand_file(temp_dir.path(), "file", 17); + evictor.clone().scan_entries(false).await; + assert_eq!( + usage.snapshot(), + CacheUsageSnapshot::Ready { + used_bytes: 17, + capacity_bytes: None, + } + ); + + gen_rand_file(temp_dir.path(), "file", 29); + evictor.clone().scan_entries(false).await; + assert_eq!( + usage.snapshot(), + CacheUsageSnapshot::Ready { + used_bytes: 29, + capacity_bytes: None, + } + ); + + std::fs::remove_file(path).unwrap(); + evictor.scan_entries(false).await; + assert_eq!( + usage.snapshot(), + CacheUsageSnapshot::Ready { + used_bytes: 0, + capacity_bytes: None, + } + ); + + let bounded = FsCacheStorage::new( + temp_dir.path().to_path_buf(), + Some(1024), + None, + Arc::new(CachedObjectStoreStats::new(&recorder)), + Arc::new(DefaultSystemClock::new()), + Arc::new(DbRand::default()), + 10, + ); + assert_eq!( + bounded.usage_snapshot(), + CacheUsageSnapshot::Initializing { + capacity_bytes: Some(1024), + } + ); + } + #[rstest::rstest] // Basic case: 2 keys, nothing picked, no exclusion #[case(&[0, 1], &[], None, &[0, 1])] diff --git a/slatedb/src/db.rs b/slatedb/src/db.rs index 825c5d0f3..8442b9963 100644 --- a/slatedb/src/db.rs +++ b/slatedb/src/db.rs @@ -657,9 +657,34 @@ impl DbInner { pub struct Db { pub(crate) inner: Arc, task_executor: Arc, + pub(crate) object_store_cache: Option>, } impl Db { + /// Returns a synchronous point-in-time cache accounting snapshot. + /// + /// This method reads only in-memory counters and never performs object-store + /// or filesystem I/O. + pub fn cache_usage_snapshot(&self) -> crate::db_cache::SlateDbCacheUsageSnapshot { + let db_cache = self.inner.table_store.cache().map_or_else( + || crate::db_cache::DbCacheUsageSnapshot { + memory: crate::db_cache::CacheUsageSnapshot::Disabled, + disk: crate::db_cache::CacheUsageSnapshot::Disabled, + }, + |cache| cache.usage_snapshot(), + ); + let object_store = self + .object_store_cache + .as_ref() + .map_or(crate::db_cache::CacheUsageSnapshot::Disabled, |cache| { + cache.usage_snapshot() + }); + crate::db_cache::SlateDbCacheUsageSnapshot { + db_cache, + object_store, + } + } + /// Open a new database with default options. /// /// ## Arguments diff --git a/slatedb/src/db/builder.rs b/slatedb/src/db/builder.rs index 741e9a502..2e1dc32d8 100644 --- a/slatedb/src/db/builder.rs +++ b/slatedb/src/db/builder.rs @@ -821,9 +821,9 @@ impl> DbBuilder

{ inner.replay_wal(replay_range).await?; // Preload cache if enabled - if let Some(cached_obj_store) = cached_object_store { + if let Some(cached_obj_store) = &cached_object_store { inner - .preload_cache(&cached_obj_store, &path_resolver) + .preload_cache(cached_obj_store, &path_resolver) .await?; } @@ -831,6 +831,7 @@ impl> DbBuilder

{ Ok(Db { inner, task_executor, + object_store_cache: cached_object_store, }) } } @@ -1863,7 +1864,7 @@ impl> DbReaderBuilder

{ BlockCachePolicy::default(), )); - let reader = DbReader::open_internal( + let mut reader = DbReader::open_internal( manifest_store, table_store, self.mode, @@ -1880,6 +1881,7 @@ impl> DbReaderBuilder

{ if let Some(cached) = &maybe_cached { reader.preload_cache(cached, path).await?; } + reader.object_store_cache = maybe_cached; Ok(reader) } diff --git a/slatedb/src/db_cache/foyer.rs b/slatedb/src/db_cache/foyer.rs index 66d7e54a7..af83148bf 100644 --- a/slatedb/src/db_cache/foyer.rs +++ b/slatedb/src/db_cache/foyer.rs @@ -31,7 +31,10 @@ //! ``` //! -use crate::db_cache::{CacheLoader, CachedEntry, CachedKey, DbCache, DEFAULT_MAX_CAPACITY}; +use crate::db_cache::{ + CacheLoader, CacheUsageSnapshot, CachedEntry, CachedKey, DbCache, DbCacheUsageSnapshot, + DEFAULT_MAX_CAPACITY, +}; use crate::error::SlateDBError; use async_trait::async_trait; use std::sync::Arc; @@ -127,6 +130,16 @@ impl DbCache for FoyerCache { 0 } + fn usage_snapshot(&self) -> DbCacheUsageSnapshot { + DbCacheUsageSnapshot { + memory: CacheUsageSnapshot::Ready { + used_bytes: self.inner.usage() as u64, + capacity_bytes: Some(self.inner.capacity() as u64), + }, + disk: CacheUsageSnapshot::Unavailable, + } + } + async fn fetch_block( &self, key: CachedKey, diff --git a/slatedb/src/db_cache/foyer_hybrid.rs b/slatedb/src/db_cache/foyer_hybrid.rs index f40e9d898..773edf383 100644 --- a/slatedb/src/db_cache/foyer_hybrid.rs +++ b/slatedb/src/db_cache/foyer_hybrid.rs @@ -71,21 +71,236 @@ //! use crate::{ - db_cache::{CacheLoader, CachedEntry, CachedKey, DbCache}, + db_cache::{ + CacheLoader, CacheUsageSnapshot, CachedEntry, CachedKey, DbCache, DbCacheUsageSnapshot, + }, error::SlateDBError, utils::format_bytes_si, }; use async_trait::async_trait; use log::info; -use std::sync::Arc; +use mixtrics::{ + metrics::{ + BoxedCounterVec, BoxedGauge, BoxedGaugeVec, BoxedHistogramVec, GaugeOps, GaugeVecOps, + RegistryOps, + }, + registry::noop::NoopMetricsRegistry, +}; +use std::{ + borrow::Cow, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, +}; + +const FOYER_BLOCK_GAUGE: &str = "foyer_storage_block_engine_block"; +const FOYER_BLOCK_SIZE_GAUGE: &str = "foyer_storage_block_engine_block_size_bytes"; + +#[derive(Debug, Default)] +struct FoyerHybridCacheMetricValues { + clean_blocks: AtomicU64, + writing_blocks: AtomicU64, + evictable_blocks: AtomicU64, + reclaiming_blocks: AtomicU64, + block_size_bytes: AtomicU64, +} + +/// Retained Foyer block-engine gauges used for synchronous cache accounting. +/// +/// Install [`Self::registry`] on the same `HybridCacheBuilder` whose cache is +/// passed to [`FoyerHybridCache::new_with_cache_and_metrics`]. +#[derive(Clone, Debug, Default)] +pub struct FoyerHybridCacheMetrics { + values: Arc, +} + +impl FoyerHybridCacheMetrics { + /// Build an empty metrics handle. + pub fn new() -> Self { + Self::default() + } + + /// Build the registry passed to `HybridCacheBuilder::with_metrics_registry`. + pub fn registry(&self) -> mixtrics::metrics::BoxedRegistry { + Box::new(FoyerCacheMetricsRegistry { + values: Arc::clone(&self.values), + }) + } + + fn disk_usage_snapshot(&self) -> CacheUsageSnapshot { + let block_size_bytes = self.values.block_size_bytes.load(Ordering::Relaxed); + if block_size_bytes == 0 { + return CacheUsageSnapshot::Unavailable; + } + let clean_blocks = self.values.clean_blocks.load(Ordering::Relaxed); + let writing_blocks = self.values.writing_blocks.load(Ordering::Relaxed); + let evictable_blocks = self.values.evictable_blocks.load(Ordering::Relaxed); + let reclaiming_blocks = self.values.reclaiming_blocks.load(Ordering::Relaxed); + let used_blocks = writing_blocks + .saturating_add(evictable_blocks) + .saturating_add(reclaiming_blocks); + let total_blocks = clean_blocks.saturating_add(used_blocks); + CacheUsageSnapshot::Ready { + used_bytes: used_blocks.saturating_mul(block_size_bytes), + capacity_bytes: Some(total_blocks.saturating_mul(block_size_bytes)), + } + } +} + +#[derive(Debug)] +struct FoyerCacheMetricsRegistry { + values: Arc, +} + +impl RegistryOps for FoyerCacheMetricsRegistry { + fn register_counter_vec( + &self, + _name: Cow<'static, str>, + _desc: Cow<'static, str>, + _label_names: &'static [&'static str], + ) -> BoxedCounterVec { + Box::new(NoopMetricsRegistry) + } + + fn register_gauge_vec( + &self, + name: Cow<'static, str>, + _desc: Cow<'static, str>, + _label_names: &'static [&'static str], + ) -> BoxedGaugeVec { + let kind = match name.as_ref() { + FOYER_BLOCK_GAUGE => TrackedGaugeKind::BlockState, + FOYER_BLOCK_SIZE_GAUGE => TrackedGaugeKind::BlockSize, + _ => return Box::new(NoopMetricsRegistry), + }; + Box::new(FoyerTrackedGaugeVec { + kind, + values: Arc::clone(&self.values), + }) + } + + fn register_histogram_vec( + &self, + _name: Cow<'static, str>, + _desc: Cow<'static, str>, + _label_names: &'static [&'static str], + ) -> BoxedHistogramVec { + Box::new(NoopMetricsRegistry) + } + + fn register_histogram_vec_with_buckets( + &self, + _name: Cow<'static, str>, + _desc: Cow<'static, str>, + _label_names: &'static [&'static str], + _buckets: Vec, + ) -> BoxedHistogramVec { + Box::new(NoopMetricsRegistry) + } +} + +#[derive(Debug, Clone, Copy)] +enum TrackedGaugeKind { + BlockState, + BlockSize, +} + +#[derive(Debug)] +struct FoyerTrackedGaugeVec { + kind: TrackedGaugeKind, + values: Arc, +} + +impl GaugeVecOps for FoyerTrackedGaugeVec { + fn gauge(&self, labels: &[Cow<'static, str>]) -> BoxedGauge { + Box::new(AtomicGauge { + value: Arc::clone(&self.values), + field: match self.kind { + TrackedGaugeKind::BlockSize => AtomicGaugeField::BlockSize, + TrackedGaugeKind::BlockState => match labels.get(1).map(Cow::as_ref) { + Some("clean") => AtomicGaugeField::Clean, + Some("writing") => AtomicGaugeField::Writing, + Some("evictable") => AtomicGaugeField::Evictable, + Some("reclaiming") => AtomicGaugeField::Reclaiming, + _ => return Box::new(NoopMetricsRegistry), + }, + }, + }) + } +} + +#[derive(Debug, Clone, Copy)] +enum AtomicGaugeField { + Clean, + Writing, + Evictable, + Reclaiming, + BlockSize, +} + +#[derive(Debug)] +struct AtomicGauge { + value: Arc, + field: AtomicGaugeField, +} + +impl AtomicGauge { + fn atomic(&self) -> &AtomicU64 { + match self.field { + AtomicGaugeField::Clean => &self.value.clean_blocks, + AtomicGaugeField::Writing => &self.value.writing_blocks, + AtomicGaugeField::Evictable => &self.value.evictable_blocks, + AtomicGaugeField::Reclaiming => &self.value.reclaiming_blocks, + AtomicGaugeField::BlockSize => &self.value.block_size_bytes, + } + } +} + +impl GaugeOps for AtomicGauge { + fn increase(&self, value: u64) { + let _ = self + .atomic() + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + Some(current.saturating_add(value)) + }); + } + + fn decrease(&self, value: u64) { + let _ = self + .atomic() + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + Some(current.saturating_sub(value)) + }); + } + + fn absolute(&self, value: u64) { + self.atomic().store(value, Ordering::Relaxed); + } +} pub struct FoyerHybridCache { inner: foyer::HybridCache, + metrics: Option, } impl FoyerHybridCache { pub fn new_with_cache(cache: foyer::HybridCache) -> Self { - Self { inner: cache } + Self { + inner: cache, + metrics: None, + } + } + + /// Build a hybrid cache with retained block-engine accounting. + pub fn new_with_cache_and_metrics( + cache: foyer::HybridCache, + metrics: FoyerHybridCacheMetrics, + ) -> Self { + Self { + inner: cache, + metrics: Some(metrics), + } } } @@ -130,6 +345,21 @@ impl DbCache for FoyerHybridCache { 0 } + fn usage_snapshot(&self) -> DbCacheUsageSnapshot { + DbCacheUsageSnapshot { + memory: CacheUsageSnapshot::Ready { + used_bytes: self.inner.memory().usage() as u64, + capacity_bytes: Some(self.inner.memory().capacity() as u64), + }, + disk: self + .metrics + .as_ref() + .map_or(CacheUsageSnapshot::Unavailable, |metrics| { + metrics.disk_usage_snapshot() + }), + } + } + async fn close(&self) -> Result<(), crate::Error> { let memory_bytes = self.inner.memory().usage(); info!( @@ -203,8 +433,11 @@ impl FoyerHybridCache { #[cfg(test)] mod tests { - use crate::db_cache::foyer_hybrid::FoyerHybridCache; - use crate::db_cache::{CachedEntry, CachedKey, DbCache}; + use super::{ + FoyerCacheMetricsRegistry, FoyerHybridCache, FoyerHybridCacheMetrics, FOYER_BLOCK_GAUGE, + FOYER_BLOCK_SIZE_GAUGE, + }; + use crate::db_cache::{CacheUsageSnapshot, CachedEntry, CachedKey, DbCache}; use crate::db_state::SsTableId; use crate::format::sst::BlockBuilder; use foyer::{ @@ -217,6 +450,50 @@ mod tests { const SST_ID: SsTableId = SsTableId::Wal(123); + #[test] + fn tracks_pinned_foyer_block_metrics() { + use mixtrics::metrics::RegistryOps; + use std::borrow::Cow; + + let metrics = FoyerHybridCacheMetrics::new(); + let registry = FoyerCacheMetricsRegistry { + values: Arc::clone(&metrics.values), + }; + let block_states = registry.register_gauge_vec( + Cow::Borrowed(FOYER_BLOCK_GAUGE), + Cow::Borrowed(""), + &["name", "type"], + ); + block_states + .gauge(&[Cow::Borrowed("test"), Cow::Borrowed("clean")]) + .absolute(5); + block_states + .gauge(&[Cow::Borrowed("test"), Cow::Borrowed("writing")]) + .absolute(1); + block_states + .gauge(&[Cow::Borrowed("test"), Cow::Borrowed("evictable")]) + .absolute(2); + block_states + .gauge(&[Cow::Borrowed("test"), Cow::Borrowed("reclaiming")]) + .absolute(1); + registry + .register_gauge_vec( + Cow::Borrowed(FOYER_BLOCK_SIZE_GAUGE), + Cow::Borrowed(""), + &["name"], + ) + .gauge(&[Cow::Borrowed("test")]) + .absolute(4096); + + assert_eq!( + metrics.disk_usage_snapshot(), + CacheUsageSnapshot::Ready { + used_bytes: 4 * 4096, + capacity_bytes: Some(9 * 4096), + } + ); + } + #[tokio::test] async fn test_hybrid_cache() { let (cache, _dir) = setup().await; diff --git a/slatedb/src/db_cache/mod.rs b/slatedb/src/db_cache/mod.rs index 82fdaf538..dabde9786 100644 --- a/slatedb/src/db_cache/mod.rs +++ b/slatedb/src/db_cache/mod.rs @@ -45,6 +45,93 @@ pub const DEFAULT_MAX_CAPACITY: u64 = 64 * 1024 * 1024; pub const DEFAULT_BLOCK_CACHE_CAPACITY: u64 = 512 * 1024 * 1024; pub const DEFAULT_META_CACHE_CAPACITY: u64 = 128 * 1024 * 1024; +/// Point-in-time byte accounting for one cache tier. +/// +/// `Unavailable` is deliberately distinct from an empty ready cache. Callers +/// must not turn an unavailable measurement into zero. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum CacheUsageSnapshot { + /// The cache tier is not configured. + Disabled, + /// This cache implementation cannot currently provide byte accounting. + #[default] + Unavailable, + /// The cache is discovering its current contents. + Initializing { + /// Configured capacity, if the cache is bounded. + capacity_bytes: Option, + }, + /// The cache has a usable byte measurement. + Ready { + /// Bytes charged by the cache implementation. + used_bytes: u64, + /// Configured or usable capacity, absent for an unbounded cache. + capacity_bytes: Option, + }, +} + +impl CacheUsageSnapshot { + fn sum_enabled<'a>( + snapshots: impl Iterator, + ) -> CacheUsageSnapshot { + let mut found_enabled = false; + let mut initializing = false; + let mut used_bytes = 0_u64; + let mut capacity_bytes = Some(0_u64); + for snapshot in snapshots { + let (child_used, child_capacity) = match snapshot { + Self::Disabled => continue, + Self::Unavailable => return Self::Unavailable, + Self::Initializing { capacity_bytes } => { + found_enabled = true; + initializing = true; + (0, capacity_bytes) + } + Self::Ready { + used_bytes, + capacity_bytes, + } => { + found_enabled = true; + (*used_bytes, capacity_bytes) + } + }; + used_bytes = used_bytes.saturating_add(child_used); + capacity_bytes = match (capacity_bytes, child_capacity) { + (Some(total), Some(child)) => Some(total.saturating_add(*child)), + _ => None, + }; + } + if !found_enabled { + Self::Disabled + } else if initializing { + Self::Initializing { capacity_bytes } + } else { + Self::Ready { + used_bytes, + capacity_bytes, + } + } + } +} + +/// Point-in-time accounting for SlateDB's memory and optional disk cache tiers. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct DbCacheUsageSnapshot { + /// Cache-accounted resident memory. + pub memory: CacheUsageSnapshot, + /// Cache-accounted local disk. + pub disk: CacheUsageSnapshot, +} + +/// Point-in-time accounting for all cache tiers owned by one SlateDB handle. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct SlateDbCacheUsageSnapshot { + /// Block and metadata cache accounting. + pub db_cache: DbCacheUsageSnapshot, + /// Optional local object-store file cache accounting. + pub object_store: CacheUsageSnapshot, +} + /// Atomic counter to generate unique scope IDs for `DbCacheWrapper` instances. /// Scope `0` belongs exclusively to cache keys serialized before scoping was /// introduced, so live wrappers start at `1` and can never alias those entries. @@ -171,6 +258,11 @@ pub trait DbCache: Send + Sync { #[allow(dead_code)] fn entry_count(&self) -> u64; + /// Return current cache byte accounting without performing I/O. + fn usage_snapshot(&self) -> DbCacheUsageSnapshot { + DbCacheUsageSnapshot::default() + } + /// Gracefully close the cache, flushing any in-memory state to disk. /// /// Implementations backed by hybrid (memory + disk) caches should use @@ -577,6 +669,20 @@ impl DbCache for SplitCache { + self.meta_cache.as_ref().map_or(0, |c| c.entry_count()) } + fn usage_snapshot(&self) -> DbCacheUsageSnapshot { + let snapshots = [self.block_cache.as_ref(), self.meta_cache.as_ref()] + .into_iter() + .flatten() + .map(|cache| cache.usage_snapshot()) + .collect::>(); + DbCacheUsageSnapshot { + memory: CacheUsageSnapshot::sum_enabled( + snapshots.iter().map(|snapshot| &snapshot.memory), + ), + disk: CacheUsageSnapshot::sum_enabled(snapshots.iter().map(|snapshot| &snapshot.disk)), + } + } + async fn close(&self) -> Result<(), crate::Error> { if let Some(ref cache) = self.block_cache { cache.close().await?; @@ -830,6 +936,10 @@ impl DbCache for DbCacheWrapper { self.cache.entry_count() } + fn usage_snapshot(&self) -> DbCacheUsageSnapshot { + self.cache.usage_snapshot() + } + async fn close(&self) -> Result<(), crate::Error> { self.cache.close().await } @@ -933,6 +1043,10 @@ impl DbCache for UnownedDbCache { self.inner.entry_count() } + fn usage_snapshot(&self) -> DbCacheUsageSnapshot { + self.inner.usage_snapshot() + } + /// The point of this type: never propagate close to a cache we don't own. async fn close(&self) -> Result<(), crate::Error> { Ok(()) @@ -1190,7 +1304,9 @@ pub(crate) mod test_utils { #[cfg(test)] mod tests { - use crate::db_cache::{CachedEntry, CachedKey, DbCache, DbCacheWrapper, SplitCache}; + use crate::db_cache::{ + CacheUsageSnapshot, CachedEntry, CachedKey, DbCache, DbCacheWrapper, SplitCache, + }; use crate::db_state::SsTableId; use crate::filter_policy::{BloomFilterPolicy, FilterPolicy, NamedFilter}; use crate::format::sst::BlockBuilder; @@ -1199,6 +1315,58 @@ mod tests { use crate::flatbuffer_types::test_utils::assert_index_clamped; use crate::db_cache::test_utils::TestCache; + + #[test] + fn usage_snapshot_sum_preserves_typed_states() { + let snapshots = [ + CacheUsageSnapshot::Disabled, + CacheUsageSnapshot::Ready { + used_bytes: 7, + capacity_bytes: Some(10), + }, + CacheUsageSnapshot::Initializing { + capacity_bytes: Some(20), + }, + ]; + assert_eq!( + CacheUsageSnapshot::sum_enabled(snapshots.iter()), + CacheUsageSnapshot::Initializing { + capacity_bytes: Some(30), + } + ); + assert_eq!( + CacheUsageSnapshot::sum_enabled( + [ + CacheUsageSnapshot::Ready { + used_bytes: 12, + capacity_bytes: None, + }, + CacheUsageSnapshot::Ready { + used_bytes: 4, + capacity_bytes: Some(8), + }, + ] + .iter() + ), + CacheUsageSnapshot::Ready { + used_bytes: 16, + capacity_bytes: None, + } + ); + assert_eq!( + CacheUsageSnapshot::sum_enabled( + [ + CacheUsageSnapshot::Ready { + used_bytes: 1, + capacity_bytes: Some(1), + }, + CacheUsageSnapshot::Unavailable, + ] + .iter() + ), + CacheUsageSnapshot::Unavailable + ); + } use crate::format::sst::{EncodedSsTable, SsTableFormat}; use crate::test_utils::build_test_sst; use crate::types::{RowEntry, ValueDeletable}; diff --git a/slatedb/src/db_reader.rs b/slatedb/src/db_reader.rs index a03bee957..21661da8e 100644 --- a/slatedb/src/db_reader.rs +++ b/slatedb/src/db_reader.rs @@ -81,6 +81,7 @@ pub enum DbReaderMode { pub struct DbReader { inner: Arc, task_executor: MessageHandlerExecutor, + pub(crate) object_store_cache: Option>, } pub(crate) struct DbReaderInner { @@ -1450,9 +1451,34 @@ impl DbReader { Ok(Self { inner, task_executor, + object_store_cache: None, }) } + /// Returns a synchronous point-in-time cache accounting snapshot. + /// + /// This method reads only in-memory counters and never performs object-store + /// or filesystem I/O. + pub fn cache_usage_snapshot(&self) -> crate::db_cache::SlateDbCacheUsageSnapshot { + let db_cache = self.inner.table_store.cache().map_or_else( + || crate::db_cache::DbCacheUsageSnapshot { + memory: crate::db_cache::CacheUsageSnapshot::Disabled, + disk: crate::db_cache::CacheUsageSnapshot::Disabled, + }, + |cache| cache.usage_snapshot(), + ); + let object_store = self + .object_store_cache + .as_ref() + .map_or(crate::db_cache::CacheUsageSnapshot::Disabled, |cache| { + cache.usage_snapshot() + }); + crate::db_cache::SlateDbCacheUsageSnapshot { + db_cache, + object_store, + } + } + /// Get a value from the database with default read options. /// /// The `Bytes` object returned contains a slice of an entire From ec429d779aef4e9097ead0418b59b63e503f8ef6 Mon Sep 17 00:00:00 2001 From: xav-db Date: Wed, 29 Jul 2026 13:25:22 +0100 Subject: [PATCH 59/63] Update SlateDB dependency lockfile --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 53af1e213..933ac3035 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3239,6 +3239,7 @@ dependencies = [ "log", "lru", "lz4_flex", + "mixtrics", "moka", "object_store", "ouroboros", From 2b99b7bcebc9e36a286ba1bd5a0999f9c51d64cc Mon Sep 17 00:00:00 2001 From: xav-db Date: Wed, 29 Jul 2026 14:01:16 +0100 Subject: [PATCH 60/63] Keep unbounded cache accounting synchronous --- slatedb/src/cached_object_store/storage_fs.rs | 211 +++++++++++++----- 1 file changed, 159 insertions(+), 52 deletions(-) diff --git a/slatedb/src/cached_object_store/storage_fs.rs b/slatedb/src/cached_object_store/storage_fs.rs index b872c38a5..bbac90611 100644 --- a/slatedb/src/cached_object_store/storage_fs.rs +++ b/slatedb/src/cached_object_store/storage_fs.rs @@ -236,18 +236,17 @@ pub(crate) struct FsCacheEntry { impl FsCacheEntry { async fn atomic_write(&self, path: std::path::PathBuf, buf: Bytes) -> object_store::Result<()> { let tmp_path = path.with_extension(format!("_tmp{}", self.make_rand_suffix())); + let bytes = buf.len(); - // try triggering evict before writing - if let Some(evictor) = &self.evictor { + let Some(write_reservation) = (if let Some(evictor) = &self.evictor { // If the evictor is backpressured, skip this cache write to avoid // stalling foreground PUTs. Cache writes are best-effort. - if !evictor - .track_entry_accessed(path.clone(), EntryAccess::Write(buf.len())) - .await - { - return Ok(()); - } - } + evictor.reserve_entry_access() + } else { + Some(WriteReservation::Untracked) + }) else { + return Ok(()); + }; // Spawn a blocking task and do synchronous I/O rather than use the tokio async apis. // Under the hood, on linux systems , tokio itself spawns a blocking task for each call to @@ -283,6 +282,10 @@ impl FsCacheEntry { .await? .map_err(wrap_io_err)?; + if let WriteReservation::Reserved(permit) = write_reservation { + permit.send((invalidate_path.clone(), EntryAccess::Write(bytes))); + } + // The rename replaced the file at `path`, so any previously cached // handle now points to the old (unlinked) inode. Invalidate it so // the next read opens the new file. @@ -529,9 +532,13 @@ impl LocalCacheEntry for FsCacheEntry { }; if let Some(evictor) = &self.evictor { - evictor - .track_entry_accessed(path, EntryAccess::Delete) - .await; + if evictor.max_cache_size_bytes.is_some() { + evictor + .track_entry_accessed(path, EntryAccess::Delete) + .await; + } else { + evictor.inner.delete_entry(path).await; + } } else { delete_cache_entry(path, self.file_handle_cache.clone()).await; } @@ -545,6 +552,11 @@ enum EntryAccess { } type FsCacheEvictorWork = (std::path::PathBuf, EntryAccess); + +enum WriteReservation<'a> { + Untracked, + Reserved(tokio::sync::mpsc::Permit<'a, FsCacheEvictorWork>), +} // Minimum time between aggregated "evictor queue is full" warnings. const QUEUE_FULL_LOG_INTERVAL_MS: i64 = 30_000; @@ -553,7 +565,6 @@ const QUEUE_FULL_LOG_INTERVAL_MS: i64 = 30_000; /// is added. #[derive(Debug)] struct FsCacheEvictor { - root_folder: std::path::PathBuf, max_cache_size_bytes: Option, scan_interval: Option, tx: tokio::sync::mpsc::Sender, @@ -563,12 +574,10 @@ struct FsCacheEvictor { last_queue_full_log_ms: AtomicI64, background_evict_handle: OnceCell>, background_scan_handle: OnceCell>, - stats: Arc, system_clock: Arc, - rand: Arc, - file_handle_cache: FileHandleCache, usage: Arc, reconcile_notify: Arc, + inner: Arc, } #[derive(Debug)] @@ -603,8 +612,20 @@ impl FsCacheEvictor { file_handle_cache: FileHandleCache, ) -> Self { let (tx, rx) = tokio::sync::mpsc::channel(100); + let usage = Arc::new(FsCacheUsage { + initialized: AtomicBool::new(false), + used_bytes: AtomicU64::new(0), + capacity_bytes: max_cache_size_bytes.map(|bytes| bytes as u64), + }); + let inner = Arc::new(FsCacheEvictorInner::new_with_usage( + root_folder.clone(), + max_cache_size_bytes, + stats.clone(), + rand.clone(), + file_handle_cache.clone(), + usage.clone(), + )); Self { - root_folder, scan_interval, max_cache_size_bytes, tx, @@ -614,28 +635,15 @@ impl FsCacheEvictor { last_queue_full_log_ms: AtomicI64::new(i64::MIN), background_evict_handle: OnceCell::new(), background_scan_handle: OnceCell::new(), - stats, system_clock, - rand, - file_handle_cache, - usage: Arc::new(FsCacheUsage { - initialized: AtomicBool::new(false), - used_bytes: AtomicU64::new(0), - capacity_bytes: max_cache_size_bytes.map(|bytes| bytes as u64), - }), + usage, reconcile_notify: Arc::new(Notify::new()), + inner, } } async fn start(&self) { - let inner = Arc::new(FsCacheEvictorInner::new_with_usage( - self.root_folder.clone(), - self.max_cache_size_bytes, - self.stats.clone(), - self.rand.clone(), - self.file_handle_cache.clone(), - self.usage.clone(), - )); + let inner = self.inner.clone(); let guard = self.rx.lock(); let rx = guard.await.take().expect("evictor already started"); @@ -671,6 +679,49 @@ impl FsCacheEvictor { self.usage.snapshot() } + fn reserve_entry_access(&self) -> Option> { + if !self.started() { + return Some(WriteReservation::Untracked); + } + match self.tx.try_reserve() { + Ok(permit) => Some(WriteReservation::Reserved(permit)), + Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => { + self.mark_accounting_dirty(); + self.record_queue_full(); + None + } + Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => { + self.mark_accounting_dirty(); + None + } + } + } + + fn mark_accounting_dirty(&self) { + self.usage.initialized.store(false, Ordering::Release); + self.reconcile_notify.notify_one(); + } + + fn record_queue_full(&self) { + self.queue_full_count.fetch_add(1, Ordering::AcqRel); + let now_ms = self.system_clock.now().timestamp_millis(); + let last_log_ms = self.last_queue_full_log_ms.load(Ordering::Acquire); + if now_ms.saturating_sub(last_log_ms) < QUEUE_FULL_LOG_INTERVAL_MS { + return; + } + if self + .last_queue_full_log_ms + .compare_exchange(last_log_ms, now_ms, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + let queue_full_count = self.queue_full_count.swap(0, Ordering::AcqRel); + warn!( + "evictor queue skipped cache write/access event because it was full {} times in the last 30s", + queue_full_count, + ); + } + } + async fn background_evict( inner: Arc, mut rx: tokio::sync::mpsc::Receiver, @@ -732,28 +783,12 @@ impl FsCacheEvictor { match self.tx.try_send((path, access)) { Ok(()) => true, Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => { - self.usage.initialized.store(false, Ordering::Release); - self.reconcile_notify.notify_one(); - self.queue_full_count.fetch_add(1, Ordering::AcqRel); - let now_ms = self.system_clock.now().timestamp_millis(); - let last_log_ms = self.last_queue_full_log_ms.load(Ordering::Acquire); - if now_ms.saturating_sub(last_log_ms) >= QUEUE_FULL_LOG_INTERVAL_MS - && self - .last_queue_full_log_ms - .compare_exchange(last_log_ms, now_ms, Ordering::AcqRel, Ordering::Acquire) - .is_ok() - { - let queue_full_count = self.queue_full_count.swap(0, Ordering::AcqRel); - warn!( - "evictor queue skipped cache write/access event because it was full {} times in the last 30s", - queue_full_count, - ); - } + self.mark_accounting_dirty(); + self.record_queue_full(); false } Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => { - self.usage.initialized.store(false, Ordering::Release); - self.reconcile_notify.notify_one(); + self.mark_accounting_dirty(); false } } @@ -1382,6 +1417,15 @@ mod tests { assert!(accepted); } + evictor.usage.initialized.store(true, Ordering::Release); + assert!(evictor.reserve_entry_access().is_none()); + assert_eq!( + evictor.usage_snapshot(), + CacheUsageSnapshot::Initializing { + capacity_bytes: Some(1024), + } + ); + let accepted = evictor .track_entry_accessed(std::path::PathBuf::from("overflow"), EntryAccess::Write(1)) .await; @@ -1521,6 +1565,69 @@ mod tests { ); } + #[tokio::test] + async fn unbounded_usage_tracks_successful_write_overwrite_and_delete() { + let temp_dir = tempfile::Builder::new() + .prefix("objstore_cache_usage_updates_") + .tempdir() + .unwrap(); + let recorder = slatedb_common::metrics::MetricsRecorderHelper::noop(); + let storage = FsCacheStorage::new( + temp_dir.path().to_path_buf(), + None, + None, + Arc::new(CachedObjectStoreStats::new(&recorder)), + Arc::new(DefaultSystemClock::new()), + Arc::new(DbRand::default()), + 10, + ); + storage.start_evictor().await; + tokio::time::timeout(Duration::from_secs(1), async { + while !matches!( + storage.usage_snapshot(), + CacheUsageSnapshot::Ready { used_bytes: 0, .. } + ) { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + + let entry = storage.entry(&Path::from("cached/object"), 1024); + entry.save_part(0, Bytes::from(vec![1; 17])).await.unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + while !matches!( + storage.usage_snapshot(), + CacheUsageSnapshot::Ready { used_bytes: 17, .. } + ) { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + + entry.save_part(0, Bytes::from(vec![2; 29])).await.unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + while !matches!( + storage.usage_snapshot(), + CacheUsageSnapshot::Ready { used_bytes: 29, .. } + ) { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + + entry.delete().await; + assert_eq!( + storage.usage_snapshot(), + CacheUsageSnapshot::Ready { + used_bytes: 0, + capacity_bytes: None, + } + ); + } + #[rstest::rstest] // Basic case: 2 keys, nothing picked, no exclusion #[case(&[0, 1], &[], None, &[0, 1])] From eaff69b34a7217251227a304cb2b6ead86c4cb79 Mon Sep 17 00:00:00 2001 From: xav-db Date: Wed, 29 Jul 2026 13:40:16 +0100 Subject: [PATCH 61/63] Add typed database-missing reader error --- slatedb/src/db_reader.rs | 27 ++++++++++++++++++++- slatedb/src/error.rs | 51 ++++++++++++++++++++++++++++++++++++++++ slatedb/src/lib.rs | 2 +- 3 files changed, 78 insertions(+), 2 deletions(-) diff --git a/slatedb/src/db_reader.rs b/slatedb/src/db_reader.rs index 21661da8e..3cd071d2a 100644 --- a/slatedb/src/db_reader.rs +++ b/slatedb/src/db_reader.rs @@ -1413,7 +1413,13 @@ impl DbReader { Self::validate_options(mode, &options)?; let manifest = - StoredManifest::load(Arc::clone(&manifest_store), system_clock.clone()).await?; + match StoredManifest::load(Arc::clone(&manifest_store), system_clock.clone()).await { + Ok(manifest) => manifest, + Err(SlateDBError::LatestTransactionalObjectVersionMissing) => { + return Err(SlateDBError::DatabaseMissing); + } + Err(error) => return Err(error), + }; if !manifest.db_state().initialized { return Err(SlateDBError::InvalidDBState); } @@ -2063,6 +2069,25 @@ mod tests { ); } + #[tokio::test] + async fn empty_database_reader_returns_typed_database_missing() { + let object_store: Arc = Arc::new(InMemory::new()); + let error = match DbReader::open( + "/tmp/test_reader_database_missing", + object_store, + None, + DbReaderOptions::default(), + ) + .await + { + Ok(_) => panic!("empty object store must not open a reader"), + Err(error) => error, + }; + + assert_eq!(error.kind(), crate::ErrorKind::Data); + assert_eq!(error.code(), Some(crate::ErrorCode::DatabaseMissing)); + } + #[tokio::test] async fn should_return_current_versioned_manifest() { let object_store: Arc = Arc::new(InMemory::new()); diff --git a/slatedb/src/error.rs b/slatedb/src/error.rs index 4327435bd..1f15e1faa 100644 --- a/slatedb/src/error.rs +++ b/slatedb/src/error.rs @@ -55,6 +55,9 @@ pub(crate) enum SlateDBError { #[error("failed to find latest transactional object (e.g. manifest) version")] LatestTransactionalObjectVersionMissing, + #[error("database does not exist")] + DatabaseMissing, + #[error("generic transactional object (e.g. manifest) error {0:?}")] TransactionalObjectError(#[from] Arc), @@ -496,6 +499,15 @@ pub enum ErrorKind { Internal, } +/// Stable machine-readable detail for public errors that require more precise +/// handling than [`ErrorKind`]. +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ErrorCode { + /// A reader cannot open because no database manifest exists. + DatabaseMissing, +} + impl From for CloseReason { fn from(kind: ErrorKind) -> Self { match kind { @@ -548,6 +560,7 @@ pub enum RetryReason { pub struct Error { msg: String, kind: ErrorKind, + code: Option, source: Option, } @@ -575,6 +588,7 @@ impl Error { Self { msg, kind: ErrorKind::Transaction, + code: None, source: None, } } @@ -584,6 +598,7 @@ impl Error { Self { msg, kind: ErrorKind::Closed(reason), + code: None, source: None, } } @@ -593,6 +608,7 @@ impl Error { Self { msg, kind: ErrorKind::Unavailable, + code: None, source: None, } } @@ -602,6 +618,7 @@ impl Error { Self { msg, kind: ErrorKind::Invalid, + code: None, source: None, } } @@ -611,6 +628,7 @@ impl Error { Self { msg, kind: ErrorKind::Data, + code: None, source: None, } } @@ -620,6 +638,7 @@ impl Error { Self { msg, kind: ErrorKind::Internal, + code: None, source: None, } } @@ -630,10 +649,20 @@ impl Error { self } + fn with_code(mut self, code: ErrorCode) -> Self { + self.code = Some(code); + self + } + /// Returns the error kind. pub fn kind(&self) -> ErrorKind { self.kind } + + /// Returns stable machine-readable detail when one is available. + pub fn code(&self) -> Option { + self.code + } } impl From for Error { @@ -727,6 +756,7 @@ impl From for Error { SlateDBError::InvalidVersion { .. } => Error::data(msg), SlateDBError::ManifestMissing(_) => Error::data(msg), SlateDBError::LatestTransactionalObjectVersionMissing => Error::data(msg), + SlateDBError::DatabaseMissing => Error::data(msg).with_code(ErrorCode::DatabaseMissing), SlateDBError::TransactionalObjectVersionExists => Error::data(msg), SlateDBError::InvalidTransactionalObjectState => Error::data(msg), SlateDBError::EmptyManifest => Error::data(msg), @@ -795,4 +825,25 @@ mod tests { assert_eq!(public_err.kind(), ErrorKind::Unavailable); } + + #[test] + fn database_missing_has_stable_code_without_changing_broad_kind() { + let public_err = Error::from(SlateDBError::DatabaseMissing); + + assert_eq!(public_err.kind(), ErrorKind::Data); + assert_eq!(public_err.code(), Some(ErrorCode::DatabaseMissing)); + } + + #[test] + fn other_data_errors_do_not_claim_database_is_missing() { + for err in [ + SlateDBError::LatestTransactionalObjectVersionMissing, + SlateDBError::ManifestMissing(7), + SlateDBError::InvalidDBState, + ] { + let public_err = Error::from(err); + assert_eq!(public_err.kind(), ErrorKind::Data); + assert_eq!(public_err.code(), None); + } + } } diff --git a/slatedb/src/lib.rs b/slatedb/src/lib.rs index 0ba56acc8..43a884464 100644 --- a/slatedb/src/lib.rs +++ b/slatedb/src/lib.rs @@ -54,7 +54,7 @@ pub use db_iter::{DbIterator, DbRecencyIterator}; pub use db_reader::{DbReader, DbReaderMode}; pub use db_snapshot::DbSnapshot; pub use db_transaction::DbTransaction; -pub use error::{CloseReason, Error, ErrorKind}; +pub use error::{CloseReason, Error, ErrorCode, ErrorKind}; pub use filter::BloomFilter; pub use filter_policy::{ BloomFilterPolicy, Filter, FilterBuilder, FilterContext, FilterPolicy, FilterQuery, From 567d6839e41b5caba278112dfd2857fd92cc1ff0 Mon Sep 17 00:00:00 2001 From: xav-db Date: Wed, 29 Jul 2026 14:47:51 +0100 Subject: [PATCH 62/63] Update missing database binding tests --- bindings/node/tests/reader.test.mjs | 2 +- bindings/python/tests/test_reader.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bindings/node/tests/reader.test.mjs b/bindings/node/tests/reader.test.mjs index ca0192841..180b85f5e 100644 --- a/bindings/node/tests/reader.test.mjs +++ b/bindings/node/tests/reader.test.mjs @@ -67,7 +67,7 @@ test("reader build fails when database is missing", async (t) => { const builder = cleanup.track(new DbReaderBuilder(TEST_DB_PATH, store), { shutdown: false }); const error = await expectError(() => builder.build(), ErrorData); - assert.match(error.message, /failed to find latest transactional object/); + assert.match(error.message, /database does not exist/); }); test("reader point reads", async (t) => { diff --git a/bindings/python/tests/test_reader.py b/bindings/python/tests/test_reader.py index 5567c1b95..f3fde2204 100644 --- a/bindings/python/tests/test_reader.py +++ b/bindings/python/tests/test_reader.py @@ -52,7 +52,7 @@ async def test_reader_build_fails_when_database_is_missing() -> None: with pytest.raises(Error.Data) as exc: await builder.build() - assert "failed to find latest transactional object" in exc.value.message + assert "database does not exist" in exc.value.message @pytest.mark.asyncio From ce9c7d2f005a9dc018d2b573c23a96fb8447da3a Mon Sep 17 00:00:00 2001 From: Matthew Sanetra <41018997+matthewsanetra@users.noreply.github.com> Date: Fri, 7 Aug 2026 16:23:00 +0100 Subject: [PATCH 63/63] Finish v0.15 reader API adaptation --- slatedb/benches/db_reader_scaling.rs | 4 +-- slatedb/src/db_reader.rs | 41 +++++++--------------------- slatedb/src/db_transaction.rs | 5 +--- slatedb/src/reader.rs | 4 +-- 4 files changed, 14 insertions(+), 40 deletions(-) diff --git a/slatedb/benches/db_reader_scaling.rs b/slatedb/benches/db_reader_scaling.rs index 278cb4ea0..c55c23c4e 100644 --- a/slatedb/benches/db_reader_scaling.rs +++ b/slatedb/benches/db_reader_scaling.rs @@ -26,7 +26,7 @@ use slatedb::config::{ Settings, WriteOptions, }; use slatedb::instrumented_object_store_stats; -use slatedb::{Db, DbReader, DbSnapshot, PrefixExtractor, PrefixTarget}; +use slatedb::{Db, DbReader, DbReaderMode, DbSnapshot, PrefixExtractor, PrefixTarget}; use slatedb_common::metrics::{lookup_metric, lookup_metric_with_labels, DefaultMetricsRecorder}; use slatedb_common::{MockSystemClock, SystemClock}; use tokio::sync::Barrier; @@ -1151,7 +1151,7 @@ async fn benchmark_fixed_reader_open(scales: &[usize], full: bool) { let object_store: Arc = store.clone(); let start = Instant::now(); let reader = DbReader::builder(path.as_str(), object_store) - .with_checkpoint_id(checkpoint.id) + .with_reader_mode(DbReaderMode::Checkpoint(checkpoint.id)) .with_options(quiet_reader_options(1)) .with_metrics_recorder(recorder.clone()) .with_db_cache_disabled() diff --git a/slatedb/src/db_reader.rs b/slatedb/src/db_reader.rs index 3cd071d2a..2b4e2f4c5 100644 --- a/slatedb/src/db_reader.rs +++ b/slatedb/src/db_reader.rs @@ -139,11 +139,7 @@ impl Drop for ReaderGenerationPermit { } impl ReaderGeneration { - fn new( - manifest_id: u64, - checkpoint: Option, - manifest: Manifest, - ) -> Arc { + fn new(manifest_id: u64, checkpoint: Option, manifest: Manifest) -> Arc { Arc::new(Self { manifest_id, checkpoint: checkpoint.map(RwLock::new), @@ -154,7 +150,9 @@ impl ReaderGeneration { } fn checkpoint(&self) -> Option { - self.checkpoint.as_ref().map(|checkpoint| checkpoint.read().clone()) + self.checkpoint + .as_ref() + .map(|checkpoint| checkpoint.read().clone()) } fn managed_checkpoint(&self) -> &RwLock { @@ -522,9 +520,7 @@ impl DbReaderInner { .await } - pub(crate) async fn snapshot_multi_get_key_value_with_options< - K: AsRef<[u8]> + Send + Sync, - >( + pub(crate) async fn snapshot_multi_get_key_value_with_options + Send + Sync>( &self, state: Arc, max_seq: u64, @@ -1040,10 +1036,7 @@ impl ManifestPoller { .id; generations.insert(checkpoint_id, Arc::downgrade(&generation)); } - let poller = Self { - inner, - generations, - }; + let poller = Self { inner, generations }; poller.report_active_checkpoints(); poller } @@ -1974,9 +1967,7 @@ fn has_not_found_object_store_error(err: &(dyn std::error::Error + 'static)) -> #[cfg(test)] mod tests { - use super::{ - DbReaderMessage, ManifestPoller, ReaderGeneration, ReaderState, ReplayMemtables, - }; + use super::{DbReaderMessage, ManifestPoller, ReaderGeneration, ReaderState, ReplayMemtables}; use crate::block_cache_policy::BlockCachePolicy; use crate::clock::MonotonicClock; use crate::config::{ @@ -2075,7 +2066,7 @@ mod tests { let error = match DbReader::open( "/tmp/test_reader_database_missing", object_store, - None, + DbReaderMode::ManagedCheckpoint, DbReaderOptions::default(), ) .await @@ -2808,13 +2799,7 @@ mod tests { ) .await .unwrap(); - let reader_checkpoint_id = inner - .state - .read() - .generation - .checkpoint() - .unwrap() - .id; + let reader_checkpoint_id = inner.state.read().generation.checkpoint().unwrap().id; // Simulate the writer's GC reaping the expired checkpoint. let mut stored_manifest = StoredManifest::load(Arc::clone(&manifest_store), clock.clone()) @@ -2836,13 +2821,7 @@ mod tests { .unwrap(); // The reader should have replaced the reaped checkpoint with a new one. - let new_checkpoint_id = inner - .state - .read() - .generation - .checkpoint() - .unwrap() - .id; + let new_checkpoint_id = inner.state.read().generation.checkpoint().unwrap().id; assert_ne!(reader_checkpoint_id, new_checkpoint_id); let latest_manifest = manifest_store.read_latest_manifest().await.unwrap(); let checkpoints = &latest_manifest.manifest.core.checkpoints; diff --git a/slatedb/src/db_transaction.rs b/slatedb/src/db_transaction.rs index 004150e18..33292182f 100644 --- a/slatedb/src/db_transaction.rs +++ b/slatedb/src/db_transaction.rs @@ -290,10 +290,7 @@ impl DbTransaction { ) .await .map_err(crate::Error::from)?; - for (key_idx, value) in reader_key_indices - .into_iter() - .zip(reader_values.into_iter()) - { + for (key_idx, value) in reader_key_indices.into_iter().zip(reader_values) { resolved[key_idx] = true; values[key_idx] = value; } diff --git a/slatedb/src/reader.rs b/slatedb/src/reader.rs index b9e436a98..6c35b6fdc 100644 --- a/slatedb/src/reader.rs +++ b/slatedb/src/reader.rs @@ -1162,9 +1162,7 @@ mod tests { self.memtable.clone() } - fn imm_memtables( - &self, - ) -> Box> + Send + '_> { + fn imm_memtables(&self) -> Box> + Send + '_> { Box::new(self.imm_memtable.iter().cloned()) }