diff --git a/downstairs/src/extent.rs b/downstairs/src/extent.rs index 1fab7b2cf..0a4c8e57d 100644 --- a/downstairs/src/extent.rs +++ b/downstairs/src/extent.rs @@ -505,6 +505,10 @@ impl Extent { self.inner.dirty().unwrap() } + pub fn read_only(&self) -> bool { + self.read_only + } + /// Close an extent, returning a tuple of `(gen, flush, dirty)` pub fn close(self) -> Result<(u64, u64, bool), CrucibleError> { let generation = self.inner.gen_number().unwrap(); diff --git a/downstairs/src/region.rs b/downstairs/src/region.rs index 01dce10d1..c4ead0864 100644 --- a/downstairs/src/region.rs +++ b/downstairs/src/region.rs @@ -7,6 +7,7 @@ use std::path::{Path, PathBuf}; use anyhow::{Result, bail}; use futures::TryStreamExt; +use rayon::prelude::*; use tracing::instrument; @@ -321,6 +322,7 @@ impl Region { }; region.open_extents()?; + region.validate_extents()?; Ok(region) } @@ -375,6 +377,59 @@ impl Region { Ok(()) } + /// Validate every raw extent by hashing all blocks and checking + /// against stored on-disk hashes. Runs up to 20 extents in + /// parallel. SQLite-backed extents (read-only snapshots) are + /// skipped. + fn validate_extents(&self) -> Result<()> { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(20) + .build() + .expect("Failed to build validation thread pool"); + + let errors: Vec<_> = pool.install(|| { + self.extents + .par_iter() + .filter_map(|e| { + let extent = match e { + ExtentState::Opened(extent) => extent, + ExtentState::Closed => { + panic!("validate on closed extent!") + } + }; + + // SQLite extents don't support validate and + // are only used for read-only snapshots. + if extent.read_only() { + return None; + } + + if let Err(err) = extent.validate() { + Some((extent.number, err)) + } else { + None + } + }) + .collect() + }); + + if !errors.is_empty() { + for (number, err) in &errors { + error!( + self.log, + "validation failed for extent {number}: {err}", + ); + } + bail!("Region failed to validate ({} extents bad)", errors.len()); + } + info!( + self.log, + "validated {} extents on startup", + self.extents.len(), + ); + Ok(()) + } + /// Creates `self.extent_count` extent files and opens them fn create_extents(&mut self, backend: Backend) -> Result<()> { let next_eid = self.extents.len() as u32;