Skip to content

Add csv output option to crucible-verify-raw #1911

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
leftwo merged 7 commits into from
Aug 4, 2026
Merged

Add csv output option to crucible-verify-raw #1911

Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
f887b8c
Add csv output option to crucible-verify-raw
leftwo Mar 18, 2026
acf4d36
Fix unquoted filename in CSV output of crucible-verify-raw
leftwo Mar 18, 2026
5b91136
also print out actual hash and expected hashes
leftwo Mar 19, 2026
a2afbf1
Add dirty and selected_slot columns to crucible-verify-raw CSV output
leftwo Mar 24, 2026
720a78e
Always populate selected_slot in crucible-verify-raw CSV output
leftwo Mar 24, 2026
afa4a2f
Merge branch 'main' into alan/raw-to-csv
Apr 20, 2026
cd6927c
Merge branch 'main' into alan/raw-to-csv
Aug 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
273 changes: 170 additions & 103 deletions verify-raw/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,26 @@ struct Args {
#[clap(long, short)]
verbose: bool,

/// Output CSV to stdout with one row per block; metadata is
/// written as #-prefixed comment lines
#[clap(long, conflicts_with = "verbose")]
verbose_csv: bool,

/// Block size in the extent (usually autodetected)
#[clap(long)]
block_size: Option<usize>,
}

fn main() -> Result<()> {
let args = Args::parse();
check_one(&args.file, args.block_size, args.verbose)
check_one(&args.file, args.block_size, args.verbose, args.verbose_csv)
}

fn check_one(
p: &std::path::Path,
block_size: Option<usize>,
verbose: bool,
verbose_csv: bool,
) -> Result<()> {
let data = std::fs::read(p)?;
let data_len = data.len() - BLOCK_META_SIZE_BYTES as usize;
Expand Down Expand Up @@ -65,28 +71,46 @@ fn check_one(
);
}
})?;
print!("bs:{block_size} bytes bc:{block_count:>6}");
println!(
" dirty:{:>5} gen:{} flush_number:{:>6} ext_ver:{} bonus_sync:{} defrag:{}",
meta.dirty,
meta.gen_number,
meta.flush_number,
meta.ext_version,
meta.bonus_sync_count,
meta.defrag_count,
);

let slot_selected = if !meta.dirty {
let mut selected = vec![];
for d in &data[data_len - block_count.div_ceil(8)..data_len] {
for i in 0..8 {
selected.push((d & (1 << i)) == 0);
}
}
Some(selected)
if verbose_csv {
println!(
"# file:{} bs:{block_size} bytes bc:{block_count} \
dirty:{} gen:{} flush_number:{} ext_ver:{} \
bonus_sync:{} defrag:{}",
p.display(),
meta.dirty,
meta.gen_number,
meta.flush_number,
meta.ext_version,
meta.bonus_sync_count,
meta.defrag_count,
);
println!(
"file,block,dirty,status,\
slot_a_result,slot_a_flush_id,slot_a_hash,\
slot_b_result,slot_b_flush_id,slot_b_hash,\
selected_slot,data_hash,all_zeros"
);
} else {
None
};
print!("bs:{block_size} bytes bc:{block_count:>6}");
println!(
" dirty:{:>5} gen:{} flush_number:{:>6} ext_ver:{} \
bonus_sync:{} defrag:{}",
meta.dirty,
meta.gen_number,
meta.flush_number,
meta.ext_version,
meta.bonus_sync_count,
meta.defrag_count,
);
}

let mut slot_selected = vec![];
for d in &data[data_len - block_count.div_ceil(8)..data_len] {
for i in 0..8 {
slot_selected.push((d & (1 << i)) == 0);
}
}

let context_slots = (0..block_count * 2)
.map(|i| {
Expand All @@ -99,122 +123,156 @@ fn check_one(
.collect::<Vec<Option<OnDiskDownstairsBlockContext>>>();
let (ctx_a, ctx_b) = context_slots.split_at(block_count);

let filename = csv_quote(&p.display().to_string());
let mut failed = false;
// Check each block and emit one output line per block
for (i, chunk) in data[..block_size * block_count]
.chunks_exact(block_size)
.enumerate()
{
let hash = integrity_hash(&[chunk]);
let mut printed = false;
let ra = check_block(chunk, hash, ctx_a[i]);
let rb = check_block(chunk, hash, ctx_b[i]);
let all_zeros = chunk.iter().all(|b| *b == 0u8);

if verbose_csv {
let sel_a = slot_selected[i];
let status = if !meta.dirty {
if (sel_a && ra.is_err()) || (!sel_a && rb.is_err()) {
"error"
} else {
"ok"
}
} else if ra.is_err() && rb.is_err() {
"error"
} else {
"ok"
};
if status == "error" {
failed = true;
}
println!(
"{filename},{i},{},{status},{},{},{},{},{},{},{},{},{all_zeros}",
meta.dirty,
ra.map_or_else(|e| format!("{e:?}"), |s| format!("{s:?}")),
ctx_a[i].map_or(String::new(), |c| c.flush_id.to_string()),
ctx_a[i].map_or(String::new(), |c| c.on_disk_hash.to_string()),
rb.map_or_else(|e| format!("{e:?}"), |s| format!("{s:?}")),
ctx_b[i].map_or(String::new(), |c| c.flush_id.to_string()),
ctx_b[i].map_or(String::new(), |c| c.on_disk_hash.to_string()),
sel_a,
hash,
);
} else {
let mut printed = false;

if let Some(slot_selected) = &slot_selected {
// If the slot selected array is valid (i.e. the extent file is not
// dirty), then it must be correct.
let s = slot_selected[i];
let ctx = if s { ctx_a[i] } else { ctx_b[i] };
let r = if s { ra } else { rb };
if r.is_err() {
if !meta.dirty {
// If the extent is not dirty, the slot selected array is
// valid and must be correct.
let s = slot_selected[i];
let ctx = if s { ctx_a[i] } else { ctx_b[i] };
let r = if s { ra } else { rb };
if r.is_err() {
failed = true;
printed = true;
print!("Error at block {:>6}:", i);
print!(
" slot {} [selected]: {r:?}{}",
if s { "A" } else { "B" },
if let Some(ctx) = ctx {
format!(", flush id: {}", ctx.flush_id)
} else {
"".to_owned()
}
);
let other_ctx = if s { ctx_b[i] } else { ctx_a[i] };
let other_r = if s { rb } else { ra };
print!(
" | slot {} [deselected]: {:?}{}",
if s { "B" } else { "A" },
other_r,
if let Some(ctx) = other_ctx {
format!(", flush id: {}", ctx.flush_id)
} else {
"".to_owned()
}
);
if all_zeros {
print!(" Block is all zeros");
}
println!();
}
} else if let Err(ea) = ra
&& let Err(eb) = rb
{
// Otherwise, both context slots are invalid, so print that
failed = true;
printed = true;
print!("Error at block {:>6}:", i);
print!("Error at block {i}:");
print!(
" slot {} [selected]: {r:?}{}",
if s { "A" } else { "B" },
if let Some(ctx) = ctx {
" slot A: {ea:?}{}",
if let Some(ctx) = ctx_a[i] {
format!(", flush id: {}", ctx.flush_id)
} else {
"".to_owned()
}
);
let other_ctx = if s { ctx_b[i] } else { ctx_a[i] };
let other_r = if s { rb } else { ra };
print!(
" | slot {} [deselected]: {:?}{}",
if s { "B" } else { "A" },
other_r,
if let Some(ctx) = other_ctx {
" | slot B: {eb:?}{}",
if let Some(ctx) = ctx_b[i] {
format!(", flush id: {}", ctx.flush_id)
} else {
"".to_owned()
}
);
if chunk.iter().all(|b| *b == 0u8) {
if all_zeros {
print!(" Block is all zeros");
}
println!();
}
} else if let Err(ea) = ra
&& let Err(eb) = rb
{
// Otherwise, both context slots are invalid, so print that
failed = true;
printed = true;
print!("Error at block {i}:");
print!(
" slot A: {ea:?}{}",
if let Some(ctx) = ctx_a[i] {
format!(", flush id: {}", ctx.flush_id)
} else {
"".to_owned()
}
);
print!(
" | slot B: {eb:?}{}",
if let Some(ctx) = ctx_b[i] {
format!(", flush id: {}", ctx.flush_id)
} else {
"".to_owned()
}
);
if chunk.iter().all(|b| *b == 0u8) {
print!(" Block is all zeros");
}
println!();
}

// Print a log line for each block if the verbose flag is set
if verbose && !printed {
print!("Success at block {i}:");
print!(
" slot A{}: {ra:?}{}",
if let Some(slot_selected) = &slot_selected {
if slot_selected[i] {
" [selected]"
// Print a log line for each block if the verbose flag is set
if verbose && !printed {
print!("Success at block {i}:");
print!(
" slot A{}: {ra:?}{}",
if !meta.dirty {
if slot_selected[i] {
" [selected]"
} else {
" [deselected]"
}
} else {
""
},
if let Some(ctx) = ctx_a[i] {
format!(", flush id: {}", ctx.flush_id)
} else {
" [deselected]"
"".to_owned()
}
} else {
""
},
if let Some(ctx) = ctx_a[i] {
format!(", flush id: {}", ctx.flush_id)
} else {
"".to_owned()
}
);
print!(
" | slot B{}: {rb:?}{}",
if let Some(slot_selected) = &slot_selected {
if !slot_selected[i] {
" [selected]"
);
print!(
" | slot B{}: {rb:?}{}",
if !meta.dirty {
if !slot_selected[i] {
" [selected]"
} else {
" [deselected]"
}
} else {
""
},
if let Some(ctx) = ctx_b[i] {
format!(", flush id: {}", ctx.flush_id)
} else {
" [deselected]"
"".to_owned()
}
} else {
""
},
if let Some(ctx) = ctx_b[i] {
format!(", flush id: {}", ctx.flush_id)
} else {
"".to_owned()
);
if all_zeros {
print!(" Block is all zeros");
}
);
if chunk.iter().all(|b| *b == 0u8) {
print!(" Block is all zeros");
println!();
}
println!();
}
}

Expand Down Expand Up @@ -254,6 +312,15 @@ enum Failure {
EmptySlotWithNonzeroData,
}

/// Wrap a string in double quotes, escaping internal double quotes for CSV
fn csv_quote(s: &str) -> String {
if s.contains([',', '"', '\n']) {
format!("\"{}\"", s.replace('"', "\"\""))
} else {
s.to_owned()
}
}

/// Brute force strategy to get block count
fn get_block_count(data_len: usize, block_size: usize) -> Option<usize> {
let estimated_block_count =
Expand Down