From 3b94eb0003e10f30fea5ecd3d62110df18160f4d Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Wed, 3 Jun 2026 14:06:44 -0700 Subject: [PATCH 1/4] Add /memory endpoint to downstairs repair server Add a GET /memory endpoint to the repair server that reports some of the heap memory usage of the downstairs's internal data structures. The report includes: * Region configuration (extent count/size, block size) * Per-extent heap cost (RawInner struct + BlockBitArrays) * Per-connection work queue stats (pending jobs, completed ranges) * High-water marks for transient I/O buffers (largest read response and write payload seen) * High-water marks for work queue capacity and completed range fragmentation * Thread counts (tokio workers, rayon pool) High-water mark tracking does add a small per-I/O cost. The rest of the sizing is computed on demand when the endpoint is called. --- downstairs-api/src/lib.rs | 9 + .../versions/src/initial/repair.rs | 39 +- downstairs-types/versions/src/latest.rs | 2 + downstairs/src/complete_jobs.rs | 5 + downstairs/src/extent.rs | 17 + downstairs/src/extent_inner_raw.rs | 10 + downstairs/src/extent_inner_sqlite.rs | 4 + downstairs/src/lib.rs | 469 ++++++++++++++++++ downstairs/src/region.rs | 22 + downstairs/src/repair.rs | 15 + 10 files changed, 591 insertions(+), 1 deletion(-) diff --git a/downstairs-api/src/lib.rs b/downstairs-api/src/lib.rs index 40496afe5..ecc447a37 100644 --- a/downstairs-api/src/lib.rs +++ b/downstairs-api/src/lib.rs @@ -122,4 +122,13 @@ pub trait CrucibleDownstairsRepairApi { async fn get_work( rqctx: RequestContext, ) -> Result, HttpError>; + + /// Memory usage report + #[endpoint { + method = GET, + path = "/memory", + }] + async fn get_memory( + rqctx: RequestContext, + ) -> Result, HttpError>; } diff --git a/downstairs-types/versions/src/initial/repair.rs b/downstairs-types/versions/src/initial/repair.rs index 1bf2cdc92..dbdbf8108 100644 --- a/downstairs-types/versions/src/initial/repair.rs +++ b/downstairs-types/versions/src/initial/repair.rs @@ -1,7 +1,7 @@ // Copyright 2026 Oxide Computer Company use schemars::JsonSchema; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; #[derive(Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] @@ -37,3 +37,40 @@ pub struct ExtentFilePath { pub struct JobPath { pub id: String, } + +/// Per-connection memory report +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct ConnectionMemoryReport { + pub pending_jobs: usize, + pub pending_jobs_bytes: usize, + pub pending_jobs_capacity_hwm: usize, + pub completed_ranges: usize, + pub completed_ranges_hwm: usize, +} + +/// Summary of downstairs memory usage +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct MemoryReport { + // Region configuration for context + pub extent_count: u32, + pub extent_size: u64, + pub block_size: u64, + + // Region memory + pub region_bytes: usize, + pub extent_meta_bytes: usize, + pub bytes_per_extent: usize, + pub dirty_extent_count: usize, + + // High-water marks for transient I/O buffers + pub read_bytes_hwm: usize, + pub write_bytes_hwm: usize, + + // Thread counts + pub tokio_worker_threads: usize, + pub rayon_threads: usize, + + // Connection memory + pub active_connections: usize, + pub connections: Vec, +} diff --git a/downstairs-types/versions/src/latest.rs b/downstairs-types/versions/src/latest.rs index e90574df4..9c2f9f19a 100644 --- a/downstairs-types/versions/src/latest.rs +++ b/downstairs-types/versions/src/latest.rs @@ -7,9 +7,11 @@ pub mod admin { } pub mod repair { + pub use crate::v1::repair::ConnectionMemoryReport; pub use crate::v1::repair::ExtentFilePath; pub use crate::v1::repair::ExtentPath; pub use crate::v1::repair::FileSpec; pub use crate::v1::repair::FileType; pub use crate::v1::repair::JobPath; + pub use crate::v1::repair::MemoryReport; } diff --git a/downstairs/src/complete_jobs.rs b/downstairs/src/complete_jobs.rs index 57af6691d..4dc804e9d 100644 --- a/downstairs/src/complete_jobs.rs +++ b/downstairs/src/complete_jobs.rs @@ -43,6 +43,11 @@ impl CompletedJobs { self.completed.contains(&id) } + /// Returns the number of contiguous ranges stored + pub fn range_count(&self) -> usize { + self.completed.iter().count() + } + /// Returns the list of completed jobs pub fn completed(&self) -> impl Iterator + use<'_> { self.completed diff --git a/downstairs/src/extent.rs b/downstairs/src/extent.rs index 1fab7b2cf..90399f411 100644 --- a/downstairs/src/extent.rs +++ b/downstairs/src/extent.rs @@ -115,6 +115,10 @@ pub(crate) trait ExtentInner: Send + Sync + Debug { &mut self, block_context: &DownstairsBlockContext, ) -> Result<(), CrucibleError>; + + /// Returns the heap bytes allocated by this extent's in-memory + /// metadata (not the on-disk file data). + fn heap_size(&self) -> usize; } /// BlockContext, with the addition of block index and on_disk_hash @@ -716,6 +720,19 @@ impl Extent { ) -> Result>, CrucibleError> { self.inner.get_block_contexts(block, count) } + + pub fn heap_size(&self) -> usize { + self.inner.heap_size() + } +} + +impl ExtentState { + pub fn heap_size(&self) -> usize { + match self { + ExtentState::Opened(extent) => extent.heap_size(), + ExtentState::Closed => 0, + } + } } /** diff --git a/downstairs/src/extent_inner_raw.rs b/downstairs/src/extent_inner_raw.rs index 1a9a052cd..84dc65025 100644 --- a/downstairs/src/extent_inner_raw.rs +++ b/downstairs/src/extent_inner_raw.rs @@ -177,6 +177,10 @@ impl BlockBitArray { self.block_count as usize } + fn heap_size(&self) -> usize { + self.data.capacity() * size_of::() + } + fn iter(&self) -> impl Iterator + '_ { (0..self.block_count).map(|i| self[i]) } @@ -687,6 +691,12 @@ impl ExtentInner for RawInner { ) -> Result>, CrucibleError> { RawInner::get_block_contexts(self, block, count) } + + fn heap_size(&self) -> usize { + size_of_val(self) + + self.active_context.0.heap_size() + + self.block_dirty.heap_size() + } } impl RawInner { diff --git a/downstairs/src/extent_inner_sqlite.rs b/downstairs/src/extent_inner_sqlite.rs index 61b4964bd..272c17344 100644 --- a/downstairs/src/extent_inner_sqlite.rs +++ b/downstairs/src/extent_inner_sqlite.rs @@ -118,6 +118,10 @@ impl ExtentInner for SqliteInner { .unwrap() .set_dirty_and_block_context(block_context) } + + fn heap_size(&self) -> usize { + 0 + } } impl SqliteInner { diff --git a/downstairs/src/lib.rs b/downstairs/src/lib.rs index 683e925a7..733134f5b 100644 --- a/downstairs/src/lib.rs +++ b/downstairs/src/lib.rs @@ -16,6 +16,7 @@ use crucible_common::{ RegionDefinition, VerboseTimeout, build_logger, integrity_hash, mkdir_for_file, }; +use crucible_downstairs_types::repair::{ConnectionMemoryReport, MemoryReport}; use crucible_protocol::{ BlockContext, CRUCIBLE_MESSAGE_VERSION, CrucibleDecoder, JobId, Message, MessageWriter, ReadBlockContext, ReconciliationId, SnapshotDetails, @@ -126,6 +127,17 @@ impl IOop { | IOop::ExtentLiveNoOp { dependencies } => dependencies, } } + + fn heap_size(&self) -> usize { + let deps_bytes = std::mem::size_of_val(self.deps()); + let payload_bytes = match self { + IOop::Write { writes, .. } + | IOop::WriteUnwritten { writes, .. } => writes.heap_size(), + IOop::Read { requests, .. } => requests.heap_size(), + _ => 0, + }; + deps_bytes + payload_bytes + } } /// Read request for a particular extent @@ -186,6 +198,10 @@ impl RegionReadRequest { fn iter(&self) -> impl Iterator { self.0.iter() } + + fn heap_size(&self) -> usize { + self.0.capacity() * size_of::() + } } impl IntoIterator for RegionReadRequest { @@ -432,6 +448,19 @@ impl RegionWrite { fn iter(&self) -> impl Iterator { self.0.iter() } + + fn heap_size(&self) -> usize { + let vec_bytes = self.0.capacity() * size_of::(); + let inner_bytes: usize = self + .0 + .iter() + .map(|req| { + req.write.block_contexts.capacity() * size_of::() + + req.write.data.len() + }) + .sum(); + vec_bytes + inner_bytes + } } impl IntoIterator for RegionWrite { @@ -1591,6 +1620,7 @@ impl ActiveConnection { } else { self.work.completed.push(new_id); } + self.work.update_completed_hwm(); cdt::work__done!(|| new_id.0); Ok(None) @@ -2244,6 +2274,8 @@ impl DownstairsBuilder { .timeout(std::time::Duration::from_secs(15)) .build() .unwrap(), + read_bytes_hwm: 0, + write_bytes_hwm: 0, }) } } @@ -2285,6 +2317,12 @@ pub struct Downstairs { // A reqwest client, to be reused when creating progenitor clients pub reqwest_client: reqwest::Client, + + /// Largest read response buffer allocated (bytes) + read_bytes_hwm: usize, + + /// Largest write data buffer received (bytes) + write_bytes_hwm: usize, } #[allow(clippy::too_many_arguments)] @@ -2550,6 +2588,47 @@ impl Downstairs { self.active_upstairs.values().cloned().collect() } + pub fn memory_report(&self) -> MemoryReport { + let def = self.region.def(); + let active = self.active_upstairs(); + let connections = active + .iter() + .map(|conn_id| { + let work = self.work(*conn_id); + ConnectionMemoryReport { + pending_jobs: work.pending_jobs.len(), + pending_jobs_bytes: work.pending_jobs_bytes(), + pending_jobs_capacity_hwm: work.pending_capacity_hwm, + completed_ranges: work.completed.range_count(), + completed_ranges_hwm: work.completed_ranges_hwm, + } + }) + .collect(); + let extent_count = def.extent_count(); + let extent_meta_bytes = self.region.extent_meta_bytes(); + MemoryReport { + extent_count, + extent_size: def.extent_size().value, + block_size: def.block_size(), + region_bytes: self.region.heap_size(), + extent_meta_bytes, + bytes_per_extent: if extent_count > 0 { + extent_meta_bytes / extent_count as usize + } else { + 0 + }, + dirty_extent_count: self.region.dirty_extent_count(), + read_bytes_hwm: self.read_bytes_hwm, + write_bytes_hwm: self.write_bytes_hwm, + tokio_worker_threads: tokio::runtime::Handle::current() + .metrics() + .num_workers(), + rayon_threads: self.region.rayon_thread_count(), + active_connections: active.len(), + connections, + } + } + /// Does one round of work for the given connection #[cfg(test)] async fn do_work_for(&mut self, conn_id: ConnectionId) -> Result<()> { @@ -2680,6 +2759,12 @@ impl Downstairs { DownstairsRequest::ShowWork => { show_work(&mut self); } + DownstairsRequest::MemoryReport { done } => { + let report = self.memory_report(); + if done.send(report).is_err() { + warn!(log, "failed to reply to MemoryReport"); + } + } DownstairsRequest::IsExtentClosed { eid, done } => { let closed = matches!( self.region.extents[eid.0 as usize], @@ -3117,12 +3202,33 @@ impl Downstairs { } /// Handles a single message, either negotiation or doing IO + fn update_hwm(&mut self, m: &Message) { + match m { + Message::Write { data, .. } + | Message::WriteUnwritten { data, .. } => { + let n = data.len(); + if n > self.write_bytes_hwm { + self.write_bytes_hwm = n; + } + } + Message::ReadRequest { count, .. } => { + let n = + *count as usize * self.region.def().block_size() as usize; + if n > self.read_bytes_hwm { + self.read_bytes_hwm = n; + } + } + _ => {} + } + } + async fn on_message_for( &mut self, id: ConnectionId, m: Message, client_state: &DownstairsClientState, ) { + self.update_hwm(&m); let Some(state) = self.connection_state.get_mut(&id) else { warn!(self.log, "got message for disconnected id {id:?}; ignoring"); return; @@ -3230,6 +3336,9 @@ enum DownstairsRequest { /// This is fire-and-forget, so there's no oneshot reply channel ShowWork, + /// Returns a memory usage report + MemoryReport { done: oneshot::Sender }, + /// A message has arrived for the given id /// /// This is fire-and-forget; if the Downstairs doesn't like the message, it @@ -3260,6 +3369,14 @@ impl DownstairsHandle { .context("could not send message on channel") } + pub async fn memory_report(&self) -> Result { + let (done, rx) = oneshot::channel(); + self.tx + .send(DownstairsRequest::MemoryReport { done }) + .context("could not send message on channel")?; + rx.await.context("could not receive result") + } + async fn new_connection( &self, id: ConnectionId, @@ -3298,6 +3415,12 @@ pub struct Work { /// Track completed jobs since the last flush completed: CompletedJobs, + /// High-water mark for pending_jobs VecDeque capacity + pending_capacity_hwm: usize, + + /// High-water mark for completed job range count + completed_ranges_hwm: usize, + log: Logger, } @@ -3306,6 +3429,8 @@ impl Work { Work { pending_jobs: VecDeque::new(), completed: CompletedJobs::new(last_flush), + pending_capacity_hwm: 0, + completed_ranges_hwm: 0, log, } } @@ -3314,9 +3439,28 @@ impl Work { self.completed.completed().collect() } + fn pending_jobs_bytes(&self) -> usize { + let deque_bytes = + self.pending_jobs.capacity() * size_of::(); + let inner_bytes: usize = + self.pending_jobs.iter().map(|job| job.io.heap_size()).sum(); + deque_bytes + inner_bytes + } + /// Pushes a new job to the back of the queue fn add_pending_job(&mut self, job: PendingJob) { self.pending_jobs.push_back(job); + let cap = self.pending_jobs.capacity(); + if cap > self.pending_capacity_hwm { + self.pending_capacity_hwm = cap; + } + } + + fn update_completed_hwm(&mut self) { + let ranges = self.completed.range_count(); + if ranges > self.completed_ranges_hwm { + self.completed_ranges_hwm = ranges; + } } /// Returns a list of pending job IDs @@ -6247,4 +6391,329 @@ mod test { assert!(jh.stop().await.is_ok()); } + + #[tokio::test] + async fn memory_report_no_connections() { + let dir = tempdir().unwrap(); + let ds = create_test_downstairs(512, 4, 3, &dir).unwrap(); + + let report = ds.memory_report(); + assert_eq!(report.extent_count, 3); + assert_eq!(report.extent_size, 4); + assert_eq!(report.block_size, 512); + assert_eq!(report.active_connections, 0); + assert!(report.connections.is_empty()); + // 3 extents, each with two BlockBitArrays of 4 blocks. + // Each BlockBitArray uses ceil(4/32) = 1 u32 = 4 bytes. + // So per extent: 2 * 4 = 8 bytes of extent inner data. + // region_bytes includes the extents Vec capacity, the + // extent inner data, the dirty_extents HashSet, and the + // dir PathBuf. + assert!(report.region_bytes > 0); + } + + #[tokio::test] + async fn memory_report_one_connection_no_work() { + let dir = tempdir().unwrap(); + let mut ds = create_test_downstairs(512, 4, 3, &dir).unwrap(); + + let upstairs_connection = UpstairsConnection { + upstairs_id: Uuid::new_v4(), + session_id: Uuid::new_v4(), + generation: 10, + }; + let conn_id = ConnectionId(0); + let (_cancel, _rx) = + ds.add_fake_connection(upstairs_connection, conn_id); + ds.promote_to_active(upstairs_connection, conn_id).unwrap(); + + let report = ds.memory_report(); + assert_eq!(report.active_connections, 1); + assert_eq!(report.connections.len(), 1); + assert_eq!(report.connections[0].pending_jobs, 0); + assert_eq!(report.connections[0].pending_jobs_bytes, 0); + } + + #[tokio::test] + async fn memory_report_one_connection_with_work() { + let dir = tempdir().unwrap(); + let mut ds = create_test_downstairs(512, 4, 3, &dir).unwrap(); + + let upstairs_connection = UpstairsConnection { + upstairs_id: Uuid::new_v4(), + session_id: Uuid::new_v4(), + generation: 10, + }; + let conn_id = ConnectionId(0); + let (_cancel, _rx) = + ds.add_fake_connection(upstairs_connection, conn_id); + ds.promote_to_active(upstairs_connection, conn_id).unwrap(); + + let rio = IOop::Read { + dependencies: Vec::new(), + requests: RegionReadRequest(vec![RegionReadReq { + extent: ExtentId(0), + offset: BlockOffset(1), + count: NonZeroUsize::new(1).unwrap(), + }]), + }; + ds.active_mut(conn_id).add_work(JobId(1000), rio); + + let rio = IOop::Read { + dependencies: vec![JobId(1000)], + requests: RegionReadRequest(vec![RegionReadReq { + extent: ExtentId(1), + offset: BlockOffset(1), + count: NonZeroUsize::new(1).unwrap(), + }]), + }; + ds.active_mut(conn_id).add_work(JobId(1001), rio); + + let report = ds.memory_report(); + assert_eq!(report.active_connections, 1); + assert_eq!(report.connections.len(), 1); + assert_eq!(report.connections[0].pending_jobs, 2); + assert!(report.connections[0].pending_jobs_bytes > 0); + } + + #[tokio::test] + async fn memory_report_region_scales_with_extents() { + // Verify that more extents means more region memory. + let dir_small = tempdir().unwrap(); + let ds_small = create_test_downstairs(512, 4, 3, &dir_small).unwrap(); + let small = ds_small.memory_report(); + + let dir_large = tempdir().unwrap(); + let ds_large = create_test_downstairs(512, 4, 30, &dir_large).unwrap(); + let large = ds_large.memory_report(); + + assert_eq!(small.extent_count, 3); + assert_eq!(large.extent_count, 30); + assert!( + large.region_bytes > small.region_bytes, + "30 extents ({}) should use more memory than 3 ({})", + large.region_bytes, + small.region_bytes, + ); + } + + #[tokio::test] + async fn memory_report_region_scales_with_extent_size() { + // Larger extents (more blocks) means bigger BlockBitArrays. + let dir_small = tempdir().unwrap(); + let ds_small = create_test_downstairs(512, 4, 3, &dir_small).unwrap(); + let small = ds_small.memory_report(); + + let dir_large = tempdir().unwrap(); + let ds_large = + create_test_downstairs(512, 1024, 3, &dir_large).unwrap(); + let large = ds_large.memory_report(); + + assert_eq!(small.extent_size, 4); + assert_eq!(large.extent_size, 1024); + assert!( + large.region_bytes > small.region_bytes, + "1024-block extents ({}) should use more memory \ + than 4-block extents ({})", + large.region_bytes, + small.region_bytes, + ); + } + + #[tokio::test] + async fn memory_report_multiple_connections() { + // Use a read-only downstairs so multiple active connections + // are allowed. + let dir = tempdir().unwrap(); + let mut region_options: crucible_common::RegionOptions = + Default::default(); + region_options.set_block_size(512); + region_options.set_extent_size(Block::new(4, 9)); + region_options.set_uuid(Uuid::new_v4()); + mkdir_for_file(dir.path()).unwrap(); + let mut region = Region::create(&dir, region_options, csl()).unwrap(); + region.extend(3, Backend::default()).unwrap(); + let path_dir = dir.as_ref().to_path_buf(); + let mut ds = Downstairs::new_builder(&path_dir, true) + .set_logger(csl()) + .build() + .unwrap(); + + // First connection with 1 pending job + let uc1 = UpstairsConnection { + upstairs_id: Uuid::new_v4(), + session_id: Uuid::new_v4(), + generation: 10, + }; + let conn1 = ConnectionId(0); + let (_cancel1, _rx1) = ds.add_fake_connection(uc1, conn1); + ds.promote_to_active(uc1, conn1).unwrap(); + + let rio = IOop::Read { + dependencies: Vec::new(), + requests: RegionReadRequest(vec![RegionReadReq { + extent: ExtentId(0), + offset: BlockOffset(1), + count: NonZeroUsize::new(1).unwrap(), + }]), + }; + ds.active_mut(conn1).add_work(JobId(1000), rio); + + // Second connection with 3 pending jobs + let uc2 = UpstairsConnection { + upstairs_id: Uuid::new_v4(), + session_id: Uuid::new_v4(), + generation: 10, + }; + let conn2 = ConnectionId(1); + let (_cancel2, _rx2) = ds.add_fake_connection(uc2, conn2); + ds.promote_to_active(uc2, conn2).unwrap(); + + for i in 0..3 { + let rio = IOop::Read { + dependencies: Vec::new(), + requests: RegionReadRequest(vec![RegionReadReq { + extent: ExtentId(0), + offset: BlockOffset(1), + count: NonZeroUsize::new(1).unwrap(), + }]), + }; + ds.active_mut(conn2).add_work(JobId(2000 + i), rio); + } + + let report = ds.memory_report(); + assert_eq!(report.active_connections, 2); + assert_eq!(report.connections.len(), 2); + + let mut pending: Vec = + report.connections.iter().map(|c| c.pending_jobs).collect(); + pending.sort(); + assert_eq!(pending, vec![1, 3]); + } + + #[tokio::test] + async fn memory_report_per_extent_cost() { + // Verify that bytes_per_extent is consistent across regions + // with different extent counts but the same extent size. + let dir = tempdir().unwrap(); + let ds = create_test_downstairs(512, 4, 10, &dir).unwrap(); + + let report = ds.memory_report(); + assert!(report.bytes_per_extent > 0); + assert_eq!( + report.extent_meta_bytes, + report.bytes_per_extent * report.extent_count as usize, + ); + + let dir2 = tempdir().unwrap(); + let ds2 = create_test_downstairs(512, 4, 20, &dir2).unwrap(); + let report2 = ds2.memory_report(); + + assert_eq!(report.bytes_per_extent, report2.bytes_per_extent); + } + + #[tokio::test] + async fn memory_report_hwm_tracking() { + let dir = tempdir().unwrap(); + let mut ds = create_test_downstairs(4096, 4, 3, &dir).unwrap(); + + // Initially zero + let report = ds.memory_report(); + assert_eq!(report.read_bytes_hwm, 0); + assert_eq!(report.write_bytes_hwm, 0); + + // Simulate a write message + let write_msg = Message::Write { + header: crucible_protocol::WriteHeader { + upstairs_id: Uuid::new_v4(), + session_id: Uuid::new_v4(), + job_id: JobId(1), + dependencies: vec![], + start: BlockIndex(0), + contexts: vec![], + }, + data: Bytes::from(vec![0u8; 4096 * 10]), + }; + ds.update_hwm(&write_msg); + + let report = ds.memory_report(); + assert_eq!(report.write_bytes_hwm, 4096 * 10); + assert_eq!(report.read_bytes_hwm, 0); + + // Smaller write doesn't change the HWM + let small_write = Message::Write { + header: crucible_protocol::WriteHeader { + upstairs_id: Uuid::new_v4(), + session_id: Uuid::new_v4(), + job_id: JobId(2), + dependencies: vec![], + start: BlockIndex(0), + contexts: vec![], + }, + data: Bytes::from(vec![0u8; 4096]), + }; + ds.update_hwm(&small_write); + assert_eq!(ds.memory_report().write_bytes_hwm, 4096 * 10); + + // Simulate a read request + let read_msg = Message::ReadRequest { + upstairs_id: Uuid::new_v4(), + session_id: Uuid::new_v4(), + job_id: JobId(3), + dependencies: vec![], + start: BlockIndex(0), + count: 20, + }; + ds.update_hwm(&read_msg); + + let report = ds.memory_report(); + assert_eq!(report.read_bytes_hwm, 4096 * 20); + assert_eq!(report.write_bytes_hwm, 4096 * 10); + } + + #[tokio::test] + async fn memory_report_work_queue_hwm() { + let dir = tempdir().unwrap(); + let mut ds = create_test_downstairs(512, 4, 3, &dir).unwrap(); + + let uc = UpstairsConnection { + upstairs_id: Uuid::new_v4(), + session_id: Uuid::new_v4(), + generation: 10, + }; + let conn_id = ConnectionId(0); + let (_cancel, _rx) = ds.add_fake_connection(uc, conn_id); + ds.promote_to_active(uc, conn_id).unwrap(); + + // HWMs start at zero + let report = ds.memory_report(); + assert_eq!(report.connections[0].pending_jobs_capacity_hwm, 0); + assert_eq!(report.connections[0].completed_ranges_hwm, 0); + + // Add several jobs to grow the VecDeque capacity + for i in 0..10 { + let rio = IOop::Read { + dependencies: Vec::new(), + requests: RegionReadRequest(vec![RegionReadReq { + extent: ExtentId(0), + offset: BlockOffset(0), + count: NonZeroUsize::new(1).unwrap(), + }]), + }; + ds.active_mut(conn_id).add_work(JobId(1000 + i), rio); + } + + let report = ds.memory_report(); + assert!(report.connections[0].pending_jobs_capacity_hwm >= 10); + + // Process all work to trigger completed range HWM + ds.do_work_for(conn_id).await.unwrap(); + + let report = ds.memory_report(); + assert_eq!(report.connections[0].pending_jobs, 0); + // Capacity HWM remains from the peak + assert!(report.connections[0].pending_jobs_capacity_hwm >= 10); + // Completed ranges HWM should be at least 1 + assert!(report.connections[0].completed_ranges_hwm >= 1); + } } diff --git a/downstairs/src/region.rs b/downstairs/src/region.rs index 01dce10d1..40178de03 100644 --- a/downstairs/src/region.rs +++ b/downstairs/src/region.rs @@ -1219,6 +1219,28 @@ impl Region { pub fn read_only(&self) -> bool { self.read_only } + + pub fn heap_size(&self) -> usize { + let extents_vec = self.extents.capacity() * size_of::(); + let extents_inner: usize = + self.extents.iter().map(|e| e.heap_size()).sum(); + let dirty_extents = + self.dirty_extents.capacity() * size_of::(); + let dir = self.dir.capacity(); + extents_vec + extents_inner + dirty_extents + dir + } + + pub fn extent_meta_bytes(&self) -> usize { + self.extents.iter().map(|e| e.heap_size()).sum() + } + + pub fn dirty_extent_count(&self) -> usize { + self.dirty_extents.len() + } + + pub fn rayon_thread_count(&self) -> usize { + self.pool.current_num_threads() + } } #[cfg(feature = "omicron-build")] diff --git a/downstairs/src/repair.rs b/downstairs/src/repair.rs index dba56f6a8..6b03258a4 100644 --- a/downstairs/src/repair.rs +++ b/downstairs/src/repair.rs @@ -196,6 +196,21 @@ impl CrucibleDownstairsRepairApi for CrucibleDownstairsRepairImpl { .map(|_| HttpResponseOk(true)) .map_err(|e| HttpError::for_internal_error(e.to_string())) } + + /// Get memory usage report. + async fn get_memory( + rqctx: RequestContext, + ) -> Result< + HttpResponseOk, + HttpError, + > { + let downstairs = &rqctx.context().downstairs; + downstairs + .memory_report() + .await + .map(HttpResponseOk) + .map_err(|e| HttpError::for_internal_error(e.to_string())) + } } async fn get_a_file( From 30764a42657273eb737b759796d015cb585447e4 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Wed, 3 Jun 2026 16:14:00 -0700 Subject: [PATCH 2/4] Bump downstairs repair service to version 2 This adds the debug /memory endpoint to the repair server. It's not needed for repair, and none of the actual repair protocol changed. --- downstairs-api/src/lib.rs | 2 + downstairs/src/repair.rs | 20 ++- ...ownstairs-repair-1.0.0-178638.json.gitstub | 1 + ...on => downstairs-repair-2.0.0-9f0059.json} | 151 +++++++++++++++++- .../downstairs-repair-latest.json | 2 +- 5 files changed, 168 insertions(+), 8 deletions(-) create mode 100644 openapi/downstairs-repair/downstairs-repair-1.0.0-178638.json.gitstub rename openapi/downstairs-repair/{downstairs-repair-1.0.0-178638.json => downstairs-repair-2.0.0-9f0059.json} (67%) diff --git a/downstairs-api/src/lib.rs b/downstairs-api/src/lib.rs index ecc447a37..05a978304 100644 --- a/downstairs-api/src/lib.rs +++ b/downstairs-api/src/lib.rs @@ -23,6 +23,7 @@ api_versions!([ // | example for the next person. // v // (next_int, IDENT), + (2, MEMORY), (1, INITIAL), ]); @@ -127,6 +128,7 @@ pub trait CrucibleDownstairsRepairApi { #[endpoint { method = GET, path = "/memory", + versions = VERSION_MEMORY.., }] async fn get_memory( rqctx: RequestContext, diff --git a/downstairs/src/repair.rs b/downstairs/src/repair.rs index 6b03258a4..d95546a38 100644 --- a/downstairs/src/repair.rs +++ b/downstairs/src/repair.rs @@ -5,8 +5,9 @@ use std::sync::Arc; use crucible_downstairs_api::*; use crucible_downstairs_types::repair::{ExtentFilePath, ExtentPath, FileType}; use dropshot::{ - Body, CompressionConfig, ConfigDropshot, HandlerTaskMode, HttpError, - HttpResponseOk, HttpServerStarter, Path, RequestContext, + Body, ClientSpecifiesVersionInHeader, CompressionConfig, ConfigDropshot, + HandlerTaskMode, HttpError, HttpResponseOk, Path, RequestContext, + VersionPolicy, }; use hyper::{Response, StatusCode}; @@ -68,10 +69,17 @@ pub fn repair_main( /* * Set up the server. */ - let server = - HttpServerStarter::new(&config_dropshot, api, context.into(), log) - .map_err(|error| format!("failed to create server: {}", error))? - .start(); + let server = dropshot::ServerBuilder::new(api, context.into(), log.clone()) + .config(config_dropshot) + .version_policy(VersionPolicy::Dynamic(Box::new( + ClientSpecifiesVersionInHeader::new( + omicron_common::api::VERSION_HEADER, + crucible_downstairs_api::latest_version(), + ), + ))) + .build_starter() + .map_err(|error| format!("failed to create server: {}", error))? + .start(); let local_addr = server.local_addr(); let h = tokio::spawn(async move { diff --git a/openapi/downstairs-repair/downstairs-repair-1.0.0-178638.json.gitstub b/openapi/downstairs-repair/downstairs-repair-1.0.0-178638.json.gitstub new file mode 100644 index 000000000..9dddbcc62 --- /dev/null +++ b/openapi/downstairs-repair/downstairs-repair-1.0.0-178638.json.gitstub @@ -0,0 +1 @@ +ef04744d2a7ba34d6036ec7c39abcc49eeec25ef:openapi/downstairs-repair/downstairs-repair-1.0.0-178638.json diff --git a/openapi/downstairs-repair/downstairs-repair-1.0.0-178638.json b/openapi/downstairs-repair/downstairs-repair-2.0.0-9f0059.json similarity index 67% rename from openapi/downstairs-repair/downstairs-repair-1.0.0-178638.json rename to openapi/downstairs-repair/downstairs-repair-2.0.0-9f0059.json index 1632ab315..0d1ed1fa2 100644 --- a/openapi/downstairs-repair/downstairs-repair-1.0.0-178638.json +++ b/openapi/downstairs-repair/downstairs-repair-2.0.0-9f0059.json @@ -6,7 +6,7 @@ "url": "https://oxide.computer", "email": "api@oxide.computer" }, - "version": "1.0.0" + "version": "2.0.0" }, "paths": { "/extent/{eid}/files": { @@ -87,6 +87,30 @@ } } }, + "/memory": { + "get": { + "summary": "Memory usage report", + "operationId": "get_memory", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MemoryReport" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/newextent/{eid}/{file_type}": { "get": { "summary": "Get a specific extent file (data, database, or log files).", @@ -219,6 +243,44 @@ "value" ] }, + "ConnectionMemoryReport": { + "description": "Per-connection memory report", + "type": "object", + "properties": { + "completed_ranges": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "completed_ranges_hwm": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "pending_jobs": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "pending_jobs_bytes": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "pending_jobs_capacity_hwm": { + "type": "integer", + "format": "uint", + "minimum": 0 + } + }, + "required": [ + "completed_ranges", + "completed_ranges_hwm", + "pending_jobs", + "pending_jobs_bytes", + "pending_jobs_capacity_hwm" + ] + }, "Error": { "description": "Error information from a response.", "type": "object", @@ -238,6 +300,93 @@ "request_id" ] }, + "MemoryReport": { + "description": "Summary of downstairs memory usage", + "type": "object", + "properties": { + "active_connections": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "block_size": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "bytes_per_extent": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "connections": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ConnectionMemoryReport" + } + }, + "dirty_extent_count": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "extent_count": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "extent_meta_bytes": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "extent_size": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "rayon_threads": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "read_bytes_hwm": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "region_bytes": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "tokio_worker_threads": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "write_bytes_hwm": { + "type": "integer", + "format": "uint", + "minimum": 0 + } + }, + "required": [ + "active_connections", + "block_size", + "bytes_per_extent", + "connections", + "dirty_extent_count", + "extent_count", + "extent_meta_bytes", + "extent_size", + "rayon_threads", + "read_bytes_hwm", + "region_bytes", + "tokio_worker_threads", + "write_bytes_hwm" + ] + }, "RegionDefinition": { "type": "object", "properties": { diff --git a/openapi/downstairs-repair/downstairs-repair-latest.json b/openapi/downstairs-repair/downstairs-repair-latest.json index 6b6b998e0..c1ad40fdf 120000 --- a/openapi/downstairs-repair/downstairs-repair-latest.json +++ b/openapi/downstairs-repair/downstairs-repair-latest.json @@ -1 +1 @@ -downstairs-repair-1.0.0-178638.json \ No newline at end of file +downstairs-repair-2.0.0-9f0059.json \ No newline at end of file From 396517c54e78f528f5983e9a950af61a190c4166 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Mon, 8 Jun 2026 15:27:08 -0700 Subject: [PATCH 3/4] Switch downstairs to jemalloc and report allocator stats Use tikv-jemallocator as the global allocator for the downstairs. This gives us visibility into allocator-level memory usage through the /memory endpoint. New fields on MemoryReport: * jemalloc_allocated: bytes actively held by the application * jemalloc_active: bytes in active allocator pages * jemalloc_resident: total RSS from the allocator * jemalloc_mapped: total virtual memory mapped * jemalloc_retained: memory freed but still mapped The gap between allocated and resident shows how much memory is lost to fragmentation and allocator overhead. --- Cargo.lock | 54 +- Cargo.toml | 2 + .../versions/src/initial/repair.rs | 7 + downstairs/Cargo.toml | 2 + downstairs/src/lib.rs | 19 + downstairs/src/main.rs | 4 + .../downstairs-repair-2.0.0-9f0059.json | 468 ------------------ .../downstairs-repair-latest.json | 2 +- workspace-hack/Cargo.toml | 6 - 9 files changed, 78 insertions(+), 486 deletions(-) delete mode 100644 openapi/downstairs-repair/downstairs-repair-2.0.0-9f0059.json diff --git a/Cargo.lock b/Cargo.lock index 8e77938e4..f79fbe498 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -926,7 +926,7 @@ dependencies = [ "terminfo", "thiserror 2.0.18", "which 8.0.0", - "windows-sys 0.59.0", + "windows-sys 0.61.1", ] [[package]] @@ -1053,7 +1053,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.1", ] [[package]] @@ -1573,6 +1573,8 @@ dependencies = [ "statistical", "tempfile", "thiserror 2.0.18", + "tikv-jemalloc-ctl", + "tikv-jemallocator", "tokio", "tokio-rustls 0.24.1", "tokio-util", @@ -1893,7 +1895,6 @@ dependencies = [ "futures-util", "generic-array", "getrandom 0.2.11", - "getrandom 0.3.1", "getrandom 0.4.1", "hex", "hyper", @@ -3777,7 +3778,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.3", "system-configuration", "tokio", "tower-layer", @@ -6486,7 +6487,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls 0.23.31", - "socket2 0.5.10", + "socket2 0.6.3", "thiserror 2.0.18", "tokio", "tracing", @@ -7147,7 +7148,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.52.0", + "windows-sys 0.61.1", ] [[package]] @@ -7250,7 +7251,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.1", ] [[package]] @@ -8086,7 +8087,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.117", @@ -8430,10 +8431,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0" dependencies = [ "fastrand", - "getrandom 0.3.1", + "getrandom 0.4.1", "once_cell", "rustix 1.1.4", - "windows-sys 0.52.0", + "windows-sys 0.61.1", ] [[package]] @@ -8442,7 +8443,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2111ef44dae28680ae9752bb89409e7310ca33a8c621ebe7b106cf5c928b3ac0" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.1", ] [[package]] @@ -8610,6 +8611,37 @@ dependencies = [ "threadpool", ] +[[package]] +name = "tikv-jemalloc-ctl" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "661f1f6a57b3a36dc9174a2c10f19513b4866816e13425d3e418b11cc37bc24c" +dependencies = [ + "libc", + "paste", + "tikv-jemalloc-sys", +] + +[[package]] +name = "tikv-jemalloc-sys" +version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd8aa5b2ab86a2cefa406d889139c162cbb230092f7d1d7cbc1716405d852a3b" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0359b4327f954e0567e69fb191cf1436617748813819c94b8cd4a431422d053a" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "time" version = "0.3.47" diff --git a/Cargo.toml b/Cargo.toml index 9ef1a01b8..9be8f902c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,6 +77,8 @@ hyper = { version = "1", features = [ "full" ] } hyper-staticfile = "0.10.1" indicatif = { version = "0.18.4", features = ["rayon"] } itertools = "0.14.0" +tikv-jemallocator = { version = "0.6", features = ["stats"] } +tikv-jemalloc-ctl = { version = "0.6", features = ["stats"] } libc = "0.2" mime_guess = "2.0.5" nbd = "0.3.1" diff --git a/downstairs-types/versions/src/initial/repair.rs b/downstairs-types/versions/src/initial/repair.rs index dbdbf8108..6f36543be 100644 --- a/downstairs-types/versions/src/initial/repair.rs +++ b/downstairs-types/versions/src/initial/repair.rs @@ -70,6 +70,13 @@ pub struct MemoryReport { pub tokio_worker_threads: usize, pub rayon_threads: usize, + // jemalloc allocator stats + pub jemalloc_allocated: usize, + pub jemalloc_active: usize, + pub jemalloc_resident: usize, + pub jemalloc_mapped: usize, + pub jemalloc_retained: usize, + // Connection memory pub active_connections: usize, pub connections: Vec, diff --git a/downstairs/Cargo.toml b/downstairs/Cargo.toml index f8425beb0..a5965b28e 100644 --- a/downstairs/Cargo.toml +++ b/downstairs/Cargo.toml @@ -60,6 +60,8 @@ tracing-subscriber.workspace = true tracing.workspace = true usdt.workspace = true uuid.workspace = true +tikv-jemallocator.workspace = true +tikv-jemalloc-ctl.workspace = true crucible-workspace-hack.workspace = true [dev-dependencies] diff --git a/downstairs/src/lib.rs b/downstairs/src/lib.rs index 733134f5b..fc8fac025 100644 --- a/downstairs/src/lib.rs +++ b/downstairs/src/lib.rs @@ -2606,6 +2606,20 @@ impl Downstairs { .collect(); let extent_count = def.extent_count(); let extent_meta_bytes = self.region.extent_meta_bytes(); + + // Advance jemalloc's stats epoch so we get fresh numbers + tikv_jemalloc_ctl::epoch::advance().ok(); + let jemalloc_allocated = + tikv_jemalloc_ctl::stats::allocated::read().unwrap_or(0); + let jemalloc_active = + tikv_jemalloc_ctl::stats::active::read().unwrap_or(0); + let jemalloc_resident = + tikv_jemalloc_ctl::stats::resident::read().unwrap_or(0); + let jemalloc_mapped = + tikv_jemalloc_ctl::stats::mapped::read().unwrap_or(0); + let jemalloc_retained = + tikv_jemalloc_ctl::stats::retained::read().unwrap_or(0); + MemoryReport { extent_count, extent_size: def.extent_size().value, @@ -2624,6 +2638,11 @@ impl Downstairs { .metrics() .num_workers(), rayon_threads: self.region.rayon_thread_count(), + jemalloc_allocated, + jemalloc_active, + jemalloc_resident, + jemalloc_mapped, + jemalloc_retained, active_connections: active.len(), connections, } diff --git a/downstairs/src/main.rs b/downstairs/src/main.rs index 5901d57ed..258d348cf 100644 --- a/downstairs/src/main.rs +++ b/downstairs/src/main.rs @@ -1,4 +1,8 @@ // Copyright 2023 Oxide Computer Company + +#[global_allocator] +static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; use std::time::Duration; diff --git a/openapi/downstairs-repair/downstairs-repair-2.0.0-9f0059.json b/openapi/downstairs-repair/downstairs-repair-2.0.0-9f0059.json deleted file mode 100644 index 0d1ed1fa2..000000000 --- a/openapi/downstairs-repair/downstairs-repair-2.0.0-9f0059.json +++ /dev/null @@ -1,468 +0,0 @@ -{ - "openapi": "3.0.3", - "info": { - "title": "Downstairs Repair", - "contact": { - "url": "https://oxide.computer", - "email": "api@oxide.computer" - }, - "version": "2.0.0" - }, - "paths": { - "/extent/{eid}/files": { - "get": { - "summary": "Get the list of files related to an extent.", - "description": "For a given extent, return a vec of strings representing the names of the files that exist for that extent.", - "operationId": "get_files_for_extent", - "parameters": [ - { - "in": "path", - "name": "eid", - "required": true, - "schema": { - "type": "integer", - "format": "uint32", - "minimum": 0 - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "title": "Array_of_String", - "type": "array", - "items": { - "type": "string" - } - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/extent/{eid}/repair-ready": { - "get": { - "summary": "Return true if the provided extent is closed or the region is read only.", - "operationId": "extent_repair_ready", - "parameters": [ - { - "in": "path", - "name": "eid", - "required": true, - "schema": { - "type": "integer", - "format": "uint32", - "minimum": 0 - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "title": "Boolean", - "type": "boolean" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/memory": { - "get": { - "summary": "Memory usage report", - "operationId": "get_memory", - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/MemoryReport" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/newextent/{eid}/{file_type}": { - "get": { - "summary": "Get a specific extent file (data, database, or log files).", - "operationId": "get_extent_file", - "parameters": [ - { - "in": "path", - "name": "eid", - "required": true, - "schema": { - "type": "integer", - "format": "uint32", - "minimum": 0 - } - }, - { - "in": "path", - "name": "file_type", - "required": true, - "schema": { - "$ref": "#/components/schemas/FileType" - } - } - ], - "responses": { - "default": { - "description": "", - "content": { - "*/*": { - "schema": {} - } - } - } - } - } - }, - "/region-info": { - "get": { - "summary": "Return the RegionDefinition describing our region.", - "operationId": "get_region_info", - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/RegionDefinition" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/region-mode": { - "get": { - "summary": "Return the region-mode describing our region.", - "operationId": "get_region_mode", - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "title": "Boolean", - "type": "boolean" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/work": { - "get": { - "summary": "Work queue", - "operationId": "get_work", - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "title": "Boolean", - "type": "boolean" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - } - }, - "components": { - "schemas": { - "Block": { - "type": "object", - "properties": { - "shift": { - "type": "integer", - "format": "uint32", - "minimum": 0 - }, - "value": { - "type": "integer", - "format": "uint64", - "minimum": 0 - } - }, - "required": [ - "shift", - "value" - ] - }, - "ConnectionMemoryReport": { - "description": "Per-connection memory report", - "type": "object", - "properties": { - "completed_ranges": { - "type": "integer", - "format": "uint", - "minimum": 0 - }, - "completed_ranges_hwm": { - "type": "integer", - "format": "uint", - "minimum": 0 - }, - "pending_jobs": { - "type": "integer", - "format": "uint", - "minimum": 0 - }, - "pending_jobs_bytes": { - "type": "integer", - "format": "uint", - "minimum": 0 - }, - "pending_jobs_capacity_hwm": { - "type": "integer", - "format": "uint", - "minimum": 0 - } - }, - "required": [ - "completed_ranges", - "completed_ranges_hwm", - "pending_jobs", - "pending_jobs_bytes", - "pending_jobs_capacity_hwm" - ] - }, - "Error": { - "description": "Error information from a response.", - "type": "object", - "properties": { - "error_code": { - "type": "string" - }, - "message": { - "type": "string" - }, - "request_id": { - "type": "string" - } - }, - "required": [ - "message", - "request_id" - ] - }, - "MemoryReport": { - "description": "Summary of downstairs memory usage", - "type": "object", - "properties": { - "active_connections": { - "type": "integer", - "format": "uint", - "minimum": 0 - }, - "block_size": { - "type": "integer", - "format": "uint64", - "minimum": 0 - }, - "bytes_per_extent": { - "type": "integer", - "format": "uint", - "minimum": 0 - }, - "connections": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ConnectionMemoryReport" - } - }, - "dirty_extent_count": { - "type": "integer", - "format": "uint", - "minimum": 0 - }, - "extent_count": { - "type": "integer", - "format": "uint32", - "minimum": 0 - }, - "extent_meta_bytes": { - "type": "integer", - "format": "uint", - "minimum": 0 - }, - "extent_size": { - "type": "integer", - "format": "uint64", - "minimum": 0 - }, - "rayon_threads": { - "type": "integer", - "format": "uint", - "minimum": 0 - }, - "read_bytes_hwm": { - "type": "integer", - "format": "uint", - "minimum": 0 - }, - "region_bytes": { - "type": "integer", - "format": "uint", - "minimum": 0 - }, - "tokio_worker_threads": { - "type": "integer", - "format": "uint", - "minimum": 0 - }, - "write_bytes_hwm": { - "type": "integer", - "format": "uint", - "minimum": 0 - } - }, - "required": [ - "active_connections", - "block_size", - "bytes_per_extent", - "connections", - "dirty_extent_count", - "extent_count", - "extent_meta_bytes", - "extent_size", - "rayon_threads", - "read_bytes_hwm", - "region_bytes", - "tokio_worker_threads", - "write_bytes_hwm" - ] - }, - "RegionDefinition": { - "type": "object", - "properties": { - "block_size": { - "description": "The size of each block in bytes. Must be a power of 2, minimum 512.", - "type": "integer", - "format": "uint64", - "minimum": 0 - }, - "database_read_version": { - "description": "The database version format for reading an extent database file.", - "type": "integer", - "format": "uint", - "minimum": 0 - }, - "database_write_version": { - "description": "The database version format for writing an extent database file.", - "type": "integer", - "format": "uint", - "minimum": 0 - }, - "encrypted": { - "description": "region data will be encrypted", - "type": "boolean" - }, - "extent_count": { - "description": "How many whole extents comprise this region?", - "type": "integer", - "format": "uint32", - "minimum": 0 - }, - "extent_size": { - "description": "How many blocks should appear in each extent?", - "allOf": [ - { - "$ref": "#/components/schemas/Block" - } - ] - }, - "uuid": { - "description": "UUID for this region", - "type": "string", - "format": "uuid" - } - }, - "required": [ - "block_size", - "database_read_version", - "database_write_version", - "encrypted", - "extent_count", - "extent_size", - "uuid" - ] - }, - "FileType": { - "type": "string", - "enum": [ - "data", - "db", - "db_shm", - "db_wal" - ] - } - }, - "responses": { - "Error": { - "description": "Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Error" - } - } - } - } - } - } -} diff --git a/openapi/downstairs-repair/downstairs-repair-latest.json b/openapi/downstairs-repair/downstairs-repair-latest.json index c1ad40fdf..7513c4bc9 120000 --- a/openapi/downstairs-repair/downstairs-repair-latest.json +++ b/openapi/downstairs-repair/downstairs-repair-latest.json @@ -1 +1 @@ -downstairs-repair-2.0.0-9f0059.json \ No newline at end of file +downstairs-repair-2.0.0-bd1bb6.json \ No newline at end of file diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 9ec612f30..b0a256a97 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -160,7 +160,6 @@ zeroize = { version = "1", features = ["std", "zeroize_derive"] } aws-lc-rs = { version = "1", features = ["prebuilt-nasm"] } dof-468e82937335b1c9 = { package = "dof", version = "0.3", default-features = false, features = ["des"] } dof-9fbad63c4bcf4a8f = { package = "dof", version = "0.4", default-features = false, features = ["des"] } -getrandom-468e82937335b1c9 = { package = "getrandom", version = "0.3", default-features = false, features = ["std"] } hyper = { version = "1", features = ["full"] } hyper-rustls = { version = "0.27", default-features = false, features = ["aws-lc-rs", "http1", "http2", "ring", "tls12", "webpki-tokio"] } hyper-util = { version = "0.1", features = ["full"] } @@ -178,7 +177,6 @@ tokio-rustls = { version = "0.26", default-features = false, features = ["aws-lc aws-lc-rs = { version = "1", features = ["prebuilt-nasm"] } dof-468e82937335b1c9 = { package = "dof", version = "0.3", default-features = false, features = ["des"] } dof-9fbad63c4bcf4a8f = { package = "dof", version = "0.4", default-features = false, features = ["des"] } -getrandom-468e82937335b1c9 = { package = "getrandom", version = "0.3", default-features = false, features = ["std"] } hyper = { version = "1", features = ["full"] } hyper-rustls = { version = "0.27", default-features = false, features = ["aws-lc-rs", "http1", "http2", "ring", "tls12", "webpki-tokio"] } hyper-util = { version = "0.1", features = ["full"] } @@ -193,7 +191,6 @@ tokio-rustls = { version = "0.26", default-features = false, features = ["aws-lc [target.aarch64-apple-darwin.dependencies] aws-lc-rs = { version = "1", features = ["prebuilt-nasm"] } -getrandom-468e82937335b1c9 = { package = "getrandom", version = "0.3", default-features = false, features = ["std"] } hyper = { version = "1", features = ["full"] } hyper-rustls = { version = "0.27", default-features = false, features = ["aws-lc-rs", "http1", "http2", "ring", "tls12", "webpki-tokio"] } hyper-util = { version = "0.1", features = ["full"] } @@ -207,7 +204,6 @@ tokio-rustls = { version = "0.26", default-features = false, features = ["aws-lc [target.aarch64-apple-darwin.build-dependencies] aws-lc-rs = { version = "1", features = ["prebuilt-nasm"] } -getrandom-468e82937335b1c9 = { package = "getrandom", version = "0.3", default-features = false, features = ["std"] } hyper = { version = "1", features = ["full"] } hyper-rustls = { version = "0.27", default-features = false, features = ["aws-lc-rs", "http1", "http2", "ring", "tls12", "webpki-tokio"] } hyper-util = { version = "0.1", features = ["full"] } @@ -222,7 +218,6 @@ tokio-rustls = { version = "0.26", default-features = false, features = ["aws-lc aws-lc-rs = { version = "1", features = ["prebuilt-nasm"] } dof-468e82937335b1c9 = { package = "dof", version = "0.3", default-features = false, features = ["des"] } dof-9fbad63c4bcf4a8f = { package = "dof", version = "0.4", default-features = false, features = ["des"] } -getrandom-468e82937335b1c9 = { package = "getrandom", version = "0.3", default-features = false, features = ["std"] } getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4", default-features = false, features = ["std", "sys_rng"] } hyper = { version = "1", features = ["full"] } hyper-rustls = { version = "0.27", default-features = false, features = ["aws-lc-rs", "http1", "http2", "ring", "tls12", "webpki-tokio"] } @@ -244,7 +239,6 @@ toml_edit = { version = "0.19", features = ["serde"] } aws-lc-rs = { version = "1", features = ["prebuilt-nasm"] } dof-468e82937335b1c9 = { package = "dof", version = "0.3", default-features = false, features = ["des"] } dof-9fbad63c4bcf4a8f = { package = "dof", version = "0.4", default-features = false, features = ["des"] } -getrandom-468e82937335b1c9 = { package = "getrandom", version = "0.3", default-features = false, features = ["std"] } getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4", default-features = false, features = ["std", "sys_rng"] } hyper = { version = "1", features = ["full"] } hyper-rustls = { version = "0.27", default-features = false, features = ["aws-lc-rs", "http1", "http2", "ring", "tls12", "webpki-tokio"] } From 9290dff7410a58b7aa30689af882236bf82cd1e8 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Thu, 11 Jun 2026 10:49:23 -0700 Subject: [PATCH 4/4] Revert "Switch downstairs to jemalloc and report allocator stats" This reverts commit 396517c54e78f528f5983e9a950af61a190c4166. It never would have worked on illumos, despite what Claude promised me. --- Cargo.lock | 54 +- Cargo.toml | 2 - .../versions/src/initial/repair.rs | 7 - downstairs/Cargo.toml | 2 - downstairs/src/lib.rs | 19 - downstairs/src/main.rs | 4 - .../downstairs-repair-2.0.0-9f0059.json | 468 ++++++++++++++++++ .../downstairs-repair-latest.json | 2 +- workspace-hack/Cargo.toml | 6 + 9 files changed, 486 insertions(+), 78 deletions(-) create mode 100644 openapi/downstairs-repair/downstairs-repair-2.0.0-9f0059.json diff --git a/Cargo.lock b/Cargo.lock index f79fbe498..8e77938e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -926,7 +926,7 @@ dependencies = [ "terminfo", "thiserror 2.0.18", "which 8.0.0", - "windows-sys 0.61.1", + "windows-sys 0.59.0", ] [[package]] @@ -1053,7 +1053,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.1", + "windows-sys 0.48.0", ] [[package]] @@ -1573,8 +1573,6 @@ dependencies = [ "statistical", "tempfile", "thiserror 2.0.18", - "tikv-jemalloc-ctl", - "tikv-jemallocator", "tokio", "tokio-rustls 0.24.1", "tokio-util", @@ -1895,6 +1893,7 @@ dependencies = [ "futures-util", "generic-array", "getrandom 0.2.11", + "getrandom 0.3.1", "getrandom 0.4.1", "hex", "hyper", @@ -3778,7 +3777,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2 0.5.10", "system-configuration", "tokio", "tower-layer", @@ -6487,7 +6486,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls 0.23.31", - "socket2 0.6.3", + "socket2 0.5.10", "thiserror 2.0.18", "tokio", "tracing", @@ -7148,7 +7147,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.1", + "windows-sys 0.52.0", ] [[package]] @@ -7251,7 +7250,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.1", + "windows-sys 0.52.0", ] [[package]] @@ -8087,7 +8086,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck 0.5.0", + "heck 0.4.1", "proc-macro2", "quote", "syn 2.0.117", @@ -8431,10 +8430,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0" dependencies = [ "fastrand", - "getrandom 0.4.1", + "getrandom 0.3.1", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.1", + "windows-sys 0.52.0", ] [[package]] @@ -8443,7 +8442,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2111ef44dae28680ae9752bb89409e7310ca33a8c621ebe7b106cf5c928b3ac0" dependencies = [ - "windows-sys 0.61.1", + "windows-sys 0.59.0", ] [[package]] @@ -8611,37 +8610,6 @@ dependencies = [ "threadpool", ] -[[package]] -name = "tikv-jemalloc-ctl" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "661f1f6a57b3a36dc9174a2c10f19513b4866816e13425d3e418b11cc37bc24c" -dependencies = [ - "libc", - "paste", - "tikv-jemalloc-sys", -] - -[[package]] -name = "tikv-jemalloc-sys" -version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8aa5b2ab86a2cefa406d889139c162cbb230092f7d1d7cbc1716405d852a3b" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "tikv-jemallocator" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0359b4327f954e0567e69fb191cf1436617748813819c94b8cd4a431422d053a" -dependencies = [ - "libc", - "tikv-jemalloc-sys", -] - [[package]] name = "time" version = "0.3.47" diff --git a/Cargo.toml b/Cargo.toml index 9be8f902c..9ef1a01b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,8 +77,6 @@ hyper = { version = "1", features = [ "full" ] } hyper-staticfile = "0.10.1" indicatif = { version = "0.18.4", features = ["rayon"] } itertools = "0.14.0" -tikv-jemallocator = { version = "0.6", features = ["stats"] } -tikv-jemalloc-ctl = { version = "0.6", features = ["stats"] } libc = "0.2" mime_guess = "2.0.5" nbd = "0.3.1" diff --git a/downstairs-types/versions/src/initial/repair.rs b/downstairs-types/versions/src/initial/repair.rs index 6f36543be..dbdbf8108 100644 --- a/downstairs-types/versions/src/initial/repair.rs +++ b/downstairs-types/versions/src/initial/repair.rs @@ -70,13 +70,6 @@ pub struct MemoryReport { pub tokio_worker_threads: usize, pub rayon_threads: usize, - // jemalloc allocator stats - pub jemalloc_allocated: usize, - pub jemalloc_active: usize, - pub jemalloc_resident: usize, - pub jemalloc_mapped: usize, - pub jemalloc_retained: usize, - // Connection memory pub active_connections: usize, pub connections: Vec, diff --git a/downstairs/Cargo.toml b/downstairs/Cargo.toml index a5965b28e..f8425beb0 100644 --- a/downstairs/Cargo.toml +++ b/downstairs/Cargo.toml @@ -60,8 +60,6 @@ tracing-subscriber.workspace = true tracing.workspace = true usdt.workspace = true uuid.workspace = true -tikv-jemallocator.workspace = true -tikv-jemalloc-ctl.workspace = true crucible-workspace-hack.workspace = true [dev-dependencies] diff --git a/downstairs/src/lib.rs b/downstairs/src/lib.rs index fc8fac025..733134f5b 100644 --- a/downstairs/src/lib.rs +++ b/downstairs/src/lib.rs @@ -2606,20 +2606,6 @@ impl Downstairs { .collect(); let extent_count = def.extent_count(); let extent_meta_bytes = self.region.extent_meta_bytes(); - - // Advance jemalloc's stats epoch so we get fresh numbers - tikv_jemalloc_ctl::epoch::advance().ok(); - let jemalloc_allocated = - tikv_jemalloc_ctl::stats::allocated::read().unwrap_or(0); - let jemalloc_active = - tikv_jemalloc_ctl::stats::active::read().unwrap_or(0); - let jemalloc_resident = - tikv_jemalloc_ctl::stats::resident::read().unwrap_or(0); - let jemalloc_mapped = - tikv_jemalloc_ctl::stats::mapped::read().unwrap_or(0); - let jemalloc_retained = - tikv_jemalloc_ctl::stats::retained::read().unwrap_or(0); - MemoryReport { extent_count, extent_size: def.extent_size().value, @@ -2638,11 +2624,6 @@ impl Downstairs { .metrics() .num_workers(), rayon_threads: self.region.rayon_thread_count(), - jemalloc_allocated, - jemalloc_active, - jemalloc_resident, - jemalloc_mapped, - jemalloc_retained, active_connections: active.len(), connections, } diff --git a/downstairs/src/main.rs b/downstairs/src/main.rs index 258d348cf..5901d57ed 100644 --- a/downstairs/src/main.rs +++ b/downstairs/src/main.rs @@ -1,8 +1,4 @@ // Copyright 2023 Oxide Computer Company - -#[global_allocator] -static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; - use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; use std::time::Duration; diff --git a/openapi/downstairs-repair/downstairs-repair-2.0.0-9f0059.json b/openapi/downstairs-repair/downstairs-repair-2.0.0-9f0059.json new file mode 100644 index 000000000..0d1ed1fa2 --- /dev/null +++ b/openapi/downstairs-repair/downstairs-repair-2.0.0-9f0059.json @@ -0,0 +1,468 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "Downstairs Repair", + "contact": { + "url": "https://oxide.computer", + "email": "api@oxide.computer" + }, + "version": "2.0.0" + }, + "paths": { + "/extent/{eid}/files": { + "get": { + "summary": "Get the list of files related to an extent.", + "description": "For a given extent, return a vec of strings representing the names of the files that exist for that extent.", + "operationId": "get_files_for_extent", + "parameters": [ + { + "in": "path", + "name": "eid", + "required": true, + "schema": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_String", + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/extent/{eid}/repair-ready": { + "get": { + "summary": "Return true if the provided extent is closed or the region is read only.", + "operationId": "extent_repair_ready", + "parameters": [ + { + "in": "path", + "name": "eid", + "required": true, + "schema": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Boolean", + "type": "boolean" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/memory": { + "get": { + "summary": "Memory usage report", + "operationId": "get_memory", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MemoryReport" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/newextent/{eid}/{file_type}": { + "get": { + "summary": "Get a specific extent file (data, database, or log files).", + "operationId": "get_extent_file", + "parameters": [ + { + "in": "path", + "name": "eid", + "required": true, + "schema": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + { + "in": "path", + "name": "file_type", + "required": true, + "schema": { + "$ref": "#/components/schemas/FileType" + } + } + ], + "responses": { + "default": { + "description": "", + "content": { + "*/*": { + "schema": {} + } + } + } + } + } + }, + "/region-info": { + "get": { + "summary": "Return the RegionDefinition describing our region.", + "operationId": "get_region_info", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RegionDefinition" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/region-mode": { + "get": { + "summary": "Return the region-mode describing our region.", + "operationId": "get_region_mode", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Boolean", + "type": "boolean" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/work": { + "get": { + "summary": "Work queue", + "operationId": "get_work", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Boolean", + "type": "boolean" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + } + }, + "components": { + "schemas": { + "Block": { + "type": "object", + "properties": { + "shift": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "value": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "shift", + "value" + ] + }, + "ConnectionMemoryReport": { + "description": "Per-connection memory report", + "type": "object", + "properties": { + "completed_ranges": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "completed_ranges_hwm": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "pending_jobs": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "pending_jobs_bytes": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "pending_jobs_capacity_hwm": { + "type": "integer", + "format": "uint", + "minimum": 0 + } + }, + "required": [ + "completed_ranges", + "completed_ranges_hwm", + "pending_jobs", + "pending_jobs_bytes", + "pending_jobs_capacity_hwm" + ] + }, + "Error": { + "description": "Error information from a response.", + "type": "object", + "properties": { + "error_code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "message", + "request_id" + ] + }, + "MemoryReport": { + "description": "Summary of downstairs memory usage", + "type": "object", + "properties": { + "active_connections": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "block_size": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "bytes_per_extent": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "connections": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ConnectionMemoryReport" + } + }, + "dirty_extent_count": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "extent_count": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "extent_meta_bytes": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "extent_size": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "rayon_threads": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "read_bytes_hwm": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "region_bytes": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "tokio_worker_threads": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "write_bytes_hwm": { + "type": "integer", + "format": "uint", + "minimum": 0 + } + }, + "required": [ + "active_connections", + "block_size", + "bytes_per_extent", + "connections", + "dirty_extent_count", + "extent_count", + "extent_meta_bytes", + "extent_size", + "rayon_threads", + "read_bytes_hwm", + "region_bytes", + "tokio_worker_threads", + "write_bytes_hwm" + ] + }, + "RegionDefinition": { + "type": "object", + "properties": { + "block_size": { + "description": "The size of each block in bytes. Must be a power of 2, minimum 512.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "database_read_version": { + "description": "The database version format for reading an extent database file.", + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "database_write_version": { + "description": "The database version format for writing an extent database file.", + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "encrypted": { + "description": "region data will be encrypted", + "type": "boolean" + }, + "extent_count": { + "description": "How many whole extents comprise this region?", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "extent_size": { + "description": "How many blocks should appear in each extent?", + "allOf": [ + { + "$ref": "#/components/schemas/Block" + } + ] + }, + "uuid": { + "description": "UUID for this region", + "type": "string", + "format": "uuid" + } + }, + "required": [ + "block_size", + "database_read_version", + "database_write_version", + "encrypted", + "extent_count", + "extent_size", + "uuid" + ] + }, + "FileType": { + "type": "string", + "enum": [ + "data", + "db", + "db_shm", + "db_wal" + ] + } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } +} diff --git a/openapi/downstairs-repair/downstairs-repair-latest.json b/openapi/downstairs-repair/downstairs-repair-latest.json index 7513c4bc9..c1ad40fdf 120000 --- a/openapi/downstairs-repair/downstairs-repair-latest.json +++ b/openapi/downstairs-repair/downstairs-repair-latest.json @@ -1 +1 @@ -downstairs-repair-2.0.0-bd1bb6.json \ No newline at end of file +downstairs-repair-2.0.0-9f0059.json \ No newline at end of file diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index b0a256a97..9ec612f30 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -160,6 +160,7 @@ zeroize = { version = "1", features = ["std", "zeroize_derive"] } aws-lc-rs = { version = "1", features = ["prebuilt-nasm"] } dof-468e82937335b1c9 = { package = "dof", version = "0.3", default-features = false, features = ["des"] } dof-9fbad63c4bcf4a8f = { package = "dof", version = "0.4", default-features = false, features = ["des"] } +getrandom-468e82937335b1c9 = { package = "getrandom", version = "0.3", default-features = false, features = ["std"] } hyper = { version = "1", features = ["full"] } hyper-rustls = { version = "0.27", default-features = false, features = ["aws-lc-rs", "http1", "http2", "ring", "tls12", "webpki-tokio"] } hyper-util = { version = "0.1", features = ["full"] } @@ -177,6 +178,7 @@ tokio-rustls = { version = "0.26", default-features = false, features = ["aws-lc aws-lc-rs = { version = "1", features = ["prebuilt-nasm"] } dof-468e82937335b1c9 = { package = "dof", version = "0.3", default-features = false, features = ["des"] } dof-9fbad63c4bcf4a8f = { package = "dof", version = "0.4", default-features = false, features = ["des"] } +getrandom-468e82937335b1c9 = { package = "getrandom", version = "0.3", default-features = false, features = ["std"] } hyper = { version = "1", features = ["full"] } hyper-rustls = { version = "0.27", default-features = false, features = ["aws-lc-rs", "http1", "http2", "ring", "tls12", "webpki-tokio"] } hyper-util = { version = "0.1", features = ["full"] } @@ -191,6 +193,7 @@ tokio-rustls = { version = "0.26", default-features = false, features = ["aws-lc [target.aarch64-apple-darwin.dependencies] aws-lc-rs = { version = "1", features = ["prebuilt-nasm"] } +getrandom-468e82937335b1c9 = { package = "getrandom", version = "0.3", default-features = false, features = ["std"] } hyper = { version = "1", features = ["full"] } hyper-rustls = { version = "0.27", default-features = false, features = ["aws-lc-rs", "http1", "http2", "ring", "tls12", "webpki-tokio"] } hyper-util = { version = "0.1", features = ["full"] } @@ -204,6 +207,7 @@ tokio-rustls = { version = "0.26", default-features = false, features = ["aws-lc [target.aarch64-apple-darwin.build-dependencies] aws-lc-rs = { version = "1", features = ["prebuilt-nasm"] } +getrandom-468e82937335b1c9 = { package = "getrandom", version = "0.3", default-features = false, features = ["std"] } hyper = { version = "1", features = ["full"] } hyper-rustls = { version = "0.27", default-features = false, features = ["aws-lc-rs", "http1", "http2", "ring", "tls12", "webpki-tokio"] } hyper-util = { version = "0.1", features = ["full"] } @@ -218,6 +222,7 @@ tokio-rustls = { version = "0.26", default-features = false, features = ["aws-lc aws-lc-rs = { version = "1", features = ["prebuilt-nasm"] } dof-468e82937335b1c9 = { package = "dof", version = "0.3", default-features = false, features = ["des"] } dof-9fbad63c4bcf4a8f = { package = "dof", version = "0.4", default-features = false, features = ["des"] } +getrandom-468e82937335b1c9 = { package = "getrandom", version = "0.3", default-features = false, features = ["std"] } getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4", default-features = false, features = ["std", "sys_rng"] } hyper = { version = "1", features = ["full"] } hyper-rustls = { version = "0.27", default-features = false, features = ["aws-lc-rs", "http1", "http2", "ring", "tls12", "webpki-tokio"] } @@ -239,6 +244,7 @@ toml_edit = { version = "0.19", features = ["serde"] } aws-lc-rs = { version = "1", features = ["prebuilt-nasm"] } dof-468e82937335b1c9 = { package = "dof", version = "0.3", default-features = false, features = ["des"] } dof-9fbad63c4bcf4a8f = { package = "dof", version = "0.4", default-features = false, features = ["des"] } +getrandom-468e82937335b1c9 = { package = "getrandom", version = "0.3", default-features = false, features = ["std"] } getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4", default-features = false, features = ["std", "sys_rng"] } hyper = { version = "1", features = ["full"] } hyper-rustls = { version = "0.27", default-features = false, features = ["aws-lc-rs", "http1", "http2", "ring", "tls12", "webpki-tokio"] }