diff --git a/TODO.md b/TODO.md index 9030a62..ee798a8 100644 --- a/TODO.md +++ b/TODO.md @@ -21,7 +21,7 @@ - [X] (bugfix) stress test found this error with num workers = 4: `database integrity violated: main key not found`. fix it. - [ ] (minor) perf analysis on queueber while being stressed - [X] (minor) make sure all storage stuff happens within a spawn_blocking or similar -- [ ] (minor) rocksdb settings tuning +- [X] (minor) rocksdb settings tuning - [X] (test) add extend to fuzz test and stress - [ ] (major) server/storage sharding - [ ] (major) fix server parallelism -- it's not right currently diff --git a/evidence_bench_after6.txt b/evidence_bench_after6.txt new file mode 100644 index 0000000..17d8fd1 --- /dev/null +++ b/evidence_bench_after6.txt @@ -0,0 +1,60 @@ + +running 15 tests +iiiiiiiiiiiiiii +test result: ok. 0 passed; 0 failed; 15 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 2 tests +ii +test result: ok. 0 passed; 0 failed; 2 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 3 tests +iii +test result: ok. 0 passed; 0 failed; 3 ignored; 0 measured; 0 filtered out; finished in 0.00s + +storage_add/add_available_500 + time: [7.1581 ms 7.3818 ms 7.6308 ms] + change: [+7.4683% +10.743% +15.614%] (p = 0.00 < 0.05) + Performance has regressed. +Found 16 outliers among 100 measurements (16.00%) + 10 (10.00%) high mild + 6 (6.00%) high severe + +storage_remove/remove_in_progress_500 + time: [14.432 ms 14.715 ms 15.034 ms] + change: [+4.0189% +5.9532% +8.1366%] (p = 0.00 < 0.05) + Performance has regressed. +Found 11 outliers among 100 measurements (11.00%) + 2 (2.00%) high mild + 9 (9.00%) high severe + +storage_poll/get_next_available_entries_500 + time: [9.1169 ms 9.3460 ms 9.5942 ms] + change: [+2.9014% +5.2808% +8.2718%] (p = 0.00 < 0.05) + Performance has regressed. +Found 16 outliers among 100 measurements (16.00%) + 9 (9.00%) high mild + 7 (7.00%) high severe + +e2e_rpc/rpc_add_batch_200 + time: [501.12 µs 507.65 µs 514.94 µs] + change: [+3.1384% +5.3169% +7.4291%] (p = 0.00 < 0.05) + Performance has regressed. +Found 4 outliers among 100 measurements (4.00%) + 2 (2.00%) low mild + 1 (1.00%) high mild + 1 (1.00%) high severe +e2e_rpc/rpc_poll_batch_200 + time: [36.257 ms 38.757 ms 41.264 ms] + change: [-7.5339% +0.4694% +9.2683%] (p = 0.91 > 0.05) + No change in performance detected. +e2e_rpc/rpc_remove_each_200 + time: [10.503 ms 10.749 ms 11.040 ms] + change: [-3.6378% -0.6125% +2.6352%] (p = 0.71 > 0.05) + No change in performance detected. +Found 12 outliers among 100 measurements (12.00%) + 8 (8.00%) high mild + 4 (4.00%) high severe + + diff --git a/evidence_bench_after7.txt b/evidence_bench_after7.txt new file mode 100644 index 0000000..db21de5 --- /dev/null +++ b/evidence_bench_after7.txt @@ -0,0 +1,59 @@ + +running 15 tests +iiiiiiiiiiiiiii +test result: ok. 0 passed; 0 failed; 15 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 2 tests +ii +test result: ok. 0 passed; 0 failed; 2 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 3 tests +iii +test result: ok. 0 passed; 0 failed; 3 ignored; 0 measured; 0 filtered out; finished in 0.00s + +storage_add/add_available_500 + time: [6.5660 ms 6.6300 ms 6.6998 ms] + change: [-13.247% -10.185% -7.2332%] (p = 0.00 < 0.05) + Performance has improved. +Found 2 outliers among 100 measurements (2.00%) + 1 (1.00%) high mild + 1 (1.00%) high severe + +storage_remove/remove_in_progress_500 + time: [13.906 ms 13.994 ms 14.101 ms] + change: [-7.0104% -4.8994% -2.9146%] (p = 0.00 < 0.05) + Performance has improved. +Found 5 outliers among 100 measurements (5.00%) + 1 (1.00%) low mild + 3 (3.00%) high mild + 1 (1.00%) high severe + +storage_poll/get_next_available_entries_500 + time: [8.6617 ms 8.6988 ms 8.7366 ms] + change: [-9.4025% -6.9253% -4.5489%] (p = 0.00 < 0.05) + Performance has improved. +Found 5 outliers among 100 measurements (5.00%) + 1 (1.00%) low mild + 4 (4.00%) high mild + +e2e_rpc/rpc_add_batch_200 + time: [481.33 µs 488.60 µs 496.74 µs] + change: [-6.0598% -4.5137% -2.9793%] (p = 0.00 < 0.05) + Performance has improved. +Found 2 outliers among 100 measurements (2.00%) + 2 (2.00%) high mild +e2e_rpc/rpc_poll_batch_200 + time: [37.996 ms 40.726 ms 43.482 ms] + change: [-4.6413% +5.0794% +15.290%] (p = 0.30 > 0.05) + No change in performance detected. +e2e_rpc/rpc_remove_each_200 + time: [10.682 ms 10.892 ms 11.119 ms] + change: [-1.9818% +1.3291% +4.5159%] (p = 0.43 > 0.05) + No change in performance detected. +Found 5 outliers among 100 measurements (5.00%) + 2 (2.00%) high mild + 3 (3.00%) high severe + + diff --git a/evidence_bench_default.txt b/evidence_bench_default.txt new file mode 100644 index 0000000..88edd25 --- /dev/null +++ b/evidence_bench_default.txt @@ -0,0 +1,59 @@ + +running 15 tests +iiiiiiiiiiiiiii +test result: ok. 0 passed; 0 failed; 15 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 2 tests +ii +test result: ok. 0 passed; 0 failed; 2 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 3 tests +iii +test result: ok. 0 passed; 0 failed; 3 ignored; 0 measured; 0 filtered out; finished in 0.00s + +storage_add/add_available_500 + time: [7.1761 ms 7.3172 ms 7.4732 ms] + change: [+8.0895% +10.365% +12.860%] (p = 0.00 < 0.05) + Performance has regressed. +Found 11 outliers among 100 measurements (11.00%) + 7 (7.00%) high mild + 4 (4.00%) high severe + +storage_remove/remove_in_progress_500 + time: [14.262 ms 14.529 ms 14.934 ms] + change: [+1.6334% +3.8238% +6.7540%] (p = 0.00 < 0.05) + Performance has regressed. +Found 8 outliers among 100 measurements (8.00%) + 6 (6.00%) high mild + 2 (2.00%) high severe + +storage_poll/get_next_available_entries_500 + time: [8.8972 ms 8.9705 ms 9.0439 ms] + change: [+2.1534% +3.1235% +4.0920%] (p = 0.00 < 0.05) + Performance has regressed. +Found 13 outliers among 100 measurements (13.00%) + 6 (6.00%) low mild + 5 (5.00%) high mild + 2 (2.00%) high severe + +e2e_rpc/rpc_add_batch_200 + time: [488.73 µs 494.18 µs 499.90 µs] + change: [+0.0410% +1.6120% +3.3056%] (p = 0.05 > 0.05) + No change in performance detected. +Found 1 outliers among 100 measurements (1.00%) + 1 (1.00%) high severe +e2e_rpc/rpc_poll_batch_200 + time: [37.163 ms 38.832 ms 40.470 ms] + change: [-11.723% -4.6505% +3.4019%] (p = 0.25 > 0.05) + No change in performance detected. +e2e_rpc/rpc_remove_each_200 + time: [11.009 ms 11.203 ms 11.416 ms] + change: [+0.0534% +2.8627% +5.6315%] (p = 0.04 < 0.05) + Change within noise threshold. +Found 9 outliers among 100 measurements (9.00%) + 5 (5.00%) high mild + 4 (4.00%) high severe + + diff --git a/evidence_bench_tuned.txt b/evidence_bench_tuned.txt new file mode 100644 index 0000000..40ec872 --- /dev/null +++ b/evidence_bench_tuned.txt @@ -0,0 +1,59 @@ + +running 15 tests +iiiiiiiiiiiiiii +test result: ok. 0 passed; 0 failed; 15 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 2 tests +ii +test result: ok. 0 passed; 0 failed; 2 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 3 tests +iii +test result: ok. 0 passed; 0 failed; 3 ignored; 0 measured; 0 filtered out; finished in 0.00s + +storage_add/add_available_500 + time: [6.7382 ms 6.7811 ms 6.8238 ms] + change: [-9.3638% -7.3265% -5.4330%] (p = 0.00 < 0.05) + Performance has improved. +Found 8 outliers among 100 measurements (8.00%) + 1 (1.00%) low severe + 4 (4.00%) low mild + 2 (2.00%) high mild + 1 (1.00%) high severe + +storage_remove/remove_in_progress_500 + time: [14.019 ms 14.090 ms 14.177 ms] + change: [-5.6906% -3.0197% -1.1119%] (p = 0.01 < 0.05) + Performance has improved. +Found 6 outliers among 100 measurements (6.00%) + 2 (2.00%) low mild + 2 (2.00%) high mild + 2 (2.00%) high severe + +storage_poll/get_next_available_entries_500 + time: [8.7392 ms 8.7829 ms 8.8274 ms] + change: [-3.0541% -2.0906% -1.1655%] (p = 0.00 < 0.05) + Performance has improved. + +e2e_rpc/rpc_add_batch_200 + time: [480.94 µs 485.13 µs 489.57 µs] + change: [-3.7387% -2.2809% -0.7758%] (p = 0.00 < 0.05) + Change within noise threshold. +Found 4 outliers among 100 measurements (4.00%) + 3 (3.00%) high mild + 1 (1.00%) high severe +e2e_rpc/rpc_poll_batch_200 + time: [36.773 ms 39.280 ms 41.768 ms] + change: [-5.7240% +1.1551% +9.2170%] (p = 0.77 > 0.05) + No change in performance detected. +e2e_rpc/rpc_remove_each_200 + time: [10.739 ms 10.926 ms 11.128 ms] + change: [-4.9157% -2.4799% -0.0714%] (p = 0.06 > 0.05) + No change in performance detected. +Found 7 outliers among 100 measurements (7.00%) + 5 (5.00%) high mild + 2 (2.00%) high severe + + diff --git a/evidence_stress_after6.txt b/evidence_stress_after6.txt new file mode 100644 index 0000000..086e59d --- /dev/null +++ b/evidence_stress_after6.txt @@ -0,0 +1,8 @@ +2025-08-27T16:28:10.207981Z  INFO queueber: using 4 worker threads +Running client with args: -p 2 -a 2 -r0 -d 10 +add: 958850 (191730.1/s), poll: 10460 (2091.6/s), remove: 10460 (2091.6/s), extend: 2 (0.4/s) +add: 728830 (145722.6/s), poll: 17430 (3485.0/s), remove: 17430 (3485.0/s), extend: 6 (1.2/s) +stress test completed +Waiting for server to exit... +Server exited with code 137 +Client exited with code 0 diff --git a/evidence_stress_after7.txt b/evidence_stress_after7.txt new file mode 100644 index 0000000..eecbb54 --- /dev/null +++ b/evidence_stress_after7.txt @@ -0,0 +1,8 @@ +2025-08-27T18:06:47.726544Z  INFO queueber: using 4 worker threads +Running client with args: -p 2 -a 2 -r0 -d 10 +add: 1084240 (216812.2/s), poll: 10330 (2065.7/s), remove: 10320 (2063.7/s), extend: 2 (0.4/s) +add: 798120 (159592.7/s), poll: 17640 (3527.3/s), remove: 17640 (3527.3/s), extend: 6 (1.2/s) +stress test completed +Waiting for server to exit... +Server exited with code 137 +Client exited with code 0 diff --git a/evidence_stress_default.txt b/evidence_stress_default.txt new file mode 100644 index 0000000..02d675f --- /dev/null +++ b/evidence_stress_default.txt @@ -0,0 +1,7 @@ +2025-08-27T18:25:06.426026Z  INFO queueber: using 4 worker threads +Running client with args: -p 2 -a 2 -r0 -d 10 +2025-08-27T18:25:12.073416Z  WARN queueber::storage: RocksDB Busy: giving up after max retries operation=get_next_available_entries_with_lease attempts=9 +add: 1058990 (211772.4/s), poll: 10400 (2079.7/s), remove: 10400 (2079.7/s), extend: 2 (0.4/s) +Waiting for server to exit... +Server exited with code 137 +Client exited with code 101 diff --git a/evidence_stress_tuned.txt b/evidence_stress_tuned.txt new file mode 100644 index 0000000..b3d3e1b --- /dev/null +++ b/evidence_stress_tuned.txt @@ -0,0 +1,8 @@ +2025-08-27T18:25:38.125770Z  INFO queueber: using 4 worker threads +Running client with args: -p 2 -a 2 -r0 -d 10 +add: 891970 (178361.0/s), poll: 10050 (2009.6/s), remove: 10050 (2009.6/s), extend: 2 (0.4/s) +add: 662090 (132388.2/s), poll: 17670 (3533.2/s), remove: 17670 (3533.2/s), extend: 6 (1.2/s) +stress test completed +Waiting for server to exit... +Server exited with code 137 +Client exited with code 0 diff --git a/src/storage.rs b/src/storage.rs index 84e9bf5..4f60130 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -67,6 +67,7 @@ pub struct Storage { impl Storage { pub fn new(path: &Path) -> Result { let mut opts = Options::default(); + // Enable prefix extractor for our namespace scheme (before creating bloom) // Optimize for prefix scans used by `prefix_iterator` across all key namespaces. // Extract the namespace prefix up to and including the first '/'. // Examples: @@ -83,6 +84,62 @@ impl Storage { Some(|_key: &[u8]| true), ); opts.set_prefix_extractor(ns_prefix); + + // Optional: apply tuned settings only when explicitly enabled + let tuning_enabled = matches!( + std::env::var("QUEUEBER_DB_TUNING").as_deref(), + Ok("1") | Ok("on") | Ok("poll") | Ok("true") + ); + if tuning_enabled { + // Parallelism and background jobs sized to CPUs for compaction/flush + let cpus = std::thread::available_parallelism() + .map(|n| n.get() as i32) + .unwrap_or(2); + // Increase parallelism also sets max_background_jobs if larger in RocksDB + opts.increase_parallelism(cpus); + // Allow more background jobs for compaction/flush to reduce write stalls + opts.set_max_background_jobs(cpus.saturating_mul(2)); + + // Block table + bloom filters (including whole-key bloom for point lookups) + // Note: newer rust-rocksdb exposes set_opt/optimize APIs on Options directly for simplicity + { + use rocksdb::BlockBasedOptions; + use rocksdb::Cache; + let mut bopts = BlockBasedOptions::default(); + // 6 bits per key reduces CPU for filters; good tradeoff in mixed workloads + bopts.set_bloom_filter(6.0, false); + bopts.set_whole_key_filtering(true); + // 32 KiB block size balances CPU and I/O + bopts.set_block_size(32 * 1024); + // Cache index+filter blocks to reduce I/O under scan/prefix iterator + bopts.set_cache_index_and_filter_blocks(true); + // Avoid partitioned filters; increases CPU on small/index-centric workloads + bopts.set_partition_filters(false); + // Provide a shared LRU cache for blocks (tune size as needed) + let cache = Cache::new_lru_cache(256usize << 20); + bopts.set_block_cache(&cache); + // Keep L0 metadata hot + bopts.set_pin_l0_filter_and_index_blocks_in_cache(true); + opts.set_block_based_table_factory(&bopts); + } + + // Heuristic tuning for point lookups; helps poll read path + opts.optimize_for_point_lookup(u64::try_from(256usize).unwrap() << 20); + + // Compaction strategy and write path improvements + opts.set_level_compaction_dynamic_level_bytes(true); + opts.set_allow_concurrent_memtable_write(true); + opts.set_enable_write_thread_adaptive_yield(true); + opts.set_enable_pipelined_write(true); + + // Smooth write I/O to reduce tail latencies + opts.set_bytes_per_sync(1 << 20); // 1 MiB + opts.set_wal_bytes_per_sync(1 << 20); // 1 MiB + + // Compression: favor speed with Snappy; keeps space reasonable without high CPU + opts.set_compression_type(rocksdb::DBCompressionType::Snappy); + } + opts.create_if_missing(true); let db = OptimisticTransactionDB::open(&opts, path)?; Ok(Self { db })