asg0451 · asg0451 · Aug 27, 2025 · Aug 27, 2025 · Aug 27, 2025 · Aug 27, 2025
diff --git a/TODO.md b/TODO.md
@@ -21,7 +21,7 @@
 - [X] (bugfix) stress test found this error with num workers = 4: `database integrity violated: main key not found`. fix it.
 - [ ] (minor) perf analysis on queueber while being stressed
 - [X] (minor) make sure all storage stuff happens within a spawn_blocking or similar
-- [ ] (minor) rocksdb settings tuning
+- [X] (minor) rocksdb settings tuning
 - [X] (test) add extend to fuzz test and stress
 - [ ] (major) server/storage sharding
 - [ ] (major) fix server parallelism -- it's not right currently

diff --git a/evidence_bench_after6.txt b/evidence_bench_after6.txt
@@ -0,0 +1,60 @@
+
+running 15 tests
+iiiiiiiiiiiiiii
+test result: ok. 0 passed; 0 failed; 15 ignored; 0 measured; 0 filtered out; finished in 0.00s
+
+
+running 2 tests
+ii
+test result: ok. 0 passed; 0 failed; 2 ignored; 0 measured; 0 filtered out; finished in 0.00s
+
+
+running 3 tests
+iii
+test result: ok. 0 passed; 0 failed; 3 ignored; 0 measured; 0 filtered out; finished in 0.00s
+
+storage_add/add_available_500
+                        time:   [7.1581 ms 7.3818 ms 7.6308 ms]
+                        change: [+7.4683% +10.743% +15.614%] (p = 0.00 < 0.05)
+                        Performance has regressed.
+Found 16 outliers among 100 measurements (16.00%)
+  10 (10.00%) high mild
+  6 (6.00%) high severe
+
+storage_remove/remove_in_progress_500
+                        time:   [14.432 ms 14.715 ms 15.034 ms]
+                        change: [+4.0189% +5.9532% +8.1366%] (p = 0.00 < 0.05)
+                        Performance has regressed.
+Found 11 outliers among 100 measurements (11.00%)
+  2 (2.00%) high mild
+  9 (9.00%) high severe
+
+storage_poll/get_next_available_entries_500
+                        time:   [9.1169 ms 9.3460 ms 9.5942 ms]
+                        change: [+2.9014% +5.2808% +8.2718%] (p = 0.00 < 0.05)
+                        Performance has regressed.
+Found 16 outliers among 100 measurements (16.00%)
+  9 (9.00%) high mild
+  7 (7.00%) high severe
+
+e2e_rpc/rpc_add_batch_200
+                        time:   [501.12 µs 507.65 µs 514.94 µs]
+                        change: [+3.1384% +5.3169% +7.4291%] (p = 0.00 < 0.05)
+                        Performance has regressed.
+Found 4 outliers among 100 measurements (4.00%)
+  2 (2.00%) low mild
+  1 (1.00%) high mild
+  1 (1.00%) high severe
+e2e_rpc/rpc_poll_batch_200
+                        time:   [36.257 ms 38.757 ms 41.264 ms]
+                        change: [-7.5339% +0.4694% +9.2683%] (p = 0.91 > 0.05)
+                        No change in performance detected.
+e2e_rpc/rpc_remove_each_200
+                        time:   [10.503 ms 10.749 ms 11.040 ms]
+                        change: [-3.6378% -0.6125% +2.6352%] (p = 0.71 > 0.05)
+                        No change in performance detected.
+Found 12 outliers among 100 measurements (12.00%)
+  8 (8.00%) high mild
+  4 (4.00%) high severe
+
+
diff --git a/evidence_bench_after7.txt b/evidence_bench_after7.txt
@@ -0,0 +1,59 @@
+
+running 15 tests
+iiiiiiiiiiiiiii
+test result: ok. 0 passed; 0 failed; 15 ignored; 0 measured; 0 filtered out; finished in 0.00s
+
+
+running 2 tests
+ii
+test result: ok. 0 passed; 0 failed; 2 ignored; 0 measured; 0 filtered out; finished in 0.00s
+
+
+running 3 tests
+iii
+test result: ok. 0 passed; 0 failed; 3 ignored; 0 measured; 0 filtered out; finished in 0.00s
+
+storage_add/add_available_500
+                        time:   [6.5660 ms 6.6300 ms 6.6998 ms]
+                        change: [-13.247% -10.185% -7.2332%] (p = 0.00 < 0.05)
+                        Performance has improved.
+Found 2 outliers among 100 measurements (2.00%)
+  1 (1.00%) high mild
+  1 (1.00%) high severe
+
+storage_remove/remove_in_progress_500
+                        time:   [13.906 ms 13.994 ms 14.101 ms]
+                        change: [-7.0104% -4.8994% -2.9146%] (p = 0.00 < 0.05)
+                        Performance has improved.
+Found 5 outliers among 100 measurements (5.00%)
+  1 (1.00%) low mild
+  3 (3.00%) high mild
+  1 (1.00%) high severe
+
+storage_poll/get_next_available_entries_500
+                        time:   [8.6617 ms 8.6988 ms 8.7366 ms]
+                        change: [-9.4025% -6.9253% -4.5489%] (p = 0.00 < 0.05)
+                        Performance has improved.
+Found 5 outliers among 100 measurements (5.00%)
+  1 (1.00%) low mild
+  4 (4.00%) high mild
+
+e2e_rpc/rpc_add_batch_200
+                        time:   [481.33 µs 488.60 µs 496.74 µs]
+                        change: [-6.0598% -4.5137% -2.9793%] (p = 0.00 < 0.05)
+                        Performance has improved.
+Found 2 outliers among 100 measurements (2.00%)
+  2 (2.00%) high mild
+e2e_rpc/rpc_poll_batch_200
+                        time:   [37.996 ms 40.726 ms 43.482 ms]
+                        change: [-4.6413% +5.0794% +15.290%] (p = 0.30 > 0.05)
+                        No change in performance detected.
+e2e_rpc/rpc_remove_each_200
+                        time:   [10.682 ms 10.892 ms 11.119 ms]
+                        change: [-1.9818% +1.3291% +4.5159%] (p = 0.43 > 0.05)
+                        No change in performance detected.
+Found 5 outliers among 100 measurements (5.00%)
+  2 (2.00%) high mild
+  3 (3.00%) high severe
+
+
diff --git a/evidence_bench_default.txt b/evidence_bench_default.txt
@@ -0,0 +1,59 @@
+
+running 15 tests
+iiiiiiiiiiiiiii
+test result: ok. 0 passed; 0 failed; 15 ignored; 0 measured; 0 filtered out; finished in 0.00s
+
+
+running 2 tests
+ii
+test result: ok. 0 passed; 0 failed; 2 ignored; 0 measured; 0 filtered out; finished in 0.00s
+
+
+running 3 tests
+iii
+test result: ok. 0 passed; 0 failed; 3 ignored; 0 measured; 0 filtered out; finished in 0.00s
+
+storage_add/add_available_500
+                        time:   [7.1761 ms 7.3172 ms 7.4732 ms]
+                        change: [+8.0895% +10.365% +12.860%] (p = 0.00 < 0.05)
+                        Performance has regressed.
+Found 11 outliers among 100 measurements (11.00%)
+  7 (7.00%) high mild
+  4 (4.00%) high severe
+
+storage_remove/remove_in_progress_500
+                        time:   [14.262 ms 14.529 ms 14.934 ms]
+                        change: [+1.6334% +3.8238% +6.7540%] (p = 0.00 < 0.05)
+                        Performance has regressed.
+Found 8 outliers among 100 measurements (8.00%)
+  6 (6.00%) high mild
+  2 (2.00%) high severe
+
+storage_poll/get_next_available_entries_500
+                        time:   [8.8972 ms 8.9705 ms 9.0439 ms]
+                        change: [+2.1534% +3.1235% +4.0920%] (p = 0.00 < 0.05)
+                        Performance has regressed.
+Found 13 outliers among 100 measurements (13.00%)
+  6 (6.00%) low mild
+  5 (5.00%) high mild
+  2 (2.00%) high severe
+
+e2e_rpc/rpc_add_batch_200
+                        time:   [488.73 µs 494.18 µs 499.90 µs]
+                        change: [+0.0410% +1.6120% +3.3056%] (p = 0.05 > 0.05)
+                        No change in performance detected.
+Found 1 outliers among 100 measurements (1.00%)
+  1 (1.00%) high severe
+e2e_rpc/rpc_poll_batch_200
+                        time:   [37.163 ms 38.832 ms 40.470 ms]
+                        change: [-11.723% -4.6505% +3.4019%] (p = 0.25 > 0.05)
+                        No change in performance detected.
+e2e_rpc/rpc_remove_each_200
+                        time:   [11.009 ms 11.203 ms 11.416 ms]
+                        change: [+0.0534% +2.8627% +5.6315%] (p = 0.04 < 0.05)
+                        Change within noise threshold.
+Found 9 outliers among 100 measurements (9.00%)
+  5 (5.00%) high mild
+  4 (4.00%) high severe
+
+
diff --git a/evidence_bench_tuned.txt b/evidence_bench_tuned.txt
@@ -0,0 +1,59 @@
+
+running 15 tests
+iiiiiiiiiiiiiii
+test result: ok. 0 passed; 0 failed; 15 ignored; 0 measured; 0 filtered out; finished in 0.00s
+
+
+running 2 tests
+ii
+test result: ok. 0 passed; 0 failed; 2 ignored; 0 measured; 0 filtered out; finished in 0.00s
+
+
+running 3 tests
+iii
+test result: ok. 0 passed; 0 failed; 3 ignored; 0 measured; 0 filtered out; finished in 0.00s
+
+storage_add/add_available_500
+                        time:   [6.7382 ms 6.7811 ms 6.8238 ms]
+                        change: [-9.3638% -7.3265% -5.4330%] (p = 0.00 < 0.05)
+                        Performance has improved.
+Found 8 outliers among 100 measurements (8.00%)
+  1 (1.00%) low severe
+  4 (4.00%) low mild
+  2 (2.00%) high mild
+  1 (1.00%) high severe
+
+storage_remove/remove_in_progress_500
+                        time:   [14.019 ms 14.090 ms 14.177 ms]
+                        change: [-5.6906% -3.0197% -1.1119%] (p = 0.01 < 0.05)
+                        Performance has improved.
+Found 6 outliers among 100 measurements (6.00%)
+  2 (2.00%) low mild
+  2 (2.00%) high mild
+  2 (2.00%) high severe
+
+storage_poll/get_next_available_entries_500
+                        time:   [8.7392 ms 8.7829 ms 8.8274 ms]
+                        change: [-3.0541% -2.0906% -1.1655%] (p = 0.00 < 0.05)
+                        Performance has improved.
+
+e2e_rpc/rpc_add_batch_200
+                        time:   [480.94 µs 485.13 µs 489.57 µs]
+                        change: [-3.7387% -2.2809% -0.7758%] (p = 0.00 < 0.05)
+                        Change within noise threshold.
+Found 4 outliers among 100 measurements (4.00%)
+  3 (3.00%) high mild
+  1 (1.00%) high severe
+e2e_rpc/rpc_poll_batch_200
+                        time:   [36.773 ms 39.280 ms 41.768 ms]
+                        change: [-5.7240% +1.1551% +9.2170%] (p = 0.77 > 0.05)
+                        No change in performance detected.
+e2e_rpc/rpc_remove_each_200
+                        time:   [10.739 ms 10.926 ms 11.128 ms]
+                        change: [-4.9157% -2.4799% -0.0714%] (p = 0.06 > 0.05)
+                        No change in performance detected.
+Found 7 outliers among 100 measurements (7.00%)
+  5 (5.00%) high mild
+  2 (2.00%) high severe
+
+
diff --git a/evidence_stress_after6.txt b/evidence_stress_after6.txt
@@ -0,0 +1,8 @@
+[2m2025-08-27T16:28:10.207981Z[0m [32m INFO[0m [2mqueueber[0m[2m:[0m using 4 worker threads
+Running client with args: -p 2 -a 2 -r0 -d 10
+add: 958850 (191730.1/s), poll: 10460 (2091.6/s), remove: 10460 (2091.6/s), extend: 2 (0.4/s)
+add: 728830 (145722.6/s), poll: 17430 (3485.0/s), remove: 17430 (3485.0/s), extend: 6 (1.2/s)
+stress test completed
+Waiting for server to exit...
+Server exited with code 137
+Client exited with code 0
diff --git a/evidence_stress_after7.txt b/evidence_stress_after7.txt
@@ -0,0 +1,8 @@
+[2m2025-08-27T18:06:47.726544Z[0m [32m INFO[0m [2mqueueber[0m[2m:[0m using 4 worker threads
+Running client with args: -p 2 -a 2 -r0 -d 10
+add: 1084240 (216812.2/s), poll: 10330 (2065.7/s), remove: 10320 (2063.7/s), extend: 2 (0.4/s)
+add: 798120 (159592.7/s), poll: 17640 (3527.3/s), remove: 17640 (3527.3/s), extend: 6 (1.2/s)
+stress test completed
+Waiting for server to exit...
+Server exited with code 137
+Client exited with code 0
diff --git a/evidence_stress_default.txt b/evidence_stress_default.txt
@@ -0,0 +1,7 @@
+[2m2025-08-27T18:25:06.426026Z[0m [32m INFO[0m [2mqueueber[0m[2m:[0m using 4 worker threads
+Running client with args: -p 2 -a 2 -r0 -d 10
+[2m2025-08-27T18:25:12.073416Z[0m [33m WARN[0m [2mqueueber::storage[0m[2m:[0m RocksDB Busy: giving up after max retries [3moperation[0m[2m=[0mget_next_available_entries_with_lease [3mattempts[0m[2m=[0m9
+add: 1058990 (211772.4/s), poll: 10400 (2079.7/s), remove: 10400 (2079.7/s), extend: 2 (0.4/s)
+Waiting for server to exit...
+Server exited with code 137
+Client exited with code 101
diff --git a/evidence_stress_tuned.txt b/evidence_stress_tuned.txt
@@ -0,0 +1,8 @@
+[2m2025-08-27T18:25:38.125770Z[0m [32m INFO[0m [2mqueueber[0m[2m:[0m using 4 worker threads
+Running client with args: -p 2 -a 2 -r0 -d 10
+add: 891970 (178361.0/s), poll: 10050 (2009.6/s), remove: 10050 (2009.6/s), extend: 2 (0.4/s)
+add: 662090 (132388.2/s), poll: 17670 (3533.2/s), remove: 17670 (3533.2/s), extend: 6 (1.2/s)
+stress test completed
+Waiting for server to exit...
+Server exited with code 137
+Client exited with code 0
diff --git a/src/storage.rs b/src/storage.rs
@@ -67,6 +67,7 @@ pub struct Storage {
 impl Storage {
     pub fn new(path: &Path) -> Result<Self> {
         let mut opts = Options::default();
+        // Enable prefix extractor for our namespace scheme (before creating bloom)
         // Optimize for prefix scans used by `prefix_iterator` across all key namespaces.
         // Extract the namespace prefix up to and including the first '/'.
         // Examples:
@@ -83,6 +84,62 @@ impl Storage {
             Some(|_key: &[u8]| true),
         );
         opts.set_prefix_extractor(ns_prefix);
+
+        // Optional: apply tuned settings only when explicitly enabled
+        let tuning_enabled = matches!(
+            std::env::var("QUEUEBER_DB_TUNING").as_deref(),
+            Ok("1") | Ok("on") | Ok("poll") | Ok("true")
+        );
+        if tuning_enabled {
+            // Parallelism and background jobs sized to CPUs for compaction/flush
+            let cpus = std::thread::available_parallelism()
+                .map(|n| n.get() as i32)
+                .unwrap_or(2);
+            // Increase parallelism also sets max_background_jobs if larger in RocksDB
+            opts.increase_parallelism(cpus);
+            // Allow more background jobs for compaction/flush to reduce write stalls
+            opts.set_max_background_jobs(cpus.saturating_mul(2));
+
+            // Block table + bloom filters (including whole-key bloom for point lookups)
+            // Note: newer rust-rocksdb exposes set_opt/optimize APIs on Options directly for simplicity
+            {
+                use rocksdb::BlockBasedOptions;
+                use rocksdb::Cache;
+                let mut bopts = BlockBasedOptions::default();
+                // 6 bits per key reduces CPU for filters; good tradeoff in mixed workloads
+                bopts.set_bloom_filter(6.0, false);
+                bopts.set_whole_key_filtering(true);
+                // 32 KiB block size balances CPU and I/O
+                bopts.set_block_size(32 * 1024);
+                // Cache index+filter blocks to reduce I/O under scan/prefix iterator
+                bopts.set_cache_index_and_filter_blocks(true);
+                // Avoid partitioned filters; increases CPU on small/index-centric workloads
+                bopts.set_partition_filters(false);
+                // Provide a shared LRU cache for blocks (tune size as needed)
+                let cache = Cache::new_lru_cache(256usize << 20);
+                bopts.set_block_cache(&cache);
+                // Keep L0 metadata hot
+                bopts.set_pin_l0_filter_and_index_blocks_in_cache(true);
+                opts.set_block_based_table_factory(&bopts);
+            }
+
+            // Heuristic tuning for point lookups; helps poll read path
+            opts.optimize_for_point_lookup(u64::try_from(256usize).unwrap() << 20);
+
+            // Compaction strategy and write path improvements
+            opts.set_level_compaction_dynamic_level_bytes(true);
+            opts.set_allow_concurrent_memtable_write(true);
+            opts.set_enable_write_thread_adaptive_yield(true);
+            opts.set_enable_pipelined_write(true);
+
+            // Smooth write I/O to reduce tail latencies
+            opts.set_bytes_per_sync(1 << 20); // 1 MiB
+            opts.set_wal_bytes_per_sync(1 << 20); // 1 MiB
+
+            // Compression: favor speed with Snappy; keeps space reasonable without high CPU
+            opts.set_compression_type(rocksdb::DBCompressionType::Snappy);
+        }
+
         opts.create_if_missing(true);
         let db = OptimisticTransactionDB::open(&opts, path)?;
         Ok(Self { db })