From e5df56aa18d8c8d724402536ef0a9321b9a734b1 Mon Sep 17 00:00:00 2001 From: Brayo Date: Sat, 13 Jun 2026 11:28:03 +0300 Subject: [PATCH 1/4] fix(deps): upgrade reqwest 0.11 -> 0.12 to fix rustls-webpki vuln reqwest 0.11.x pins rustls 0.21.x, which only allows the vulnerable rustls-webpki 0.101.7 (flagged by Dependabot, GHSA for rustls-webpki). Upgrading to reqwest 0.12 pulls in rustls 0.23 and rustls-webpki 0.103.13, removing the vulnerable version from the tree. The rustls-tls-native-roots feature is kept (required for Android builds). No source changes were needed; the reqwest API surface used by aw-client-rust and aw-sync is unchanged between 0.11 and 0.12. Closes #592 --- Cargo.lock | 517 ++++++++++++++++++++++++++++++-------- aw-client-rust/Cargo.toml | 2 +- aw-sync/Cargo.toml | 2 +- 3 files changed, 411 insertions(+), 110 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7673fa53..951cd269 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -181,6 +181,12 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.0" @@ -341,9 +347,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.7" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "binascii" @@ -366,12 +372,6 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - [[package]] name = "bitflags" version = "2.9.1" @@ -450,7 +450,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -567,6 +567,16 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -713,7 +723,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b035a542cf7abf01f2e3c4d5a7acbaebfefe120ae4efc7bde3df98186e4b8af7" dependencies = [ - "bitflags 2.9.1", + "bitflags", "proc-macro2", "proc-macro2-diagnostics", "quote", @@ -1006,8 +1016,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.11.1+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -1017,9 +1029,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasi 0.14.2+wasi-0.2.4", + "wasm-bindgen", ] [[package]] @@ -1053,6 +1067,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.3.1", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "2.6.0" @@ -1133,6 +1166,29 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.3.1", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http 1.3.1", + "http-body 1.0.1", + "pin-project-lite", +] + [[package]] name = "httparse" version = "1.10.1" @@ -1155,9 +1211,9 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.3.27", "http 0.2.12", - "http-body", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -1169,31 +1225,82 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2 0.4.14", + "http 1.3.1", + "http-body 1.0.1", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + [[package]] name = "hyper-rustls" -version = "0.24.2" +version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "futures-util", - "http 0.2.12", - "hyper", + "http 1.3.1", + "hyper 1.10.1", + "hyper-util", "rustls", + "rustls-native-certs", "tokio", "tokio-rustls", + "tower-service", ] [[package]] name = "hyper-tls" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", - "hyper", + "http-body-util", + "hyper 1.10.1", + "hyper-util", "native-tls", "tokio", "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "hyper 1.10.1", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2 0.6.0", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", ] [[package]] @@ -1350,7 +1457,7 @@ version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" dependencies = [ - "bitflags 2.9.1", + "bitflags", "cfg-if", "libc", ] @@ -1483,7 +1590,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3" dependencies = [ - "bitflags 2.9.1", + "bitflags", "libc", ] @@ -1552,6 +1659,12 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "matchers" version = "0.2.0" @@ -1630,10 +1743,10 @@ dependencies = [ "libc", "log", "openssl", - "openssl-probe", + "openssl-probe 0.1.6", "openssl-sys", "schannel", - "security-framework", + "security-framework 2.11.1", "security-framework-sys", "tempfile", ] @@ -1644,7 +1757,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.9.1", + "bitflags", "cfg-if", "cfg_aliases", "libc", @@ -1717,7 +1830,7 @@ version = "0.10.80" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" dependencies = [ - "bitflags 2.9.1", + "bitflags", "cfg-if", "foreign-types", "libc", @@ -1742,6 +1855,12 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "openssl-src" version = "300.5.1+3.5.1" @@ -1839,7 +1958,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared", - "rand", + "rand 0.8.5", ] [[package]] @@ -1969,6 +2088,61 @@ dependencies = [ "yansi", ] +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2 0.6.0", + "thiserror 2.0.12", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.3", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.12", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2 0.6.0", + "tracing", + "windows-sys 0.59.0", +] + [[package]] name = "quote" version = "1.0.40" @@ -1991,8 +2165,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", ] [[package]] @@ -2002,7 +2186,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", ] [[package]] @@ -2014,6 +2208,15 @@ dependencies = [ "getrandom 0.2.16", ] +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.3", +] + [[package]] name = "rayon" version = "1.10.0" @@ -2049,7 +2252,7 @@ version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" dependencies = [ - "bitflags 2.9.1", + "bitflags", ] [[package]] @@ -2114,46 +2317,48 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.11.27" +version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64", "bytes", "encoding_rs", + "futures-channel", "futures-core", "futures-util", - "h2", - "http 0.2.12", - "http-body", - "hyper", + "h2 0.4.14", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "hyper 1.10.1", "hyper-rustls", "hyper-tls", - "ipnet", + "hyper-util", "js-sys", "log", "mime", "native-tls", - "once_cell", "percent-encoding", "pin-project-lite", + "quinn", "rustls", "rustls-native-certs", - "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", - "system-configuration", "tokio", "tokio-native-tls", "tokio-rustls", + "tower", + "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "winreg", ] [[package]] @@ -2191,7 +2396,7 @@ dependencies = [ "num_cpus", "parking_lot", "pin-project-lite", - "rand", + "rand 0.8.5", "ref-cast", "rocket_codegen", "rocket_http", @@ -2252,7 +2457,7 @@ dependencies = [ "either", "futures", "http 0.2.12", - "hyper", + "hyper 0.14.32", "indexmap", "log", "memchr", @@ -2275,7 +2480,7 @@ version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d" dependencies = [ - "bitflags 2.9.1", + "bitflags", "chrono", "fallible-iterator", "fallible-streaming-iterator", @@ -2326,13 +2531,19 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + [[package]] name = "rustix" version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" dependencies = [ - "bitflags 2.9.1", + "bitflags", "errno", "libc", "linux-raw-sys", @@ -2341,44 +2552,48 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.12" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ - "log", + "once_cell", "ring", + "rustls-pki-types", "rustls-webpki", - "sct", + "subtle", + "zeroize", ] [[package]] name = "rustls-native-certs" -version = "0.6.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" dependencies = [ - "openssl-probe", - "rustls-pemfile", + "openssl-probe 0.2.1", + "rustls-pki-types", "schannel", - "security-framework", + "security-framework 3.6.0", ] [[package]] -name = "rustls-pemfile" -version = "1.0.4" +name = "rustls-pki-types" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ - "base64", + "web-time", + "zeroize", ] [[package]] name = "rustls-webpki" -version = "0.101.7" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "ring", + "rustls-pki-types", "untrusted", ] @@ -2449,16 +2664,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "sd-notify" version = "0.4.5" @@ -2474,8 +2679,21 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.9.1", - "core-foundation", + "bitflags", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d17b898a6d6948c3a8ee4372c17cb384f90d2e6e912ef00895b14fd7ab54ec38" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -2483,9 +2701,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.14.0" +version = "2.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" dependencies = [ "core-foundation-sys", "libc", @@ -2702,9 +2920,12 @@ dependencies = [ [[package]] name = "sync_wrapper" -version = "0.1.2" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] [[package]] name = "synstructure" @@ -2719,20 +2940,20 @@ dependencies = [ [[package]] name = "system-configuration" -version = "0.5.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 1.3.2", - "core-foundation", + "bitflags", + "core-foundation 0.9.4", "system-configuration-sys", ] [[package]] name = "system-configuration-sys" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" dependencies = [ "core-foundation-sys", "libc", @@ -2851,6 +3072,21 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.47.1" @@ -2893,9 +3129,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.24.1" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ "rustls", "tokio", @@ -2979,6 +3215,45 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -3279,6 +3554,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi-util" version = "0.1.9" @@ -3305,9 +3590,9 @@ checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ "windows-implement", "windows-interface", - "windows-link", - "windows-result", - "windows-strings", + "windows-link 0.1.3", + "windows-result 0.3.4", + "windows-strings 0.4.2", ] [[package]] @@ -3338,13 +3623,39 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + [[package]] name = "windows-result" version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" dependencies = [ - "windows-link", + "windows-link 0.1.3", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link 0.2.1", ] [[package]] @@ -3353,25 +3664,25 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" dependencies = [ - "windows-link", + "windows-link 0.1.3", ] [[package]] -name = "windows-sys" -version = "0.45.0" +name = "windows-strings" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ - "windows-targets 0.42.2", + "windows-link 0.2.1", ] [[package]] name = "windows-sys" -version = "0.48.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows-targets 0.48.5", + "windows-targets 0.42.2", ] [[package]] @@ -3579,23 +3890,13 @@ dependencies = [ "memchr", ] -[[package]] -name = "winreg" -version = "0.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "wit-bindgen-rt" version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags 2.9.1", + "bitflags", ] [[package]] diff --git a/aw-client-rust/Cargo.toml b/aw-client-rust/Cargo.toml index 8b0bc624..83ca0ad2 100644 --- a/aw-client-rust/Cargo.toml +++ b/aw-client-rust/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" authors = ["Johan Bjäreholt "] [dependencies] -reqwest = { version = "0.11", default-features = false, features = ["json", "blocking", "rustls-tls-native-roots"] } +reqwest = { version = "0.12", default-features = false, features = ["json", "blocking", "rustls-tls-native-roots"] } gethostname = "0.4" serde = { version = "1.0", features = ["derive"] } phf = { version = "0.11", features = ["macros"] } diff --git a/aw-sync/Cargo.toml b/aw-sync/Cargo.toml index 924e1cd9..4bd53ca5 100644 --- a/aw-sync/Cargo.toml +++ b/aw-sync/Cargo.toml @@ -19,7 +19,7 @@ toml = "0.8" chrono = { version = "0.4", features = ["serde"] } serde = "1.0" serde_json = "1.0" -reqwest = { version = "0.11", features = ["json", "blocking"] } +reqwest = { version = "0.12", features = ["json", "blocking"] } dirs = "6" gethostname = "0.4.3" From fb4d81c7f4977874e18e35b8ae7c387c8854936f Mon Sep 17 00:00:00 2001 From: Brayo Date: Sat, 13 Jun 2026 12:36:50 +0300 Subject: [PATCH 2/4] feat(datastore): transparent zstd compression of event data (#493) Add an opt-in 'compression-zstd' feature that transparently compresses the events.data column with zstd. ActivityWatch events are tiny and their redundancy is across rows (repeated app names, JSON keys, window titles), so a dictionary is trained on a sample of the database's own events, stored once, and used to compress every row. On a real ~545k-event database this shrinks the stored event JSON by ~47% (file after VACUUM: 77.4 MB -> 52.3 MB) with negligible read overhead (~0.5 us/event). Details: - New compression module: dictionary-compressed blobs are marked with a 0xCC prefix; rows that would not shrink (or before a dictionary exists) are stored as raw JSON, so a row is never larger than plain JSON. A single reusable compressor/decompressor is kept per connection. - Event data column migrated from TEXT to BLOB (db v6) so binary compresses without any encoding overhead. - The dictionary is trained once at startup when the database has enough events (and on upgrade), then existing rows are recompressed in a single transaction; new events are compressed on insert. - Feature is off by default; a database with compressed rows requires a build with the feature enabled to read it. Also enable foreign_keys and add ON DELETE CASCADE to events.bucketrow so deleting a bucket removes its events; the v6 migration drops pre-existing orphan events whose bucket no longer exists. Tested with and without the feature, including a full migrate+train+backfill roundtrip on a real database. --- Cargo.lock | 41 +++++ aw-datastore/Cargo.toml | 6 + aw-datastore/src/compression.rs | 258 ++++++++++++++++++++++++++++++++ aw-datastore/src/datastore.rs | 229 +++++++++++++++++++++++++++- aw-datastore/src/lib.rs | 1 + aw-datastore/src/worker.rs | 7 + aw-datastore/tests/datastore.rs | 94 +++++++++++- 7 files changed, 628 insertions(+), 8 deletions(-) create mode 100644 aw-datastore/src/compression.rs diff --git a/Cargo.lock b/Cargo.lock index 951cd269..93be1883 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -232,6 +232,7 @@ dependencies = [ "serde", "serde_json", "zeroize", + "zstd", ] [[package]] @@ -417,6 +418,8 @@ version = "1.2.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7" dependencies = [ + "jobserver", + "libc", "shlex", ] @@ -1556,6 +1559,16 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.3", + "libc", +] + [[package]] name = "js-sys" version = "0.3.77" @@ -4017,3 +4030,31 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/aw-datastore/Cargo.toml b/aw-datastore/Cargo.toml index e00bed4f..4f6c29ca 100644 --- a/aw-datastore/Cargo.toml +++ b/aw-datastore/Cargo.toml @@ -14,6 +14,11 @@ bundled = ["rusqlite/bundled"] encryption = ["rusqlite/bundled-sqlcipher", "dep:zeroize"] # Like 'encryption' but also vendors OpenSSL (fully self-contained) encryption-vendored = ["rusqlite/bundled-sqlcipher-vendored-openssl", "dep:zeroize"] +# Enable transparent zstd compression of event data, using a dictionary trained +# on the database's own events (~47% smaller event JSON on real data). +# Opt-in: once rows are compressed, the database can only be read by a build with +# this feature enabled. +compression-zstd = ["dep:zstd"] legacy_import_tests = [] [dependencies] @@ -25,6 +30,7 @@ rusqlite = { version = "0.30", features = ["chrono", "serde_json"] } mpsc_requests = "0.3" log = "0.4" zeroize = { version = "1", optional = true, features = ["alloc"] } +zstd = { version = "0.13", optional = true } aw-models = { path = "../aw-models" } aw-transform = { path = "../aw-transform" } diff --git a/aw-datastore/src/compression.rs b/aw-datastore/src/compression.rs new file mode 100644 index 00000000..eac7a155 --- /dev/null +++ b/aw-datastore/src/compression.rs @@ -0,0 +1,258 @@ +// Transparent compression of event data using zstd with a shared trained dictionary. +// +// ActivityWatch events are tiny (most < 128 bytes) and their redundancy is +// *across* rows: the same app names, JSON keys and window titles repeat +// thousands of times, while a single row holds little repetition for zstd to +// exploit on its own. To capture that cross-row redundancy we train one +// dictionary on a sample of the data, store it once in the database, and +// compress every row against it (the same idea as sqlite-zstd). On real data +// this reduces the stored event JSON by roughly 47%. +// +// Stored BLOB format: +// [0xCC][u32 LE original_len][zstd frame] -> dictionary-compressed +// -> stored uncompressed +// +// A row is stored uncompressed when compression would not make it smaller (or +// when no dictionary exists yet), so the worst case is never larger than the +// plain JSON. JSON event data always starts with '{' (0x7B), so the 0xCC marker +// is unambiguous. +// +// Tradeoff: once rows are compressed, the database can only be read by a build +// with this feature enabled (and with the stored dictionary intact). That is +// why the feature is opt-in at build time. + +/// Marker byte prefixed to dictionary-compressed blobs. +const COMPRESSION_MARKER: u8 = 0xCC; + +/// Target size of the trained dictionary. 64 KiB was the sweet spot in +/// benchmarks (larger dictionaries started to hurt the ratio). +#[cfg(feature = "compression-zstd")] +const DICT_SIZE: usize = 64 * 1024; + +/// zstd compression level. 6 is a good balance for this data: the dictionary +/// provides essentially all of the savings, and higher levels add cost for only +/// ~1% extra reduction. +#[cfg(feature = "compression-zstd")] +const COMPRESSION_LEVEL: i32 = 6; + +/// Minimum number of events before it is worth training a dictionary. Below +/// this the database is small enough that the savings are negligible, and there +/// is too little data to train a good dictionary. +pub const MIN_EVENTS_TO_TRAIN: i64 = 2000; + +/// Holds the reusable zstd compressor/decompressor (with the loaded +/// dictionary, if any) for the lifetime of a datastore connection. Lives on the +/// single-threaded datastore worker, so the compressor/decompressor — which +/// need `&mut` per call — are wrapped in `RefCell` and reused across calls +/// rather than reallocated each time (allocating a fresh context per row is the +/// dominant cost when compressing hundreds of thousands of tiny events). +pub struct CompressionContext { + #[cfg(feature = "compression-zstd")] + dict: Option, +} + +#[cfg(feature = "compression-zstd")] +struct Codec { + compressor: std::cell::RefCell>, + decompressor: std::cell::RefCell>, +} + +impl CompressionContext { + /// A context with no dictionary: writes are stored uncompressed, reads still + /// transparently handle both uncompressed and (if a dictionary is later + /// available) compressed data. + pub fn empty() -> Self { + CompressionContext { + #[cfg(feature = "compression-zstd")] + dict: None, + } + } + + /// Build a context from raw trained-dictionary bytes. Falls back to an empty + /// (no-compression) context if the dictionary can't be loaded. + #[cfg(feature = "compression-zstd")] + pub fn from_dictionary(dict_bytes: &[u8]) -> Self { + // with_dictionary copies the dictionary into the (de)compression + // context, so the returned values own it and are 'static. + match ( + zstd::bulk::Compressor::with_dictionary(COMPRESSION_LEVEL, dict_bytes), + zstd::bulk::Decompressor::with_dictionary(dict_bytes), + ) { + (Ok(compressor), Ok(decompressor)) => CompressionContext { + dict: Some(Codec { + compressor: std::cell::RefCell::new(compressor), + decompressor: std::cell::RefCell::new(decompressor), + }), + }, + _ => { + warn!("Failed to load compression dictionary; storing events uncompressed"); + CompressionContext::empty() + } + } + } + + #[cfg(not(feature = "compression-zstd"))] + pub fn from_dictionary(_dict_bytes: &[u8]) -> Self { + CompressionContext {} + } + + pub fn has_dictionary(&self) -> bool { + #[cfg(feature = "compression-zstd")] + { + self.dict.is_some() + } + #[cfg(not(feature = "compression-zstd"))] + { + false + } + } + + /// Compress a JSON string into the stored blob representation. + /// + /// Never fails: if there is no dictionary, the feature is disabled, or + /// compression would not shrink the row, the raw UTF-8 bytes are returned. + pub fn compress(&self, json: &str) -> Vec { + #[cfg(feature = "compression-zstd")] + { + if let Some(codec) = &self.dict { + if let Ok(frame) = codec.compressor.borrow_mut().compress(json.as_bytes()) { + // [marker][u32 LE original length][zstd frame] + if 5 + frame.len() < json.len() { + let mut blob = Vec::with_capacity(5 + frame.len()); + blob.push(COMPRESSION_MARKER); + blob.extend_from_slice(&(json.len() as u32).to_le_bytes()); + blob.extend_from_slice(&frame); + return blob; + } + } + } + } + json.as_bytes().to_vec() + } + + /// Decompress a stored blob back into its JSON string. + pub fn decompress(&self, bytes: &[u8]) -> Result { + if bytes.first() == Some(&COMPRESSION_MARKER) { + #[cfg(feature = "compression-zstd")] + { + if bytes.len() < 5 { + return Err("compressed blob too short to contain a length header".to_string()); + } + let orig_len = + u32::from_le_bytes([bytes[1], bytes[2], bytes[3], bytes[4]]) as usize; + let frame = &bytes[5..]; + let codec = self + .dict + .as_ref() + .ok_or("event data is compressed but no dictionary is loaded")?; + let out = codec + .decompressor + .borrow_mut() + .decompress(frame, orig_len) + .map_err(|e| format!("failed to decompress event data: {e}"))?; + String::from_utf8(out) + .map_err(|e| format!("decompressed data is not valid UTF-8: {e}")) + } + #[cfg(not(feature = "compression-zstd"))] + { + Err( + "event data is zstd-compressed but the compression-zstd feature is disabled" + .to_string(), + ) + } + } else { + String::from_utf8(bytes.to_vec()) + .map_err(|e| format!("event data is not valid UTF-8: {e}")) + } + } +} + +/// Train a zstd dictionary from a set of JSON samples. Returns `None` if there +/// is not enough data to train a usable dictionary. +#[cfg(feature = "compression-zstd")] +pub fn train_dictionary(samples: &[&[u8]]) -> Option> { + if (samples.len() as i64) < MIN_EVENTS_TO_TRAIN { + return None; + } + match zstd::dict::from_samples(samples, DICT_SIZE) { + Ok(dict) => Some(dict), + Err(e) => { + warn!("Failed to train zstd dictionary: {e}"); + None + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_uncompressed_roundtrip() { + // empty() context stores raw and reads it back + let ctx = CompressionContext::empty(); + let json = r#"{"app":"Firefox","title":"Hello"}"#; + let stored = ctx.compress(json); + assert_eq!(stored, json.as_bytes()); + assert_eq!(ctx.decompress(&stored).unwrap(), json); + } + + #[cfg(feature = "compression-zstd")] + #[test] + fn test_dictionary_roundtrip_and_size() { + // Build a realistic corpus with heavy cross-row redundancy. + let mut samples: Vec = Vec::new(); + for i in 0..5000 { + let app = ["Firefox", "Terminal", "Code", "Slack"][i % 4]; + samples.push(format!( + r#"{{"app":"{app}","title":"Some window title number {}"}}"#, + i % 50 + )); + } + let sample_refs: Vec<&[u8]> = samples.iter().map(|s| s.as_bytes()).collect(); + let dict = train_dictionary(&sample_refs).expect("training should succeed"); + let ctx = CompressionContext::from_dictionary(&dict); + + let mut total_raw = 0usize; + let mut total_stored = 0usize; + for s in &samples { + let blob = ctx.compress(s); + // roundtrip is exact + assert_eq!(ctx.decompress(&blob).unwrap(), *s); + total_raw += s.len(); + total_stored += blob.len(); + } + // dictionary compression must save substantial space on this corpus + // (without a dictionary, per-row zstd saves ~0% on data this small) + assert!( + total_stored * 10 < total_raw * 6, + "expected >40% reduction, got {total_stored} vs {total_raw}" + ); + } + + #[cfg(feature = "compression-zstd")] + #[test] + fn test_incompressible_row_stored_raw() { + // A context with a dictionary still stores tiny/incompressible rows raw + // when compression would not help, so a row is never larger than raw. + let samples: Vec = (0..2000) + .map(|i| format!(r#"{{"app":"A","title":"{i}"}}"#)) + .collect(); + let refs: Vec<&[u8]> = samples.iter().map(|s| s.as_bytes()).collect(); + let dict = train_dictionary(&refs).expect("training should succeed"); + let ctx = CompressionContext::from_dictionary(&dict); + + let tiny = r#"{"a":1}"#; + let blob = ctx.compress(tiny); + assert!(blob.len() <= tiny.len()); + assert_eq!(ctx.decompress(&blob).unwrap(), tiny); + } + + #[cfg(feature = "compression-zstd")] + #[test] + fn test_too_few_samples_no_dict() { + let samples: Vec = (0..10).map(|i| format!("{{\"x\":{i}}}")).collect(); + let refs: Vec<&[u8]> = samples.iter().map(|s| s.as_bytes()).collect(); + assert!(train_dictionary(&refs).is_none()); + } +} diff --git a/aw-datastore/src/datastore.rs b/aw-datastore/src/datastore.rs index 20cd3634..17a7fba4 100644 --- a/aw-datastore/src/datastore.rs +++ b/aw-datastore/src/datastore.rs @@ -15,6 +15,7 @@ use aw_models::Event; use rusqlite::params; use rusqlite::types::ToSql; +use super::compression; use super::DatastoreError; fn _get_db_version(conn: &Connection) -> i32 { @@ -22,6 +23,23 @@ fn _get_db_version(conn: &Connection) -> i32 { .unwrap() } +/// Load the stored zstd compression dictionary, if one has been trained. +fn _load_dictionary(conn: &Connection) -> Option> { + conn.query_row( + "SELECT dict FROM compression_dict WHERE id = 0", + [], + |row| row.get::<_, Vec>(0), + ) + .ok() +} + +/// Count of events in the database (used to decide whether to train a dictionary). +#[cfg(feature = "compression-zstd")] +fn _event_count(conn: &Connection) -> i64 { + conn.query_row("SELECT count(*) FROM events", [], |row| row.get(0)) + .unwrap_or(0) +} + /* * ### Database version changelog ### * 0: Uninitialized database @@ -30,8 +48,9 @@ fn _get_db_version(conn: &Connection) -> i32 { * 3: see: https://github.com/ActivityWatch/aw-server-rust/pull/52 * 4: Added 'key_value' table for storing key - value pairs * 5: Replaced single-column events indexes with a composite index + * 6: Enable zstd compression for event data column */ -static NEWEST_DB_VERSION: i32 = 5; +static NEWEST_DB_VERSION: i32 = 6; fn _create_tables(conn: &Connection, version: i32) -> bool { let mut first_init = false; @@ -57,6 +76,10 @@ fn _create_tables(conn: &Connection, version: i32) -> bool { _migrate_v4_to_v5(conn); } + if version < 6 { + _migrate_v5_to_v6(conn); + } + first_init } @@ -207,10 +230,64 @@ fn _migrate_v4_to_v5(conn: &Connection) { .expect("Failed to run v5 migration transaction"); } +fn _migrate_v5_to_v6(conn: &Connection) { + info!("Upgrading database to v6, converting event data column to BLOB for zstd compression"); + + // The data column becomes a BLOB so compressed binary can be stored without + // any text-encoding overhead. The actual dictionary training and + // compression of existing rows happens once at startup in + // `ensure_compression`, which is shared with the path that compresses + // databases that grow large enough after a fresh install. + // + // CAST(data AS BLOB) reinterprets each TEXT value as its raw UTF-8 bytes, + // converting the whole column in a single statement. The table is rebuilt + // (rather than altered in place) so the column's declared type — and thus + // the type returned to readers — is BLOB. + // + // The new table's foreign key uses ON DELETE CASCADE so deleting a bucket + // removes its events automatically (foreign keys are enforced on the + // connection, see work_loop). Older databases can contain orphan events + // whose bucket was already deleted; those are dropped here (the WHERE + // clause copies only events with an existing bucket) rather than carried + // forward, which also lets the copy satisfy the foreign key. + conn.execute_batch( + " + BEGIN EXCLUSIVE TRANSACTION; + CREATE TABLE events_v6 ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + bucketrow INTEGER NOT NULL, + starttime INTEGER NOT NULL, + endtime INTEGER NOT NULL, + data BLOB NOT NULL, + FOREIGN KEY (bucketrow) REFERENCES buckets(id) ON DELETE CASCADE + ); + INSERT INTO events_v6 (id, bucketrow, starttime, endtime, data) + SELECT id, bucketrow, starttime, endtime, CAST(data AS BLOB) + FROM events + WHERE bucketrow IN (SELECT id FROM buckets); + DROP INDEX IF EXISTS events_bucketrow_starttime_endtime_index; + DROP TABLE events; + ALTER TABLE events_v6 RENAME TO events; + CREATE INDEX events_bucketrow_starttime_endtime_index + ON events(bucketrow, starttime DESC, endtime); + CREATE TABLE IF NOT EXISTS compression_dict ( + id INTEGER PRIMARY KEY CHECK (id = 0), + dict BLOB NOT NULL + ); + PRAGMA user_version = 6; + COMMIT; + ", + ) + .expect("Failed to complete v6 migration transaction"); + + info!("Database upgrade to v6 complete"); +} + pub struct DatastoreInstance { buckets_cache: HashMap, first_init: bool, pub db_version: i32, + compression: compression::CompressionContext, } impl DatastoreInstance { @@ -239,11 +316,123 @@ impl DatastoreInstance { buckets_cache: HashMap::new(), first_init, db_version, + compression: compression::CompressionContext::empty(), }; + ds.ensure_compression(conn, migrate_enabled); ds.get_stored_buckets(conn)?; Ok(ds) } + /// Load the compression dictionary, training and applying one if the + /// database is large enough and doesn't have one yet. + /// + /// Runs once at startup. Loading a dictionary always happens (so compressed + /// rows can be read); training/back-filling only happens when migration is + /// enabled (we must not write to a read-only/migration-disabled datastore). + fn ensure_compression(&mut self, conn: &Connection, migrate_enabled: bool) { + // The compression_dict table only exists from v6 onward. + if self.db_version < 6 { + return; + } + if let Some(dict) = _load_dictionary(conn) { + self.compression = compression::CompressionContext::from_dictionary(&dict); + return; + } + #[cfg(feature = "compression-zstd")] + { + if migrate_enabled && _event_count(conn) >= compression::MIN_EVENTS_TO_TRAIN { + if let Err(e) = self.train_and_backfill(conn) { + warn!("Failed to set up event-data compression: {e:?}"); + } + } + } + #[cfg(not(feature = "compression-zstd"))] + { + let _ = migrate_enabled; + } + } + + /// Train a dictionary from existing events, store it, and recompress all + /// rows against it. Only called once, when a database first has enough data. + #[cfg(feature = "compression-zstd")] + fn train_and_backfill(&mut self, conn: &Connection) -> Result<(), DatastoreError> { + // Load every row's data (raw, uncompressed at this point). + let mut stmt = conn + .prepare("SELECT id, data FROM events") + .map_err(|e| DatastoreError::InternalError(e.to_string()))?; + let rows: Vec<(i64, Vec)> = stmt + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .map_err(|e| DatastoreError::InternalError(e.to_string()))? + .collect::, _>>() + .map_err(|e| DatastoreError::InternalError(e.to_string()))?; + drop(stmt); + + // Decompress (a no-op for already-raw rows) to recover the JSON to train on. + let jsons: Vec = rows + .iter() + .filter_map(|(_, bytes)| self.compression.decompress(bytes).ok()) + .collect(); + let sample_refs: Vec<&[u8]> = jsons.iter().map(|s| s.as_bytes()).collect(); + + let dict = match compression::train_dictionary(&sample_refs) { + Some(dict) => dict, + None => return Ok(()), // not enough usable data; leave uncompressed + }; + + info!( + "Trained {}-byte compression dictionary, recompressing {} events", + dict.len(), + rows.len() + ); + + self.compression = compression::CompressionContext::from_dictionary(&dict); + + // Store the dictionary and recompress every row in a single transaction. + // Without it, each statement would auto-commit and, with + // synchronous=FULL, fsync — turning a one-time backfill into hundreds of + // thousands of disk syncs. + conn.execute_batch("BEGIN IMMEDIATE TRANSACTION;") + .map_err(|e| DatastoreError::InternalError(e.to_string()))?; + + let result = (|| -> Result<(), DatastoreError> { + conn.execute( + "INSERT OR REPLACE INTO compression_dict (id, dict) VALUES (0, ?1)", + params![dict], + ) + .map_err(|e| DatastoreError::InternalError(e.to_string()))?; + + let mut update = conn + .prepare("UPDATE events SET data = ?1 WHERE id = ?2") + .map_err(|e| DatastoreError::InternalError(e.to_string()))?; + for (id, bytes) in &rows { + let json = match self.compression.decompress(bytes) { + Ok(json) => json, + Err(_) => continue, + }; + let compressed = self.compression.compress(&json); + update + .execute(params![compressed, id]) + .map_err(|e| DatastoreError::InternalError(e.to_string()))?; + } + Ok(()) + })(); + + match result { + Ok(()) => { + conn.execute_batch("COMMIT;") + .map_err(|e| DatastoreError::InternalError(e.to_string()))?; + Ok(()) + } + Err(e) => { + let _ = conn.execute_batch("ROLLBACK;"); + // Revert to no compression so reads don't expect compressed rows + // that were rolled back. + self.compression = compression::CompressionContext::empty(); + Err(e) + } + } + } + fn get_stored_buckets(&mut self, conn: &Connection) -> Result<(), DatastoreError> { let mut stmt = match conn.prepare_cached( " @@ -491,7 +680,8 @@ impl DatastoreInstance { } }; let endtime_nanos = starttime_nanos + duration_nanos; - let data = serde_json::to_string(&event.data).unwrap(); + let json_data = serde_json::to_string(&event.data).unwrap(); + let data = self.compression.compress(&json_data); let res = stmt.execute([ &bucket.bid.unwrap(), &event.id as &dyn ToSql, @@ -619,7 +809,8 @@ impl DatastoreInstance { } }; let endtime_nanos = starttime_nanos + duration_nanos; - let data = serde_json::to_string(&event.data).unwrap(); + let json_data = serde_json::to_string(&event.data).unwrap(); + let data = self.compression.compress(&json_data); match stmt.execute([ &bucket.bid.unwrap(), &starttime_nanos, @@ -721,17 +912,30 @@ impl DatastoreInstance { }; // TODO: Refactor to share row-parsing logic with get_events + let compression = &self.compression; let row = match stmt.query_row([&bucket.bid.unwrap(), &event_id], |row| { let id = row.get(0)?; let starttime_ns: i64 = row.get(1)?; let endtime_ns: i64 = row.get(2)?; - let data_str: String = row.get(3)?; + let data_bytes: Vec = row.get(3)?; let time_seconds: i64 = starttime_ns / 1_000_000_000; let time_subnanos: u32 = (starttime_ns % 1_000_000_000) as u32; let duration_ns = endtime_ns - starttime_ns; + + let decompressed_data_str = match compression.decompress(&data_bytes) { + Ok(decompressed) => decompressed, + Err(e) => { + warn!( + "Failed to decompress event data: {}. Attempting to parse as uncompressed.", + e + ); + String::from_utf8_lossy(&data_bytes).to_string() + } + }; + let data: serde_json::map::Map = - serde_json::from_str(&data_str).unwrap(); + serde_json::from_str(&decompressed_data_str).unwrap(); Ok(Event { id: Some(id), @@ -781,6 +985,8 @@ impl DatastoreInstance { None => -1, }; + let compression = &self.compression; + let mut stmt = match conn.prepare_cached( " SELECT id, starttime, endtime, data @@ -811,7 +1017,7 @@ impl DatastoreInstance { let id = row.get(0)?; let mut starttime_ns: i64 = row.get(1)?; let mut endtime_ns: i64 = row.get(2)?; - let data_str: String = row.get(3)?; + let data_bytes: Vec = row.get(3)?; if clip_to_query_range { if starttime_ns < starttime_filter_ns { @@ -825,8 +1031,17 @@ impl DatastoreInstance { let time_seconds: i64 = starttime_ns / 1_000_000_000; let time_subnanos: u32 = (starttime_ns % 1_000_000_000) as u32; + + let decompressed_data_str = match compression.decompress(&data_bytes) { + Ok(decompressed) => decompressed, + Err(e) => { + warn!("Failed to decompress event data: {}. Attempting to parse as uncompressed.", e); + String::from_utf8_lossy(&data_bytes).to_string() + } + }; + let data: serde_json::map::Map = - serde_json::from_str(&data_str).unwrap(); + serde_json::from_str(&decompressed_data_str).unwrap(); Ok(Event { id: Some(id), diff --git a/aw-datastore/src/lib.rs b/aw-datastore/src/lib.rs index 37df34ea..e857bd0a 100644 --- a/aw-datastore/src/lib.rs +++ b/aw-datastore/src/lib.rs @@ -17,6 +17,7 @@ macro_rules! json_map { }}; } +mod compression; mod datastore; mod legacy_import; mod privacy_filter; diff --git a/aw-datastore/src/worker.rs b/aw-datastore/src/worker.rs index e9481da1..9edd1b77 100644 --- a/aw-datastore/src/worker.rs +++ b/aw-datastore/src/worker.rs @@ -156,6 +156,13 @@ impl DatastoreWorker { conn.pragma_update(None, "synchronous", "FULL") .expect("Failed to set synchronous=FULL"); + // Enforce foreign keys so deleting a bucket cascades to its events + // (events.bucketrow REFERENCES buckets(id) ON DELETE CASCADE). SQLite + // defaults this off per connection; set it before migrations run and + // outside any transaction so it takes effect. + conn.pragma_update(None, "foreign_keys", "ON") + .expect("Failed to enable foreign_keys"); + let mut ds = DatastoreInstance::new(&conn, true).unwrap(); // Ensure legacy import diff --git a/aw-datastore/tests/datastore.rs b/aw-datastore/tests/datastore.rs index fe353e2a..d36a1300 100644 --- a/aw-datastore/tests/datastore.rs +++ b/aw-datastore/tests/datastore.rs @@ -538,7 +538,8 @@ mod datastore_tests { let version: i32 = conn .pragma_query_value(None, "user_version", |row| row.get(0)) .unwrap(); - assert_eq!(version, 5); + // Database should be upgraded to v6 (compression) automatically + assert_eq!(version, 6); let old_indexes: i64 = conn .query_row( "SELECT count(*) FROM sqlite_master WHERE type = 'index' AND name IN @@ -667,4 +668,95 @@ mod datastore_tests { let _ = fs::remove_file(&db_path); } + + /// With the compression feature, a database that grows past the training + /// threshold should get a dictionary on the next open, recompress its rows, + /// and still return every event's data intact. + #[test] + #[cfg(feature = "compression-zstd")] + fn test_compression_dictionary_roundtrip() { + use rusqlite::Connection; + + let mut db_path = get_cache_dir().unwrap(); + db_path.push("datastore-unittest-compression.db"); + let db_path_str = db_path.to_str().unwrap().to_string(); + if db_path.exists() { + std::fs::remove_file(&db_path).unwrap(); + } + + let bucket = test_bucket(); + // Realistic, highly-repetitive event data, well above MIN_EVENTS_TO_TRAIN. + let n = 4000; + let make_event = |i: usize| Event { + id: None, + timestamp: Utc::now() + Duration::milliseconds(i as i64), + duration: Duration::seconds(1), + data: json_map! { + "app": json!(["Firefox", "Terminal", "Code", "Slack"][i % 4]), + "title": json!(format!("Working on window {}", i % 100)) + }, + }; + + // Session 1: no dictionary exists yet (DB started empty), rows stored raw. + { + let ds = Datastore::new(db_path_str.clone(), false); + ds.create_bucket(&bucket).unwrap(); + let events: Vec = (0..n).map(make_event).collect(); + ds.insert_events(&bucket.id, &events).unwrap(); + ds.force_commit().unwrap(); + ds.close(); + } + + // No dictionary should have been trained during session 1. + { + let conn = Connection::open(&db_path).unwrap(); + let dict_rows: i64 = conn + .query_row("SELECT count(*) FROM compression_dict", [], |r| r.get(0)) + .unwrap(); + assert_eq!(dict_rows, 0, "no dictionary expected before reopen"); + } + + // Session 2: reopening trains a dictionary and recompresses existing rows. + { + let ds = Datastore::new(db_path_str.clone(), false); + let events = ds.get_events(&bucket.id, None, None, None).unwrap(); + assert_eq!(events.len(), n); + // Data must survive the train + recompress roundtrip exactly. + for e in &events { + let i = e.data["title"] + .as_str() + .unwrap() + .strip_prefix("Working on window ") + .unwrap() + .parse::() + .unwrap(); + assert!(i < 100); + assert!(e.data["app"].is_string()); + } + // Inserting more after the dictionary exists must also roundtrip. + ds.insert_events(&bucket.id, &[make_event(99999)]).unwrap(); + ds.force_commit().unwrap(); + ds.close(); + } + + // A dictionary now exists and at least some rows are stored compressed. + { + let conn = Connection::open(&db_path).unwrap(); + let dict_rows: i64 = conn + .query_row("SELECT count(*) FROM compression_dict", [], |r| r.get(0)) + .unwrap(); + assert_eq!(dict_rows, 1, "dictionary should exist after reopen"); + // 0xCC marker is the first byte of compressed blobs. + let compressed: i64 = conn + .query_row( + "SELECT count(*) FROM events WHERE hex(substr(data, 1, 1)) = 'CC'", + [], + |r| r.get(0), + ) + .unwrap(); + assert!(compressed > 0, "expected some rows to be stored compressed"); + } + + std::fs::remove_file(&db_path).unwrap(); + } } From e7a9ea01fd1bbfbfe1a8189d5a33b090dd8dfa40 Mon Sep 17 00:00:00 2001 From: Brayo Date: Sat, 13 Jun 2026 12:40:45 +0300 Subject: [PATCH 3/4] fix: expose compression-zstd in aw-server and remove dead code --- aw-datastore/src/compression.rs | 11 ----------- aw-server/Cargo.toml | 2 ++ 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/aw-datastore/src/compression.rs b/aw-datastore/src/compression.rs index eac7a155..f1ba66e8 100644 --- a/aw-datastore/src/compression.rs +++ b/aw-datastore/src/compression.rs @@ -96,17 +96,6 @@ impl CompressionContext { CompressionContext {} } - pub fn has_dictionary(&self) -> bool { - #[cfg(feature = "compression-zstd")] - { - self.dict.is_some() - } - #[cfg(not(feature = "compression-zstd"))] - { - false - } - } - /// Compress a JSON string into the stored blob representation. /// /// Never fails: if there is no dictionary, the feature is disabled, or diff --git a/aw-server/Cargo.toml b/aw-server/Cargo.toml index 814a6215..4674e7ea 100644 --- a/aw-server/Cargo.toml +++ b/aw-server/Cargo.toml @@ -22,6 +22,8 @@ bundled = ["aw-datastore/bundled"] encryption = ["aw-datastore/encryption"] # Enable SQLCipher encryption with vendored OpenSSL (fully self-contained) encryption-vendored = ["aw-datastore/encryption-vendored"] +# Enable zstd compression for events data +compression-zstd = ["aw-datastore/compression-zstd"] [dependencies] rocket = { version = "0.5.0", features = ["json"] } From 3c5a5aa731dae6a40c6ca6695ea8b06c24185a94 Mon Sep 17 00:00:00 2001 From: Brayo Date: Sat, 13 Jun 2026 15:26:37 +0300 Subject: [PATCH 4/4] fix: resolve cargo build warnings --- aw-datastore/src/compression.rs | 1 + aw-datastore/src/legacy_import.rs | 2 ++ aw-datastore/src/privacy_filter.rs | 2 ++ aw-models/src/tryvec.rs | 1 + aw-query/src/ast.rs | 1 + aw-sync/src/accessmethod.rs | 7 ------- aw-sync/src/dirs.rs | 1 + aw-sync/src/sync.rs | 1 + aw-sync/src/util.rs | 1 + 9 files changed, 10 insertions(+), 7 deletions(-) diff --git a/aw-datastore/src/compression.rs b/aw-datastore/src/compression.rs index f1ba66e8..ed87bfeb 100644 --- a/aw-datastore/src/compression.rs +++ b/aw-datastore/src/compression.rs @@ -38,6 +38,7 @@ const COMPRESSION_LEVEL: i32 = 6; /// Minimum number of events before it is worth training a dictionary. Below /// this the database is small enough that the savings are negligible, and there /// is too little data to train a good dictionary. +#[cfg(feature = "compression-zstd")] pub const MIN_EVENTS_TO_TRAIN: i64 = 2000; /// Holds the reusable zstd compressor/decompressor (with the loaded diff --git a/aw-datastore/src/legacy_import.rs b/aw-datastore/src/legacy_import.rs index be6f5d28..a4043488 100644 --- a/aw-datastore/src/legacy_import.rs +++ b/aw-datastore/src/legacy_import.rs @@ -3,7 +3,9 @@ use rusqlite::Connection; #[derive(Debug, Clone)] pub enum LegacyDatastoreImportError { + #[allow(dead_code)] SQLPrepareError(String), + #[allow(dead_code)] SQLMapError(String), } diff --git a/aw-datastore/src/privacy_filter.rs b/aw-datastore/src/privacy_filter.rs index 4d53136b..634e8693 100644 --- a/aw-datastore/src/privacy_filter.rs +++ b/aw-datastore/src/privacy_filter.rs @@ -113,6 +113,7 @@ impl PrivacyFilterEngine { /// Example rules for common sensitive data patterns. /// Not applied automatically — use `new()` with these rules to opt in. + #[allow(dead_code)] pub fn with_defaults() -> Self { let rules = vec![ PrivacyFilterRule { @@ -173,6 +174,7 @@ impl PrivacyFilterEngine { } /// Serialize rules to JSON string. + #[allow(dead_code)] pub fn to_json(&self) -> Result { serde_json::to_string(&self.rules) .map_err(|e| format!("Failed to serialize privacy filter rules: {e}")) diff --git a/aw-models/src/tryvec.rs b/aw-models/src/tryvec.rs index 60716ce4..4b3bf1a1 100644 --- a/aw-models/src/tryvec.rs +++ b/aw-models/src/tryvec.rs @@ -10,6 +10,7 @@ use std::fmt::Debug; #[derive(Debug, Clone)] pub enum TryParse { Parsed(T), + #[allow(dead_code)] Unparsed(Value), NotPresent, } diff --git a/aw-query/src/ast.rs b/aw-query/src/ast.rs index bba16557..0a45d78d 100644 --- a/aw-query/src/ast.rs +++ b/aw-query/src/ast.rs @@ -9,6 +9,7 @@ pub struct Program { #[derive(Debug, Clone)] pub struct Expr { + #[allow(dead_code)] pub span: Span, pub node: Expr_, } diff --git a/aw-sync/src/accessmethod.rs b/aw-sync/src/accessmethod.rs index 91825140..497b2ac0 100644 --- a/aw-sync/src/accessmethod.rs +++ b/aw-sync/src/accessmethod.rs @@ -22,7 +22,6 @@ pub trait AccessMethod: std::fmt::Debug { fn insert_events(&self, bucket_id: &str, events: Vec) -> Result<(), String>; fn get_event_count(&self, bucket_id: &str) -> Result; fn heartbeat(&self, bucket_id: &str, event: Event, duration: f64) -> Result<(), String>; - fn close(&self); } impl AccessMethod for Datastore { @@ -59,9 +58,6 @@ impl AccessMethod for Datastore { fn get_event_count(&self, bucket_id: &str) -> Result { Ok(Datastore::get_event_count(self, bucket_id, None, None).unwrap()) } - fn close(&self) { - Datastore::close(self); - } } impl AccessMethod for AwClient { @@ -105,7 +101,4 @@ impl AccessMethod for AwClient { fn heartbeat(&self, bucket_id: &str, event: Event, duration: f64) -> Result<(), String> { AwClient::heartbeat(self, bucket_id, &event, duration).map_err(|e| format!("{e:?}")) } - fn close(&self) { - // NOP - } } diff --git a/aw-sync/src/dirs.rs b/aw-sync/src/dirs.rs index 2744bba6..1e072ae0 100644 --- a/aw-sync/src/dirs.rs +++ b/aw-sync/src/dirs.rs @@ -17,6 +17,7 @@ pub fn get_config_dir() -> Result> { } #[cfg(not(target_os = "android"))] +#[allow(dead_code)] pub fn get_server_config_path(testing: bool) -> Result { let dir = aw_server::dirs::get_config_dir()?; Ok(dir.join(if testing { diff --git a/aw-sync/src/sync.rs b/aw-sync/src/sync.rs index 64618561..b8753e49 100644 --- a/aw-sync/src/sync.rs +++ b/aw-sync/src/sync.rs @@ -42,6 +42,7 @@ pub struct SyncSpec { /// Bucket IDs to sync pub buckets: Option>, /// Start of time range to sync + #[allow(dead_code)] pub start: Option>, } diff --git a/aw-sync/src/util.rs b/aw-sync/src/util.rs index 5681d9ef..b18df29a 100644 --- a/aw-sync/src/util.rs +++ b/aw-sync/src/util.rs @@ -7,6 +7,7 @@ use std::path::{Path, PathBuf}; /// Returns the port of the local aw-server instance #[cfg(not(target_os = "android"))] +#[allow(dead_code)] pub fn get_server_port(testing: bool) -> Result> { // TODO: get aw-server config more reliably let aw_server_conf = crate::dirs::get_server_config_path(testing)