diff --git a/Cargo.lock b/Cargo.lock index d0d0394..d35d17a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,12 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "aho-corasick" version = "1.1.4" @@ -325,6 +331,12 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + [[package]] name = "bytemuck" version = "1.25.0" @@ -365,6 +377,15 @@ dependencies = [ "shlex", ] +[[package]] +name = "cedarwood" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0524a528a6a0288df1863c3c20fe92c301875b4941e7b6c4b394ab08c5a4c55" +dependencies = [ + "smallvec", +] + [[package]] name = "cexpr" version = "0.6.0" @@ -788,6 +809,12 @@ dependencies = [ "syn", ] +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" + [[package]] name = "dashmap" version = "6.1.0" @@ -1233,6 +1260,8 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ + "allocator-api2", + "equivalent", "foldhash 0.2.0", ] @@ -1625,6 +1654,39 @@ dependencies = [ "quick-error", ] +[[package]] +name = "include-flate" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23e233413926ef735f7d87024466cfda5a4b87467730846bd82ea7d504121347" +dependencies = [ + "include-flate-codegen", + "include-flate-compress", +] + +[[package]] +name = "include-flate-codegen" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e7148f24ef8922cc0e5574ebb908729ccdd3a110c440a45165733fedadd9969" +dependencies = [ + "include-flate-compress", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "include-flate-compress" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74783a9ed407e844e99d5e7a57bd650acbfa124cf6e97ffd790ba59d8ab8e7ff" +dependencies = [ + "libflate", + "zstd", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -1705,6 +1767,30 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jieba-macros" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46adade69b634535a8f495cf87710ed893cff53e1dbc9dd750c2ab81c5defb82" +dependencies = [ + "phf_codegen", +] + +[[package]] +name = "jieba-rs" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11b53580aaa8ec8b713da271da434f8947409242c537a9ab3f7b76bdbb19e8a9" +dependencies = [ + "bytecount", + "cedarwood", + "include-flate", + "jieba-macros", + "phf", + "regex", + "rustc-hash", +] + [[package]] name = "jni" version = "0.22.4" @@ -1807,6 +1893,30 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libflate" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd96e993e5f3368b0cb8497dae6c860c22af8ff18388c61c6c0b86c58d86b5df" +dependencies = [ + "adler32", + "crc32fast", + "dary_heap", + "libflate_lz77", + "no_std_io2", +] + +[[package]] +name = "libflate_lz77" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff7a10e427698aef6eef269482776debfef63384d30f13aad39a1a95e0e098fd" +dependencies = [ + "hashbrown 0.16.1", + "no_std_io2", + "rle-decode-fast", +] + [[package]] name = "libm" version = "0.2.16" @@ -2025,6 +2135,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" +[[package]] +name = "no_std_io2" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418abd1b6d34fbf6cae440dc874771b0525a604428704c76e48b29a5e67b8003" +dependencies = [ + "memchr", +] + [[package]] name = "nom" version = "7.1.3" @@ -2278,6 +2397,45 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_codegen" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.11" @@ -2390,6 +2548,28 @@ dependencies = [ "syn", ] +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -2832,6 +3012,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rle-decode-fast" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" + [[package]] name = "rocksdb" version = "0.24.0" @@ -3160,6 +3346,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + [[package]] name = "sketches-ddsketch" version = "0.3.1" @@ -3873,6 +4065,7 @@ dependencies = [ "bytes", "dashmap", "futures", + "jieba-rs", "object_store", "rand 0.9.3", "rocksdb", @@ -4916,6 +5109,34 @@ version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "zune-core" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index 43f6460..fb0a4ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -98,6 +98,9 @@ thiserror = "2" anyhow = "1" tempfile = "3" regex = "1" +# Chinese/CJK word segmentation for full-text search. Pure Rust, embedded +# (compressed) dictionary; used only by vela-store's search tokenizer. +jieba-rs = "0.10" serde_yaml = "0.9" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } wiremock = "0.6" diff --git a/tools/deploy/vela.toml.example b/tools/deploy/vela.toml.example index 9ca515b..f7deb05 100644 --- a/tools/deploy/vela.toml.example +++ b/tools/deploy/vela.toml.example @@ -71,6 +71,14 @@ backend = "fs" # prefix = "media" # allow_http = false +[search] +# Full-text message search (POST /v3/search). When enabled, vela keeps a +# RocksDB inverted index on the event write path, tokenized with jieba so +# Chinese/CJK text is searchable word-by-word (not just whitespace-split). +# Turn off to skip all indexing. After enabling on a server with existing +# history, run the `!reindex-search` admin command once to backfill. +enabled = true + [room_defaults] # Server-side encryption-by-default for new rooms. When `/createRoom` # omits `m.room.encryption` in `initial_state`, vela injects it for diff --git a/tools/testing/smoketest-rs/Cargo.lock b/tools/testing/smoketest-rs/Cargo.lock index 67cd1a0..80e5ca4 100644 --- a/tools/testing/smoketest-rs/Cargo.lock +++ b/tools/testing/smoketest-rs/Cargo.lock @@ -8,6 +8,12 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "aho-corasick" version = "1.1.4" @@ -17,6 +23,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -244,6 +256,12 @@ version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + [[package]] name = "bytemuck" version = "1.25.0" @@ -284,6 +302,15 @@ dependencies = [ "shlex", ] +[[package]] +name = "cedarwood" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0524a528a6a0288df1863c3c20fe92c301875b4941e7b6c4b394ab08c5a4c55" +dependencies = [ + "smallvec", +] + [[package]] name = "cexpr" version = "0.6.0" @@ -498,6 +525,12 @@ dependencies = [ "syn", ] +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" + [[package]] name = "dashmap" version = "6.1.0" @@ -667,6 +700,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -868,7 +907,18 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "foldhash", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", ] [[package]] @@ -1236,6 +1286,39 @@ dependencies = [ "quick-error", ] +[[package]] +name = "include-flate" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23e233413926ef735f7d87024466cfda5a4b87467730846bd82ea7d504121347" +dependencies = [ + "include-flate-codegen", + "include-flate-compress", +] + +[[package]] +name = "include-flate-codegen" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e7148f24ef8922cc0e5574ebb908729ccdd3a110c440a45165733fedadd9969" +dependencies = [ + "include-flate-compress", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "include-flate-compress" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74783a9ed407e844e99d5e7a57bd650acbfa124cf6e97ffd790ba59d8ab8e7ff" +dependencies = [ + "libflate", + "zstd", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -1294,6 +1377,30 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jieba-macros" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46adade69b634535a8f495cf87710ed893cff53e1dbc9dd750c2ab81c5defb82" +dependencies = [ + "phf_codegen", +] + +[[package]] +name = "jieba-rs" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11b53580aaa8ec8b713da271da434f8947409242c537a9ab3f7b76bdbb19e8a9" +dependencies = [ + "bytecount", + "cedarwood", + "include-flate", + "jieba-macros", + "phf", + "regex", + "rustc-hash", +] + [[package]] name = "jni" version = "0.22.4" @@ -1418,6 +1525,30 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libflate" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd96e993e5f3368b0cb8497dae6c860c22af8ff18388c61c6c0b86c58d86b5df" +dependencies = [ + "adler32", + "crc32fast", + "dary_heap", + "libflate_lz77", + "no_std_io2", +] + +[[package]] +name = "libflate_lz77" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff7a10e427698aef6eef269482776debfef63384d30f13aad39a1a95e0e098fd" +dependencies = [ + "hashbrown 0.16.1", + "no_std_io2", + "rle-decode-fast", +] + [[package]] name = "librocksdb-sys" version = "0.17.3+10.4.2" @@ -1579,6 +1710,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" +[[package]] +name = "no_std_io2" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418abd1b6d34fbf6cae440dc874771b0525a604428704c76e48b29a5e67b8003" +dependencies = [ + "memchr", +] + [[package]] name = "nom" version = "7.1.3" @@ -1698,6 +1838,45 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_codegen" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1793,6 +1972,28 @@ dependencies = [ "toml_edit", ] +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -2101,6 +2302,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rle-decode-fast" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" + [[package]] name = "rocksdb" version = "0.24.0" @@ -2587,6 +2794,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + [[package]] name = "slab" version = "0.4.12" @@ -3104,6 +3317,7 @@ dependencies = [ "bytes", "dashmap", "futures", + "jieba-rs", "object_store", "rand 0.9.4", "rocksdb", @@ -3778,6 +3992,34 @@ version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "zune-core" version = "0.5.1" diff --git a/vela-api/src/admin/mod.rs b/vela-api/src/admin/mod.rs index e70cac0..04f7fee 100644 --- a/vela-api/src/admin/mod.rs +++ b/vela-api/src/admin/mod.rs @@ -929,6 +929,7 @@ async fn handle_command( "tokens" => cmd_tokens(state).await?, "create-user" => cmd_create_user(state, sender_nid, rest).await?, "rotate-signing-key" => cmd_rotate_signing_key(state).await?, + "reindex-search" => cmd_reindex_search(state).await?, "reports" => cmd_reports(state, rest).await?, "as" => cmd_appservice(state, body).await, other => Reply::plain(format!( @@ -985,6 +986,7 @@ fn cmd_help() -> Reply { !token revoke delete a registration token\n\ !create-user [pw] force-create a local user (bypasses UIA + tokens)\n\ !rotate-signing-key generate a new server signing key (restart required)\n\ + !reindex-search rebuild the full-text search index from history\n\ !reports [N] last N user-submitted abuse reports (default 20)\n\ !as register register an Application Service (paste YAML)\n\ !as list list registered Application Services\n\ @@ -1383,6 +1385,26 @@ async fn cmd_create_user( // for the new key to actually be used. Until restart, vela keeps // signing with the previous key (which is now in old_verify_keys // and so still verifiable by peers). +/// `!reindex-search` — rebuild the full-text search index from the live +/// timeline. Used to backfill history after enabling search on an existing +/// server (new events index automatically on persist). Runs off the async +/// executor since it walks every room's timeline. +async fn cmd_reindex_search(state: &AppState) -> Result { + if !state.db.is_search_indexing_enabled() { + return Ok(Reply::plain( + "search indexing is off ([search] enabled = false); nothing to reindex.".to_string(), + )); + } + let db = state.db.clone(); + let indexed = tokio::task::spawn_blocking(move || db.reindex_search()) + .await + .map_err(|e| ApiError(VelaError::Store(format!("reindex task join: {e}"))))? + .map_err(|e| ApiError(VelaError::Store(e.to_string())))?; + Ok(Reply::plain(format!( + "reindexed {indexed} events into the search index." + ))) +} + async fn cmd_rotate_signing_key(state: &AppState) -> Result { // Refuse a second rotation before restart. The DB's stored active // key_id will have advanced past the in-memory one in that case. diff --git a/vela-server/src/main.rs b/vela-server/src/main.rs index e836c12..705772e 100644 --- a/vela-server/src/main.rs +++ b/vela-server/src/main.rs @@ -79,6 +79,25 @@ struct Config { support: SupportSection, #[serde(default)] extensions: ExtensionsSection, + #[serde(default)] + search: SearchSection, +} + +/// `[search]` section — full-text message search (`POST /v3/search`). +/// Enabled by default; when on, vela maintains a RocksDB inverted index on +/// the event write path (jieba-tokenized, CJK-aware). Turn off to skip all +/// indexing (the jieba dictionary then never loads). After enabling on an +/// existing server, run the `!reindex-search` admin command to backfill. +#[derive(Debug, Deserialize)] +#[serde(default)] +struct SearchSection { + enabled: bool, +} + +impl Default for SearchSection { + fn default() -> Self { + Self { enabled: true } + } } /// `[support]` section — drives `.well-known/matrix/support` (MSC1929 / @@ -1266,8 +1285,21 @@ fn main() -> anyhow::Result<()> { let db_path = PathBuf::from(&config.database.path); let db = Database::open(&db_path) .map_err(|e| anyhow::anyhow!("failed to open database: {e}"))?; + // Full-text search: when enabled, the event write/redaction paths + // maintain the inverted index. Off → zero indexing cost, jieba + // never loads. Warm the dictionary here (at boot, off the request + // path) so the first message after a restart doesn't eat the + // one-time load latency inline on its persist. + db.set_search_indexing_enabled(config.search.enabled); + if config.search.enabled { + vela_store::search::warm(); + } - info!(path = %config.database.path, "database opened"); + info!( + path = %config.database.path, + search = config.search.enabled, + "database opened" + ); // Initialize media store. Filesystem is the single-pod default; // S3 (or any S3-compatible: MinIO, Cloudflare R2, B2) is for diff --git a/vela-store/Cargo.toml b/vela-store/Cargo.toml index 740e7e6..165ab49 100644 --- a/vela-store/Cargo.toml +++ b/vela-store/Cargo.toml @@ -24,6 +24,7 @@ futures = { workspace = true } object_store = { workspace = true } tokio-util = { workspace = true } uuid = { workspace = true } +jieba-rs = { workspace = true } [dev-dependencies] tempfile = { workspace = true } diff --git a/vela-store/src/cf.rs b/vela-store/src/cf.rs index c3dc3cb..e8cd008 100644 --- a/vela-store/src/cf.rs +++ b/vela-store/src/cf.rs @@ -320,4 +320,13 @@ pub const COLUMN_FAMILIES: &[&str] = &[ // value `u64 BE` = total (key+value) bytes that plugin stores. Updated // incrementally on set/delete and recomputed (self-healing) by the sweep. "wasm_kv_quota", + // Full-text search inverted index (see `search` module). + // Key: `[room_nid:8 BE] | token(UTF-8) | 0xFF | [stream_pos:8 BE]`. + // 0xFF never appears in valid UTF-8, so it cleanly delimits the + // variable-length token; the fixed 8-byte stream_pos suffix sorts + // postings chronologically (reverse scan = newest first). + // Value: `[event_nid:8 BE] | [field:1] | [tf:1]` — field 0=body 1=name + // 2=topic (for the CS-API `keys` filter); tf = term frequency + // (capped 255) summed into the relevance `rank`. + "search_index", ]; diff --git a/vela-store/src/db.rs b/vela-store/src/db.rs index c93d013..31d5d4c 100644 --- a/vela-store/src/db.rs +++ b/vela-store/src/db.rs @@ -1,7 +1,7 @@ use std::collections::BTreeSet; use std::path::Path; use std::sync::Mutex; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use rocksdb::{ColumnFamilyDescriptor, DB, Direction, IteratorMode, Options, WriteBatch}; use serde_json::Value; @@ -208,6 +208,12 @@ pub struct Database { /// `(user_nid, pos)`). Shared across users so positions are unique; /// recovered from the CF at open. pub(crate) notifications_stream_counter: AtomicU64, + /// Whether to maintain the full-text `search_index` on the event + /// write/redaction paths. Off by default; vela-server flips it on + /// from `[search] enabled` at boot. Tests opt in explicitly. When + /// off, `persist_event_kind`/`mark_redacted_by` skip all search work + /// (and the jieba dictionary is never loaded). + pub(crate) search_indexing_enabled: AtomicBool, } impl Database { @@ -260,6 +266,7 @@ impl Database { presence_stream_counter: AtomicU64::new(1), notifications_stream_counter: AtomicU64::new(1), snapshot_nid_counter: PersistedCounter::ephemeral(1), + search_indexing_enabled: AtomicBool::new(false), }) } @@ -389,6 +396,7 @@ impl Database { receipts_stream_counter: AtomicU64::new(receipts_stream_counter), presence_stream_counter: AtomicU64::new(presence_stream_counter), notifications_stream_counter: AtomicU64::new(notifications_stream_counter), + search_indexing_enabled: AtomicBool::new(false), }; // Migrate any pre-per-room outbox entries left over from before // the federation_sender refactor. No-op on fresh DBs. @@ -2759,6 +2767,15 @@ impl Database { keys::encode_u64(event_nid), keys::encode_u64(stream_pos), ); + // Full-text search index, in the same batch → atomic with the + // event. Parses body/name/topic from event_json and tokenizes + // (jieba). Skipped entirely when indexing is off, so the + // dictionary never loads and the hot path pays nothing. + if self.search_indexing_enabled.load(Ordering::Relaxed) { + self.index_event_into_batch( + &mut batch, room_nid, stream_pos, event_nid, event_json, + ); + } } // event_depth CF @@ -3680,7 +3697,15 @@ impl Database { &cf, keys::encode_u64(target_event_nid), keys::encode_u64(redactor_event_nid), - ) + )?; + // Drop the redacted event from the search index so it stops + // matching. Best-effort (see `deindex_event_search`); the query + // path also filters redacted hits, so a lingering posting is never + // a leak. + if self.search_indexing_enabled.load(Ordering::Relaxed) { + self.deindex_event_search(target_event_nid); + } + Ok(()) } /// Returns the redactor event NID if `target_event_nid` has been redacted. @@ -3692,6 +3717,170 @@ impl Database { } } + // --- Full-text search index (see the `search` module) --- + + /// Turn full-text search indexing on/off at runtime. vela-server calls + /// this at boot from `[search] enabled`; tests opt in explicitly. + pub fn set_search_indexing_enabled(&self, on: bool) { + self.search_indexing_enabled.store(on, Ordering::Relaxed); + } + + /// Whether the event write/redaction paths maintain the search index. + pub fn is_search_indexing_enabled(&self) -> bool { + self.search_indexing_enabled.load(Ordering::Relaxed) + } + + /// Add this event's inverted-index entries to `batch` so they commit + /// atomically with the event. Returns whether anything was indexed + /// (false for events with no searchable text). Called from + /// `persist_event_kind` for timeline events when indexing is on. + fn index_event_into_batch( + &self, + batch: &mut WriteBatch, + room_nid: u64, + stream_pos: u64, + event_nid: u64, + event_json: &[u8], + ) -> bool { + let Ok(json) = serde_json::from_slice::(event_json) else { + return false; + }; + let Some((field, text)) = crate::search::searchable_field(&json) else { + return false; + }; + let cf = self.db.cf_handle("search_index").unwrap(); + let mut any = false; + for (token, tf) in crate::search::token_frequencies(&text) { + let key = crate::search::index_key(room_nid, &token, stream_pos); + let val = crate::search::encode_value(event_nid, field, tf.min(255) as u8); + batch.put_cf(&cf, key, val); + any = true; + } + any + } + + /// Remove an event's entries from the search index. Reconstructs the + /// keys from the event's own content — this store keeps the original + /// bytes after a redaction marker (redaction doesn't scrub in place), so + /// reconstruction is exact. The query path also filters redacted hits, + /// so even a missed posting is never a leak. Called from + /// `mark_redacted_by` when indexing is on. + fn deindex_event_search(&self, event_nid: u64) { + let Ok(Some((_hdr, json_bytes))) = self.get_event(event_nid) else { + return; + }; + let Ok(json) = serde_json::from_slice::(&json_bytes) else { + return; + }; + let Some((_field, text)) = crate::search::searchable_field(&json) else { + return; + }; + let Some(room_id) = json.get("room_id").and_then(|v| v.as_str()) else { + return; + }; + let (Ok(Some(room_nid)), Ok(Some(stream_pos))) = + (self.get_nid(room_id), self.event_timeline_pos_of(event_nid)) + else { + return; + }; + let cf = self.db.cf_handle("search_index").unwrap(); + let mut batch = WriteBatch::default(); + for token in crate::search::token_frequencies(&text).into_keys() { + batch.delete_cf(&cf, crate::search::index_key(room_nid, &token, stream_pos)); + } + let _ = self.db.write(batch); + } + + /// All postings for `(room_nid, token)`, newest stream position first. + /// The building block the `/search` query intersects across the query's + /// tokens. Returns at most `limit` postings, newest stream position + /// first. Bounded so a very common token in a large room can't load an + /// unbounded posting range into memory. + pub fn search_room_token( + &self, + room_nid: u64, + token: &str, + limit: usize, + ) -> Vec { + let cf = self.db.cf_handle("search_index").unwrap(); + let prefix = crate::search::token_prefix(room_nid, token); + // Seek to `prefix ++ max stream_pos` and walk backwards: postings for + // one token are contiguous and all ≤ this bound, so a reverse scan + // yields newest-first directly — no load-everything-then-sort. + let mut upper = prefix.clone(); + upper.extend_from_slice(&u64::MAX.to_be_bytes()); + let mut out = Vec::new(); + let iter = self + .db + .iterator_cf(&cf, IteratorMode::From(&upper, Direction::Reverse)); + for item in iter { + if out.len() >= limit { + break; + } + let Ok((key, val)) = item else { break }; + if !key.starts_with(&prefix) { + break; + } + let (Some(stream_pos), Some((event_nid, field, tf))) = ( + crate::search::key_stream_pos(&key), + crate::search::decode_value(&val), + ) else { + continue; + }; + out.push(crate::search::Posting { + stream_pos, + event_nid, + field, + tf, + }); + } + out + } + + /// Rebuild the search index for every room by walking the live + /// timeline. Backfills history after enabling search on an existing DB + /// (the `!reindex-search` admin command). Additive and idempotent: + /// re-indexing an event overwrites its own keys. Returns the number of + /// events indexed. + pub fn reindex_search(&self) -> Result { + let cf_tl = self.db.cf_handle("room_timeline").unwrap(); + let mut indexed: u64 = 0; + let mut batch = WriteBatch::default(); + let mut pending: u64 = 0; + for item in self.db.iterator_cf(&cf_tl, IteratorMode::Start) { + let (key, val) = item?; + // room_timeline key = room_nid(8 BE) ++ stream_pos(8 BE); val = event_nid. + if key.len() < 16 { + continue; + } + let room_nid = u64::from_be_bytes(key[..8].try_into().unwrap()); + let stream_pos = u64::from_be_bytes(key[8..16].try_into().unwrap()); + let event_nid = keys::decode_u64(&val); + // Don't re-index redacted events — a redaction de-indexed them, + // and re-adding their tokens would just be filtered again at + // query time. Keeps a rebuild consistent with the live path. + if matches!(self.get_redacted_by(event_nid), Ok(Some(_))) { + continue; + } + let Ok(Some((_hdr, json_bytes))) = self.get_event(event_nid) else { + continue; + }; + if self.index_event_into_batch(&mut batch, room_nid, stream_pos, event_nid, &json_bytes) + { + indexed += 1; + pending += 1; + } + if pending >= 1000 { + self.db.write(std::mem::take(&mut batch))?; + pending = 0; + } + } + if pending > 0 { + self.db.write(batch)?; + } + Ok(indexed) + } + /// Park a redaction whose target event_id isn't on disk yet. Read /// back when the target arrives via a later /send/{txn} so the /// marker fires retroactively. Multiple redactions racing for the @@ -7411,7 +7600,7 @@ fn configure_cf(opts: &mut Options, name: &str) { opts.set_bloom_locality(10); opts.optimize_for_point_lookup(64 * 1024 * 1024); } - "room_timeline" | "room_state" | "user_rooms" => { + "room_timeline" | "room_state" | "user_rooms" | "search_index" => { opts.set_prefix_extractor(rocksdb::SliceTransform::create_fixed_prefix(8)); } _ => {} @@ -8069,6 +8258,162 @@ mod stream_recovery_tests { ); } + /// End-to-end: a persisted message is indexed (English + CJK tokens, + /// with term frequency), found by `search_room_token`, and dropped from + /// the index on redaction. Exercises the real persist → prefix-scan → + /// redact path against RocksDB (validates the prefix-extractor scan). + #[test] + fn search_index_roundtrip_and_deindex() { + let tmp = tempfile::tempdir().unwrap(); + let db = Database::open(tmp.path()).unwrap(); + db.set_search_indexing_enabled(true); + + let room_id = "!r:example.com"; + let room_nid = db.get_or_create_nid(room_id).unwrap(); + let type_nid = db.get_or_create_nid("m.room.message").unwrap(); + let sender_nid = db.get_or_create_nid("@alice:example.com").unwrap(); + let event_nid = db.next_nid().unwrap(); + let json = serde_json::json!({ + "type": "m.room.message", + "room_id": room_id, + "sender": "@alice:example.com", + "content": {"msgtype": "m.text", "body": "hello 世界 hello"}, + "origin_server_ts": 1, + "depth": 1, + }); + let bytes = serde_json::to_vec(&json).unwrap(); + let pos = db + .persist_event( + event_nid, + "$evt1:example.com", + room_nid, + type_nid, + sender_nid, + 0, + 1, + 1, + &bytes, + &[], + &[], + false, + false, + ) + .unwrap(); + + // English token: aggregated term frequency = 2. + let hits = db.search_room_token(room_nid, "hello", 100); + assert_eq!(hits.len(), 1, "one posting for 'hello'"); + assert_eq!(hits[0].event_nid, event_nid); + assert_eq!(hits[0].stream_pos, pos); + assert_eq!(hits[0].field, crate::search::FIELD_BODY); + assert_eq!(hits[0].tf, 2, "'hello' appears twice"); + + // CJK: jieba segmented 世界 into its own token, so it's findable — + // the whole point of the feature. + assert_eq!(db.search_room_token(room_nid, "世界", 100).len(), 1); + // A prefix of an indexed token must NOT match (0xFF separator). + assert!(db.search_room_token(room_nid, "hell", 100).is_empty()); + // An absent word: no hits. + assert!(db.search_room_token(room_nid, "goodbye", 100).is_empty()); + + // Cross-room isolation: the same token in a different room must not + // leak into this room's scan (the room_nid key prefix separates + // them). Index "hello" in room 2 and confirm room 1 still sees one. + let room2_nid = db.get_or_create_nid("!r2:example.com").unwrap(); + let e2 = db.next_nid().unwrap(); + let json2 = serde_json::json!({ + "type": "m.room.message", "room_id": "!r2:example.com", + "sender": "@bob:example.com", + "content": {"msgtype": "m.text", "body": "hello there"}, + "origin_server_ts": 2, "depth": 1, + }); + db.persist_event( + e2, + "$e2:example.com", + room2_nid, + type_nid, + db.get_or_create_nid("@bob:example.com").unwrap(), + 0, + 2, + 1, + &serde_json::to_vec(&json2).unwrap(), + &[], + &[], + false, + false, + ) + .unwrap(); + assert_eq!( + db.search_room_token(room_nid, "hello", 100).len(), + 1, + "room 1 unaffected" + ); + assert_eq!( + db.search_room_token(room2_nid, "hello", 100).len(), + 1, + "room 2 has its own" + ); + assert_eq!( + db.search_room_token(room2_nid, "世界", 100).len(), + 0, + "room 2 has no CJK" + ); + + // Redaction removes it from the index. + let redactor = db.next_nid().unwrap(); + db.mark_redacted_by(event_nid, redactor).unwrap(); + assert!( + db.search_room_token(room_nid, "hello", 100).is_empty(), + "redacted event must leave the index" + ); + assert!(db.search_room_token(room_nid, "世界", 100).is_empty()); + } + + /// Indexing is off by default: a persisted message produces no postings + /// until `set_search_indexing_enabled(true)`. Guards the hot path from + /// paying for jieba when search is disabled. + #[test] + fn search_index_off_by_default() { + let tmp = tempfile::tempdir().unwrap(); + let db = Database::open(tmp.path()).unwrap(); + assert!(!db.is_search_indexing_enabled()); + + let room_id = "!r:example.com"; + let room_nid = db.get_or_create_nid(room_id).unwrap(); + let type_nid = db.get_or_create_nid("m.room.message").unwrap(); + let sender_nid = db.get_or_create_nid("@alice:example.com").unwrap(); + let event_nid = db.next_nid().unwrap(); + let json = serde_json::json!({ + "type": "m.room.message", "room_id": room_id, "sender": "@alice:example.com", + "content": {"msgtype": "m.text", "body": "hello world"}, + "origin_server_ts": 1, "depth": 1, + }); + let bytes = serde_json::to_vec(&json).unwrap(); + db.persist_event( + event_nid, + "$e:example.com", + room_nid, + type_nid, + sender_nid, + 0, + 1, + 1, + &bytes, + &[], + &[], + false, + false, + ) + .unwrap(); + assert!(db.search_room_token(room_nid, "hello", 100).is_empty()); + + // reindex_search backfills it (as the admin command does). + db.set_search_indexing_enabled(true); + let n = db.reindex_search().unwrap(); + assert_eq!(n, 1, "one searchable event backfilled"); + assert_eq!(db.search_room_token(room_nid, "hello", 100).len(), 1); + } + /// Stream positions allocated by real-world write paths /// (set_account_data, set_membership) across two restarts are /// strictly monotonic — never reused, never decreasing. diff --git a/vela-store/src/lib.rs b/vela-store/src/lib.rs index 868767c..1a168f7 100644 --- a/vela-store/src/lib.rs +++ b/vela-store/src/lib.rs @@ -3,6 +3,7 @@ pub mod db; pub mod keys; pub mod media; pub mod nid; +pub mod search; pub mod store; // Re-exports so the trait is reachable without the `store::` path. diff --git a/vela-store/src/search.rs b/vela-store/src/search.rs new file mode 100644 index 0000000..17aac2e --- /dev/null +++ b/vela-store/src/search.rs @@ -0,0 +1,264 @@ +//! Full-text search tokenizer. +//! +//! The tokenizer is the load-bearing piece for CJK search: it wraps jieba +//! word segmentation so Chinese/Japanese text — written without spaces — +//! yields real word tokens (你好世界 → 你好 / 世界) instead of one giant +//! unmatchable token (the failure mode in Synapse and conduwuit). +//! +//! The SAME function must run at index time and query time or nothing +//! matches, so it lives in the store, shared by the write path +//! (`Database::persist_event_kind`) and the query path (vela-api's +//! `/search` handler via `Database`). + +use std::collections::HashMap; +use std::sync::OnceLock; + +use jieba_rs::Jieba; +use serde_json::Value; + +/// Key separator between the variable-length token and the fixed 8-byte +/// stream-position suffix. `0xFF` is never a byte in valid UTF-8, so a +/// token can never contain it — the delimiter is unambiguous, and the +/// prefix `room ++ token ++ 0xFF` matches only that exact token (never a +/// longer token that happens to share a prefix). +pub const SEP: u8 = 0xFF; + +/// Field tags stored in the index value (which of the searchable CS-API +/// keys this posting came from), so a query can honor the `keys` filter. +pub const FIELD_BODY: u8 = 0; +pub const FIELD_NAME: u8 = 1; +pub const FIELD_TOPIC: u8 = 2; + +/// One hit for a (room, token): where it is and how strongly it matched. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Posting { + pub stream_pos: u64, + pub event_nid: u64, + pub field: u8, + /// Term frequency in the event (capped at 255); summed into `rank`. + pub tf: u8, +} + +/// `search_index` key: `room_nid ++ token ++ 0xFF ++ stream_pos`. +pub fn index_key(room_nid: u64, token: &str, stream_pos: u64) -> Vec { + let mut k = Vec::with_capacity(8 + token.len() + 1 + 8); + k.extend_from_slice(&room_nid.to_be_bytes()); + k.extend_from_slice(token.as_bytes()); + k.push(SEP); + k.extend_from_slice(&stream_pos.to_be_bytes()); + k +} + +/// Prefix selecting every posting for (room, token): `room ++ token ++ 0xFF`. +pub fn token_prefix(room_nid: u64, token: &str) -> Vec { + let mut p = Vec::with_capacity(8 + token.len() + 1); + p.extend_from_slice(&room_nid.to_be_bytes()); + p.extend_from_slice(token.as_bytes()); + p.push(SEP); + p +} + +/// `search_index` value: `event_nid ++ field ++ tf`. +pub fn encode_value(event_nid: u64, field: u8, tf: u8) -> [u8; 10] { + let mut v = [0u8; 10]; + v[..8].copy_from_slice(&event_nid.to_be_bytes()); + v[8] = field; + v[9] = tf; + v +} + +/// Decode a `search_index` value into `(event_nid, field, tf)`. +pub fn decode_value(v: &[u8]) -> Option<(u64, u8, u8)> { + if v.len() < 10 { + return None; + } + let event_nid = u64::from_be_bytes(v[..8].try_into().ok()?); + Some((event_nid, v[8], v[9])) +} + +/// The trailing 8-byte stream position of a `search_index` key. +pub fn key_stream_pos(key: &[u8]) -> Option { + if key.len() < 8 { + return None; + } + Some(u64::from_be_bytes(key[key.len() - 8..].try_into().ok()?)) +} + +/// The searchable text of an event and which CS-API key it is, or `None` +/// when the event carries nothing searchable. Mirrors the spec's supported +/// keys: `content.body` (m.room.message), `content.name` (m.room.name), +/// `content.topic` plus the `text/plain` body of the extensible +/// `content['m.topic']` (m.room.topic). +pub fn searchable_field(event_json: &Value) -> Option<(u8, String)> { + let ty = event_json.get("type")?.as_str()?; + let content = event_json.get("content")?; + match ty { + "m.room.message" => content + .get("body") + .and_then(Value::as_str) + .filter(|b| !b.is_empty()) + .map(|b| (FIELD_BODY, b.to_string())), + "m.room.name" => content + .get("name") + .and_then(Value::as_str) + .filter(|n| !n.is_empty()) + .map(|n| (FIELD_NAME, n.to_string())), + "m.room.topic" => { + let mut text = content + .get("topic") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + // Fold in the text/plain representation of MSC3765 m.topic. + if let Some(reps) = content + .get("m.topic") + .and_then(|t| t.get("m.text")) + .and_then(Value::as_array) + { + for rep in reps { + let mimetype = rep + .get("mimetype") + .and_then(Value::as_str) + .unwrap_or("text/plain"); + if mimetype == "text/plain" + && let Some(b) = rep.get("body").and_then(Value::as_str) + && !text.contains(b) + { + if !text.is_empty() { + text.push(' '); + } + text.push_str(b); + } + } + } + (!text.trim().is_empty()).then_some((FIELD_TOPIC, text)) + } + _ => None, + } +} + +/// Cap on an indexed token's UTF-8 length. Bounds search-index key size; +/// longer runs (URLs, base64 blobs, pathological input) are dropped from +/// the index rather than bloating it. Query tokens over the cap likewise +/// never match — acceptable, since real words are far shorter. +pub const TOKEN_MAX_BYTES: usize = 60; + +/// Process-wide jieba instance. `Jieba::new` parses the embedded dictionary +/// (~tens of ms, tens of MB resident), so build it once and lazily — a +/// server with search disabled never pays for it. +fn jieba() -> &'static Jieba { + static JIEBA: OnceLock = OnceLock::new(); + JIEBA.get_or_init(Jieba::new) +} + +/// Force the jieba dictionary to load now. Called once at boot (off the +/// request path) when search is enabled, so the first indexed message +/// doesn't pay the one-time dictionary-load latency inline on a persist. +pub fn warm() { + let _ = jieba(); +} + +/// Split `text` into lowercased search tokens. +/// +/// jieba `cut_for_search` segments CJK into words (and further splits long +/// words, which improves recall) while passing Latin runs through on +/// whitespace/punctuation boundaries. Segments with no alphanumeric +/// character (punctuation, whitespace) and tokens over [`TOKEN_MAX_BYTES`] +/// are dropped. Deterministic and identical at index and query time. +/// +/// Duplicates are preserved so the caller can count term frequency; use +/// [`token_frequencies`] when you want the aggregated form. +pub fn tokenize(text: &str) -> Vec { + jieba() + .cut_for_search(text, true) + .into_iter() + .map(|tok| tok.word) + .filter(|seg| seg.chars().any(char::is_alphanumeric)) + .map(str::to_lowercase) + .filter(|tok| tok.len() <= TOKEN_MAX_BYTES) + .collect() +} + +/// Tokenize `text` and count how often each distinct token appears. The +/// count is the term frequency stored in the inverted index and summed +/// into the relevance `rank`. +pub fn token_frequencies(text: &str) -> HashMap { + let mut freq: HashMap = HashMap::new(); + for tok in tokenize(text) { + *freq.entry(tok).or_insert(0) += 1; + } + freq +} + +/// The distinct query tokens for `search_term`, deduplicated (order not +/// significant — the query intersects postings across them). +pub fn query_tokens(search_term: &str) -> Vec { + let mut seen = std::collections::HashSet::new(); + tokenize(search_term) + .into_iter() + .filter(|t| seen.insert(t.clone())) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn chinese_is_segmented_into_words() { + // The whole point: a space-less Chinese run must become real word + // tokens, not one unmatchable blob. + let toks = tokenize("你好世界"); + assert!(toks.contains(&"你好".to_string()), "got {toks:?}"); + assert!(toks.contains(&"世界".to_string()), "got {toks:?}"); + // And a query for one of the words tokenizes to that same word, so + // it will match the indexed document. + assert!(query_tokens("世界").contains(&"世界".to_string())); + } + + #[test] + fn mixed_cjk_and_latin() { + let toks = tokenize("hello 世界 Rust"); + assert!(toks.contains(&"hello".to_string()), "got {toks:?}"); + assert!(toks.contains(&"世界".to_string()), "got {toks:?}"); + // Latin is lowercased. + assert!(toks.contains(&"rust".to_string()), "got {toks:?}"); + } + + #[test] + fn latin_lowercased_and_punctuation_dropped() { + let toks = tokenize("Hello, WORLD! foo-bar."); + assert!(toks.contains(&"hello".to_string()), "got {toks:?}"); + assert!(toks.contains(&"world".to_string()), "got {toks:?}"); + // No pure-punctuation tokens survive. + assert!(toks.iter().all(|t| t.chars().any(char::is_alphanumeric))); + } + + #[test] + fn over_long_tokens_dropped() { + let long = "a".repeat(TOKEN_MAX_BYTES + 1); + assert!(tokenize(&long).is_empty(), "over-cap token must be dropped"); + let ok = "a".repeat(TOKEN_MAX_BYTES); + assert_eq!(tokenize(&ok), vec![ok]); + } + + #[test] + fn term_frequency_counts_repeats() { + let freq = token_frequencies("cat cat dog"); + assert_eq!(freq.get("cat"), Some(&2)); + assert_eq!(freq.get("dog"), Some(&1)); + } + + #[test] + fn query_tokens_are_deduped() { + let q = query_tokens("cat cat cat"); + assert_eq!(q, vec!["cat".to_string()]); + } + + #[test] + fn empty_and_whitespace_yield_no_tokens() { + assert!(tokenize("").is_empty()); + assert!(tokenize(" \n\t ").is_empty()); + assert!(tokenize("!!! ??? ...").is_empty()); + } +}