From 3696e7cecc5b4486631b743695c5a46a5b762972 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 9 Jun 2026 18:19:57 -0500 Subject: [PATCH 01/55] Splat based on #19824 --- Cargo.lock | 855 ++++++++++++++++++++++++-- rust/Cargo.toml | 7 + rust/src/config/mod.rs | 36 ++ rust/src/handlers/mod.rs | 81 +++ rust/src/handlers/versions.rs | 176 ++++++ rust/src/lib.rs | 4 + rust/src/storage/db/mod.rs | 47 ++ rust/src/storage/db/python_db_pool.rs | 260 ++++++++ rust/src/storage/db/rust_db_pool.rs | 86 +++ rust/src/storage/mod.rs | 17 + rust/src/storage/store.rs | 72 +++ synapse/rest/client/versions.py | 140 +---- synapse/server.py | 4 + 13 files changed, 1601 insertions(+), 184 deletions(-) create mode 100644 rust/src/config/mod.rs create mode 100644 rust/src/handlers/mod.rs create mode 100644 rust/src/handlers/versions.rs create mode 100644 rust/src/storage/db/mod.rs create mode 100644 rust/src/storage/db/python_db_pool.rs create mode 100644 rust/src/storage/db/rust_db_pool.rs create mode 100644 rust/src/storage/mod.rs create mode 100644 rust/src/storage/store.rs diff --git a/Cargo.lock b/Cargo.lock index 761e99b3e65..5faf827e1db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,6 +23,17 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -35,6 +46,30 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bb8" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89aabfae550a5c44b43ab941844ffcd2e993cb6900b342debf59e9ea74acdb8" +dependencies = [ + "async-trait", + "futures-util", + "parking_lot", + "tokio", +] + +[[package]] +name = "bb8-postgres" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ac82c42eb30889b5c4ee4763a24b8c566518171ebea648cd7e3bc532c60680" +dependencies = [ + "async-trait", + "bb8", + "tokio", + "tokio-postgres", +] + [[package]] name = "bitflags" version = "2.9.1" @@ -47,7 +82,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -59,12 +94,27 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bumpalo" version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.11.1" @@ -92,6 +142,39 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -126,6 +209,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -136,17 +228,47 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.6", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.0", + "const-oid", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -170,12 +292,55 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -305,11 +470,25 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasi 0.14.2+wasi-0.2.4", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasip2", + "wasip3", +] + [[package]] name = "h2" version = "0.4.11" @@ -334,6 +513,9 @@ name = "hashbrown" version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" +dependencies = [ + "foldhash", +] [[package]] name = "headers" @@ -371,6 +553,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "http" version = "1.4.0" @@ -416,6 +607,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.6.0" @@ -609,6 +809,12 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5360a2fbe97f617c4f8b944356dedb36d423f7da7f13c070995cf89e59f01220" +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + [[package]] name = "idna" version = "1.0.3" @@ -638,6 +844,7 @@ checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" dependencies = [ "equivalent", "hashbrown", + "serde", ] [[package]] @@ -673,10 +880,12 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "js-sys" -version = "0.3.77" +version = "0.3.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -687,11 +896,17 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "libc" -version = "0.2.174" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libm" @@ -699,12 +914,36 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +[[package]] +name = "libredox" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" +dependencies = [ + "libc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.29" @@ -717,6 +956,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "memchr" version = "2.7.5" @@ -740,36 +989,197 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework 2.11.1", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + [[package]] name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "openssl" +version = "0.10.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "openssl-sys" +version = "0.9.116" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + [[package]] name = "percent-encoding" version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "portable-atomic" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +[[package]] +name = "postgres-native-tls" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fef4de47bb81477e0c3deaf153a1b10ae176484713ff1640969f4cb96b653ebc" +dependencies = [ + "native-tls", + "tokio", + "tokio-native-tls", + "tokio-postgres", +] + +[[package]] +name = "postgres-protocol" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56201207dac53e2f38e848e31b4b91616a6bb6e0c7205b77718994a7f49e70fc" +dependencies = [ + "base64", + "byteorder", + "bytes", + "fallible-iterator", + "hmac", + "md-5", + "memchr", + "rand 0.10.1", + "sha2 0.11.0", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dc729a129e682e8d24170cd30ae1aa01b336b096cbb56df6d534ffec133d186" +dependencies = [ + "bytes", + "fallible-iterator", + "postgres-protocol", +] + [[package]] name = "potential_utf" version = "0.1.2" @@ -789,6 +1199,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro2" version = "1.0.95" @@ -909,7 +1329,7 @@ dependencies = [ "bytes", "getrandom 0.3.3", "lru-slab", - "rand", + "rand 0.9.4", "ring", "rustc-hash", "rustls", @@ -950,6 +1370,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rand" version = "0.9.4" @@ -957,7 +1383,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha", - "rand_core", + "rand_core 0.9.3", +] + +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", ] [[package]] @@ -967,7 +1404,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.3", ] [[package]] @@ -979,6 +1416,21 @@ dependencies = [ "getrandom 0.3.3", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.12.3" @@ -1079,6 +1531,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" version = "0.23.31" @@ -1102,7 +1567,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 3.2.0", ] [[package]] @@ -1147,6 +1612,25 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + [[package]] name = "security-framework" version = "3.2.0" @@ -1154,7 +1638,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ "bitflags", - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -1238,8 +1722,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", ] [[package]] @@ -1249,8 +1733,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -1259,6 +1754,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + [[package]] name = "slab" version = "0.4.11" @@ -1297,6 +1798,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "subtle" version = "2.6.1" @@ -1305,9 +1817,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.104" +version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ "proc-macro2", "quote", @@ -1319,7 +1831,10 @@ name = "synapse" version = "0.1.0" dependencies = [ "anyhow", + "async-trait", "base64", + "bb8", + "bb8-postgres", "blake2", "bytes", "futures", @@ -1333,6 +1848,7 @@ dependencies = [ "log", "mime", "once_cell", + "postgres-native-tls", "pyo3", "pyo3-log", "pythonize", @@ -1341,7 +1857,7 @@ dependencies = [ "rustc_version", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "tokio", "ulid", ] @@ -1372,6 +1888,19 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "2.0.12" @@ -1426,11 +1955,48 @@ dependencies = [ "bytes", "libc", "mio", + "parking_lot", "pin-project-lite", "socket2 0.6.0", "windows-sys 0.61.2", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-postgres" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dd8df5ef180f6364759a6f00f7aadda4fbbac86cdee37480826a6ff9f3574ce" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand 0.10.1", + "socket2 0.6.0", + "tokio", + "tokio-util", + "whoami", +] + [[package]] name = "tokio-rustls" version = "0.26.2" @@ -1526,9 +2092,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" -version = "1.18.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "ulid" @@ -1536,16 +2102,43 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" dependencies = [ - "rand", + "rand 0.9.4", "web-time", ] +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "untrusted" version = "0.9.0" @@ -1569,6 +2162,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -1600,49 +2199,60 @@ dependencies = [ ] [[package]] -name = "wasm-bindgen" -version = "0.2.100" +name = "wasip2" +version = "1.0.3+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", + "wit-bindgen 0.57.1", ] [[package]] -name = "wasm-bindgen-backend" -version = "0.2.100" +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn", + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.2+wasi-0.2.4", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.50" +version = "0.4.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" dependencies = [ - "cfg-if", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.100" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1650,26 +2260,48 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.100" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" dependencies = [ + "bumpalo", "proc-macro2", "quote", "syn", - "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.100" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + [[package]] name = "wasm-streams" version = "0.4.2" @@ -1683,11 +2315,23 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown", + "indexmap", + "semver", +] + [[package]] name = "web-sys" -version = "0.3.77" +version = "0.3.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" dependencies = [ "js-sys", "wasm-bindgen", @@ -1703,6 +2347,19 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "whoami" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "998767ef88740d1f5b0682a9c53c24431453923962269c2db68ee43788c5a40d" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite", + "web-sys", +] + [[package]] name = "windows-link" version = "0.2.1" @@ -1800,6 +2457,32 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + [[package]] name = "wit-bindgen-rt" version = "0.39.0" @@ -1809,6 +2492,74 @@ dependencies = [ "bitflags", ] +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + [[package]] name = "writeable" version = "0.6.1" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 162bc981820..1e16b8769c8 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -22,6 +22,7 @@ crate-type = ["lib", "cdylib"] name = "synapse.synapse_rust" [dependencies] +async-trait = "0.1.89" anyhow = "1.0.63" base64 = "0.22.1" bytes = "1.6.0" @@ -64,6 +65,12 @@ tokio = { version = "1.44.2", features = ["rt", "rt-multi-thread"] } once_cell = "1.18.0" itertools = "0.14.0" +# TODO: Remove: These are just used to make sure a tokio-postgres backed database pool makes sense +# with our interfaces +bb8 = "0.8.3" +bb8-postgres = "0.8.1" +postgres-native-tls = "0.5.0" + [build-dependencies] blake2 = "0.10.4" hex = "0.4.3" diff --git a/rust/src/config/mod.rs b/rust/src/config/mod.rs new file mode 100644 index 00000000000..763b9fcb1f1 --- /dev/null +++ b/rust/src/config/mod.rs @@ -0,0 +1,36 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use pyo3::prelude::*; + +#[derive(FromPyObject, Clone)] +pub struct SynapseConfig { + pub experimental: ExperimentalConfig, +} + +// #[derive(FromPyObject)] +// #[serde(rename_all = "snake_case")] +// pub enum RoomCreationPreset { +// PrviateChat, +// PublicChat, +// TrustedPrivateChat, +// } + +#[derive(FromPyObject, Clone)] +pub struct ExperimentalConfig { + pub msc3881_enabled: bool, + pub msc3575_enabled: bool, + pub msc4222_enabled: bool, +} diff --git a/rust/src/handlers/mod.rs b/rust/src/handlers/mod.rs new file mode 100644 index 00000000000..cc650a9d658 --- /dev/null +++ b/rust/src/handlers/mod.rs @@ -0,0 +1,81 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use std::sync::Arc; + +use pyo3::{ + prelude::*, + types::{PyAnyMethods, PyModule, PyModuleMethods}, + Bound, PyResult, Python, +}; + +use crate::config::SynapseConfig; +use crate::storage::db::python_db_pool::PythonDatabasePoolWrapper; +use crate::storage::store::Store; +use crate::UnwrapInfallible; + +pub mod versions; + +#[pyclass] +struct RustHandlers { + versions: versions::VersionsHandler, +} + +#[pymethods] +impl RustHandlers { + #[new] + #[pyo3(signature = (homeserver))] + pub fn py_new(py: Python<'_>, homeserver: &Bound<'_, PyAny>) -> PyResult { + let config: SynapseConfig = homeserver.getattr("config")?.extract()?; + + // hs.get_datastores().main.db_pool + let db_pool: PythonDatabasePoolWrapper = homeserver + .call_method0("get_datastores")? + .into_pyobject(py) + .unwrap_infallible() + .getattr("main")? + .getattr("db_pool")? + .extract()?; + + // Store is shared across all of the handlers so let's use an `Arc` + let store = Arc::new(Store { + config: config.clone(), + db_pool: Box::new(db_pool), + }); + + Ok(RustHandlers { + versions: versions::VersionsHandler { + config: config.clone(), + store: Arc::clone(&store), + }, + }) + } +} + +/// Called when registering modules with python. +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let child_module = PyModule::new(py, "handlers")?; + child_module.add_class::()?; + + m.add_submodule(&child_module)?; + + // We need to manually add the module to sys.modules to make `from + // synapse.synapse_rust import push` work. + py.import("sys")? + .getattr("modules")? + .set_item("synapse.synapse_rust.handlers", child_module)?; + + Ok(()) +} diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs new file mode 100644 index 00000000000..322d3a06205 --- /dev/null +++ b/rust/src/handlers/versions.rs @@ -0,0 +1,176 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +use crate::config::SynapseConfig; +use crate::storage::store::{PerUserExperimentalFeature, Store}; + +/// `GET /_matrix/client/versions` response +#[derive(Serialize, Deserialize, Clone, Debug)] +struct VersionsResponse { + versions: Vec, + /// as per MSC1497 + unstable_features: std::collections::BTreeMap, +} + +pub struct VersionsHandler { + pub config: SynapseConfig, + pub store: Arc, +} + +impl VersionsHandler { + /// Assemble a `/versions` response + async fn get_versions(&self, user_id: Option<&str>) -> Result { + let msc3881_enabled = match user_id { + Some(user_id) => { + self.store + .is_feature_enabled(user_id, PerUserExperimentalFeature::MSC3881) + .await? + } + None => PerUserExperimentalFeature::MSC3881.is_globally_enabled(&self.config), + }; + + let msc3575_enabled = match user_id { + Some(user_id) => { + self.store + .is_feature_enabled(user_id, PerUserExperimentalFeature::MSC3575) + .await? + } + None => PerUserExperimentalFeature::MSC3575.is_globally_enabled(&self.config), + }; + + // TODO: Calculate these once since they shouldn't change after start-up. + // e2ee_forced_public = ( + // RoomCreationPreset.PUBLIC_CHAT + // in config.room.encryption_enabled_by_default_for_room_presets + // ); + // e2ee_forced_private = ( + // RoomCreationPreset.PRIVATE_CHAT + // in config.room.encryption_enabled_by_default_for_room_presets + // ); + // e2ee_forced_trusted_private = ( + // RoomCreationPreset.TRUSTED_PRIVATE_CHAT + // in config.room.encryption_enabled_by_default_for_room_presets + // ); + + return Ok(VersionsResponse { + versions: Vec::from([ + // XXX: at some point we need to decide whether we need to include + // the previous version numbers, given we've defined r0.3.0 to be + // backwards compatible with r0.2.0. But need to check how + // conscientious we've been in compatibility, and decide whether the + // middle number is the major revision when at 0.X.Y (as opposed to + // X.Y.Z). And we need to decide whether it's fair to make clients + // parse the version string to figure out what's going on. + "r0.0.1".to_string(), + "r0.1.0".to_string(), + "r0.2.0".to_string(), + "r0.3.0".to_string(), + "r0.4.0".to_string(), + "r0.5.0".to_string(), + "r0.6.0".to_string(), + "r0.6.1".to_string(), + "v1.1".to_string(), + "v1.2".to_string(), + "v1.3".to_string(), + "v1.4".to_string(), + "v1.5".to_string(), + "v1.6".to_string(), + "v1.7".to_string(), + "v1.8".to_string(), + "v1.9".to_string(), + "v1.10".to_string(), + "v1.11".to_string(), + "v1.12".to_string(), + ]), + unstable_features: std::collections::BTreeMap::from([ + // // Implements support for label-based filtering as described in + // // MSC2326. + // ("org.matrix.label_based_filtering".to_string(), true), + // // Implements support for cross signing as described in MSC1756 + // ("org.matrix.e2e_cross_signing".to_string(), true), + // // Implements additional endpoints as described in MSC2432 + // ("org.matrix.msc2432".to_string(), true), + // // Implements additional endpoints as described in MSC2666 + // ("uk.half-shot.msc2666.query_mutual_rooms.stable".to_string(), true), + // // Whether new rooms will be set to encrypted or not (based on presets). + // ("io.element.e2ee_forced.public".to_string(), e2ee_forced_public), + // ("io.element.e2ee_forced.private".to_string(), e2ee_forced_private), + // ("io.element.e2ee_forced.trusted_private".to_string(), e2ee_forced_trusted_private), + // // Supports the busy presence state described in MSC3026. + // ("org.matrix.msc3026.busy_presence".to_string(), config.experimental.msc3026_enabled), + // // Supports receiving private read receipts as per MSC2285 + // ("org.matrix.msc2285.stable".to_string(), true), // TODO: Remove when MSC2285 becomes a part of the spec + // // Supports filtering of /publicRooms by room type as per MSC3827 + // ("org.matrix.msc3827.stable".to_string(), true), + // // Adds support for thread relations, per MSC3440. + // ("org.matrix.msc3440.stable".to_string(), true), // TODO: remove when "v1.3" is added above + // // Support for thread read receipts & notification counts. + // ("org.matrix.msc3771".to_string(), true), + // ("org.matrix.msc3773".to_string(), config.experimental.msc3773_enabled), + // // Allows moderators to fetch redacted event content as described in MSC2815 + // ("fi.mau.msc2815".to_string(), config.experimental.msc2815_enabled), + // // Adds a ping endpoint for appservices to check HS->AS connection + // ("fi.mau.msc2659.stable".to_string(), true), // TODO: remove when "v1.7" is added above + // // TODO: this is no longer needed once unstable MSC3882 does not need to be supported: + // ("org.matrix.msc3882".to_string(), config.auth.login_via_existing_enabled), + // Adds support for remotely enabling/disabling pushers, as per MSC3881 + ("org.matrix.msc3881".to_string(), msc3881_enabled), + // // Adds support for filtering /messages by event relation. + // ("org.matrix.msc3874".to_string(), config.experimental.msc3874_enabled), + // // Adds support for relation-based redactions as per MSC3912. + // ("org.matrix.msc3912".to_string(), config.experimental.msc3912_enabled), + // // Whether recursively provide relations is supported. + // // TODO This is no longer needed once unstable MSC3981 does not need to be supported. + // ("org.matrix.msc3981".to_string(), true), + // // Adds support for deleting account data. + // ("org.matrix.msc3391".to_string(), config.experimental.msc3391_enabled), + // // Allows clients to inhibit profile update propagation. + // ("org.matrix.msc4069".to_string(), config.experimental.msc4069_profile_inhibit_propagation), + // // Allows clients to handle push for encrypted events. + // ("org.matrix.msc4028".to_string(), config.experimental.msc4028_push_encrypted_events), + // // MSC4108: Mechanism to allow OIDC sign in and E2EE set up via QR code - 2024 version + // ("org.matrix.msc4108".to_string(), ( + // config.experimental.msc4108_enabled + // or ( + // config.experimental.msc4108_delegation_endpoint + // is not None + // ) + // )), + // // MSC4140: Delayed events + // ("org.matrix.msc4140".to_string(), bool(config.server.max_event_delay_ms)), + // Simplified sliding sync + ("org.matrix.simplified_msc3575".to_string(), msc3575_enabled), + // // Arbitrary key-value profile fields. + // ("uk.tcpip.msc4133".to_string(), config.experimental.msc4133_enabled), + // ("uk.tcpip.msc4133.stable".to_string(), true), + // // MSC4155: Invite filtering + // ("org.matrix.msc4155".to_string(), config.experimental.msc4155_enabled), + // // MSC4306: Support for thread subscriptions + // ("org.matrix.msc4306".to_string(), config.experimental.msc4306_enabled), + // // MSC4169: Backwards-compatible redaction sending using `/send` + // ("com.beeper.msc4169".to_string(), config.experimental.msc4169_enabled), + // // MSC4354: Sticky events + // ("org.matrix.msc4354".to_string(), config.experimental.msc4354_enabled), + // // MSC4380: Invite blocking + // ("org.matrix.msc4380.stable".to_string(), true), + // // MSC4445: Sync timeline order + // ("org.matrix.msc4445.initial_sync_timeline_topological_ordering".to_string(), true), + ]), + }); + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index bf96422cbb0..4ffddb9af06 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -6,9 +6,11 @@ use pyo3_log::ResetHandle; pub mod acl; pub mod canonical_json; +pub mod config; pub mod duration; pub mod errors; pub mod events; +pub mod handlers; pub mod http; pub mod http_client; pub mod identifier; @@ -19,6 +21,7 @@ pub mod push; pub mod rendezvous; pub mod room_versions; pub mod segmenter; +pub mod storage; pub mod types; lazy_static! { @@ -67,6 +70,7 @@ fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { acl::register_module(py, m)?; push::register_module(py, m)?; events::register_module(py, m)?; + handlers::register_module(py, m)?; http_client::register_module(py, m)?; rendezvous::register_module(py, m)?; msc4388_rendezvous::register_module(py, m)?; diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs new file mode 100644 index 00000000000..a068732326f --- /dev/null +++ b/rust/src/storage/db/mod.rs @@ -0,0 +1,47 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +pub mod python_db_pool; +pub mod rust_db_pool; + +// Using `Send + Sync` traits so this can stored in the `#[pyclass]` just fine +#[async_trait::async_trait] +pub trait DatabasePool: Send + Sync { + /// Starts a transaction on the database and runs a given function + /// + /// The given `func` may be called multiple times under certain failure modes (like + /// serialization and deadlock errors). + fn run_interaction<'txn, R, F>( + &'txn self, + name: &'static str, + func: F, + ) -> impl Future> + 'txn + where + R: Send + Sync + 'static, + // TODO: Can we use `&'f mut dyn Txn` here? + F: for<'f> Fn(&'f mut dyn Transaction<'f>) -> BoxFuture<'f, anyhow::Result> + + Send + + 'static; +} + +/// A [`tokio_postgres::Transaction`] looking thing that we can use on the Rust side to +/// interact with the database +#[async_trait::async_trait] +pub trait Transaction { + async fn query(&self, sql: &str, args: &[&str]) -> Result, anyhow::Error>; + async fn commit(self) -> Result<(), anyhow::Error>; +} + +pub type Row = Vec; diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs new file mode 100644 index 00000000000..9dbd7cc8a5f --- /dev/null +++ b/rust/src/storage/db/python_db_pool.rs @@ -0,0 +1,260 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +//! We have these three main classes: +//! - Database pool [`PythonDatabasePoolWrapper`] which creates +//! - connections [`LoggingDatabaseConnectionWrapper`] which creates +//! - transactions [`LoggingTransactionWrapper`] + +use pyo3::{intern, prelude::*, types::PyCFunction, types::PyList}; + +use crate::storage::db::{DatabaseConnection, DatabasePool, Row, Transaction}; + +/// The database engines we support in the Python side of Synapse +#[derive(Copy, Clone, Debug)] +pub enum DatabaseEngine { + Sqlite, + Postgres, +} + +impl DatabaseEngine { + //[inline] + pub fn supports_using_any_list(&self) -> bool { + match self { + DatabaseEngine::Sqlite => false, + DatabaseEngine::Postgres => true, + } + } +} + +/// Wrapper for a `DatabasePool` from the Python side of Synapse. +pub struct PythonDatabasePoolWrapper { + /// The underlying `DatabasePool` + database_pool_py: Py, +} + +impl<'a, 'py> FromPyObject<'a, 'py> for PythonDatabasePoolWrapper { + type Error = PyErr; + + /// Extract from a Python `DatabasePool` passed as an argument. + fn extract(database_pool_py: Borrowed<'a, 'py, PyAny>) -> PyResult { + Ok(Self { + database_pool_py: database_pool_py.to_owned().unbind(), + }) + } +} + +#[async_trait::async_trait] +impl DatabasePool for PythonDatabasePoolWrapper { + async fn get_connection(&self) -> Result, anyhow::Error> { + let callback_func = + PyCFunction::new_closure(py, None, None, move |args, _| -> PyResult> { + // TODO: Error if already called + + let py = args.py(); + // We found our `LoggingDatabaseConnection` + let conn_py = args.get_item(0)?; + tx.send(conn_py.unbind()); + }); + + let execute_fn = self + .database_pool_py + .getattr(intern!(self.database_pool_py.py(), "runWithConnection"))?; + execute_fn.call1((callback_func,))?; + + Ok(Box::new(LoggingDatabaseConnectionWrapper { + database_pool_py: self.database_pool_py, + logging_database_connection_py: connection, + })) + } +} + +/// Wrapper for a `LoggingDatabaseConnection` from the Python side of Synapse. +pub struct LoggingDatabaseConnectionWrapper { + /// The underlying `DatabasePool` + /// + /// We purposely avoid `Bound<'py, PyAny>` so it can be stored and moved freely + /// across threads (like to extract it from the `runWithConnection(...)` callback). + /// You will need to acquire your own `py` and bind it using + /// `database_pool_py.bind(py)` to do anything useful. + database_pool_py: Py, + /// The underlying `LoggingDatabaseConnection` + logging_database_connection_py: Py, +} + +#[async_trait::async_trait] +impl DatabaseConnection for LoggingDatabaseConnectionWrapper { + async fn get_transaction( + &self, + description: &str, + ) -> Result, anyhow::Error> { + // Synapse's `DatabasePool.new_transaction` has built-in retry functionality and + // can call this function multiple times under certain failure modes. Normally, + // everything in the transaction happens in the callback but since we have a + // little bit of a different API surface, we instead extract the transaction for + // us to use outside. + // + // Re-using `runInteraction`, means we get all of the logging, metrics, etc for + // free. + let callback_func = + PyCFunction::new_closure(py, None, None, move |args, _| -> PyResult> { + // TODO: Error if already called + + let py = args.py(); + let txn: LoggingTransactionWrapper = args.get_item(0)?; + tx.send(txn); + + // Wait until we see the signal that we're `done_with_txn_rx` + }); + + let execute_fn = self + .database_pool_py + .getattr(intern!(self.database_pool_py.py(), "new_transaction"))?; + execute_fn.call1(( + self.logging_database_connection_py, + description, + [], + [], + [], + callback_func, + ))?; + + Ok(Box::new(txn)) + } +} + +fn detect_engine(txn_py: &Bound<'_, PyAny>) -> PyResult { + let name = txn_py + .getattr("database_engine") + .expect("`LoggingTransaction` must have `database_engine` attr") + .get_type() + .name() + .expect("`database_engine` type must have a name") + .to_str() + .expect("`database_engine` type name must be valid UTF-8") + .to_owned(); + + Ok(match name.as_str() { + "PostgresEngine" => DatabaseEngine::Postgres, + "Sqlite3Engine" => DatabaseEngine::Sqlite, + other => unimplemented!( + "Unknown database engine {other:?}. This is a Synapse programming error." + ), + }) +} + +/// Wrapper for a `LoggingTransaction` from the Python side of Synapse. +pub struct LoggingTransactionWrapper { + /// The underlying `LoggingTransaction` + /// + /// We purposely avoid `Bound<'py, PyAny>` so it can be stored and moved freely + /// across threads (like to extract it from the `new_transaction(...)` callback). + /// You will need to acquire your own `py` and bind it using + /// `logging_transaction_py.bind(py)` to do anything useful. + logging_transaction_py: Py, + + /// Disambiguate which underlying database engine we're working with + pub database_engine: DatabaseEngine, +} + +impl<'a, 'py> FromPyObject<'a, 'py> for LoggingTransactionWrapper { + type Error = PyErr; + + /// Extract from a Python `LoggingTransaction` passed as an argument. + fn extract(logging_transaction_py: Borrowed<'a, 'py, PyAny>) -> PyResult { + let database_engine = detect_engine(&logging_transaction_py.to_owned())?; + Ok(Self { + logging_transaction_py: logging_transaction_py.to_owned().unbind(), + database_engine, + }) + } +} + +pub trait ValidDatabaseFieldType {} +pub trait ValidDatabaseReturnType {} +impl ValidDatabaseFieldType for String {} +impl ValidDatabaseFieldType for usize {} +impl ValidDatabaseFieldType for Option {} +impl ValidDatabaseReturnType for (T0,) {} +impl ValidDatabaseReturnType for (T0, T1) {} +impl + ValidDatabaseReturnType for (T0, T1, T2) +{ +} +impl< + T0: ValidDatabaseFieldType, + T1: ValidDatabaseFieldType, + T2: ValidDatabaseFieldType, + T3: ValidDatabaseFieldType, + > ValidDatabaseReturnType for (T0, T1, T2, T3) +{ +} + +impl LoggingTransactionWrapper { + pub fn execute<'py>( + &mut self, + py: Python<'py>, + sql: &str, + args: &Bound<'py, PyAny>, + ) -> PyResult<()> { + let execute_fn = self + .logging_transaction_py + .bind(py) + .getattr(intern!(py, "execute"))?; + execute_fn.call1((sql, args))?; + Ok(()) + } + + pub fn fetchall<'py, T: FromPyObjectOwned<'py> + ValidDatabaseReturnType>( + &mut self, + py: Python<'py>, + ) -> anyhow::Result> { + let fetch_fn = self + .logging_transaction_py + .bind(py) + .getattr(intern!(py, "fetchall"))?; + Ok(fetch_fn.call0()?.extract()?) + } +} + +#[async_trait::async_trait] +impl Transaction for LoggingTransactionWrapper { + async fn query(&self, sql: &str, args: &[&str]) -> Result, anyhow::Error> { + Python::attach(|py| -> PyResult> { + // Convert the Rust `&[&str]` of SQL parameters into a Python sequence so it + // can be passed through to the Python-side `execute`. + let args = PyList::new(py, args)?; + // Run the query + self.execute(py, sql, args.as_any())?; + // Get the results + let rows = self.fetchall(py)?; + + Ok(rows) + }) + .map_err(anyhow::Error::from) + } + + async fn commit(&self) -> Result<(), anyhow::Error> { + // In Synapse, `commit` is part of `LoggingDatabaseConnection` and will be + // called as part of the `new_transaction(...)` machinery we used to create the + // transaction in the first place. + // + // We just need to send the proper signal which will finish the txn callback and + // have it run. + done_with_txn_tx.send(()) + // TODO: How can we guarantee that `commit` was run? Perhaps we have to wait for + // `new_transaction(...)` to complete successfully + } +} diff --git a/rust/src/storage/db/rust_db_pool.rs b/rust/src/storage/db/rust_db_pool.rs new file mode 100644 index 00000000000..bdf7ed375b6 --- /dev/null +++ b/rust/src/storage/db/rust_db_pool.rs @@ -0,0 +1,86 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +// TODO: remove. This is just here to make sure our `DatabasePool`/`Transaction` +// interfaces are compatible with `tokio-postgres`. + +use anyhow::Context; +use bb8_postgres::PostgresConnectionManager; +use postgres_native_tls::MakeTlsConnector; + +use crate::storage::db::{DatabaseConnection, DatabasePool, Row, Transaction}; + +/// Native Rust database access backed by `tokio-postgres` (for use in synapse-rust-apps) +pub struct RustDatabasePool { + db_pool: bb8::Pool>, +} + +#[async_trait::async_trait] +impl DatabasePool for RustDatabasePool { + async fn get_connection(&self) -> Result, anyhow::Error> { + let mut conn = self + .db_pool + .get() + // .instrument(tracing::info_span!("acquire database connection")) + .await + .context("Failed to acquire database connection")?; + + Ok(Box::new(RustConnection { connection: conn })) + } +} + +pub struct RustConnection<'a> { + connection: bb8::PooledConnection<'a, PostgresConnectionManager>, +} + +impl DatabaseConnection for RustConnection<'_> { + async fn get_transaction( + &self, + _description: &str, + ) -> Result, anyhow::Error> { + // TODO: Set repeatable-read isolation level (like Synapse) + let txn = self + .connection + .transaction() + // .instrument(tracing::info_span!("start transaction")) + .await + .context("Failed to start transaction")?; + + Ok(Box::new(TokioPostgresTransaction { txn })) + } +} + +struct TokioPostgresTransaction<'a> { + txn: bb8_postgres::tokio_postgres::Transaction<'a>, +} + +#[async_trait::async_trait] +impl Transaction for TokioPostgresTransaction<'_> { + async fn query(&self, sql: &str, args: &[&str]) -> Result, anyhow::Error> { + // TODO: Convert `?` SQL param style to `tokio-postgres` compatible + + let rows = self.txn.query(sql, args).await?; + + Ok(rows) + } + + async fn commit(self) -> Result<(), anyhow::Error> { + self.txn + .commit() + // .instrument(tracing::info_span!("commit transaction")) + .await + .context("Failed to commit transaction") + } +} diff --git a/rust/src/storage/mod.rs b/rust/src/storage/mod.rs new file mode 100644 index 00000000000..735fcecb266 --- /dev/null +++ b/rust/src/storage/mod.rs @@ -0,0 +1,17 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +pub mod db; +pub mod store; diff --git a/rust/src/storage/store.rs b/rust/src/storage/store.rs new file mode 100644 index 00000000000..041403f39f6 --- /dev/null +++ b/rust/src/storage/store.rs @@ -0,0 +1,72 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use sha2::digest::typenum::Len; + +use crate::{config::SynapseConfig, storage::db::DatabasePool}; + +/// Currently supported per-user features +pub enum PerUserExperimentalFeature { + MSC3881, + MSC3575, + MSC4222, +} + +impl PerUserExperimentalFeature { + pub fn is_globally_enabled(&self, config: &SynapseConfig) -> bool { + match self { + PerUserExperimentalFeature::MSC3881 => config.experimental.msc3881_enabled, + PerUserExperimentalFeature::MSC3575 => config.experimental.msc3575_enabled, + PerUserExperimentalFeature::MSC4222 => config.experimental.msc4222_enabled, + } + } +} + +pub struct Store { + pub config: SynapseConfig, + pub db_pool: Box, +} + +impl Store { + pub async fn is_feature_enabled( + &self, + user_id: &str, + feature: PerUserExperimentalFeature, + ) -> Result { + if feature.is_globally_enabled(self.config) { + return Ok(true); + } + + let txn = self.db_pool.get_transaction("is_feature_enabled").await; + let rows = txn + .query( + r#" + SELECT enabled + FROM per_user_experimental_features + WHERE user_id = ? AND feature = ? + "#, + &[user_id, feature], + ) + .await; + + match (rows.len(), rows.first()) { + (1, Some(enabled)) => Ok(enabled), + (0, None) => Ok(false), + _ => { + panic!("Synapse programming error"); + } + } + } +} diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index bb1711f2cf2..d8b0a0545db 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -25,11 +25,10 @@ import re from typing import TYPE_CHECKING -from synapse.api.constants import RoomCreationPreset from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet from synapse.http.site import SynapseRequest -from synapse.rest.admin.experimental_features import ExperimentalFeature +from synapse.synapse_rust.handlers.versions import get_versions from synapse.types import JsonDict if TYPE_CHECKING: @@ -47,25 +46,10 @@ def __init__(self, hs: "HomeServer"): self.config = hs.config self.auth = hs.get_auth() self.store = hs.get_datastores().main - - # Calculate these once since they shouldn't change after start-up. - self.e2ee_forced_public = ( - RoomCreationPreset.PUBLIC_CHAT - in self.config.room.encryption_enabled_by_default_for_room_presets - ) - self.e2ee_forced_private = ( - RoomCreationPreset.PRIVATE_CHAT - in self.config.room.encryption_enabled_by_default_for_room_presets - ) - self.e2ee_forced_trusted_private = ( - RoomCreationPreset.TRUSTED_PRIVATE_CHAT - in self.config.room.encryption_enabled_by_default_for_room_presets - ) + self.rust_handlers = hs.get_rust_handlers() async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: - msc3881_enabled = self.config.experimental.msc3881_enabled - msc3575_enabled = self.config.experimental.msc3575_enabled - + user_id = None if self.auth.has_access_token(request): requester = await self.auth.get_user_by_req( request, @@ -74,13 +58,6 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: allow_expired=True, ) user_id = requester.user.to_string() - - msc3881_enabled = await self.store.is_feature_enabled( - user_id, ExperimentalFeature.MSC3881 - ) - msc3575_enabled = await self.store.is_feature_enabled( - user_id, ExperimentalFeature.MSC3575 - ) else: # Allow caching of unauthenticated responses, as they only depend # on server configuration which rarely changes. @@ -102,114 +79,13 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: # authenticated responses are not served from cache. request.setHeader(b"Vary", b"Authorization") + versions_response_body = await self.rust_handlers.versions.get_versions( + user_id, self.config + ) + return ( 200, - { - "versions": [ - # XXX: at some point we need to decide whether we need to include - # the previous version numbers, given we've defined r0.3.0 to be - # backwards compatible with r0.2.0. But need to check how - # conscientious we've been in compatibility, and decide whether the - # middle number is the major revision when at 0.X.Y (as opposed to - # X.Y.Z). And we need to decide whether it's fair to make clients - # parse the version string to figure out what's going on. - "r0.0.1", - "r0.1.0", - "r0.2.0", - "r0.3.0", - "r0.4.0", - "r0.5.0", - "r0.6.0", - "r0.6.1", - "v1.1", - "v1.2", - "v1.3", - "v1.4", - "v1.5", - "v1.6", - "v1.7", - "v1.8", - "v1.9", - "v1.10", - "v1.11", - "v1.12", - ], - # as per MSC1497: - "unstable_features": { - # Implements support for label-based filtering as described in - # MSC2326. - "org.matrix.label_based_filtering": True, - # Implements support for cross signing as described in MSC1756 - "org.matrix.e2e_cross_signing": True, - # Implements additional endpoints as described in MSC2432 - "org.matrix.msc2432": True, - # Implements additional endpoints as described in MSC2666 - "uk.half-shot.msc2666.query_mutual_rooms.stable": True, - # Whether new rooms will be set to encrypted or not (based on presets). - "io.element.e2ee_forced.public": self.e2ee_forced_public, - "io.element.e2ee_forced.private": self.e2ee_forced_private, - "io.element.e2ee_forced.trusted_private": self.e2ee_forced_trusted_private, - # Supports the busy presence state described in MSC3026. - "org.matrix.msc3026.busy_presence": self.config.experimental.msc3026_enabled, - # Supports receiving private read receipts as per MSC2285 - "org.matrix.msc2285.stable": True, # TODO: Remove when MSC2285 becomes a part of the spec - # Supports filtering of /publicRooms by room type as per MSC3827 - "org.matrix.msc3827.stable": True, - # Adds support for thread relations, per MSC3440. - "org.matrix.msc3440.stable": True, # TODO: remove when "v1.3" is added above - # Support for thread read receipts & notification counts. - "org.matrix.msc3771": True, - "org.matrix.msc3773": self.config.experimental.msc3773_enabled, - # Allows moderators to fetch redacted event content as described in MSC2815 - "fi.mau.msc2815": self.config.experimental.msc2815_enabled, - # Adds a ping endpoint for appservices to check HS->AS connection - "fi.mau.msc2659.stable": True, # TODO: remove when "v1.7" is added above - # TODO: this is no longer needed once unstable MSC3882 does not need to be supported: - "org.matrix.msc3882": self.config.auth.login_via_existing_enabled, - # Adds support for remotely enabling/disabling pushers, as per MSC3881 - "org.matrix.msc3881": msc3881_enabled, - # Adds support for filtering /messages by event relation. - "org.matrix.msc3874": self.config.experimental.msc3874_enabled, - # Adds support for relation-based redactions as per MSC3912. - "org.matrix.msc3912": self.config.experimental.msc3912_enabled, - # Whether recursively provide relations is supported. - # TODO This is no longer needed once unstable MSC3981 does not need to be supported. - "org.matrix.msc3981": True, - # Adds support for deleting account data. - "org.matrix.msc3391": self.config.experimental.msc3391_enabled, - # Allows clients to inhibit profile update propagation. - "org.matrix.msc4069": self.config.experimental.msc4069_profile_inhibit_propagation, - # Allows clients to handle push for encrypted events. - "org.matrix.msc4028": self.config.experimental.msc4028_push_encrypted_events, - # MSC4108: Mechanism to allow OIDC sign in and E2EE set up via QR code - 2024 version - "org.matrix.msc4108": ( - self.config.experimental.msc4108_enabled - or ( - self.config.experimental.msc4108_delegation_endpoint - is not None - ) - ), - # MSC4140: Delayed events - "org.matrix.msc4140": bool(self.config.server.max_event_delay_ms), - # Simplified sliding sync - "org.matrix.simplified_msc3575": msc3575_enabled, - # Arbitrary key-value profile fields. - "uk.tcpip.msc4133": self.config.experimental.msc4133_enabled, - "uk.tcpip.msc4133.stable": True, - # MSC4155: Invite filtering - "org.matrix.msc4155": self.config.experimental.msc4155_enabled, - # MSC4306: Support for thread subscriptions - "org.matrix.msc4306": self.config.experimental.msc4306_enabled, - # MSC4169: Backwards-compatible redaction sending using `/send` - "com.beeper.msc4169": self.config.experimental.msc4169_enabled, - # MSC4354: Sticky events - "org.matrix.msc4354": self.config.experimental.msc4354_enabled, - # MSC4380: Invite blocking - "org.matrix.msc4380.stable": True, - # MSC4445: Sync timeline order - "org.matrix.msc4445.initial_sync_timeline_topological_ordering": True, - }, - }, + versions_response_body, ) diff --git a/synapse/server.py b/synapse/server.py index 8bf19f11b5d..f46f5845e4b 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -963,6 +963,10 @@ def get_send_email_handler(self) -> SendEmailHandler: def get_set_password_handler(self) -> SetPasswordHandler: return SetPasswordHandler(self) + @cache_in_self + def get_rust_handlers(self) -> RustHandlers: + return RustHandlers(self) + @cache_in_self def get_event_sources(self) -> EventSources: return EventSources(self) From c63b40e756ec5798c7125295a0d53ed679715b2d Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 9 Jun 2026 19:09:12 -0500 Subject: [PATCH 02/55] A little bit of iterating --- rust/src/storage/db/mod.rs | 8 +- rust/src/storage/db/python_db_pool.rs | 110 ++++++-------------------- rust/src/storage/db/rust_db_pool.rs | 8 -- rust/src/storage/store.rs | 66 +++++++++++----- 4 files changed, 74 insertions(+), 118 deletions(-) diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs index a068732326f..8555ec0fb7a 100644 --- a/rust/src/storage/db/mod.rs +++ b/rust/src/storage/db/mod.rs @@ -13,6 +13,8 @@ * */ +use std::future::Future; + pub mod python_db_pool; pub mod rust_db_pool; @@ -30,10 +32,7 @@ pub trait DatabasePool: Send + Sync { ) -> impl Future> + 'txn where R: Send + Sync + 'static, - // TODO: Can we use `&'f mut dyn Txn` here? - F: for<'f> Fn(&'f mut dyn Transaction<'f>) -> BoxFuture<'f, anyhow::Result> - + Send - + 'static; + F: for<'f> Fn(&'f mut dyn Transaction) -> BoxFuture<'f, anyhow::Result> + Send + 'static; } /// A [`tokio_postgres::Transaction`] looking thing that we can use on the Rust side to @@ -41,7 +40,6 @@ pub trait DatabasePool: Send + Sync { #[async_trait::async_trait] pub trait Transaction { async fn query(&self, sql: &str, args: &[&str]) -> Result, anyhow::Error>; - async fn commit(self) -> Result<(), anyhow::Error>; } pub type Row = Vec; diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index 9dbd7cc8a5f..0522e58bc1f 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -58,80 +58,30 @@ impl<'a, 'py> FromPyObject<'a, 'py> for PythonDatabasePoolWrapper { #[async_trait::async_trait] impl DatabasePool for PythonDatabasePoolWrapper { - async fn get_connection(&self) -> Result, anyhow::Error> { - let callback_func = - PyCFunction::new_closure(py, None, None, move |args, _| -> PyResult> { - // TODO: Error if already called - - let py = args.py(); - // We found our `LoggingDatabaseConnection` - let conn_py = args.get_item(0)?; - tx.send(conn_py.unbind()); - }); - - let execute_fn = self - .database_pool_py - .getattr(intern!(self.database_pool_py.py(), "runWithConnection"))?; - execute_fn.call1((callback_func,))?; - - Ok(Box::new(LoggingDatabaseConnectionWrapper { - database_pool_py: self.database_pool_py, - logging_database_connection_py: connection, - })) - } -} - -/// Wrapper for a `LoggingDatabaseConnection` from the Python side of Synapse. -pub struct LoggingDatabaseConnectionWrapper { - /// The underlying `DatabasePool` - /// - /// We purposely avoid `Bound<'py, PyAny>` so it can be stored and moved freely - /// across threads (like to extract it from the `runWithConnection(...)` callback). - /// You will need to acquire your own `py` and bind it using - /// `database_pool_py.bind(py)` to do anything useful. - database_pool_py: Py, - /// The underlying `LoggingDatabaseConnection` - logging_database_connection_py: Py, -} - -#[async_trait::async_trait] -impl DatabaseConnection for LoggingDatabaseConnectionWrapper { - async fn get_transaction( - &self, - description: &str, - ) -> Result, anyhow::Error> { - // Synapse's `DatabasePool.new_transaction` has built-in retry functionality and - // can call this function multiple times under certain failure modes. Normally, - // everything in the transaction happens in the callback but since we have a - // little bit of a different API surface, we instead extract the transaction for - // us to use outside. - // - // Re-using `runInteraction`, means we get all of the logging, metrics, etc for - // free. - let callback_func = - PyCFunction::new_closure(py, None, None, move |args, _| -> PyResult> { - // TODO: Error if already called - - let py = args.py(); - let txn: LoggingTransactionWrapper = args.get_item(0)?; - tx.send(txn); - - // Wait until we see the signal that we're `done_with_txn_rx` - }); - - let execute_fn = self - .database_pool_py - .getattr(intern!(self.database_pool_py.py(), "new_transaction"))?; - execute_fn.call1(( - self.logging_database_connection_py, - description, - [], - [], - [], - callback_func, - ))?; - - Ok(Box::new(txn)) + fn run_interaction<'txn, R, F>( + &'txn self, + name: &'static str, + func: F, + ) -> impl Future> + 'txn + where + R: Send + Sync + 'static, + F: for<'f> Fn(&'f mut dyn Transaction) -> BoxFuture<'f, anyhow::Result> + Send + 'static, + { + Python::attach(|py| -> PyResult> { + let callback_func = + PyCFunction::new_closure(py, None, None, move |args, _| -> PyResult> { + // We found our `LoggingTransactionWrapper` + let txn: LoggingTransactionWrapper = args.get_item(0)?; + func(txn); + }); + + let execute_fn = self + .database_pool_py + .bind(py) + .getattr(intern!(py, "runInteraction"))?; + let results = execute_fn.call1((callback_func,))?; + results + }) } } @@ -245,16 +195,4 @@ impl Transaction for LoggingTransactionWrapper { }) .map_err(anyhow::Error::from) } - - async fn commit(&self) -> Result<(), anyhow::Error> { - // In Synapse, `commit` is part of `LoggingDatabaseConnection` and will be - // called as part of the `new_transaction(...)` machinery we used to create the - // transaction in the first place. - // - // We just need to send the proper signal which will finish the txn callback and - // have it run. - done_with_txn_tx.send(()) - // TODO: How can we guarantee that `commit` was run? Perhaps we have to wait for - // `new_transaction(...)` to complete successfully - } } diff --git a/rust/src/storage/db/rust_db_pool.rs b/rust/src/storage/db/rust_db_pool.rs index bdf7ed375b6..79a54be20ec 100644 --- a/rust/src/storage/db/rust_db_pool.rs +++ b/rust/src/storage/db/rust_db_pool.rs @@ -75,12 +75,4 @@ impl Transaction for TokioPostgresTransaction<'_> { Ok(rows) } - - async fn commit(self) -> Result<(), anyhow::Error> { - self.txn - .commit() - // .instrument(tracing::info_span!("commit transaction")) - .await - .context("Failed to commit transaction") - } } diff --git a/rust/src/storage/store.rs b/rust/src/storage/store.rs index 041403f39f6..a4929c435fe 100644 --- a/rust/src/storage/store.rs +++ b/rust/src/storage/store.rs @@ -13,14 +13,18 @@ * */ -use sha2::digest::typenum::Len; +use serde::{Serialize}; use crate::{config::SynapseConfig, storage::db::DatabasePool}; /// Currently supported per-user features +#[derive(Serialize)] pub enum PerUserExperimentalFeature { + #[serde(rename = "msc3881")] MSC3881, + #[serde(rename = "msc3575")] MSC3575, + #[serde(rename = "msc4222")] MSC4222, } @@ -34,6 +38,24 @@ impl PerUserExperimentalFeature { } } +impl std::fmt::Display for PerUserExperimentalFeature { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "{}", + // Serialize so we can use the serde name of the variant as the source of truth + serde_json::to_string(self) + .unwrap_or_else(|err| format!( + "", + self, err + )) + // Remove the surrounding quotes from JSON serialization + .trim_matches('"') + ) + } +} + + pub struct Store { pub config: SynapseConfig, pub db_pool: Box, @@ -45,28 +67,34 @@ impl Store { user_id: &str, feature: PerUserExperimentalFeature, ) -> Result { - if feature.is_globally_enabled(self.config) { + if feature.is_globally_enabled(&self.config) { return Ok(true); } - let txn = self.db_pool.get_transaction("is_feature_enabled").await; - let rows = txn - .query( - r#" - SELECT enabled - FROM per_user_experimental_features - WHERE user_id = ? AND feature = ? - "#, - &[user_id, feature], - ) - .await; + let is_feature_enabled_for_user = self.db_pool.run_interaction("is_feature_enabled_for_user", |txn| { + async move { + let rows = txn + .query( + r#" + SELECT enabled + FROM per_user_experimental_features + WHERE user_id = ? AND feature = ? + "#, + &[user_id, &feature.to_string()], + ) + .await; - match (rows.len(), rows.first()) { - (1, Some(enabled)) => Ok(enabled), - (0, None) => Ok(false), - _ => { - panic!("Synapse programming error"); + match (rows.len(), rows.first()) { + (1, Some(enabled)) => enabled, + (0, None) => false, + _ => { + panic!("Synapse programming error"); + } + } } - } + .boxed() + }).await; + + Ok(is_feature_enabled_for_user) } } From bb4c55546bd3a0333ba6aaf003eba1e856bf1251 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 9 Jun 2026 19:13:42 -0500 Subject: [PATCH 03/55] Adjust comment --- rust/src/storage/db/python_db_pool.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index 0522e58bc1f..0c9059ab3a0 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -13,10 +13,9 @@ * */ -//! We have these three main classes: -//! - Database pool [`PythonDatabasePoolWrapper`] which creates -//! - connections [`LoggingDatabaseConnectionWrapper`] which creates -//! - transactions [`LoggingTransactionWrapper`] +//! We have these main classes: +//! - Database pool [`PythonDatabasePoolWrapper`] which allows you to start a... +//! - transaction [`LoggingTransactionWrapper`] and query the database use pyo3::{intern, prelude::*, types::PyCFunction, types::PyList}; From 8f0768d491083c12bb0b2bc0b9a5aa8856e8bb78 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 9 Jun 2026 20:22:29 -0500 Subject: [PATCH 04/55] LLM attempt 1 --- rust/src/handlers/mod.rs | 29 ++- rust/src/handlers/versions.rs | 65 ++++++- rust/src/http_client.rs | 2 +- rust/src/storage/db/mod.rs | 106 +++++++++-- rust/src/storage/db/python_db_pool.rs | 259 +++++++++++++++++++------- rust/src/storage/db/rust_db_pool.rs | 123 ++++++++---- rust/src/storage/store.rs | 72 ++++--- synapse/rest/client/versions.py | 5 +- synapse/server.py | 1 + synapse/synapse_rust/handlers.pyi | 35 ++++ 10 files changed, 532 insertions(+), 165 deletions(-) create mode 100644 synapse/synapse_rust/handlers.pyi diff --git a/rust/src/handlers/mod.rs b/rust/src/handlers/mod.rs index cc650a9d658..e42952ec1ab 100644 --- a/rust/src/handlers/mod.rs +++ b/rust/src/handlers/mod.rs @@ -24,13 +24,12 @@ use pyo3::{ use crate::config::SynapseConfig; use crate::storage::db::python_db_pool::PythonDatabasePoolWrapper; use crate::storage::store::Store; -use crate::UnwrapInfallible; pub mod versions; #[pyclass] struct RustHandlers { - versions: versions::VersionsHandler, + versions: Py, } #[pymethods] @@ -40,14 +39,17 @@ impl RustHandlers { pub fn py_new(py: Python<'_>, homeserver: &Bound<'_, PyAny>) -> PyResult { let config: SynapseConfig = homeserver.getattr("config")?.extract()?; + // The Twisted reactor, used both to drive our Tokio runtime and to + // marshal database work back onto the reactor thread. + let reactor: Py = homeserver.call_method0("get_reactor")?.unbind(); + // hs.get_datastores().main.db_pool - let db_pool: PythonDatabasePoolWrapper = homeserver + let db_pool_py: Py = homeserver .call_method0("get_datastores")? - .into_pyobject(py) - .unwrap_infallible() .getattr("main")? .getattr("db_pool")? - .extract()?; + .unbind(); + let db_pool = PythonDatabasePoolWrapper::new(db_pool_py, reactor.clone_ref(py)); // Store is shared across all of the handlers so let's use an `Arc` let store = Arc::new(Store { @@ -55,12 +57,21 @@ impl RustHandlers { db_pool: Box::new(db_pool), }); - Ok(RustHandlers { - versions: versions::VersionsHandler { + let versions = Py::new( + py, + versions::VersionsHandler { config: config.clone(), store: Arc::clone(&store), + reactor: reactor.clone_ref(py), }, - }) + )?; + + Ok(RustHandlers { versions }) + } + + #[getter] + fn versions(&self, py: Python<'_>) -> Py { + self.versions.clone_ref(py) } } diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index 322d3a06205..d9297599393 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -13,10 +13,14 @@ * */ -use serde::{Deserialize, Serialize}; use std::sync::Arc; +use pyo3::prelude::*; +use pythonize::{pythonize, PythonizeError}; +use serde::{Deserialize, Serialize}; + use crate::config::SynapseConfig; +use crate::http_client::create_deferred; use crate::storage::store::{PerUserExperimentalFeature, Store}; /// `GET /_matrix/client/versions` response @@ -27,30 +31,73 @@ struct VersionsResponse { unstable_features: std::collections::BTreeMap, } +impl<'py> IntoPyObject<'py> for VersionsResponse { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PythonizeError; + + fn into_pyobject(self, py: Python<'py>) -> Result { + pythonize(py, &self) + } +} + +#[pyclass] pub struct VersionsHandler { pub config: SynapseConfig, pub store: Arc, + /// The Twisted reactor, used to bridge our `async` response back into a + /// Twisted deferred that Python can `await`. + pub reactor: Py, } +#[pymethods] impl VersionsHandler { - /// Assemble a `/versions` response - async fn get_versions(&self, user_id: Option<&str>) -> Result { + /// Assemble a `/versions` response, returning a Twisted deferred that + /// resolves to the response body (a dict). + #[pyo3(signature = (user_id=None))] + fn get_versions<'py>( + &self, + py: Python<'py>, + user_id: Option, + ) -> PyResult> { + let store = Arc::clone(&self.store); + let config = self.config.clone(); + + create_deferred(py, self.reactor.bind(py), async move { + build_versions_response(&store, &config, user_id.as_deref()) + .await + .map_err(|err| { + pyo3::exceptions::PyRuntimeError::new_err(format!( + "Failed to build /versions response: {err:#}" + )) + }) + }) + } +} + +/// Assemble a `/versions` response body. +async fn build_versions_response( + store: &Store, + config: &SynapseConfig, + user_id: Option<&str>, +) -> Result { + { let msc3881_enabled = match user_id { Some(user_id) => { - self.store + store .is_feature_enabled(user_id, PerUserExperimentalFeature::MSC3881) .await? } - None => PerUserExperimentalFeature::MSC3881.is_globally_enabled(&self.config), + None => PerUserExperimentalFeature::MSC3881.is_globally_enabled(config), }; let msc3575_enabled = match user_id { Some(user_id) => { - self.store + store .is_feature_enabled(user_id, PerUserExperimentalFeature::MSC3575) .await? } - None => PerUserExperimentalFeature::MSC3575.is_globally_enabled(&self.config), + None => PerUserExperimentalFeature::MSC3575.is_globally_enabled(config), }; // TODO: Calculate these once since they shouldn't change after start-up. @@ -67,7 +114,7 @@ impl VersionsHandler { // in config.room.encryption_enabled_by_default_for_room_presets // ); - return Ok(VersionsResponse { + Ok(VersionsResponse { versions: Vec::from([ // XXX: at some point we need to decide whether we need to include // the previous version numbers, given we've defined r0.3.0 to be @@ -171,6 +218,6 @@ impl VersionsHandler { // // MSC4445: Sync timeline order // ("org.matrix.msc4445.initial_sync_timeline_topological_ordering".to_string(), true), ]), - }); + }) } } diff --git a/rust/src/http_client.rs b/rust/src/http_client.rs index 398ba9041f2..ecd5c7b55e3 100644 --- a/rust/src/http_client.rs +++ b/rust/src/http_client.rs @@ -267,7 +267,7 @@ impl HttpClient { /// tokio runtime. /// /// Does not handle deferred cancellation or contextvars. -fn create_deferred<'py, F, O>( +pub(crate) fn create_deferred<'py, F, O>( py: Python<'py>, reactor: &Bound<'py, PyAny>, fut: F, diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs index 8555ec0fb7a..804c8ac9370 100644 --- a/rust/src/storage/db/mod.rs +++ b/rust/src/storage/db/mod.rs @@ -13,33 +13,113 @@ * */ +use std::any::Any; use std::future::Future; +use futures::future::BoxFuture; + pub mod python_db_pool; pub mod rust_db_pool; -// Using `Send + Sync` traits so this can stored in the `#[pyclass]` just fine +/// A single database row, represented as the textual value of each column. +/// +/// This is intentionally a lossy, engine-agnostic representation: it is the +/// lowest common denominator that both the Python (`LoggingTransaction`) and +/// native `tokio-postgres` backends can produce. Callers are responsible for +/// parsing the strings into richer types as needed. +pub type Row = Vec; + +/// The type-erased result of a `run_interaction` callback. +/// +/// We box the result as `dyn Any` so that the [`DatabasePool`] trait can stay +/// object-safe (and therefore usable as `Box`) while still +/// allowing callbacks to return an arbitrary `R`. The ergonomic, generic +/// [`DatabasePoolExt::run_interaction`] downcasts this back to the concrete +/// type for the caller. +pub type AnyResult = anyhow::Result>; + +/// A type-erased `run_interaction` callback. +/// +/// The callback is given a [`Transaction`] and returns a boxed future +/// resolving to a type-erased result. It may be invoked multiple times under +/// certain failure modes (serialization and deadlock errors), so it is `Fn` +/// rather than `FnOnce`. +pub type InteractionFn = + Box Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, AnyResult> + Send>; + +/// A database connection pool. +/// +/// We use a `Box` so the same code can run against either the +/// Python-backed pool (in Synapse, see [`python_db_pool`]) or a native +/// `tokio-postgres` pool (in `synapse-rust-apps`, see [`rust_db_pool`]). To keep +/// the trait object-safe, the only required method is the type-erased +/// [`Self::run_interaction_erased`]; prefer the generic +/// [`DatabasePoolExt::run_interaction`] at call sites. +/// +/// `Send + Sync` so it can be stored in a `#[pyclass]` and shared across threads. #[async_trait::async_trait] pub trait DatabasePool: Send + Sync { - /// Starts a transaction on the database and runs a given function + /// Starts a transaction on the database and runs the given (type-erased) + /// function. /// - /// The given `func` may be called multiple times under certain failure modes (like - /// serialization and deadlock errors). - fn run_interaction<'txn, R, F>( - &'txn self, + /// The given `func` may be called multiple times under certain failure + /// modes (like serialization and deadlock errors). + async fn run_interaction_erased( + &self, + name: &'static str, + func: InteractionFn, + ) -> AnyResult; +} + +/// Ergonomic, generic extension to [`DatabasePool`]. +/// +/// This is automatically implemented for every `DatabasePool` (including +/// `dyn DatabasePool`) via the blanket impl below, and provides the typed +/// `run_interaction` that callers actually use. It lives in a separate trait +/// (rather than on `DatabasePool` directly) because a generic method would make +/// `DatabasePool` no longer object-safe. +pub trait DatabasePoolExt: DatabasePool { + /// Starts a transaction on the database and runs the given function, + /// returning its result. + /// + /// The given `func` may be called multiple times under certain failure + /// modes (like serialization and deadlock errors). + fn run_interaction( + &self, name: &'static str, func: F, - ) -> impl Future> + 'txn + ) -> impl Future> + Send where - R: Send + Sync + 'static, - F: for<'f> Fn(&'f mut dyn Transaction) -> BoxFuture<'f, anyhow::Result> + Send + 'static; + R: Send + 'static, + F: for<'txn> Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, anyhow::Result> + + Send + + Sync + + 'static, + { + // Erase the concrete return type `R` into `Box` so we can call + // through the object-safe `run_interaction_erased`. + let erased: InteractionFn = Box::new(move |txn| { + let fut = func(txn); + Box::pin(async move { + let value = fut.await?; + Ok(Box::new(value) as Box) + }) + }); + + async move { + let boxed = self.run_interaction_erased(name, erased).await?; + Ok(*boxed + .downcast::() + .expect("run_interaction return type mismatch (this is a Synapse programming error)")) + } + } } +impl DatabasePoolExt for T {} + /// A [`tokio_postgres::Transaction`] looking thing that we can use on the Rust side to /// interact with the database #[async_trait::async_trait] -pub trait Transaction { - async fn query(&self, sql: &str, args: &[&str]) -> Result, anyhow::Error>; +pub trait Transaction: Send { + async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error>; } - -pub type Row = Vec; diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index 0c9059ab3a0..1939fe37490 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -17,9 +17,12 @@ //! - Database pool [`PythonDatabasePoolWrapper`] which allows you to start a... //! - transaction [`LoggingTransactionWrapper`] and query the database -use pyo3::{intern, prelude::*, types::PyCFunction, types::PyList}; +use std::sync::{Arc, Mutex}; -use crate::storage::db::{DatabaseConnection, DatabasePool, Row, Transaction}; +use pyo3::{exceptions::PyRuntimeError, intern, prelude::*, types::PyCFunction, types::PyList}; +use tokio::sync::oneshot; + +use crate::storage::db::{AnyResult, DatabasePool, InteractionFn, Row, Transaction}; /// The database engines we support in the Python side of Synapse #[derive(Copy, Clone, Debug)] @@ -40,48 +43,174 @@ impl DatabaseEngine { /// Wrapper for a `DatabasePool` from the Python side of Synapse. pub struct PythonDatabasePoolWrapper { - /// The underlying `DatabasePool` + /// The underlying Python `DatabasePool` database_pool_py: Py, -} -impl<'a, 'py> FromPyObject<'a, 'py> for PythonDatabasePoolWrapper { - type Error = PyErr; + /// The Twisted reactor. We need this to marshal back onto the reactor thread + /// (via `callFromThread`) when starting transactions, since Twisted's thread + /// pool machinery must be driven from there. + reactor: Py, +} - /// Extract from a Python `DatabasePool` passed as an argument. - fn extract(database_pool_py: Borrowed<'a, 'py, PyAny>) -> PyResult { - Ok(Self { - database_pool_py: database_pool_py.to_owned().unbind(), - }) +impl PythonDatabasePoolWrapper { + /// Build a wrapper around the Python `DatabasePool` (e.g. + /// `hs.get_datastores().main.db_pool`) and the Twisted `reactor`. + pub fn new(database_pool_py: Py, reactor: Py) -> Self { + Self { + database_pool_py, + reactor, + } } } #[async_trait::async_trait] impl DatabasePool for PythonDatabasePoolWrapper { - fn run_interaction<'txn, R, F>( - &'txn self, - name: &'static str, - func: F, - ) -> impl Future> + 'txn - where - R: Send + Sync + 'static, - F: for<'f> Fn(&'f mut dyn Transaction) -> BoxFuture<'f, anyhow::Result> + Send + 'static, - { - Python::attach(|py| -> PyResult> { - let callback_func = - PyCFunction::new_closure(py, None, None, move |args, _| -> PyResult> { - // We found our `LoggingTransactionWrapper` - let txn: LoggingTransactionWrapper = args.get_item(0)?; - func(txn); - }); - - let execute_fn = self - .database_pool_py + async fn run_interaction_erased(&self, name: &'static str, func: InteractionFn) -> AnyResult { + // We bridge the Python-side `runInteraction` (a coroutine, run on the + // Twisted reactor + thread pool) back into our `async` Rust world using a + // oneshot channel that resolves when the resulting deferred fires. + let (tx, rx) = oneshot::channel::(); + + // `runInteraction` calls `func` with a `LoggingTransaction` on a DB + // thread and expects a synchronous return value. Since we can't + // round-trip an arbitrary Rust `R` back out through Python, the callback + // stashes the result here and we pick it up once the deferred fires. + let result_slot: Arc>> = Arc::new(Mutex::new(None)); + + Python::attach(|py| -> PyResult<()> { + // (1) The callback that Python's `runInteraction` invokes on a DB + // thread with a `LoggingTransaction`. We drive `func` to completion + // here. The Python query path is synchronous under the hood, so it's + // safe to block this dedicated DB thread until the future resolves. + let callback_slot = Arc::clone(&result_slot); + let callback = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let py = args.py(); + let txn_py = args.get_item(0)?; + let mut txn = txn_py.extract::()?; + + let result = futures::executor::block_on(func(&mut txn)); + + match result { + Ok(value) => { + *callback_slot.lock().unwrap() = Some(Ok(value)); + Ok(py.None()) + } + Err(err) => { + // Re-raise into Python so `runInteraction` rolls the + // transaction back (and can apply its retry logic for + // serialization/deadlock errors). + let py_err = anyhow_to_pyerr(&err); + *callback_slot.lock().unwrap() = Some(Err(err)); + Err(py_err) + } + } + }, + )? + .unbind(); + + // The oneshot sender, shared between the success and error callbacks + // (only one of which ever fires). + let sender = Arc::new(Mutex::new(Some(tx))); + + // (2a) Fired when the transaction succeeds: hand the stashed result + // back to the awaiting task. + let success_slot = Arc::clone(&result_slot); + let success_sender = Arc::clone(&sender); + let on_success = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let result = success_slot.lock().unwrap().take().unwrap_or_else(|| { + Err(anyhow::anyhow!("run_interaction produced no result")) + }); + if let Some(tx) = success_sender.lock().unwrap().take() { + let _ = tx.send(result); + } + Ok(args.py().None()) + }, + )? + .unbind(); + + // (2b) Fired when the transaction fails. Prefer the original error + // captured in the callback (it carries the Rust context); otherwise + // fall back to the Twisted `Failure` text. + let error_slot = Arc::clone(&result_slot); + let error_sender = Arc::clone(&sender); + let on_error = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let result = error_slot.lock().unwrap().take().unwrap_or_else(|| { + let description = args + .get_item(0) + .and_then(|failure| failure.str()) + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|_| "".to_owned()); + Err(anyhow::anyhow!("run_interaction failed: {description}")) + }); + if let Some(tx) = error_sender.lock().unwrap().take() { + let _ = tx.send(result); + } + Ok(args.py().None()) + }, + )? + .unbind(); + + // (3) Kick off `runInteraction` on the reactor thread. It's a + // coroutine, so we wrap it with `ensureDeferred` and attach our + // callbacks. + let database_pool_py = self.database_pool_py.clone_ref(py); + let starter = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let py = args.py(); + let coro = database_pool_py.bind(py).call_method1( + intern!(py, "runInteraction"), + (name, callback.bind(py)), + )?; + let deferred = py + .import("twisted.internet.defer")? + .call_method1(intern!(py, "ensureDeferred"), (coro,))?; + deferred.call_method1( + intern!(py, "addCallbacks"), + (on_success.bind(py), on_error.bind(py)), + )?; + Ok(py.None()) + }, + )?; + + self.reactor .bind(py) - .getattr(intern!(py, "runInteraction"))?; - let results = execute_fn.call1((callback_func,))?; - results + .call_method1(intern!(py, "callFromThread"), (starter,))?; + + Ok(()) }) + .map_err(anyhow::Error::from)?; + + rx.await + .map_err(|_| anyhow::anyhow!("run_interaction channel closed before completing"))? + } +} + +/// Convert an [`anyhow::Error`] into a [`PyErr`] to re-raise into Python. +/// +/// If the error wraps an original Python exception (e.g. a database error +/// surfaced through [`Transaction::query`]), we re-raise *that* exception so +/// Synapse's transaction machinery can apply its retry logic +/// (serialization/deadlock detection) on the real error. +fn anyhow_to_pyerr(err: &anyhow::Error) -> PyErr { + if let Some(py_err) = err.downcast_ref::() { + return Python::attach(|py| py_err.clone_ref(py)); } + PyRuntimeError::new_err(format!("{err:#}")) } fn detect_engine(txn_py: &Bound<'_, PyAny>) -> PyResult { @@ -131,28 +260,8 @@ impl<'a, 'py> FromPyObject<'a, 'py> for LoggingTransactionWrapper { } } -pub trait ValidDatabaseFieldType {} -pub trait ValidDatabaseReturnType {} -impl ValidDatabaseFieldType for String {} -impl ValidDatabaseFieldType for usize {} -impl ValidDatabaseFieldType for Option {} -impl ValidDatabaseReturnType for (T0,) {} -impl ValidDatabaseReturnType for (T0, T1) {} -impl - ValidDatabaseReturnType for (T0, T1, T2) -{ -} -impl< - T0: ValidDatabaseFieldType, - T1: ValidDatabaseFieldType, - T2: ValidDatabaseFieldType, - T3: ValidDatabaseFieldType, - > ValidDatabaseReturnType for (T0, T1, T2, T3) -{ -} - impl LoggingTransactionWrapper { - pub fn execute<'py>( + fn execute<'py>( &mut self, py: Python<'py>, sql: &str, @@ -165,30 +274,38 @@ impl LoggingTransactionWrapper { execute_fn.call1((sql, args))?; Ok(()) } - - pub fn fetchall<'py, T: FromPyObjectOwned<'py> + ValidDatabaseReturnType>( - &mut self, - py: Python<'py>, - ) -> anyhow::Result> { - let fetch_fn = self - .logging_transaction_py - .bind(py) - .getattr(intern!(py, "fetchall"))?; - Ok(fetch_fn.call0()?.extract()?) - } } #[async_trait::async_trait] impl Transaction for LoggingTransactionWrapper { - async fn query(&self, sql: &str, args: &[&str]) -> Result, anyhow::Error> { + async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error> { Python::attach(|py| -> PyResult> { - // Convert the Rust `&[&str]` of SQL parameters into a Python sequence so it - // can be passed through to the Python-side `execute`. + // Convert the Rust `&[&str]` of SQL parameters into a Python sequence + // so it can be passed through to the Python-side `execute`. Note that + // `LoggingTransaction.execute` converts `?` placeholders into the + // appropriate param style for the underlying engine, so we pass + // `?`-style SQL. let args = PyList::new(py, args)?; - // Run the query self.execute(py, sql, args.as_any())?; - // Get the results - let rows = self.fetchall(py)?; + + // Pull the rows back out. Each cell is converted to its textual + // representation so we have a single engine-agnostic `Row` type; + // callers parse the strings into richer types as needed. + let rows_py = self + .logging_transaction_py + .bind(py) + .call_method0(intern!(py, "fetchall"))?; + + let mut rows: Vec = Vec::new(); + for row_py in rows_py.try_iter()? { + let row_py = row_py?; + let mut row: Row = Vec::new(); + for cell in row_py.try_iter()? { + let cell = cell?; + row.push(cell.str()?.to_string_lossy().into_owned()); + } + rows.push(row); + } Ok(rows) }) diff --git a/rust/src/storage/db/rust_db_pool.rs b/rust/src/storage/db/rust_db_pool.rs index 79a54be20ec..addddd489dc 100644 --- a/rust/src/storage/db/rust_db_pool.rs +++ b/rust/src/storage/db/rust_db_pool.rs @@ -17,10 +17,11 @@ // interfaces are compatible with `tokio-postgres`. use anyhow::Context; +use bb8_postgres::tokio_postgres::{self, types::ToSql, IsolationLevel}; use bb8_postgres::PostgresConnectionManager; use postgres_native_tls::MakeTlsConnector; -use crate::storage::db::{DatabaseConnection, DatabasePool, Row, Transaction}; +use crate::storage::db::{AnyResult, DatabasePool, InteractionFn, Row, Transaction}; /// Native Rust database access backed by `tokio-postgres` (for use in synapse-rust-apps) pub struct RustDatabasePool { @@ -29,50 +30,106 @@ pub struct RustDatabasePool { #[async_trait::async_trait] impl DatabasePool for RustDatabasePool { - async fn get_connection(&self) -> Result, anyhow::Error> { - let mut conn = self - .db_pool - .get() - // .instrument(tracing::info_span!("acquire database connection")) - .await - .context("Failed to acquire database connection")?; + async fn run_interaction_erased(&self, _name: &'static str, func: InteractionFn) -> AnyResult { + // Like Synapse's `runInteraction`, retry the whole transaction on + // serialization/deadlock errors (which can happen under repeatable-read). + loop { + let mut conn = self + .db_pool + .get() + .await + .context("Failed to acquire database connection")?; - Ok(Box::new(RustConnection { connection: conn })) - } -} + // Repeatable-read isolation level (like Synapse). + let txn = conn + .build_transaction() + .isolation_level(IsolationLevel::RepeatableRead) + .start() + .await + .context("Failed to start transaction")?; -pub struct RustConnection<'a> { - connection: bb8::PooledConnection<'a, PostgresConnectionManager>, + let mut wrapper = TokioPostgresTransaction { txn }; + match func(&mut wrapper).await { + Ok(value) => { + wrapper + .txn + .commit() + .await + .context("Failed to commit transaction")?; + return Ok(value); + } + Err(err) => { + // The transaction is rolled back implicitly when dropped, but + // be explicit about it before deciding whether to retry. + let _ = wrapper.txn.rollback().await; + if is_retryable(&err) { + continue; + } + return Err(err); + } + } + } + } } -impl DatabaseConnection for RustConnection<'_> { - async fn get_transaction( - &self, - _description: &str, - ) -> Result, anyhow::Error> { - // TODO: Set repeatable-read isolation level (like Synapse) - let txn = self - .connection - .transaction() - // .instrument(tracing::info_span!("start transaction")) - .await - .context("Failed to start transaction")?; - - Ok(Box::new(TokioPostgresTransaction { txn })) - } +/// Whether a failed transaction should be retried (serialization/deadlock errors). +fn is_retryable(err: &anyhow::Error) -> bool { + err.downcast_ref::() + .and_then(|e| e.code()) + .map(|code| { + *code == tokio_postgres::error::SqlState::T_R_SERIALIZATION_FAILURE + || *code == tokio_postgres::error::SqlState::T_R_DEADLOCK_DETECTED + }) + .unwrap_or(false) } struct TokioPostgresTransaction<'a> { - txn: bb8_postgres::tokio_postgres::Transaction<'a>, + txn: tokio_postgres::Transaction<'a>, } #[async_trait::async_trait] impl Transaction for TokioPostgresTransaction<'_> { - async fn query(&self, sql: &str, args: &[&str]) -> Result, anyhow::Error> { - // TODO: Convert `?` SQL param style to `tokio-postgres` compatible + async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error> { + // Synapse SQL uses `?` placeholders; `tokio-postgres` uses `$1`, `$2`, ... + let sql = convert_param_style(sql); + + let params: Vec<&(dyn ToSql + Sync)> = + args.iter().map(|arg| arg as &(dyn ToSql + Sync)).collect(); + let rows = self + .txn + .query(&sql, ¶ms) + .await + .context("Failed to run query")?; - let rows = self.txn.query(sql, args).await?; + let mut out: Vec = Vec::with_capacity(rows.len()); + for row in rows { + let mut cells: Row = Vec::with_capacity(row.len()); + for i in 0..row.len() { + // Best-effort textual extraction to match the engine-agnostic + // `Row` type. A real implementation would map column types + // properly; this only exists to prove the interface fits. + let value: String = row.try_get(i).unwrap_or_default(); + cells.push(value); + } + out.push(cells); + } + + Ok(out) + } +} - Ok(rows) +/// Convert `?`-style placeholders into `tokio-postgres`'s `$1`, `$2`, ... style. +fn convert_param_style(sql: &str) -> String { + let mut out = String::with_capacity(sql.len()); + let mut n = 0; + for ch in sql.chars() { + if ch == '?' { + n += 1; + out.push('$'); + out.push_str(&n.to_string()); + } else { + out.push(ch); + } } + out } diff --git a/rust/src/storage/store.rs b/rust/src/storage/store.rs index a4929c435fe..76d092efc04 100644 --- a/rust/src/storage/store.rs +++ b/rust/src/storage/store.rs @@ -13,12 +13,16 @@ * */ -use serde::{Serialize}; +use futures::FutureExt; +use serde::Serialize; -use crate::{config::SynapseConfig, storage::db::DatabasePool}; +use crate::{ + config::SynapseConfig, + storage::db::{DatabasePool, DatabasePoolExt}, +}; /// Currently supported per-user features -#[derive(Serialize)] +#[derive(Serialize, Debug)] pub enum PerUserExperimentalFeature { #[serde(rename = "msc3881")] MSC3881, @@ -71,30 +75,48 @@ impl Store { return Ok(true); } - let is_feature_enabled_for_user = self.db_pool.run_interaction("is_feature_enabled_for_user", |txn| { - async move { - let rows = txn - .query( - r#" - SELECT enabled - FROM per_user_experimental_features - WHERE user_id = ? AND feature = ? - "#, - &[user_id, &feature.to_string()], - ) - .await; + // It's not enabled globally, so check whether it's enabled per-user. + // + // Owned copies so the callback can be `'static` (it may be moved to + // another thread and called multiple times under retries). + let user_id = user_id.to_string(); + let feature = feature.to_string(); - match (rows.len(), rows.first()) { - (1, Some(enabled)) => enabled, - (0, None) => false, - _ => { - panic!("Synapse programming error"); - } + let is_feature_enabled_for_user = self + .db_pool + .run_interaction("is_feature_enabled_for_user", move |txn| { + let user_id = user_id.clone(); + let feature = feature.clone(); + async move { + let rows = txn + .query( + "SELECT enabled \ + FROM per_user_experimental_features \ + WHERE user_id = ? AND feature = ?", + &[user_id.as_str(), feature.as_str()], + ) + .await?; + + // `None` (no row) and a falsy value are treated the same. + let enabled = rows + .first() + .and_then(|row| row.first()) + .is_some_and(|value| parse_db_bool(value)); + + Ok(enabled) } - } - .boxed() - }).await; - + .boxed() + }) + .await?; + Ok(is_feature_enabled_for_user) } } + +/// Parse a boolean as returned by either database engine. +/// +/// Postgres renders `BOOLEAN` columns as `"True"`/`"False"` while SQLite stores +/// them as integers (`"1"`/`"0"`). +fn parse_db_bool(value: &str) -> bool { + matches!(value, "True" | "true" | "t" | "1") +} diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index d8b0a0545db..9312a616dab 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -28,7 +28,6 @@ from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet from synapse.http.site import SynapseRequest -from synapse.synapse_rust.handlers.versions import get_versions from synapse.types import JsonDict if TYPE_CHECKING: @@ -79,9 +78,7 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: # authenticated responses are not served from cache. request.setHeader(b"Vary", b"Authorization") - versions_response_body = await self.rust_handlers.versions.get_versions( - user_id, self.config - ) + versions_response_body = await self.rust_handlers.versions.get_versions(user_id) return ( 200, diff --git a/synapse/server.py b/synapse/server.py index f46f5845e4b..635e444eb63 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -174,6 +174,7 @@ from synapse.storage import Databases from synapse.storage.controllers import StorageControllers from synapse.streams.events import EventSources +from synapse.synapse_rust.handlers import RustHandlers from synapse.synapse_rust.msc4388_rendezvous import MSC4388RendezvousHandler from synapse.synapse_rust.rendezvous import RendezvousHandler from synapse.types import DomainSpecificString, ISynapseReactor diff --git a/synapse/synapse_rust/handlers.pyi b/synapse/synapse_rust/handlers.pyi new file mode 100644 index 00000000000..4c4f71b7bdb --- /dev/null +++ b/synapse/synapse_rust/handlers.pyi @@ -0,0 +1,35 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from typing import TYPE_CHECKING, Optional + +from twisted.internet.defer import Deferred + +from synapse.types import JsonDict + +if TYPE_CHECKING: + from synapse.server import HomeServer + +class VersionsHandler: + def get_versions(self, user_id: Optional[str] = None) -> Deferred[JsonDict]: + """ + Assemble a `/versions` response. + + The returned deferred follows Synapse logcontext rules. + """ + +class RustHandlers: + """The collection of Rust-implemented request handlers.""" + + def __init__(self, homeserver: "HomeServer") -> None: ... + @property + def versions(self) -> VersionsHandler: ... From d9a111bdfaf51cd4cadfdc071f9956eec47aea43 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 10 Jun 2026 15:44:15 -0500 Subject: [PATCH 05/55] LLM attempt at simplifying --- rust/src/handlers/mod.rs | 2 +- rust/src/handlers/versions.rs | 5 +- rust/src/http_client.rs | 117 ++++++++++++++++++- rust/src/storage/db/mod.rs | 77 ++----------- rust/src/storage/db/python_db_pool.rs | 156 +++++++++----------------- rust/src/storage/db/rust_db_pool.rs | 13 ++- rust/src/storage/store.rs | 12 +- 7 files changed, 195 insertions(+), 187 deletions(-) diff --git a/rust/src/handlers/mod.rs b/rust/src/handlers/mod.rs index e42952ec1ab..e5243f87e1f 100644 --- a/rust/src/handlers/mod.rs +++ b/rust/src/handlers/mod.rs @@ -54,7 +54,7 @@ impl RustHandlers { // Store is shared across all of the handlers so let's use an `Arc` let store = Arc::new(Store { config: config.clone(), - db_pool: Box::new(db_pool), + db_pool, }); let versions = Py::new( diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index d9297599393..78a80116e1a 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -21,6 +21,7 @@ use serde::{Deserialize, Serialize}; use crate::config::SynapseConfig; use crate::http_client::create_deferred; +use crate::storage::db::python_db_pool::PythonDatabasePoolWrapper; use crate::storage::store::{PerUserExperimentalFeature, Store}; /// `GET /_matrix/client/versions` response @@ -44,7 +45,7 @@ impl<'py> IntoPyObject<'py> for VersionsResponse { #[pyclass] pub struct VersionsHandler { pub config: SynapseConfig, - pub store: Arc, + pub store: Arc>, /// The Twisted reactor, used to bridge our `async` response back into a /// Twisted deferred that Python can `await`. pub reactor: Py, @@ -77,7 +78,7 @@ impl VersionsHandler { /// Assemble a `/versions` response body. async fn build_versions_response( - store: &Store, + store: &Store, config: &SynapseConfig, user_id: Option<&str>, ) -> Result { diff --git a/rust/src/http_client.rs b/rust/src/http_client.rs index ecd5c7b55e3..be3bf1c1bfb 100644 --- a/rust/src/http_client.rs +++ b/rust/src/http_client.rs @@ -12,14 +12,22 @@ * . */ -use std::{collections::HashMap, future::Future, sync::OnceLock}; +use std::{ + collections::HashMap, + future::Future, + sync::{Arc, Mutex, OnceLock}, +}; use anyhow::Context; use http_body_util::BodyExt; use once_cell::sync::OnceCell; -use pyo3::{create_exception, exceptions::PyException, prelude::*}; +use pyo3::{ + create_exception, exceptions::PyException, exceptions::PyRuntimeError, intern, prelude::*, + types::PyCFunction, +}; use reqwest::RequestBuilder; use tokio::runtime::Runtime; +use tokio::sync::oneshot; use crate::errors::HttpResponseException; @@ -322,6 +330,111 @@ where make_deferred_yieldable(py, &deferred) } +/// Runs `make_deferred` on the Twisted reactor thread to obtain a Deferred (or +/// coroutine), then resolves once that Deferred fires. +/// +/// This is the inverse of [`create_deferred`]: where that turns a Rust future +/// into a Twisted Deferred, this turns a Twisted Deferred into an awaitable Rust +/// future. +/// +/// We're called on a Tokio worker thread, but Twisted `Deferred`s (and the +/// coroutine that `ensureDeferred` drives) are not thread-safe and Synapse's +/// logcontext is thread-local, so the coroutine must both start and resume on +/// the reactor thread. The `callFromThread` hop is what gets us there for the +/// kickoff; the deferred's own callbacks then fire on the reactor thread too. +/// (Note this is unrelated to offloading the DB work onto a thread — that's +/// handled internally by whatever `make_deferred` calls, e.g. `runInteraction`.) +/// +/// `make_deferred` is invoked on the reactor thread and may return either a +/// coroutine or a `Deferred`; `ensureDeferred` normalises both to a `Deferred`. +pub(crate) async fn await_deferred(reactor: Py, make_deferred: F) -> PyResult> +where + F: for<'py> Fn(Python<'py>) -> PyResult> + Send + 'static, +{ + // Resolves when the deferred fires; carries the resolved value or the error. + let (tx, rx) = oneshot::channel::>>(); + // Shared between the success and error callbacks (only one ever fires). + let sender = Arc::new(Mutex::new(Some(tx))); + + Python::attach(|py| -> PyResult<()> { + let success_sender = Arc::clone(&sender); + let on_success = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let value = args.get_item(0)?.unbind(); + if let Some(tx) = success_sender.lock().unwrap().take() { + let _ = tx.send(Ok(value)); + } + Ok(args.py().None()) + }, + )? + .unbind(); + + let error_sender = Arc::clone(&sender); + let on_error = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let err = failure_to_pyerr(&args.get_item(0)?); + if let Some(tx) = error_sender.lock().unwrap().take() { + let _ = tx.send(Err(err)); + } + Ok(args.py().None()) + }, + )? + .unbind(); + + let starter = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let py = args.py(); + let deferred = defer(py)? + .call_method1(intern!(py, "ensureDeferred"), (make_deferred(py)?,))?; + deferred.call_method1( + intern!(py, "addCallbacks"), + (on_success.bind(py), on_error.bind(py)), + )?; + Ok(py.None()) + }, + )?; + + reactor + .bind(py) + .call_method1(intern!(py, "callFromThread"), (starter,))?; + + Ok(()) + })?; + + match rx.await { + Ok(result) => result, + Err(_) => Err(PyRuntimeError::new_err( + "await_deferred channel closed before the deferred fired", + )), + } +} + +/// Convert a Twisted `Failure` (as passed to an errback) into a [`PyErr`]. +/// +/// A `Failure` carries the original exception instance in its `.value` +/// attribute, which we re-raise so callers see the real error. If that can't be +/// reached, fall back to the `Failure`'s textual representation. +fn failure_to_pyerr(failure: &Bound<'_, PyAny>) -> PyErr { + match failure.getattr(intern!(failure.py(), "value")) { + Ok(value) => PyErr::from_value(value), + Err(_) => PyRuntimeError::new_err( + failure + .str() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|_| "".to_owned()), + ), + } +} + static MAKE_DEFERRED_YIELDABLE: OnceLock> = OnceLock::new(); /// Given a deferred, make it follow the Synapse logcontext rules diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs index 804c8ac9370..bdbf43a18eb 100644 --- a/rust/src/storage/db/mod.rs +++ b/rust/src/storage/db/mod.rs @@ -13,7 +13,6 @@ * */ -use std::any::Any; use std::future::Future; use futures::future::BoxFuture; @@ -29,61 +28,21 @@ pub mod rust_db_pool; /// parsing the strings into richer types as needed. pub type Row = Vec; -/// The type-erased result of a `run_interaction` callback. -/// -/// We box the result as `dyn Any` so that the [`DatabasePool`] trait can stay -/// object-safe (and therefore usable as `Box`) while still -/// allowing callbacks to return an arbitrary `R`. The ergonomic, generic -/// [`DatabasePoolExt::run_interaction`] downcasts this back to the concrete -/// type for the caller. -pub type AnyResult = anyhow::Result>; - -/// A type-erased `run_interaction` callback. -/// -/// The callback is given a [`Transaction`] and returns a boxed future -/// resolving to a type-erased result. It may be invoked multiple times under -/// certain failure modes (serialization and deadlock errors), so it is `Fn` -/// rather than `FnOnce`. -pub type InteractionFn = - Box Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, AnyResult> + Send>; - /// A database connection pool. /// -/// We use a `Box` so the same code can run against either the -/// Python-backed pool (in Synapse, see [`python_db_pool`]) or a native -/// `tokio-postgres` pool (in `synapse-rust-apps`, see [`rust_db_pool`]). To keep -/// the trait object-safe, the only required method is the type-erased -/// [`Self::run_interaction_erased`]; prefer the generic -/// [`DatabasePoolExt::run_interaction`] at call sites. +/// Code is written against this trait so the same store can run against either +/// the Python-backed pool (in Synapse, see [`python_db_pool`]) or a native +/// `tokio-postgres` pool (in `synapse-rust-apps`, see [`rust_db_pool`]). The pool +/// type is fixed within any given binary, so callers are generic over the pool +/// (e.g. `Store`) rather than using dynamic dispatch. /// /// `Send + Sync` so it can be stored in a `#[pyclass]` and shared across threads. -#[async_trait::async_trait] pub trait DatabasePool: Send + Sync { - /// Starts a transaction on the database and runs the given (type-erased) - /// function. - /// - /// The given `func` may be called multiple times under certain failure - /// modes (like serialization and deadlock errors). - async fn run_interaction_erased( - &self, - name: &'static str, - func: InteractionFn, - ) -> AnyResult; -} - -/// Ergonomic, generic extension to [`DatabasePool`]. -/// -/// This is automatically implemented for every `DatabasePool` (including -/// `dyn DatabasePool`) via the blanket impl below, and provides the typed -/// `run_interaction` that callers actually use. It lives in a separate trait -/// (rather than on `DatabasePool` directly) because a generic method would make -/// `DatabasePool` no longer object-safe. -pub trait DatabasePoolExt: DatabasePool { /// Starts a transaction on the database and runs the given function, /// returning its result. /// - /// The given `func` may be called multiple times under certain failure - /// modes (like serialization and deadlock errors). + /// `func` may be called multiple times under certain failure modes (like + /// serialization and deadlock errors), so it is `Fn` rather than `FnOnce`. fn run_interaction( &self, name: &'static str, @@ -94,29 +53,9 @@ pub trait DatabasePoolExt: DatabasePool { F: for<'txn> Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, anyhow::Result> + Send + Sync - + 'static, - { - // Erase the concrete return type `R` into `Box` so we can call - // through the object-safe `run_interaction_erased`. - let erased: InteractionFn = Box::new(move |txn| { - let fut = func(txn); - Box::pin(async move { - let value = fut.await?; - Ok(Box::new(value) as Box) - }) - }); - - async move { - let boxed = self.run_interaction_erased(name, erased).await?; - Ok(*boxed - .downcast::() - .expect("run_interaction return type mismatch (this is a Synapse programming error)")) - } - } + + 'static; } -impl DatabasePoolExt for T {} - /// A [`tokio_postgres::Transaction`] looking thing that we can use on the Rust side to /// interact with the database #[async_trait::async_trait] diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index 1939fe37490..dcd537c3d2f 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -19,10 +19,11 @@ use std::sync::{Arc, Mutex}; +use futures::future::BoxFuture; use pyo3::{exceptions::PyRuntimeError, intern, prelude::*, types::PyCFunction, types::PyList}; -use tokio::sync::oneshot; -use crate::storage::db::{AnyResult, DatabasePool, InteractionFn, Row, Transaction}; +use crate::http_client::await_deferred; +use crate::storage::db::{DatabasePool, Row, Transaction}; /// The database engines we support in the Python side of Synapse #[derive(Copy, Clone, Debug)] @@ -63,26 +64,33 @@ impl PythonDatabasePoolWrapper { } } -#[async_trait::async_trait] impl DatabasePool for PythonDatabasePoolWrapper { - async fn run_interaction_erased(&self, name: &'static str, func: InteractionFn) -> AnyResult { - // We bridge the Python-side `runInteraction` (a coroutine, run on the - // Twisted reactor + thread pool) back into our `async` Rust world using a - // oneshot channel that resolves when the resulting deferred fires. - let (tx, rx) = oneshot::channel::(); - + async fn run_interaction(&self, name: &'static str, func: F) -> anyhow::Result + where + R: Send + 'static, + F: for<'txn> Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, anyhow::Result> + + Send + + Sync + + 'static, + { // `runInteraction` calls `func` with a `LoggingTransaction` on a DB // thread and expects a synchronous return value. Since we can't // round-trip an arbitrary Rust `R` back out through Python, the callback // stashes the result here and we pick it up once the deferred fires. - let result_slot: Arc>> = Arc::new(Mutex::new(None)); - - Python::attach(|py| -> PyResult<()> { - // (1) The callback that Python's `runInteraction` invokes on a DB - // thread with a `LoggingTransaction`. We drive `func` to completion - // here. The Python query path is synchronous under the hood, so it's - // safe to block this dedicated DB thread until the future resolves. - let callback_slot = Arc::clone(&result_slot); + // + // Note the callback may run more than once (`runInteraction` retries on + // serialization/deadlock errors), so we only trust this slot once the + // deferred has fired, i.e. once the transaction has finally committed or + // failed. + let result_slot: Arc>>> = Arc::new(Mutex::new(None)); + + // Build the callback that Python's `runInteraction` invokes on a DB + // thread with a `LoggingTransaction`, plus owned handles we can move onto + // the reactor thread. We drive `func` to completion in the callback; the + // Python query path is synchronous under the hood, so it's safe to block + // this dedicated DB thread until the future resolves. + let callback_slot = Arc::clone(&result_slot); + let (callback, database_pool_py, reactor) = Python::attach(|py| -> PyResult<_> { let callback = PyCFunction::new_closure( py, None, @@ -92,9 +100,7 @@ impl DatabasePool for PythonDatabasePoolWrapper { let txn_py = args.get_item(0)?; let mut txn = txn_py.extract::()?; - let result = futures::executor::block_on(func(&mut txn)); - - match result { + match futures::executor::block_on(func(&mut txn)) { Ok(value) => { *callback_slot.lock().unwrap() = Some(Ok(value)); Ok(py.None()) @@ -112,91 +118,37 @@ impl DatabasePool for PythonDatabasePoolWrapper { )? .unbind(); - // The oneshot sender, shared between the success and error callbacks - // (only one of which ever fires). - let sender = Arc::new(Mutex::new(Some(tx))); - - // (2a) Fired when the transaction succeeds: hand the stashed result - // back to the awaiting task. - let success_slot = Arc::clone(&result_slot); - let success_sender = Arc::clone(&sender); - let on_success = PyCFunction::new_closure( - py, - None, - None, - move |args, _kwargs| -> PyResult> { - let result = success_slot.lock().unwrap().take().unwrap_or_else(|| { - Err(anyhow::anyhow!("run_interaction produced no result")) - }); - if let Some(tx) = success_sender.lock().unwrap().take() { - let _ = tx.send(result); - } - Ok(args.py().None()) - }, - )? - .unbind(); - - // (2b) Fired when the transaction fails. Prefer the original error - // captured in the callback (it carries the Rust context); otherwise - // fall back to the Twisted `Failure` text. - let error_slot = Arc::clone(&result_slot); - let error_sender = Arc::clone(&sender); - let on_error = PyCFunction::new_closure( - py, - None, - None, - move |args, _kwargs| -> PyResult> { - let result = error_slot.lock().unwrap().take().unwrap_or_else(|| { - let description = args - .get_item(0) - .and_then(|failure| failure.str()) - .map(|s| s.to_string_lossy().into_owned()) - .unwrap_or_else(|_| "".to_owned()); - Err(anyhow::anyhow!("run_interaction failed: {description}")) - }); - if let Some(tx) = error_sender.lock().unwrap().take() { - let _ = tx.send(result); - } - Ok(args.py().None()) - }, - )? - .unbind(); - - // (3) Kick off `runInteraction` on the reactor thread. It's a - // coroutine, so we wrap it with `ensureDeferred` and attach our - // callbacks. - let database_pool_py = self.database_pool_py.clone_ref(py); - let starter = PyCFunction::new_closure( - py, - None, - None, - move |args, _kwargs| -> PyResult> { - let py = args.py(); - let coro = database_pool_py.bind(py).call_method1( - intern!(py, "runInteraction"), - (name, callback.bind(py)), - )?; - let deferred = py - .import("twisted.internet.defer")? - .call_method1(intern!(py, "ensureDeferred"), (coro,))?; - deferred.call_method1( - intern!(py, "addCallbacks"), - (on_success.bind(py), on_error.bind(py)), - )?; - Ok(py.None()) - }, - )?; - - self.reactor - .bind(py) - .call_method1(intern!(py, "callFromThread"), (starter,))?; - - Ok(()) + Ok(( + callback, + self.database_pool_py.clone_ref(py), + self.reactor.clone_ref(py), + )) }) .map_err(anyhow::Error::from)?; - rx.await - .map_err(|_| anyhow::anyhow!("run_interaction channel closed before completing"))? + // Await `runInteraction` directly. `runInteraction` offloads the actual + // DB work onto a thread itself, so we don't; `await_deferred` only has to + // start the coroutine on the reactor thread and bridge its deferred back + // into our `async` world. + let outcome = await_deferred(reactor, move |py| { + database_pool_py + .bind(py) + .call_method1(intern!(py, "runInteraction"), (name, callback.bind(py))) + }) + .await; + + // Prefer the result captured by the callback (it carries the Rust + // context). If the slot is empty, `runInteraction` failed before ever + // invoking the callback (e.g. it couldn't acquire a connection), so + // surface the deferred's outcome instead. + let captured = result_slot.lock().unwrap().take(); + match captured { + Some(result) => result, + None => match outcome { + Ok(_) => Err(anyhow::anyhow!("run_interaction produced no result")), + Err(py_err) => Err(anyhow::Error::from(py_err)), + }, + } } } diff --git a/rust/src/storage/db/rust_db_pool.rs b/rust/src/storage/db/rust_db_pool.rs index addddd489dc..284bd189243 100644 --- a/rust/src/storage/db/rust_db_pool.rs +++ b/rust/src/storage/db/rust_db_pool.rs @@ -19,18 +19,25 @@ use anyhow::Context; use bb8_postgres::tokio_postgres::{self, types::ToSql, IsolationLevel}; use bb8_postgres::PostgresConnectionManager; +use futures::future::BoxFuture; use postgres_native_tls::MakeTlsConnector; -use crate::storage::db::{AnyResult, DatabasePool, InteractionFn, Row, Transaction}; +use crate::storage::db::{DatabasePool, Row, Transaction}; /// Native Rust database access backed by `tokio-postgres` (for use in synapse-rust-apps) pub struct RustDatabasePool { db_pool: bb8::Pool>, } -#[async_trait::async_trait] impl DatabasePool for RustDatabasePool { - async fn run_interaction_erased(&self, _name: &'static str, func: InteractionFn) -> AnyResult { + async fn run_interaction(&self, _name: &'static str, func: F) -> anyhow::Result + where + R: Send + 'static, + F: for<'txn> Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, anyhow::Result> + + Send + + Sync + + 'static, + { // Like Synapse's `runInteraction`, retry the whole transaction on // serialization/deadlock errors (which can happen under repeatable-read). loop { diff --git a/rust/src/storage/store.rs b/rust/src/storage/store.rs index 76d092efc04..9d1d3f25345 100644 --- a/rust/src/storage/store.rs +++ b/rust/src/storage/store.rs @@ -16,10 +16,7 @@ use futures::FutureExt; use serde::Serialize; -use crate::{ - config::SynapseConfig, - storage::db::{DatabasePool, DatabasePoolExt}, -}; +use crate::{config::SynapseConfig, storage::db::DatabasePool}; /// Currently supported per-user features #[derive(Serialize, Debug)] @@ -59,13 +56,12 @@ impl std::fmt::Display for PerUserExperimentalFeature { } } - -pub struct Store { +pub struct Store { pub config: SynapseConfig, - pub db_pool: Box, + pub db_pool: P, } -impl Store { +impl Store

{ pub async fn is_feature_enabled( &self, user_id: &str, From 6b5e9f257785a9fe54e49c32706ebe0f2186a0cf Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 17 Jun 2026 17:29:43 -0500 Subject: [PATCH 06/55] Partial `Row` --- rust/src/storage/db/mod.rs | 28 ++++++++++++++++++--------- rust/src/storage/db/python_db_pool.rs | 9 +++++++++ rust/src/storage/store.rs | 22 +++++++++------------ 3 files changed, 37 insertions(+), 22 deletions(-) diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs index bdbf43a18eb..79001f5b8cc 100644 --- a/rust/src/storage/db/mod.rs +++ b/rust/src/storage/db/mod.rs @@ -14,20 +14,13 @@ */ use std::future::Future; +use std::str::FromStr; use futures::future::BoxFuture; pub mod python_db_pool; pub mod rust_db_pool; -/// A single database row, represented as the textual value of each column. -/// -/// This is intentionally a lossy, engine-agnostic representation: it is the -/// lowest common denominator that both the Python (`LoggingTransaction`) and -/// native `tokio-postgres` backends can produce. Callers are responsible for -/// parsing the strings into richer types as needed. -pub type Row = Vec; - /// A database connection pool. /// /// Code is written against this trait so the same store can run against either @@ -60,5 +53,22 @@ pub trait DatabasePool: Send + Sync { /// interact with the database #[async_trait::async_trait] pub trait Transaction: Send { - async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error>; + async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error>; +} + +/// A row of data returned from the database by a query. +pub trait Row { + /// Returns the number of values in the row. + fn len(&self) -> usize; + + /// Deserializes a value from the row. + /// + /// The value can be specified by its numeric index in the row. + fn try_get(&self, index: usize) -> Result; +} + +impl std::fmt::Debug for dyn Row { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // TODO + } } diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index dcd537c3d2f..8005247c267 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -17,6 +17,7 @@ //! - Database pool [`PythonDatabasePoolWrapper`] which allows you to start a... //! - transaction [`LoggingTransactionWrapper`] and query the database +use std::any::Any; use std::sync::{Arc, Mutex}; use futures::future::BoxFuture; @@ -264,3 +265,11 @@ impl Transaction for LoggingTransactionWrapper { .map_err(anyhow::Error::from) } } + +struct PythonRow { + // TODO +} + +impl Row for PythonRow { + // TODO +} diff --git a/rust/src/storage/store.rs b/rust/src/storage/store.rs index 9d1d3f25345..02191bf5193 100644 --- a/rust/src/storage/store.rs +++ b/rust/src/storage/store.rs @@ -93,11 +93,15 @@ impl Store

{ ) .await?; - // `None` (no row) and a falsy value are treated the same. - let enabled = rows - .first() - .and_then(|row| row.first()) - .is_some_and(|value| parse_db_bool(value)); + let enabled = match &rows[..] { + // If there is no row, default to disabled + [] => false, + // Found an entry for the user + [row] => row.try_get(0), + _ => { + panic!("Programming error") + } + }; Ok(enabled) } @@ -108,11 +112,3 @@ impl Store

{ Ok(is_feature_enabled_for_user) } } - -/// Parse a boolean as returned by either database engine. -/// -/// Postgres renders `BOOLEAN` columns as `"True"`/`"False"` while SQLite stores -/// them as integers (`"1"`/`"0"`). -fn parse_db_bool(value: &str) -> bool { - matches!(value, "True" | "true" | "t" | "1") -} From d3af0c4e181126d922c7f222b6d02bc4a536821e Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 17 Jun 2026 17:48:35 -0500 Subject: [PATCH 07/55] LLM further `Row` --- rust/src/storage/db/mod.rs | 113 ++++++++++++++++++++++++-- rust/src/storage/db/python_db_pool.rs | 76 +++++++++++++---- rust/src/storage/db/rust_db_pool.rs | 97 ++++++++++++++++++---- rust/src/storage/store.rs | 2 +- 4 files changed, 249 insertions(+), 39 deletions(-) diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs index 79001f5b8cc..aa18c93a79d 100644 --- a/rust/src/storage/db/mod.rs +++ b/rust/src/storage/db/mod.rs @@ -14,7 +14,6 @@ */ use std::future::Future; -use std::str::FromStr; use futures::future::BoxFuture; @@ -53,22 +52,118 @@ pub trait DatabasePool: Send + Sync { /// interact with the database #[async_trait::async_trait] pub trait Transaction: Send { - async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error>; + async fn query(&mut self, sql: &str, args: &[&str]) + -> Result>, anyhow::Error>; +} + +/// A single backend-agnostic value within a [`Row`]. +/// +/// Each pool maps the values its database driver hands back into this common +/// set, so callers can work with one representation regardless of engine. +#[derive(Debug, Clone, PartialEq)] +pub enum Value { + /// A SQL `NULL`. + Null, + Bool(bool), + Int(i64), + Float(f64), + Text(String), + Bytes(Vec), } /// A row of data returned from the database by a query. -pub trait Row { +/// +/// Modelled after [`tokio_postgres::Row`]: values are pulled out by their numeric +/// index with [`get`](Self::get) / [`try_get`](Self::try_get). Each database pool +/// implements this trait for its own native row type — the Python pool inspects +/// the Python type of each cell, while the `tokio-postgres` pool reuses that +/// crate's own `FromSql` machinery — converting cells into the engine-agnostic +/// [`Value`] returned by [`Row::value`]. +pub trait Row: std::fmt::Debug + Send { /// Returns the number of values in the row. fn len(&self) -> usize; - /// Deserializes a value from the row. + /// Returns whether the row contains no values. + fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns the backend-agnostic [`Value`] at `index`, converting from the + /// pool's native representation. /// - /// The value can be specified by its numeric index in the row. - fn try_get(&self, index: usize) -> Result; + /// Errors if `index` is out of bounds. + fn value(&self, index: usize) -> Result; +} + +impl dyn Row { + /// Deserializes a value from the row, specified by its numeric index, + /// returning an error if the index is out of bounds or the value cannot be + /// converted into `T`. + pub fn try_get(&self, index: usize) -> Result { + T::from_value(self.value(index)?) + } +} + +/// Converts a backend-agnostic [`Value`] into a concrete Rust type, analogous to +/// `tokio-postgres`'s `FromSql`. +pub trait FromValue: Sized { + fn from_value(value: Value) -> Result; +} + +impl FromValue for bool { + fn from_value(value: Value) -> Result { + match value { + Value::Bool(b) => Ok(b), + // SQLite has no native boolean type and stores them as integers. + Value::Int(i) => Ok(i != 0), + other => anyhow::bail!("cannot read {other:?} as bool"), + } + } +} + +impl FromValue for i64 { + fn from_value(value: Value) -> Result { + match value { + Value::Int(i) => Ok(i), + Value::Bool(b) => Ok(b as i64), + other => anyhow::bail!("cannot read {other:?} as i64"), + } + } +} + +impl FromValue for f64 { + fn from_value(value: Value) -> Result { + match value { + Value::Float(f) => Ok(f), + Value::Int(i) => Ok(i as f64), + other => anyhow::bail!("cannot read {other:?} as f64"), + } + } +} + +impl FromValue for String { + fn from_value(value: Value) -> Result { + match value { + Value::Text(s) => Ok(s), + other => anyhow::bail!("cannot read {other:?} as String"), + } + } +} + +impl FromValue for Vec { + fn from_value(value: Value) -> Result { + match value { + Value::Bytes(b) => Ok(b), + other => anyhow::bail!("cannot read {other:?} as bytes"), + } + } } -impl std::fmt::Debug for dyn Row { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - // TODO +impl FromValue for Option { + fn from_value(value: Value) -> Result { + match value { + Value::Null => Ok(None), + other => Ok(Some(T::from_value(other)?)), + } } } diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index 8005247c267..f97694e2482 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -21,10 +21,15 @@ use std::any::Any; use std::sync::{Arc, Mutex}; use futures::future::BoxFuture; -use pyo3::{exceptions::PyRuntimeError, intern, prelude::*, types::PyCFunction, types::PyList}; +use pyo3::{ + exceptions::{PyRuntimeError, PyTypeError}, + intern, + prelude::*, + types::{PyBool, PyBytes, PyCFunction, PyFloat, PyInt, PyList, PyString}, +}; use crate::http_client::await_deferred; -use crate::storage::db::{DatabasePool, Row, Transaction}; +use crate::storage::db::{DatabasePool, Row, Transaction, Value}; /// The database engines we support in the Python side of Synapse #[derive(Copy, Clone, Debug)] @@ -231,8 +236,12 @@ impl LoggingTransactionWrapper { #[async_trait::async_trait] impl Transaction for LoggingTransactionWrapper { - async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error> { - Python::attach(|py| -> PyResult> { + async fn query( + &mut self, + sql: &str, + args: &[&str], + ) -> Result>, anyhow::Error> { + Python::attach(|py| -> PyResult>> { // Convert the Rust `&[&str]` of SQL parameters into a Python sequence // so it can be passed through to the Python-side `execute`. Note that // `LoggingTransaction.execute` converts `?` placeholders into the @@ -241,23 +250,21 @@ impl Transaction for LoggingTransactionWrapper { let args = PyList::new(py, args)?; self.execute(py, sql, args.as_any())?; - // Pull the rows back out. Each cell is converted to its textual - // representation so we have a single engine-agnostic `Row` type; - // callers parse the strings into richer types as needed. + // Pull the rows back out, converting each cell from its Python type + // into the engine-agnostic `Value` representation as we go. let rows_py = self .logging_transaction_py .bind(py) .call_method0(intern!(py, "fetchall"))?; - let mut rows: Vec = Vec::new(); + let mut rows: Vec> = Vec::new(); for row_py in rows_py.try_iter()? { let row_py = row_py?; - let mut row: Row = Vec::new(); + let mut values: Vec = Vec::new(); for cell in row_py.try_iter()? { - let cell = cell?; - row.push(cell.str()?.to_string_lossy().into_owned()); + values.push(py_cell_to_value(&cell?)?); } - rows.push(row); + rows.push(Box::new(PythonRow { values })); } Ok(rows) @@ -266,10 +273,51 @@ impl Transaction for LoggingTransactionWrapper { } } +/// Convert a single cell from a Python row into a backend-agnostic [`Value`] by +/// inspecting its Python type (the pyo3 equivalent of `isinstance` checks). +fn py_cell_to_value(cell: &Bound<'_, PyAny>) -> PyResult { + // `None` maps to SQL `NULL`. + if cell.is_none() { + return Ok(Value::Null); + } + + // A `bool` *is* an `int` in Python, so ensure we try `bool` first. + if let Ok(b) = cell.cast::() { + Ok(Value::Bool(b.extract()?)) + } else if let Ok(i) = cell.cast::() { + Ok(Value::Int(i.extract()?)) + } else if let Ok(f) = cell.cast::() { + Ok(Value::Float(f.extract()?)) + } else if let Ok(s) = cell.cast::() { + Ok(Value::Text(s.to_string())) + } else if let Ok(bytes) = cell.cast::() { + Ok(Value::Bytes(bytes.as_bytes().to_vec())) + } else { + Err(PyTypeError::new_err(format!( + "unsupported column type {} returned from the database", + cell.get_type().name()? + ))) + } +} + +/// A [`Row`] backed by values pulled out of a Python `LoggingTransaction`. +#[derive(Debug)] struct PythonRow { - // TODO + /// Each cell, already converted from its Python type into a [`Value`]. + values: Vec, } impl Row for PythonRow { - // TODO + fn len(&self) -> usize { + self.values.len() + } + + fn value(&self, index: usize) -> Result { + self.values.get(index).cloned().ok_or_else(|| { + anyhow::anyhow!( + "tried to get column {index} but the row only has {} column(s)", + self.values.len() + ) + }) + } } diff --git a/rust/src/storage/db/rust_db_pool.rs b/rust/src/storage/db/rust_db_pool.rs index 284bd189243..252bfa52159 100644 --- a/rust/src/storage/db/rust_db_pool.rs +++ b/rust/src/storage/db/rust_db_pool.rs @@ -17,12 +17,16 @@ // interfaces are compatible with `tokio-postgres`. use anyhow::Context; -use bb8_postgres::tokio_postgres::{self, types::ToSql, IsolationLevel}; +use bb8_postgres::tokio_postgres::{ + self, + types::{ToSql, Type}, + IsolationLevel, +}; use bb8_postgres::PostgresConnectionManager; use futures::future::BoxFuture; use postgres_native_tls::MakeTlsConnector; -use crate::storage::db::{DatabasePool, Row, Transaction}; +use crate::storage::db::{DatabasePool, Row, Transaction, Value}; /// Native Rust database access backed by `tokio-postgres` (for use in synapse-rust-apps) pub struct RustDatabasePool { @@ -96,7 +100,11 @@ struct TokioPostgresTransaction<'a> { #[async_trait::async_trait] impl Transaction for TokioPostgresTransaction<'_> { - async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error> { + async fn query( + &mut self, + sql: &str, + args: &[&str], + ) -> Result>, anyhow::Error> { // Synapse SQL uses `?` placeholders; `tokio-postgres` uses `$1`, `$2`, ... let sql = convert_param_style(sql); @@ -108,23 +116,82 @@ impl Transaction for TokioPostgresTransaction<'_> { .await .context("Failed to run query")?; - let mut out: Vec = Vec::with_capacity(rows.len()); - for row in rows { - let mut cells: Row = Vec::with_capacity(row.len()); - for i in 0..row.len() { - // Best-effort textual extraction to match the engine-agnostic - // `Row` type. A real implementation would map column types - // properly; this only exists to prove the interface fits. - let value: String = row.try_get(i).unwrap_or_default(); - cells.push(value); - } - out.push(cells); - } + let out = rows + .into_iter() + .map(|row| Box::new(TokioPostgresRow { row }) as Box) + .collect(); Ok(out) } } +/// A [`Row`] backed by a native [`tokio_postgres::Row`]. +#[derive(Debug)] +struct TokioPostgresRow { + row: tokio_postgres::Row, +} + +impl Row for TokioPostgresRow { + fn len(&self) -> usize { + self.row.len() + } + + fn value(&self, index: usize) -> Result { + let column = self.row.columns().get(index).ok_or_else(|| { + anyhow::anyhow!( + "tried to get column {index} but the row only has {} column(s)", + self.row.len() + ) + })?; + + // Dispatch on the column's Postgres type, leaning on `tokio-postgres`'s + // own `FromSql` impls to read the cell. Everything is read as + // `Option<_>` so a SQL `NULL` becomes `Value::Null`. + let ty = column.type_(); + let value = if *ty == Type::BOOL { + self.row + .try_get::<_, Option>(index)? + .map_or(Value::Null, Value::Bool) + } else if *ty == Type::INT2 { + self.row + .try_get::<_, Option>(index)? + .map_or(Value::Null, |v| Value::Int(v.into())) + } else if *ty == Type::INT4 { + self.row + .try_get::<_, Option>(index)? + .map_or(Value::Null, |v| Value::Int(v.into())) + } else if *ty == Type::INT8 { + self.row + .try_get::<_, Option>(index)? + .map_or(Value::Null, Value::Int) + } else if *ty == Type::FLOAT4 { + self.row + .try_get::<_, Option>(index)? + .map_or(Value::Null, |v| Value::Float(v.into())) + } else if *ty == Type::FLOAT8 { + self.row + .try_get::<_, Option>(index)? + .map_or(Value::Null, Value::Float) + } else if *ty == Type::TEXT + || *ty == Type::VARCHAR + || *ty == Type::BPCHAR + || *ty == Type::NAME + { + self.row + .try_get::<_, Option>(index)? + .map_or(Value::Null, Value::Text) + } else if *ty == Type::BYTEA { + self.row + .try_get::<_, Option>>(index)? + .map_or(Value::Null, Value::Bytes) + } else { + anyhow::bail!("unsupported Postgres column type `{ty}` at column {index}"); + }; + + Ok(value) + } +} + /// Convert `?`-style placeholders into `tokio-postgres`'s `$1`, `$2`, ... style. fn convert_param_style(sql: &str) -> String { let mut out = String::with_capacity(sql.len()); diff --git a/rust/src/storage/store.rs b/rust/src/storage/store.rs index 02191bf5193..2f1927fb734 100644 --- a/rust/src/storage/store.rs +++ b/rust/src/storage/store.rs @@ -97,7 +97,7 @@ impl Store

{ // If there is no row, default to disabled [] => false, // Found an entry for the user - [row] => row.try_get(0), + [row] => row.try_get(0)?, _ => { panic!("Programming error") } From 65ba851f2cfcd86f901d81e32cff17b37eeb2d48 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 17 Jun 2026 18:25:07 -0500 Subject: [PATCH 08/55] Concrete `Row` type --- rust/src/storage/db/mod.rs | 60 ++++++--------- rust/src/storage/db/python_db_pool.rs | 45 ++--------- rust/src/storage/db/rust_db_pool.rs | 107 +++++++------------------- rust/src/storage/store.rs | 5 +- 4 files changed, 65 insertions(+), 152 deletions(-) diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs index aa18c93a79d..9729bc39531 100644 --- a/rust/src/storage/db/mod.rs +++ b/rust/src/storage/db/mod.rs @@ -52,8 +52,7 @@ pub trait DatabasePool: Send + Sync { /// interact with the database #[async_trait::async_trait] pub trait Transaction: Send { - async fn query(&mut self, sql: &str, args: &[&str]) - -> Result>, anyhow::Error>; + async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error>; } /// A single backend-agnostic value within a [`Row`]. @@ -68,39 +67,37 @@ pub enum Value { Int(i64), Float(f64), Text(String), - Bytes(Vec), } /// A row of data returned from the database by a query. /// -/// Modelled after [`tokio_postgres::Row`]: values are pulled out by their numeric -/// index with [`get`](Self::get) / [`try_get`](Self::try_get). Each database pool -/// implements this trait for its own native row type — the Python pool inspects -/// the Python type of each cell, while the `tokio-postgres` pool reuses that -/// crate's own `FromSql` machinery — converting cells into the engine-agnostic -/// [`Value`] returned by [`Row::value`]. -pub trait Row: std::fmt::Debug + Send { - /// Returns the number of values in the row. - fn len(&self) -> usize; - - /// Returns whether the row contains no values. - fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Returns the backend-agnostic [`Value`] at `index`, converting from the - /// pool's native representation. - /// - /// Errors if `index` is out of bounds. - fn value(&self, index: usize) -> Result; -} +/// Each pool converts the cells its database driver hands back into the +/// engine-agnostic [`Value`] representation, so a row is simply a list of them. +/// Values are pulled out by their numeric index with [`RowExt::try_get`]. +pub type Row = Vec; -impl dyn Row { +/// Extension methods for reading typed values out of a [`Row`]. +/// +/// Modelled after [`tokio_postgres::Row`]'s `try_get`: [`try_get`](Self::try_get) +/// converts the [`Value`] at a given index into the requested type via +/// [`FromValue`] (our analogue of `tokio-postgres`'s `FromSql`). +pub trait RowExt { /// Deserializes a value from the row, specified by its numeric index, /// returning an error if the index is out of bounds or the value cannot be /// converted into `T`. - pub fn try_get(&self, index: usize) -> Result { - T::from_value(self.value(index)?) + fn try_get(&self, index: usize) -> Result; +} + +impl RowExt for Row { + fn try_get(&self, index: usize) -> Result { + let value = self.get(index).cloned().ok_or_else(|| { + anyhow::anyhow!( + "tried to get column {index} but the row only has {} column(s)", + self.len() + ) + })?; + + T::from_value(value) } } @@ -150,15 +147,6 @@ impl FromValue for String { } } -impl FromValue for Vec { - fn from_value(value: Value) -> Result { - match value { - Value::Bytes(b) => Ok(b), - other => anyhow::bail!("cannot read {other:?} as bytes"), - } - } -} - impl FromValue for Option { fn from_value(value: Value) -> Result { match value { diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index f97694e2482..07fef298b23 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -17,7 +17,6 @@ //! - Database pool [`PythonDatabasePoolWrapper`] which allows you to start a... //! - transaction [`LoggingTransactionWrapper`] and query the database -use std::any::Any; use std::sync::{Arc, Mutex}; use futures::future::BoxFuture; @@ -25,7 +24,7 @@ use pyo3::{ exceptions::{PyRuntimeError, PyTypeError}, intern, prelude::*, - types::{PyBool, PyBytes, PyCFunction, PyFloat, PyInt, PyList, PyString}, + types::{PyBool, PyCFunction, PyFloat, PyInt, PyList, PyString}, }; use crate::http_client::await_deferred; @@ -236,12 +235,8 @@ impl LoggingTransactionWrapper { #[async_trait::async_trait] impl Transaction for LoggingTransactionWrapper { - async fn query( - &mut self, - sql: &str, - args: &[&str], - ) -> Result>, anyhow::Error> { - Python::attach(|py| -> PyResult>> { + async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error> { + Python::attach(|py| -> PyResult> { // Convert the Rust `&[&str]` of SQL parameters into a Python sequence // so it can be passed through to the Python-side `execute`. Note that // `LoggingTransaction.execute` converts `?` placeholders into the @@ -257,14 +252,14 @@ impl Transaction for LoggingTransactionWrapper { .bind(py) .call_method0(intern!(py, "fetchall"))?; - let mut rows: Vec> = Vec::new(); + let mut rows: Vec = Vec::new(); for row_py in rows_py.try_iter()? { let row_py = row_py?; - let mut values: Vec = Vec::new(); + let mut row: Row = Vec::new(); for cell in row_py.try_iter()? { - values.push(py_cell_to_value(&cell?)?); + row.push(py_cell_to_value(&cell?)?); } - rows.push(Box::new(PythonRow { values })); + rows.push(row); } Ok(rows) @@ -281,7 +276,7 @@ fn py_cell_to_value(cell: &Bound<'_, PyAny>) -> PyResult { return Ok(Value::Null); } - // A `bool` *is* an `int` in Python, so ensure we try `bool` first. + // A `bool` *is* an `int` in SQLite, so ensure we try `bool` first. if let Ok(b) = cell.cast::() { Ok(Value::Bool(b.extract()?)) } else if let Ok(i) = cell.cast::() { @@ -290,8 +285,6 @@ fn py_cell_to_value(cell: &Bound<'_, PyAny>) -> PyResult { Ok(Value::Float(f.extract()?)) } else if let Ok(s) = cell.cast::() { Ok(Value::Text(s.to_string())) - } else if let Ok(bytes) = cell.cast::() { - Ok(Value::Bytes(bytes.as_bytes().to_vec())) } else { Err(PyTypeError::new_err(format!( "unsupported column type {} returned from the database", @@ -299,25 +292,3 @@ fn py_cell_to_value(cell: &Bound<'_, PyAny>) -> PyResult { ))) } } - -/// A [`Row`] backed by values pulled out of a Python `LoggingTransaction`. -#[derive(Debug)] -struct PythonRow { - /// Each cell, already converted from its Python type into a [`Value`]. - values: Vec, -} - -impl Row for PythonRow { - fn len(&self) -> usize { - self.values.len() - } - - fn value(&self, index: usize) -> Result { - self.values.get(index).cloned().ok_or_else(|| { - anyhow::anyhow!( - "tried to get column {index} but the row only has {} column(s)", - self.values.len() - ) - }) - } -} diff --git a/rust/src/storage/db/rust_db_pool.rs b/rust/src/storage/db/rust_db_pool.rs index 252bfa52159..be280c1f9bb 100644 --- a/rust/src/storage/db/rust_db_pool.rs +++ b/rust/src/storage/db/rust_db_pool.rs @@ -17,11 +17,7 @@ // interfaces are compatible with `tokio-postgres`. use anyhow::Context; -use bb8_postgres::tokio_postgres::{ - self, - types::{ToSql, Type}, - IsolationLevel, -}; +use bb8_postgres::tokio_postgres::{self, types::ToSql, IsolationLevel}; use bb8_postgres::PostgresConnectionManager; use futures::future::BoxFuture; use postgres_native_tls::MakeTlsConnector; @@ -100,11 +96,7 @@ struct TokioPostgresTransaction<'a> { #[async_trait::async_trait] impl Transaction for TokioPostgresTransaction<'_> { - async fn query( - &mut self, - sql: &str, - args: &[&str], - ) -> Result>, anyhow::Error> { + async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error> { // Synapse SQL uses `?` placeholders; `tokio-postgres` uses `$1`, `$2`, ... let sql = convert_param_style(sql); @@ -116,79 +108,38 @@ impl Transaction for TokioPostgresTransaction<'_> { .await .context("Failed to run query")?; - let out = rows - .into_iter() - .map(|row| Box::new(TokioPostgresRow { row }) as Box) - .collect(); - - Ok(out) + rows.iter().map(tokio_row_to_row).collect() } } -/// A [`Row`] backed by a native [`tokio_postgres::Row`]. -#[derive(Debug)] -struct TokioPostgresRow { - row: tokio_postgres::Row, +/// Convert a native [`tokio_postgres::Row`] into our engine-agnostic [`Row`]. +fn tokio_row_to_row(row: &tokio_postgres::Row) -> Result { + (0..row.len()) + .map(|index| tokio_cell_to_value(row, index)) + .collect() } -impl Row for TokioPostgresRow { - fn len(&self) -> usize { - self.row.len() - } - - fn value(&self, index: usize) -> Result { - let column = self.row.columns().get(index).ok_or_else(|| { - anyhow::anyhow!( - "tried to get column {index} but the row only has {} column(s)", - self.row.len() - ) - })?; - - // Dispatch on the column's Postgres type, leaning on `tokio-postgres`'s - // own `FromSql` impls to read the cell. Everything is read as - // `Option<_>` so a SQL `NULL` becomes `Value::Null`. - let ty = column.type_(); - let value = if *ty == Type::BOOL { - self.row - .try_get::<_, Option>(index)? - .map_or(Value::Null, Value::Bool) - } else if *ty == Type::INT2 { - self.row - .try_get::<_, Option>(index)? - .map_or(Value::Null, |v| Value::Int(v.into())) - } else if *ty == Type::INT4 { - self.row - .try_get::<_, Option>(index)? - .map_or(Value::Null, |v| Value::Int(v.into())) - } else if *ty == Type::INT8 { - self.row - .try_get::<_, Option>(index)? - .map_or(Value::Null, Value::Int) - } else if *ty == Type::FLOAT4 { - self.row - .try_get::<_, Option>(index)? - .map_or(Value::Null, |v| Value::Float(v.into())) - } else if *ty == Type::FLOAT8 { - self.row - .try_get::<_, Option>(index)? - .map_or(Value::Null, Value::Float) - } else if *ty == Type::TEXT - || *ty == Type::VARCHAR - || *ty == Type::BPCHAR - || *ty == Type::NAME - { - self.row - .try_get::<_, Option>(index)? - .map_or(Value::Null, Value::Text) - } else if *ty == Type::BYTEA { - self.row - .try_get::<_, Option>>(index)? - .map_or(Value::Null, Value::Bytes) - } else { - anyhow::bail!("unsupported Postgres column type `{ty}` at column {index}"); - }; - - Ok(value) +/// Convert a single cell of a [`tokio_postgres::Row`] into a [`Value`]. +/// +/// Dispatch on the column's Postgres type, leaning on `tokio-postgres`'s own +/// `FromSql` impls to read the cell. Everything is read as `Option<_>` so a SQL +/// `NULL` becomes `Value::Null`. +fn tokio_cell_to_value(row: &tokio_postgres::Row, index: usize) -> Result { + if let Ok(value) = row.try_get::<_, Option>(index) { + Ok(value.map_or(Value::Null, Value::Bool)) + } else if let Ok(value) = row.try_get::<_, Option>(index) { + Ok(value.map_or(Value::Null, Value::Int)) + } else if let Ok(value) = row.try_get::<_, Option>(index) { + Ok(value.map_or(Value::Null, Value::Float)) + } else if let Ok(value) = row.try_get::<_, Option>(index) { + Ok(value.map_or(Value::Null, Value::Text)) + } else { + let ty = row.columns()[index].type_(); + anyhow::bail!( + "Unsupported `tokio-postgres` type {} encountered when trying to convert it \ + to our generic database `Value` type. You probably just need to implement it in `tokio_cell_to_value(...)`.", + ty + ) } } diff --git a/rust/src/storage/store.rs b/rust/src/storage/store.rs index 2f1927fb734..f57f94e5b02 100644 --- a/rust/src/storage/store.rs +++ b/rust/src/storage/store.rs @@ -16,7 +16,10 @@ use futures::FutureExt; use serde::Serialize; -use crate::{config::SynapseConfig, storage::db::DatabasePool}; +use crate::{ + config::SynapseConfig, + storage::db::{DatabasePool, RowExt}, +}; /// Currently supported per-user features #[derive(Serialize, Debug)] From 0226a67858a2df27905a468f57000d5b1121b411 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 17 Jun 2026 18:45:47 -0500 Subject: [PATCH 09/55] Add comments why row expectations --- rust/src/storage/store.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/src/storage/store.rs b/rust/src/storage/store.rs index f57f94e5b02..1b27b6036d0 100644 --- a/rust/src/storage/store.rs +++ b/rust/src/storage/store.rs @@ -97,9 +97,9 @@ impl Store

{ .await?; let enabled = match &rows[..] { - // If there is no row, default to disabled + // If there is no row for this user, default to disabled [] => false, - // Found an entry for the user + // Otherwise, we should only find a single row for this (user, feature) [row] => row.try_get(0)?, _ => { panic!("Programming error") From 7bf4b6abd3d29ae15d3e1f0c62ee188000c660ae Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 17 Jun 2026 19:10:39 -0500 Subject: [PATCH 10/55] Non-working: Add `test_versions.py` --- tests/rest/client/test_versions.py | 146 +++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 tests/rest/client/test_versions.py diff --git a/tests/rest/client/test_versions.py b/tests/rest/client/test_versions.py new file mode 100644 index 00000000000..c38696d6949 --- /dev/null +++ b/tests/rest/client/test_versions.py @@ -0,0 +1,146 @@ +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +import logging + +from twisted.internet.testing import MemoryReactor + +from synapse.rest import admin +from synapse.rest.client import login, versions +from synapse.server import HomeServer +from synapse.util.clock import Clock +from synapse.types import JsonDict + +from tests import unittest + +logger = logging.getLogger(__name__) + + +class VersionsTestCase(unittest.HomeserverTestCase): + """ + Test `VersionsRestServlet` + """ + + servlets = [ + admin.register_servlets, + login.register_servlets, + versions.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.admin_user = self.register_user("admin", "pass", admin=True) + self.admin_user_tok = self.login("admin", "pass") + + def test_unauthenticated(self) -> None: + channel = self.make_request( + "GET", + "/_matrix/client/versions", + content={}, + ) + self.assertEqual(channel.code, 200, channel.result) + self._sanity_check_versions_response(channel.json_body) + + def test_authenticated(self) -> None: + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + channel = self.make_request( + "GET", + "/_matrix/client/versions", + content={}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self._sanity_check_versions_response(channel.json_body) + + def test_authenticated_with_per_user_feature(self) -> None: + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Sanity check that the experimental feature should not be enabled yet + channel = self.make_request( + "GET", + "/_matrix/client/versions", + content={}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self._sanity_check_versions_response(channel.json_body) + self.assertEqual( + channel.json_body["unstable_features"]["org.matrix.msc3881"], + False, + channel.json_body, + ) + + # Enable the feature for this specific user + self._enable_experimental_feature_for_user(target_user_id=user1_id) + + # The experimental feature should be enabled for this user + channel = self.make_request( + "GET", + "/_matrix/client/versions", + content={}, + access_token=user1_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self._sanity_check_versions_response(channel.json_body) + self.assertEqual( + channel.json_body["unstable_features"]["org.matrix.msc3881"], + True, + channel.json_body, + ) + + # But not for other users + channel = self.make_request( + "GET", + "/_matrix/client/versions", + content={}, + access_token=user2_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self._sanity_check_versions_response(channel.json_body) + self.assertEqual( + channel.json_body["unstable_features"]["org.matrix.msc3881"], + False, + channel.json_body, + ) + + def _sanity_check_versions_response(self, versions_response: JsonDict) -> None: + """ + Make sure this looks like a `/_matrix/client/versions` response + """ + self.assertIsInstance( + versions_response["versions"], + list, + f"Expected `versions` to be a list of strings but saw {versions_response}", + ) + self.assertIsInstance( + versions_response["unstable_features"], + dict, + f"Expected `unstable_features` to be a dict mapping feature name to a bool but saw {versions_response}", + ) + + def _enable_experimental_feature_for_user(self, *, target_user_id: str) -> None: + """ + Use the admin API to enable an experimental feature for a specific user + """ + channel = self.make_request( + "PUT", + f"/_synapse/admin/v1/experimental_features/{target_user_id}", + content={ + "features": {"msc3881": True}, + }, + access_token=self.admin_user_tok, + ) + self.assertEqual(channel.code, 200) From 8270087c3b6c2fa989e215d754a74dca0874c36d Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 17 Jun 2026 19:55:20 -0500 Subject: [PATCH 11/55] Fix test failing with `RuntimeError: Tokio runtime is not running` `_trial_temp/test.log` ``` 2026-06-17 19:17:57-0500 [-] synapse.http.server - 147 - ERROR - GET-3 - Failed handle request via 'VersionsRestServlet': Traceback (most recent call last): File "/home/eric/Documents/github/element/synapse/synapse/http/server.py", line 335, in _async_render_wrapper callback_return = await self._async_render(request) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/eric/Documents/github/element/synapse/synapse/http/server.py", line 576, in _async_render callback_return = await raw_callback_return ^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/eric/Documents/github/element/synapse/synapse/rest/client/versions.py", line 81, in on_GET versions_response_body = await self.rust_handlers.versions.get_versions(user_id) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ RuntimeError: Tokio runtime is not running ``` --- tests/rest/client/test_versions.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/rest/client/test_versions.py b/tests/rest/client/test_versions.py index c38696d6949..f1aecce44a8 100644 --- a/tests/rest/client/test_versions.py +++ b/tests/rest/client/test_versions.py @@ -19,6 +19,7 @@ from synapse.server import HomeServer from synapse.util.clock import Clock from synapse.types import JsonDict +from synapse.synapse_rust.http_client import HttpClient from tests import unittest @@ -36,6 +37,34 @@ class VersionsTestCase(unittest.HomeserverTestCase): versions.register_servlets, ] + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + hs = self.setup_test_homeserver() + + # XXX: We must create the Rust HTTP client before we call `reactor.run()` below. + # Twisted's `MemoryReactor` doesn't invoke `callWhenRunning` callbacks if it's + # already running and we rely on that to start the Tokio thread pool in Rust. In + # the future, this may not matter, see https://github.com/twisted/twisted/pull/12514 + self._http_client = hs.get_proxied_http_client() + _ = HttpClient( + reactor=hs.get_reactor(), + user_agent=self._http_client.user_agent.decode("utf8"), + ) + + # This triggers the server startup hooks, which starts the Tokio thread pool + reactor.run() + + return hs + + def tearDown(self) -> None: + # MemoryReactor doesn't trigger the shutdown phases, and we want the + # Tokio thread pool to be stopped + # XXX: This logic should probably get moved somewhere else + shutdown_triggers = self.reactor.triggers.get("shutdown", {}) + for phase in ["before", "during", "after"]: + triggers = shutdown_triggers.get(phase, []) + for callbable, args, kwargs in triggers: + callbable(*args, **kwargs) + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.admin_user = self.register_user("admin", "pass", admin=True) self.admin_user_tok = self.login("admin", "pass") From 0b6a9733a19785d2d51ec8ff92085ed8656db500 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 17 Jun 2026 20:33:02 -0500 Subject: [PATCH 12/55] Passing `tests_versions.py`: Wait real time for tokio thread pool to work on our async Rust function Previously, we were running into this problem: ``` SYNAPSE_TEST_LOG_LEVEL=INFO poetry run trial tests.rest.client.test_versions.VersionsTestCase.test_authenticated tests.rest.client.test_versions VersionsTestCase test_authenticated ... [ERROR] =============================================================================== [ERROR] Traceback (most recent call last): File "/home/eric/Documents/github/element/synapse/tests/rest/client/test_versions.py", line 85, in test_authenticated channel = self.make_request( File "/home/eric/Documents/github/element/synapse/tests/unittest.py", line 619, in make_request return make_request( File "/home/eric/Documents/github/element/synapse/tests/server.py", line 486, in make_request channel.await_result() File "/home/eric/Documents/github/element/synapse/tests/server.py", line 310, in await_result raise TimedOutException("Timed out waiting for request to finish.") tests.server.TimedOutException: Timed out waiting for request to finish. tests.rest.client.test_versions.VersionsTestCase.test_authenticated ------------------------------------------------------------------------------- Ran 1 tests in 0.182s FAILED (errors=1) ``` --- tests/rest/client/test_versions.py | 64 +++++++++++++++++------------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/tests/rest/client/test_versions.py b/tests/rest/client/test_versions.py index f1aecce44a8..bbd91ade60c 100644 --- a/tests/rest/client/test_versions.py +++ b/tests/rest/client/test_versions.py @@ -11,17 +11,19 @@ # . import logging +import time from twisted.internet.testing import MemoryReactor from synapse.rest import admin from synapse.rest.client import login, versions from synapse.server import HomeServer -from synapse.util.clock import Clock -from synapse.types import JsonDict from synapse.synapse_rust.http_client import HttpClient +from synapse.types import JsonDict +from synapse.util.clock import Clock from tests import unittest +from tests.server import FakeChannel, TimedOutException logger = logging.getLogger(__name__) @@ -69,12 +71,40 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.admin_user = self.register_user("admin", "pass", admin=True) self.admin_user_tok = self.login("admin", "pass") - def test_unauthenticated(self) -> None: + def _request_versions(self, access_token: str | None = None) -> FakeChannel: + """ + Make a `GET /_matrix/client/versions` request and wait for it to finish. + + The endpoint is served by a Rust handler that is async and relies on the Tokio + thread pool (posting back via `reactor.callFromThread`). The default + `make_request` pump only advances the reactor's *virtual* clock in a tight loop, + never yielding real wall-clock time, so the Tokio threads never get a chance to + run and the request times out. Instead we pump the reactor while also sleeping a + little real time each iteration, the same way `wait_on_thread` and + `tests.synapse_rust.test_http_client.till_deferred_has_result` do. + """ channel = self.make_request( "GET", "/_matrix/client/versions", content={}, + access_token=access_token, + # We will pump it ourselves just below + await_result=False, ) + + deadline = time.time() + 10 + while not channel.is_finished(): + if time.time() > deadline: + raise TimedOutException("Timed out waiting for request to finish.") + # Drain anything the Tokio threads have posted back... + self.reactor.advance(0.1) + # ...then give them real time to make more progress. + time.sleep(0.01) + + return channel + + def test_unauthenticated(self) -> None: + channel = self._request_versions() self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) @@ -82,12 +112,7 @@ def test_authenticated(self) -> None: user1_id = self.register_user("user1", "pass") user1_tok = self.login(user1_id, "pass") - channel = self.make_request( - "GET", - "/_matrix/client/versions", - content={}, - access_token=user1_tok, - ) + channel = self._request_versions(access_token=user1_tok) self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) @@ -98,12 +123,7 @@ def test_authenticated_with_per_user_feature(self) -> None: user2_tok = self.login(user2_id, "pass") # Sanity check that the experimental feature should not be enabled yet - channel = self.make_request( - "GET", - "/_matrix/client/versions", - content={}, - access_token=user1_tok, - ) + channel = self._request_versions(access_token=user1_tok) self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) self.assertEqual( @@ -116,12 +136,7 @@ def test_authenticated_with_per_user_feature(self) -> None: self._enable_experimental_feature_for_user(target_user_id=user1_id) # The experimental feature should be enabled for this user - channel = self.make_request( - "GET", - "/_matrix/client/versions", - content={}, - access_token=user1_tok, - ) + channel = self._request_versions(access_token=user1_tok) self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) self.assertEqual( @@ -131,12 +146,7 @@ def test_authenticated_with_per_user_feature(self) -> None: ) # But not for other users - channel = self.make_request( - "GET", - "/_matrix/client/versions", - content={}, - access_token=user2_tok, - ) + channel = self._request_versions(access_token=user2_tok) self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) self.assertEqual( From 158b4a01d091bf93789e0a3336da36684a1ab0fb Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 17 Jun 2026 20:33:59 -0500 Subject: [PATCH 13/55] Revert "Passing `tests_versions.py`: Wait real time for tokio thread pool to work on our async Rust function" This reverts commit 0b6a9733a19785d2d51ec8ff92085ed8656db500. --- tests/rest/client/test_versions.py | 64 +++++++++++++----------------- 1 file changed, 27 insertions(+), 37 deletions(-) diff --git a/tests/rest/client/test_versions.py b/tests/rest/client/test_versions.py index bbd91ade60c..f1aecce44a8 100644 --- a/tests/rest/client/test_versions.py +++ b/tests/rest/client/test_versions.py @@ -11,19 +11,17 @@ # . import logging -import time from twisted.internet.testing import MemoryReactor from synapse.rest import admin from synapse.rest.client import login, versions from synapse.server import HomeServer -from synapse.synapse_rust.http_client import HttpClient -from synapse.types import JsonDict from synapse.util.clock import Clock +from synapse.types import JsonDict +from synapse.synapse_rust.http_client import HttpClient from tests import unittest -from tests.server import FakeChannel, TimedOutException logger = logging.getLogger(__name__) @@ -71,40 +69,12 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.admin_user = self.register_user("admin", "pass", admin=True) self.admin_user_tok = self.login("admin", "pass") - def _request_versions(self, access_token: str | None = None) -> FakeChannel: - """ - Make a `GET /_matrix/client/versions` request and wait for it to finish. - - The endpoint is served by a Rust handler that is async and relies on the Tokio - thread pool (posting back via `reactor.callFromThread`). The default - `make_request` pump only advances the reactor's *virtual* clock in a tight loop, - never yielding real wall-clock time, so the Tokio threads never get a chance to - run and the request times out. Instead we pump the reactor while also sleeping a - little real time each iteration, the same way `wait_on_thread` and - `tests.synapse_rust.test_http_client.till_deferred_has_result` do. - """ + def test_unauthenticated(self) -> None: channel = self.make_request( "GET", "/_matrix/client/versions", content={}, - access_token=access_token, - # We will pump it ourselves just below - await_result=False, ) - - deadline = time.time() + 10 - while not channel.is_finished(): - if time.time() > deadline: - raise TimedOutException("Timed out waiting for request to finish.") - # Drain anything the Tokio threads have posted back... - self.reactor.advance(0.1) - # ...then give them real time to make more progress. - time.sleep(0.01) - - return channel - - def test_unauthenticated(self) -> None: - channel = self._request_versions() self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) @@ -112,7 +82,12 @@ def test_authenticated(self) -> None: user1_id = self.register_user("user1", "pass") user1_tok = self.login(user1_id, "pass") - channel = self._request_versions(access_token=user1_tok) + channel = self.make_request( + "GET", + "/_matrix/client/versions", + content={}, + access_token=user1_tok, + ) self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) @@ -123,7 +98,12 @@ def test_authenticated_with_per_user_feature(self) -> None: user2_tok = self.login(user2_id, "pass") # Sanity check that the experimental feature should not be enabled yet - channel = self._request_versions(access_token=user1_tok) + channel = self.make_request( + "GET", + "/_matrix/client/versions", + content={}, + access_token=user1_tok, + ) self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) self.assertEqual( @@ -136,7 +116,12 @@ def test_authenticated_with_per_user_feature(self) -> None: self._enable_experimental_feature_for_user(target_user_id=user1_id) # The experimental feature should be enabled for this user - channel = self._request_versions(access_token=user1_tok) + channel = self.make_request( + "GET", + "/_matrix/client/versions", + content={}, + access_token=user1_tok, + ) self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) self.assertEqual( @@ -146,7 +131,12 @@ def test_authenticated_with_per_user_feature(self) -> None: ) # But not for other users - channel = self._request_versions(access_token=user2_tok) + channel = self.make_request( + "GET", + "/_matrix/client/versions", + content={}, + access_token=user2_tok, + ) self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) self.assertEqual( From e2ec3f1198e5a869939322865dbd8e0934fe9dae Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 17 Jun 2026 20:51:23 -0500 Subject: [PATCH 14/55] `FakeChannel.await_rust_result` Fix Tokio thread waiting a different way --- tests/rest/client/test_versions.py | 10 ++++++++++ tests/server.py | 28 ++++++++++++++++++++++++++++ tests/unittest.py | 4 ++++ 3 files changed, 42 insertions(+) diff --git a/tests/rest/client/test_versions.py b/tests/rest/client/test_versions.py index f1aecce44a8..223f8490b8f 100644 --- a/tests/rest/client/test_versions.py +++ b/tests/rest/client/test_versions.py @@ -74,7 +74,9 @@ def test_unauthenticated(self) -> None: "GET", "/_matrix/client/versions", content={}, + await_result=False, ) + channel.await_rust_result() self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) @@ -87,7 +89,9 @@ def test_authenticated(self) -> None: "/_matrix/client/versions", content={}, access_token=user1_tok, + await_result=False, ) + channel.await_rust_result() self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) @@ -103,7 +107,9 @@ def test_authenticated_with_per_user_feature(self) -> None: "/_matrix/client/versions", content={}, access_token=user1_tok, + await_result=False, ) + channel.await_rust_result() self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) self.assertEqual( @@ -121,7 +127,9 @@ def test_authenticated_with_per_user_feature(self) -> None: "/_matrix/client/versions", content={}, access_token=user1_tok, + await_result=False, ) + channel.await_rust_result() self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) self.assertEqual( @@ -136,7 +144,9 @@ def test_authenticated_with_per_user_feature(self) -> None: "/_matrix/client/versions", content={}, access_token=user2_tok, + await_result=False, ) + channel.await_rust_result() self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) self.assertEqual( diff --git a/tests/server.py b/tests/server.py index 20fcc420815..c749c4b534d 100644 --- a/tests/server.py +++ b/tests/server.py @@ -311,6 +311,34 @@ def await_result(self, timeout_ms: int = 1000) -> None: self._reactor.advance(0.1) + def await_rust_result(self, timeout_ms: int = 1000) -> None: + """ + Wait until the request is finished (a request that includes async Rust work). + + This is separate from `await_result` because we don't want to slow down the + entire test suite with real sleeps. Prefer `await_result` if possible. + + The default `await_result` only advances the reactor's *virtual* clock in a + tight loop, never yielding real wall-clock time, so the Tokio threads never get + a chance to run and the request times out. Instead we pump the reactor while + also sleeping a little real time each iteration, the same way + `HomeserverTestCase.wait_on_thread()` does. + """ + end_time = self._reactor.seconds() + timeout_ms / 1000.0 + self._reactor.run() + + while not self.is_finished(): + if self._reactor.seconds() > end_time: + raise TimedOutException("Timed out waiting for request to finish.") + + # Pump the Twisted reactor + self._reactor.advance(0.1) + + # Give some real wall-clock time for other threads to do work. This could be + # things spawned on the Twisted reactor threadpool but this is primarily + # meant for the Tokio thread pool (async Rust code). + time.sleep(0.01) + def extract_cookies(self, cookies: MutableMapping[str, str]) -> None: """Process the contents of any Set-Cookie headers in the response diff --git a/tests/unittest.py b/tests/unittest.py index 03db9a42827..5fdec2f9ea8 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -483,7 +483,11 @@ def wait_on_thread(self, deferred: Deferred, timeout: int = 10) -> None: while not deferred.called: if start_time + timeout < time.time(): raise ValueError("Timed out waiting for threadpool") + # Pump the Twisted reactor self.reactor.advance(0.01) + # Give some real wall-clock time for other threads to do work. This could be + # things spawned on the Twisted reactor threadpool or Tokio thread pool + # (async Rust code). time.sleep(0.01) def wait_for_background_updates(self) -> None: From 1dcf17d9d88c90424820a5ba2643be3355be28db Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 18 Jun 2026 18:29:20 -0500 Subject: [PATCH 15/55] Rename: `await_rust_result` -> `await_result_with_rust` --- tests/rest/client/test_versions.py | 10 +++++----- tests/server.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/rest/client/test_versions.py b/tests/rest/client/test_versions.py index 223f8490b8f..535d8f1a121 100644 --- a/tests/rest/client/test_versions.py +++ b/tests/rest/client/test_versions.py @@ -76,7 +76,7 @@ def test_unauthenticated(self) -> None: content={}, await_result=False, ) - channel.await_rust_result() + channel.await_result_with_rust() self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) @@ -91,7 +91,7 @@ def test_authenticated(self) -> None: access_token=user1_tok, await_result=False, ) - channel.await_rust_result() + channel.await_result_with_rust() self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) @@ -109,7 +109,7 @@ def test_authenticated_with_per_user_feature(self) -> None: access_token=user1_tok, await_result=False, ) - channel.await_rust_result() + channel.await_result_with_rust() self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) self.assertEqual( @@ -129,7 +129,7 @@ def test_authenticated_with_per_user_feature(self) -> None: access_token=user1_tok, await_result=False, ) - channel.await_rust_result() + channel.await_result_with_rust() self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) self.assertEqual( @@ -146,7 +146,7 @@ def test_authenticated_with_per_user_feature(self) -> None: access_token=user2_tok, await_result=False, ) - channel.await_rust_result() + channel.await_result_with_rust() self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) self.assertEqual( diff --git a/tests/server.py b/tests/server.py index c749c4b534d..dfbe1f7bba7 100644 --- a/tests/server.py +++ b/tests/server.py @@ -311,7 +311,7 @@ def await_result(self, timeout_ms: int = 1000) -> None: self._reactor.advance(0.1) - def await_rust_result(self, timeout_ms: int = 1000) -> None: + def await_result_with_rust(self, timeout_ms: int = 1000) -> None: """ Wait until the request is finished (a request that includes async Rust work). From 357860cb8eb634e1afdc2a068bfa8400f4ce19ad Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 18 Jun 2026 19:37:03 -0500 Subject: [PATCH 16/55] Specify features to enable --- tests/rest/client/test_versions.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/rest/client/test_versions.py b/tests/rest/client/test_versions.py index 535d8f1a121..b21c5e8271e 100644 --- a/tests/rest/client/test_versions.py +++ b/tests/rest/client/test_versions.py @@ -119,7 +119,9 @@ def test_authenticated_with_per_user_feature(self) -> None: ) # Enable the feature for this specific user - self._enable_experimental_feature_for_user(target_user_id=user1_id) + self._enable_experimental_feature_for_user( + target_user_id=user1_id, features={"msc3881": True} + ) # The experimental feature should be enabled for this user channel = self.make_request( @@ -170,7 +172,9 @@ def _sanity_check_versions_response(self, versions_response: JsonDict) -> None: f"Expected `unstable_features` to be a dict mapping feature name to a bool but saw {versions_response}", ) - def _enable_experimental_feature_for_user(self, *, target_user_id: str) -> None: + def _enable_experimental_feature_for_user( + self, *, target_user_id: str, features: dict[str, bool] + ) -> None: """ Use the admin API to enable an experimental feature for a specific user """ @@ -178,7 +182,7 @@ def _enable_experimental_feature_for_user(self, *, target_user_id: str) -> None: "PUT", f"/_synapse/admin/v1/experimental_features/{target_user_id}", content={ - "features": {"msc3881": True}, + "features": features, }, access_token=self.admin_user_tok, ) From ca1371e1a386ac055cd5f62b720321a7c21a89e3 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 18 Jun 2026 19:55:00 -0500 Subject: [PATCH 17/55] Refactor `deferred` and `tokio_runtime` to their own crates --- rust/src/deferred.rs | 252 ++++++++++++++++++++ rust/src/handlers/versions.rs | 2 +- rust/src/http_client.rs | 322 +------------------------- rust/src/lib.rs | 3 + rust/src/storage/db/python_db_pool.rs | 2 +- rust/src/tokio_runtime.rs | 107 +++++++++ 6 files changed, 368 insertions(+), 320 deletions(-) create mode 100644 rust/src/deferred.rs create mode 100644 rust/src/tokio_runtime.rs diff --git a/rust/src/deferred.rs b/rust/src/deferred.rs new file mode 100644 index 00000000000..50898ab0f64 --- /dev/null +++ b/rust/src/deferred.rs @@ -0,0 +1,252 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use std::{ + future::Future, + sync::{Arc, Mutex, OnceLock}, +}; + +use once_cell::sync::OnceCell; +use pyo3::{ + create_exception, exceptions::PyException, exceptions::PyRuntimeError, intern, prelude::*, + types::PyCFunction, +}; +use tokio::sync::oneshot; + +use crate::tokio_runtime::runtime; + +create_exception!( + synapse.synapse_rust.http_client, + RustPanicError, + PyException, + "A panic which happened in a Rust future" +); + +impl RustPanicError { + fn from_panic(panic_err: &(dyn std::any::Any + Send + 'static)) -> PyErr { + // Apparently this is how you extract the panic message from a panic + let panic_message = if let Some(str_slice) = panic_err.downcast_ref::<&str>() { + str_slice + } else if let Some(string) = panic_err.downcast_ref::() { + string + } else { + "unknown error" + }; + Self::new_err(panic_message.to_owned()) + } +} + +/// A reference to the `twisted.internet.defer` module. +static DEFER: OnceCell> = OnceCell::new(); + +/// Access to the `twisted.internet.defer` module. +fn defer(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + Ok(DEFER + .get_or_try_init(|| py.import("twisted.internet.defer").map(Into::into))? + .bind(py)) +} + +/// Creates a twisted deferred from the given future, spawning the task on the +/// tokio runtime. +/// +/// Does not handle deferred cancellation or contextvars. +pub(crate) fn create_deferred<'py, F, O>( + py: Python<'py>, + reactor: &Bound<'py, PyAny>, + fut: F, +) -> PyResult> +where + F: Future> + Send + 'static, + for<'a> O: IntoPyObject<'a> + Send + 'static, +{ + let deferred = defer(py)?.call_method0("Deferred")?; + let deferred_callback = deferred.getattr("callback")?.unbind(); + let deferred_errback = deferred.getattr("errback")?.unbind(); + + let rt = runtime(reactor)?; + let handle = rt.handle()?; + let task = handle.spawn(fut); + + // Unbind the reactor so that we can pass it to the task + let reactor = reactor.clone().unbind(); + handle.spawn(async move { + let res = task.await; + + Python::attach(move |py| { + // Flatten the panic into standard python error + let res = match res { + Ok(r) => r, + Err(join_err) => match join_err.try_into_panic() { + Ok(panic_err) => Err(RustPanicError::from_panic(&panic_err)), + Err(err) => Err(PyException::new_err(format!("Task cancelled: {err}"))), + }, + }; + + // Re-bind the reactor + let reactor = reactor.bind(py); + + // Send the result to the deferred, via `.callback(..)` or `.errback(..)` + match res { + Ok(obj) => { + reactor + .call_method("callFromThread", (deferred_callback, obj), None) + .expect("callFromThread should not fail"); // There's nothing we can really do with errors here + } + Err(err) => { + reactor + .call_method("callFromThread", (deferred_errback, err), None) + .expect("callFromThread should not fail"); // There's nothing we can really do with errors here + } + } + }); + }); + + // Make the deferred follow the Synapse logcontext rules + make_deferred_yieldable(py, &deferred) +} + +/// Runs `make_deferred` on the Twisted reactor thread to obtain a Deferred (or +/// coroutine), then resolves once that Deferred fires. +/// +/// This is the inverse of [`create_deferred`]: where that turns a Rust future +/// into a Twisted Deferred, this turns a Twisted Deferred into an awaitable Rust +/// future. +/// +/// We're called on a Tokio worker thread, but Twisted `Deferred`s (and the +/// coroutine that `ensureDeferred` drives) are not thread-safe and Synapse's +/// logcontext is thread-local, so the coroutine must both start and resume on +/// the reactor thread. The `callFromThread` hop is what gets us there for the +/// kickoff; the deferred's own callbacks then fire on the reactor thread too. +/// (Note this is unrelated to offloading the DB work onto a thread — that's +/// handled internally by whatever `make_deferred` calls, e.g. `runInteraction`.) +/// +/// `make_deferred` is invoked on the reactor thread and may return either a +/// coroutine or a `Deferred`; `ensureDeferred` normalises both to a `Deferred`. +pub(crate) async fn await_deferred(reactor: Py, make_deferred: F) -> PyResult> +where + F: for<'py> Fn(Python<'py>) -> PyResult> + Send + 'static, +{ + // Resolves when the deferred fires; carries the resolved value or the error. + let (tx, rx) = oneshot::channel::>>(); + // Shared between the success and error callbacks (only one ever fires). + let sender = Arc::new(Mutex::new(Some(tx))); + + Python::attach(|py| -> PyResult<()> { + let success_sender = Arc::clone(&sender); + let on_success = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let value = args.get_item(0)?.unbind(); + if let Some(tx) = success_sender.lock().unwrap().take() { + let _ = tx.send(Ok(value)); + } + Ok(args.py().None()) + }, + )? + .unbind(); + + let error_sender = Arc::clone(&sender); + let on_error = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let err = failure_to_pyerr(&args.get_item(0)?); + if let Some(tx) = error_sender.lock().unwrap().take() { + let _ = tx.send(Err(err)); + } + Ok(args.py().None()) + }, + )? + .unbind(); + + let starter = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let py = args.py(); + let deferred = defer(py)? + .call_method1(intern!(py, "ensureDeferred"), (make_deferred(py)?,))?; + deferred.call_method1( + intern!(py, "addCallbacks"), + (on_success.bind(py), on_error.bind(py)), + )?; + Ok(py.None()) + }, + )?; + + reactor + .bind(py) + .call_method1(intern!(py, "callFromThread"), (starter,))?; + + Ok(()) + })?; + + match rx.await { + Ok(result) => result, + Err(_) => Err(PyRuntimeError::new_err( + "await_deferred channel closed before the deferred fired", + )), + } +} + +/// Convert a Twisted `Failure` (as passed to an errback) into a [`PyErr`]. +/// +/// A `Failure` carries the original exception instance in its `.value` +/// attribute, which we re-raise so callers see the real error. If that can't be +/// reached, fall back to the `Failure`'s textual representation. +fn failure_to_pyerr(failure: &Bound<'_, PyAny>) -> PyErr { + match failure.getattr(intern!(failure.py(), "value")) { + Ok(value) => PyErr::from_value(value), + Err(_) => PyRuntimeError::new_err( + failure + .str() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|_| "".to_owned()), + ), + } +} + +static MAKE_DEFERRED_YIELDABLE: OnceLock> = OnceLock::new(); + +/// Given a deferred, make it follow the Synapse logcontext rules +fn make_deferred_yieldable<'py>( + py: Python<'py>, + deferred: &Bound<'py, PyAny>, +) -> PyResult> { + let make_deferred_yieldable = MAKE_DEFERRED_YIELDABLE.get_or_init(|| { + let sys = PyModule::import(py, "synapse.logging.context").unwrap(); + let func = sys.getattr("make_deferred_yieldable").unwrap().unbind(); + func + }); + + make_deferred_yieldable + .call1(py, (deferred,))? + .extract(py) + .map_err(Into::into) +} + +/// Called when registering modules with python. +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + _ = m; + + // Make sure we fail early if we can't load some modules + defer(py)?; + + Ok(()) +} diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index 78a80116e1a..77faaf2c146 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -20,7 +20,7 @@ use pythonize::{pythonize, PythonizeError}; use serde::{Deserialize, Serialize}; use crate::config::SynapseConfig; -use crate::http_client::create_deferred; +use crate::deferred::create_deferred; use crate::storage::db::python_db_pool::PythonDatabasePoolWrapper; use crate::storage::store::{PerUserExperimentalFeature, Store}; diff --git a/rust/src/http_client.rs b/rust/src/http_client.rs index be3bf1c1bfb..aaa1066a676 100644 --- a/rust/src/http_client.rs +++ b/rust/src/http_client.rs @@ -12,153 +12,22 @@ * . */ -use std::{ - collections::HashMap, - future::Future, - sync::{Arc, Mutex, OnceLock}, -}; +use std::collections::HashMap; use anyhow::Context; use http_body_util::BodyExt; -use once_cell::sync::OnceCell; -use pyo3::{ - create_exception, exceptions::PyException, exceptions::PyRuntimeError, intern, prelude::*, - types::PyCFunction, -}; +use pyo3::prelude::*; use reqwest::RequestBuilder; -use tokio::runtime::Runtime; -use tokio::sync::oneshot; +use crate::deferred::create_deferred; use crate::errors::HttpResponseException; - -create_exception!( - synapse.synapse_rust.http_client, - RustPanicError, - PyException, - "A panic which happened in a Rust future" -); - -impl RustPanicError { - fn from_panic(panic_err: &(dyn std::any::Any + Send + 'static)) -> PyErr { - // Apparently this is how you extract the panic message from a panic - let panic_message = if let Some(str_slice) = panic_err.downcast_ref::<&str>() { - str_slice - } else if let Some(string) = panic_err.downcast_ref::() { - string - } else { - "unknown error" - }; - Self::new_err(panic_message.to_owned()) - } -} - -/// This is the name of the attribute where we store the runtime on the reactor -static TOKIO_RUNTIME_ATTR: &str = "__synapse_rust_tokio_runtime"; - -/// A Python wrapper around a Tokio runtime. -/// -/// This allows us to 'store' the runtime on the reactor instance, starting it -/// when the reactor starts, and stopping it when the reactor shuts down. -#[pyclass] -struct PyTokioRuntime { - runtime: Option, -} - -#[pymethods] -impl PyTokioRuntime { - fn start(&mut self) -> PyResult<()> { - // TODO: allow customization of the runtime like the number of threads - let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(4) - .enable_all() - .build()?; - - self.runtime = Some(runtime); - - Ok(()) - } - - fn shutdown(&mut self) -> PyResult<()> { - let runtime = self - .runtime - .take() - .context("Runtime was already shutdown")?; - - // Dropping the runtime will shut it down - drop(runtime); - - Ok(()) - } -} - -impl PyTokioRuntime { - /// Get the handle to the Tokio runtime, if it is running. - fn handle(&self) -> PyResult<&tokio::runtime::Handle> { - let handle = self - .runtime - .as_ref() - .context("Tokio runtime is not running")? - .handle(); - - Ok(handle) - } -} - -/// Get a handle to the Tokio runtime stored on the reactor instance, or create -/// a new one. -fn runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult> { - if !reactor.hasattr(TOKIO_RUNTIME_ATTR)? { - install_runtime(reactor)?; - } - - get_runtime(reactor) -} - -/// Install a new Tokio runtime on the reactor instance. -fn install_runtime(reactor: &Bound) -> PyResult<()> { - let py = reactor.py(); - let runtime = PyTokioRuntime { runtime: None }; - let runtime = runtime.into_pyobject(py)?; - - // Attach the runtime to the reactor, starting it when the reactor is - // running, stopping it when the reactor is shutting down - reactor.call_method1("callWhenRunning", (runtime.getattr("start")?,))?; - reactor.call_method1( - "addSystemEventTrigger", - ("after", "shutdown", runtime.getattr("shutdown")?), - )?; - reactor.setattr(TOKIO_RUNTIME_ATTR, runtime)?; - - Ok(()) -} - -/// Get a reference to a Tokio runtime handle stored on the reactor instance. -fn get_runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult> { - // This will raise if `TOKIO_RUNTIME_ATTR` is not set or if it is - // not a `Runtime`. Careful that this could happen if the user sets it - // manually, or if multiple versions of `pyo3-twisted` are used! - let runtime: Bound = reactor.getattr(TOKIO_RUNTIME_ATTR)?.extract()?; - Ok(runtime.borrow()) -} - -/// A reference to the `twisted.internet.defer` module. -static DEFER: OnceCell> = OnceCell::new(); - -/// Access to the `twisted.internet.defer` module. -fn defer(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { - Ok(DEFER - .get_or_try_init(|| py.import("twisted.internet.defer").map(Into::into))? - .bind(py)) -} +use crate::tokio_runtime::runtime; /// Called when registering modules with python. pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { let child_module: Bound<'_, PyModule> = PyModule::new(py, "http_client")?; child_module.add_class::()?; - // Make sure we fail early if we can't load some modules - defer(py)?; - m.add_submodule(&child_module)?; // We need to manually add the module to sys.modules to make `from @@ -270,186 +139,3 @@ impl HttpClient { }) } } - -/// Creates a twisted deferred from the given future, spawning the task on the -/// tokio runtime. -/// -/// Does not handle deferred cancellation or contextvars. -pub(crate) fn create_deferred<'py, F, O>( - py: Python<'py>, - reactor: &Bound<'py, PyAny>, - fut: F, -) -> PyResult> -where - F: Future> + Send + 'static, - for<'a> O: IntoPyObject<'a> + Send + 'static, -{ - let deferred = defer(py)?.call_method0("Deferred")?; - let deferred_callback = deferred.getattr("callback")?.unbind(); - let deferred_errback = deferred.getattr("errback")?.unbind(); - - let rt = runtime(reactor)?; - let handle = rt.handle()?; - let task = handle.spawn(fut); - - // Unbind the reactor so that we can pass it to the task - let reactor = reactor.clone().unbind(); - handle.spawn(async move { - let res = task.await; - - Python::attach(move |py| { - // Flatten the panic into standard python error - let res = match res { - Ok(r) => r, - Err(join_err) => match join_err.try_into_panic() { - Ok(panic_err) => Err(RustPanicError::from_panic(&panic_err)), - Err(err) => Err(PyException::new_err(format!("Task cancelled: {err}"))), - }, - }; - - // Re-bind the reactor - let reactor = reactor.bind(py); - - // Send the result to the deferred, via `.callback(..)` or `.errback(..)` - match res { - Ok(obj) => { - reactor - .call_method("callFromThread", (deferred_callback, obj), None) - .expect("callFromThread should not fail"); // There's nothing we can really do with errors here - } - Err(err) => { - reactor - .call_method("callFromThread", (deferred_errback, err), None) - .expect("callFromThread should not fail"); // There's nothing we can really do with errors here - } - } - }); - }); - - // Make the deferred follow the Synapse logcontext rules - make_deferred_yieldable(py, &deferred) -} - -/// Runs `make_deferred` on the Twisted reactor thread to obtain a Deferred (or -/// coroutine), then resolves once that Deferred fires. -/// -/// This is the inverse of [`create_deferred`]: where that turns a Rust future -/// into a Twisted Deferred, this turns a Twisted Deferred into an awaitable Rust -/// future. -/// -/// We're called on a Tokio worker thread, but Twisted `Deferred`s (and the -/// coroutine that `ensureDeferred` drives) are not thread-safe and Synapse's -/// logcontext is thread-local, so the coroutine must both start and resume on -/// the reactor thread. The `callFromThread` hop is what gets us there for the -/// kickoff; the deferred's own callbacks then fire on the reactor thread too. -/// (Note this is unrelated to offloading the DB work onto a thread — that's -/// handled internally by whatever `make_deferred` calls, e.g. `runInteraction`.) -/// -/// `make_deferred` is invoked on the reactor thread and may return either a -/// coroutine or a `Deferred`; `ensureDeferred` normalises both to a `Deferred`. -pub(crate) async fn await_deferred(reactor: Py, make_deferred: F) -> PyResult> -where - F: for<'py> Fn(Python<'py>) -> PyResult> + Send + 'static, -{ - // Resolves when the deferred fires; carries the resolved value or the error. - let (tx, rx) = oneshot::channel::>>(); - // Shared between the success and error callbacks (only one ever fires). - let sender = Arc::new(Mutex::new(Some(tx))); - - Python::attach(|py| -> PyResult<()> { - let success_sender = Arc::clone(&sender); - let on_success = PyCFunction::new_closure( - py, - None, - None, - move |args, _kwargs| -> PyResult> { - let value = args.get_item(0)?.unbind(); - if let Some(tx) = success_sender.lock().unwrap().take() { - let _ = tx.send(Ok(value)); - } - Ok(args.py().None()) - }, - )? - .unbind(); - - let error_sender = Arc::clone(&sender); - let on_error = PyCFunction::new_closure( - py, - None, - None, - move |args, _kwargs| -> PyResult> { - let err = failure_to_pyerr(&args.get_item(0)?); - if let Some(tx) = error_sender.lock().unwrap().take() { - let _ = tx.send(Err(err)); - } - Ok(args.py().None()) - }, - )? - .unbind(); - - let starter = PyCFunction::new_closure( - py, - None, - None, - move |args, _kwargs| -> PyResult> { - let py = args.py(); - let deferred = defer(py)? - .call_method1(intern!(py, "ensureDeferred"), (make_deferred(py)?,))?; - deferred.call_method1( - intern!(py, "addCallbacks"), - (on_success.bind(py), on_error.bind(py)), - )?; - Ok(py.None()) - }, - )?; - - reactor - .bind(py) - .call_method1(intern!(py, "callFromThread"), (starter,))?; - - Ok(()) - })?; - - match rx.await { - Ok(result) => result, - Err(_) => Err(PyRuntimeError::new_err( - "await_deferred channel closed before the deferred fired", - )), - } -} - -/// Convert a Twisted `Failure` (as passed to an errback) into a [`PyErr`]. -/// -/// A `Failure` carries the original exception instance in its `.value` -/// attribute, which we re-raise so callers see the real error. If that can't be -/// reached, fall back to the `Failure`'s textual representation. -fn failure_to_pyerr(failure: &Bound<'_, PyAny>) -> PyErr { - match failure.getattr(intern!(failure.py(), "value")) { - Ok(value) => PyErr::from_value(value), - Err(_) => PyRuntimeError::new_err( - failure - .str() - .map(|s| s.to_string_lossy().into_owned()) - .unwrap_or_else(|_| "".to_owned()), - ), - } -} - -static MAKE_DEFERRED_YIELDABLE: OnceLock> = OnceLock::new(); - -/// Given a deferred, make it follow the Synapse logcontext rules -fn make_deferred_yieldable<'py>( - py: Python<'py>, - deferred: &Bound<'py, PyAny>, -) -> PyResult> { - let make_deferred_yieldable = MAKE_DEFERRED_YIELDABLE.get_or_init(|| { - let sys = PyModule::import(py, "synapse.logging.context").unwrap(); - let func = sys.getattr("make_deferred_yieldable").unwrap().unbind(); - func - }); - - make_deferred_yieldable - .call1(py, (deferred,))? - .extract(py) - .map_err(Into::into) -} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 4ffddb9af06..28783afbbac 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -7,6 +7,7 @@ use pyo3_log::ResetHandle; pub mod acl; pub mod canonical_json; pub mod config; +pub mod deferred; pub mod duration; pub mod errors; pub mod events; @@ -22,6 +23,7 @@ pub mod rendezvous; pub mod room_versions; pub mod segmenter; pub mod storage; +pub mod tokio_runtime; pub mod types; lazy_static! { @@ -68,6 +70,7 @@ fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(reset_logging_config, m)?)?; acl::register_module(py, m)?; + deferred::register_module(py, m)?; push::register_module(py, m)?; events::register_module(py, m)?; handlers::register_module(py, m)?; diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index 07fef298b23..a5f95a04ef8 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -27,7 +27,7 @@ use pyo3::{ types::{PyBool, PyCFunction, PyFloat, PyInt, PyList, PyString}, }; -use crate::http_client::await_deferred; +use crate::deferred::await_deferred; use crate::storage::db::{DatabasePool, Row, Transaction, Value}; /// The database engines we support in the Python side of Synapse diff --git a/rust/src/tokio_runtime.rs b/rust/src/tokio_runtime.rs new file mode 100644 index 00000000000..6be8a4e21f1 --- /dev/null +++ b/rust/src/tokio_runtime.rs @@ -0,0 +1,107 @@ +/* + * This file is licensed under the Affero General Public License (AGPL) version 3. + * + * Copyright (C) 2026 Element Creations Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * See the GNU Affero General Public License for more details: + * . + * + */ + +use anyhow::Context; +use pyo3::prelude::*; +use tokio::runtime::Runtime; + +/// This is the name of the attribute where we store the runtime on the reactor +static TOKIO_RUNTIME_ATTR: &str = "__synapse_rust_tokio_runtime"; + +/// A Python wrapper around a Tokio runtime. +/// +/// This allows us to 'store' the runtime on the reactor instance, starting it +/// when the reactor starts, and stopping it when the reactor shuts down. +#[pyclass] +pub struct PyTokioRuntime { + runtime: Option, +} + +#[pymethods] +impl PyTokioRuntime { + fn start(&mut self) -> PyResult<()> { + // TODO: allow customization of the runtime like the number of threads + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build()?; + + self.runtime = Some(runtime); + + Ok(()) + } + + fn shutdown(&mut self) -> PyResult<()> { + let runtime = self + .runtime + .take() + .context("Runtime was already shutdown")?; + + // Dropping the runtime will shut it down + drop(runtime); + + Ok(()) + } +} + +impl PyTokioRuntime { + /// Get the handle to the Tokio runtime, if it is running. + pub fn handle(&self) -> PyResult<&tokio::runtime::Handle> { + let handle = self + .runtime + .as_ref() + .context("Tokio runtime is not running")? + .handle(); + + Ok(handle) + } +} + +/// Install a new Tokio runtime on the reactor instance. +fn install_runtime(reactor: &Bound) -> PyResult<()> { + let py = reactor.py(); + let runtime = PyTokioRuntime { runtime: None }; + let runtime = runtime.into_pyobject(py)?; + + // Attach the runtime to the reactor, starting it when the reactor is + // running, stopping it when the reactor is shutting down + reactor.call_method1("callWhenRunning", (runtime.getattr("start")?,))?; + reactor.call_method1( + "addSystemEventTrigger", + ("after", "shutdown", runtime.getattr("shutdown")?), + )?; + reactor.setattr(TOKIO_RUNTIME_ATTR, runtime)?; + + Ok(()) +} + +/// Get a reference to a Tokio runtime handle stored on the reactor instance. +fn get_runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult> { + // This will raise if `TOKIO_RUNTIME_ATTR` is not set or if it is + // not a `Runtime`. Careful that this could happen if the user sets it + // manually, or if multiple versions of `pyo3-twisted` are used! + let runtime: Bound = reactor.getattr(TOKIO_RUNTIME_ATTR)?.extract()?; + Ok(runtime.borrow()) +} + +/// Get a handle to the Tokio runtime stored on the reactor instance, or create +/// a new one. +pub fn runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult> { + if !reactor.hasattr(TOKIO_RUNTIME_ATTR)? { + install_runtime(reactor)?; + } + + get_runtime(reactor) +} From f76c8c160b20224fae792869f11a1fd7ef3a2f45 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 18 Jun 2026 21:03:48 -0500 Subject: [PATCH 18/55] Refactor to remove `SynapseConfig` from shared code --- rust/src/handlers/mod.rs | 11 +- rust/src/handlers/versions.rs | 312 +++++++++++++++++++--------------- rust/src/storage/store.rs | 28 +-- 3 files changed, 191 insertions(+), 160 deletions(-) diff --git a/rust/src/handlers/mod.rs b/rust/src/handlers/mod.rs index e5243f87e1f..1f04b65e109 100644 --- a/rust/src/handlers/mod.rs +++ b/rust/src/handlers/mod.rs @@ -52,15 +52,16 @@ impl RustHandlers { let db_pool = PythonDatabasePoolWrapper::new(db_pool_py, reactor.clone_ref(py)); // Store is shared across all of the handlers so let's use an `Arc` - let store = Arc::new(Store { - config: config.clone(), - db_pool, - }); + let store = Arc::new(Store { db_pool }); + + let global_unstable_feature_map = Arc::new( + versions::synapse_config_to_global_unstable_feature_map(&config), + ); let versions = Py::new( py, versions::VersionsHandler { - config: config.clone(), + global_unstable_feature_map: Arc::clone(&global_unstable_feature_map), store: Arc::clone(&store), reactor: reactor.clone_ref(py), }, diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index 77faaf2c146..e2d8cd1b7d1 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -44,7 +44,7 @@ impl<'py> IntoPyObject<'py> for VersionsResponse { #[pyclass] pub struct VersionsHandler { - pub config: SynapseConfig, + pub global_unstable_feature_map: Arc, pub store: Arc>, /// The Twisted reactor, used to bridge our `async` response back into a /// Twisted deferred that Python can `await`. @@ -62,10 +62,10 @@ impl VersionsHandler { user_id: Option, ) -> PyResult> { let store = Arc::clone(&self.store); - let config = self.config.clone(); + let global_unstable_feature_map = Arc::clone(&self.global_unstable_feature_map); create_deferred(py, self.reactor.bind(py), async move { - build_versions_response(&store, &config, user_id.as_deref()) + build_versions_response(&store, &global_unstable_feature_map, user_id.as_deref()) .await .map_err(|err| { pyo3::exceptions::PyRuntimeError::new_err(format!( @@ -77,148 +77,190 @@ impl VersionsHandler { } /// Assemble a `/versions` response body. +/// +/// Args: +/// * store +/// * global_unstable_feature_map: The global values before any per-user overrides +/// * user_id: The user making the request async fn build_versions_response( store: &Store, - config: &SynapseConfig, + global_unstable_feature_map: &UnstableFeatureMap, user_id: Option<&str>, ) -> Result { - { - let msc3881_enabled = match user_id { - Some(user_id) => { - store - .is_feature_enabled(user_id, PerUserExperimentalFeature::MSC3881) + let msc3881_enabled = match user_id { + Some(user_id) => { + // Don't both looking anything up if it's enabled for everyone + if global_unstable_feature_map.msc3881 { + true + } else { + // Look up whether it's explicitly enabled/disabled for this user + match store + .is_feature_enabled_for_user(user_id, PerUserExperimentalFeature::MSC3881) .await? + { + Some(enabled) => enabled, + // Default to false if there is no entry for this user + None => false, + } } - None => PerUserExperimentalFeature::MSC3881.is_globally_enabled(config), - }; + } + None => global_unstable_feature_map.msc3881, + }; - let msc3575_enabled = match user_id { - Some(user_id) => { - store - .is_feature_enabled(user_id, PerUserExperimentalFeature::MSC3575) + let msc3575_enabled = match user_id { + Some(user_id) => { + // Don't both looking anything up if it's enabled for everyone + if global_unstable_feature_map.msc3575 { + true + } else { + // Look up whether it's explicitly enabled/disabled for this user + match store + .is_feature_enabled_for_user(user_id, PerUserExperimentalFeature::MSC3575) .await? + { + Some(enabled) => enabled, + // Default to false if there is no entry for this user + None => false, + } } - None => PerUserExperimentalFeature::MSC3575.is_globally_enabled(config), - }; + } + None => global_unstable_feature_map.msc3575, + }; - // TODO: Calculate these once since they shouldn't change after start-up. - // e2ee_forced_public = ( - // RoomCreationPreset.PUBLIC_CHAT - // in config.room.encryption_enabled_by_default_for_room_presets - // ); - // e2ee_forced_private = ( - // RoomCreationPreset.PRIVATE_CHAT - // in config.room.encryption_enabled_by_default_for_room_presets - // ); - // e2ee_forced_trusted_private = ( - // RoomCreationPreset.TRUSTED_PRIVATE_CHAT - // in config.room.encryption_enabled_by_default_for_room_presets - // ); + // TODO: Calculate these once since they shouldn't change after start-up. + // e2ee_forced_public = ( + // RoomCreationPreset.PUBLIC_CHAT + // in config.room.encryption_enabled_by_default_for_room_presets + // ); + // e2ee_forced_private = ( + // RoomCreationPreset.PRIVATE_CHAT + // in config.room.encryption_enabled_by_default_for_room_presets + // ); + // e2ee_forced_trusted_private = ( + // RoomCreationPreset.TRUSTED_PRIVATE_CHAT + // in config.room.encryption_enabled_by_default_for_room_presets + // ); - Ok(VersionsResponse { - versions: Vec::from([ - // XXX: at some point we need to decide whether we need to include - // the previous version numbers, given we've defined r0.3.0 to be - // backwards compatible with r0.2.0. But need to check how - // conscientious we've been in compatibility, and decide whether the - // middle number is the major revision when at 0.X.Y (as opposed to - // X.Y.Z). And we need to decide whether it's fair to make clients - // parse the version string to figure out what's going on. - "r0.0.1".to_string(), - "r0.1.0".to_string(), - "r0.2.0".to_string(), - "r0.3.0".to_string(), - "r0.4.0".to_string(), - "r0.5.0".to_string(), - "r0.6.0".to_string(), - "r0.6.1".to_string(), - "v1.1".to_string(), - "v1.2".to_string(), - "v1.3".to_string(), - "v1.4".to_string(), - "v1.5".to_string(), - "v1.6".to_string(), - "v1.7".to_string(), - "v1.8".to_string(), - "v1.9".to_string(), - "v1.10".to_string(), - "v1.11".to_string(), - "v1.12".to_string(), - ]), - unstable_features: std::collections::BTreeMap::from([ - // // Implements support for label-based filtering as described in - // // MSC2326. - // ("org.matrix.label_based_filtering".to_string(), true), - // // Implements support for cross signing as described in MSC1756 - // ("org.matrix.e2e_cross_signing".to_string(), true), - // // Implements additional endpoints as described in MSC2432 - // ("org.matrix.msc2432".to_string(), true), - // // Implements additional endpoints as described in MSC2666 - // ("uk.half-shot.msc2666.query_mutual_rooms.stable".to_string(), true), - // // Whether new rooms will be set to encrypted or not (based on presets). - // ("io.element.e2ee_forced.public".to_string(), e2ee_forced_public), - // ("io.element.e2ee_forced.private".to_string(), e2ee_forced_private), - // ("io.element.e2ee_forced.trusted_private".to_string(), e2ee_forced_trusted_private), - // // Supports the busy presence state described in MSC3026. - // ("org.matrix.msc3026.busy_presence".to_string(), config.experimental.msc3026_enabled), - // // Supports receiving private read receipts as per MSC2285 - // ("org.matrix.msc2285.stable".to_string(), true), // TODO: Remove when MSC2285 becomes a part of the spec - // // Supports filtering of /publicRooms by room type as per MSC3827 - // ("org.matrix.msc3827.stable".to_string(), true), - // // Adds support for thread relations, per MSC3440. - // ("org.matrix.msc3440.stable".to_string(), true), // TODO: remove when "v1.3" is added above - // // Support for thread read receipts & notification counts. - // ("org.matrix.msc3771".to_string(), true), - // ("org.matrix.msc3773".to_string(), config.experimental.msc3773_enabled), - // // Allows moderators to fetch redacted event content as described in MSC2815 - // ("fi.mau.msc2815".to_string(), config.experimental.msc2815_enabled), - // // Adds a ping endpoint for appservices to check HS->AS connection - // ("fi.mau.msc2659.stable".to_string(), true), // TODO: remove when "v1.7" is added above - // // TODO: this is no longer needed once unstable MSC3882 does not need to be supported: - // ("org.matrix.msc3882".to_string(), config.auth.login_via_existing_enabled), - // Adds support for remotely enabling/disabling pushers, as per MSC3881 - ("org.matrix.msc3881".to_string(), msc3881_enabled), - // // Adds support for filtering /messages by event relation. - // ("org.matrix.msc3874".to_string(), config.experimental.msc3874_enabled), - // // Adds support for relation-based redactions as per MSC3912. - // ("org.matrix.msc3912".to_string(), config.experimental.msc3912_enabled), - // // Whether recursively provide relations is supported. - // // TODO This is no longer needed once unstable MSC3981 does not need to be supported. - // ("org.matrix.msc3981".to_string(), true), - // // Adds support for deleting account data. - // ("org.matrix.msc3391".to_string(), config.experimental.msc3391_enabled), - // // Allows clients to inhibit profile update propagation. - // ("org.matrix.msc4069".to_string(), config.experimental.msc4069_profile_inhibit_propagation), - // // Allows clients to handle push for encrypted events. - // ("org.matrix.msc4028".to_string(), config.experimental.msc4028_push_encrypted_events), - // // MSC4108: Mechanism to allow OIDC sign in and E2EE set up via QR code - 2024 version - // ("org.matrix.msc4108".to_string(), ( - // config.experimental.msc4108_enabled - // or ( - // config.experimental.msc4108_delegation_endpoint - // is not None - // ) - // )), - // // MSC4140: Delayed events - // ("org.matrix.msc4140".to_string(), bool(config.server.max_event_delay_ms)), - // Simplified sliding sync - ("org.matrix.simplified_msc3575".to_string(), msc3575_enabled), - // // Arbitrary key-value profile fields. - // ("uk.tcpip.msc4133".to_string(), config.experimental.msc4133_enabled), - // ("uk.tcpip.msc4133.stable".to_string(), true), - // // MSC4155: Invite filtering - // ("org.matrix.msc4155".to_string(), config.experimental.msc4155_enabled), - // // MSC4306: Support for thread subscriptions - // ("org.matrix.msc4306".to_string(), config.experimental.msc4306_enabled), - // // MSC4169: Backwards-compatible redaction sending using `/send` - // ("com.beeper.msc4169".to_string(), config.experimental.msc4169_enabled), - // // MSC4354: Sticky events - // ("org.matrix.msc4354".to_string(), config.experimental.msc4354_enabled), - // // MSC4380: Invite blocking - // ("org.matrix.msc4380.stable".to_string(), true), - // // MSC4445: Sync timeline order - // ("org.matrix.msc4445.initial_sync_timeline_topological_ordering".to_string(), true), - ]), - }) + Ok(VersionsResponse { + versions: Vec::from([ + // XXX: at some point we need to decide whether we need to include + // the previous version numbers, given we've defined r0.3.0 to be + // backwards compatible with r0.2.0. But need to check how + // conscientious we've been in compatibility, and decide whether the + // middle number is the major revision when at 0.X.Y (as opposed to + // X.Y.Z). And we need to decide whether it's fair to make clients + // parse the version string to figure out what's going on. + "r0.0.1".to_string(), + "r0.1.0".to_string(), + "r0.2.0".to_string(), + "r0.3.0".to_string(), + "r0.4.0".to_string(), + "r0.5.0".to_string(), + "r0.6.0".to_string(), + "r0.6.1".to_string(), + "v1.1".to_string(), + "v1.2".to_string(), + "v1.3".to_string(), + "v1.4".to_string(), + "v1.5".to_string(), + "v1.6".to_string(), + "v1.7".to_string(), + "v1.8".to_string(), + "v1.9".to_string(), + "v1.10".to_string(), + "v1.11".to_string(), + "v1.12".to_string(), + ]), + unstable_features: std::collections::BTreeMap::from([ + // // Implements support for label-based filtering as described in + // // MSC2326. + // ("org.matrix.label_based_filtering".to_string(), true), + // // Implements support for cross signing as described in MSC1756 + // ("org.matrix.e2e_cross_signing".to_string(), true), + // // Implements additional endpoints as described in MSC2432 + // ("org.matrix.msc2432".to_string(), true), + // // Implements additional endpoints as described in MSC2666 + // ("uk.half-shot.msc2666.query_mutual_rooms.stable".to_string(), true), + // // Whether new rooms will be set to encrypted or not (based on presets). + // ("io.element.e2ee_forced.public".to_string(), e2ee_forced_public), + // ("io.element.e2ee_forced.private".to_string(), e2ee_forced_private), + // ("io.element.e2ee_forced.trusted_private".to_string(), e2ee_forced_trusted_private), + // // Supports the busy presence state described in MSC3026. + // ("org.matrix.msc3026.busy_presence".to_string(), config.experimental.msc3026_enabled), + // // Supports receiving private read receipts as per MSC2285 + // ("org.matrix.msc2285.stable".to_string(), true), // TODO: Remove when MSC2285 becomes a part of the spec + // // Supports filtering of /publicRooms by room type as per MSC3827 + // ("org.matrix.msc3827.stable".to_string(), true), + // // Adds support for thread relations, per MSC3440. + // ("org.matrix.msc3440.stable".to_string(), true), // TODO: remove when "v1.3" is added above + // // Support for thread read receipts & notification counts. + // ("org.matrix.msc3771".to_string(), true), + // ("org.matrix.msc3773".to_string(), config.experimental.msc3773_enabled), + // // Allows moderators to fetch redacted event content as described in MSC2815 + // ("fi.mau.msc2815".to_string(), config.experimental.msc2815_enabled), + // // Adds a ping endpoint for appservices to check HS->AS connection + // ("fi.mau.msc2659.stable".to_string(), true), // TODO: remove when "v1.7" is added above + // // TODO: this is no longer needed once unstable MSC3882 does not need to be supported: + // ("org.matrix.msc3882".to_string(), config.auth.login_via_existing_enabled), + // Adds support for remotely enabling/disabling pushers, as per MSC3881 + ("org.matrix.msc3881".to_string(), msc3881_enabled), + // // Adds support for filtering /messages by event relation. + // ("org.matrix.msc3874".to_string(), config.experimental.msc3874_enabled), + // // Adds support for relation-based redactions as per MSC3912. + // ("org.matrix.msc3912".to_string(), config.experimental.msc3912_enabled), + // // Whether recursively provide relations is supported. + // // TODO This is no longer needed once unstable MSC3981 does not need to be supported. + // ("org.matrix.msc3981".to_string(), true), + // // Adds support for deleting account data. + // ("org.matrix.msc3391".to_string(), config.experimental.msc3391_enabled), + // // Allows clients to inhibit profile update propagation. + // ("org.matrix.msc4069".to_string(), config.experimental.msc4069_profile_inhibit_propagation), + // // Allows clients to handle push for encrypted events. + // ("org.matrix.msc4028".to_string(), config.experimental.msc4028_push_encrypted_events), + // // MSC4108: Mechanism to allow OIDC sign in and E2EE set up via QR code - 2024 version + // ("org.matrix.msc4108".to_string(), ( + // config.experimental.msc4108_enabled + // or ( + // config.experimental.msc4108_delegation_endpoint + // is not None + // ) + // )), + // // MSC4140: Delayed events + // ("org.matrix.msc4140".to_string(), bool(config.server.max_event_delay_ms)), + // Simplified sliding sync + ("org.matrix.simplified_msc3575".to_string(), msc3575_enabled), + // // Arbitrary key-value profile fields. + // ("uk.tcpip.msc4133".to_string(), config.experimental.msc4133_enabled), + // ("uk.tcpip.msc4133.stable".to_string(), true), + // // MSC4155: Invite filtering + // ("org.matrix.msc4155".to_string(), config.experimental.msc4155_enabled), + // // MSC4306: Support for thread subscriptions + // ("org.matrix.msc4306".to_string(), config.experimental.msc4306_enabled), + // // MSC4169: Backwards-compatible redaction sending using `/send` + // ("com.beeper.msc4169".to_string(), config.experimental.msc4169_enabled), + // // MSC4354: Sticky events + // ("org.matrix.msc4354".to_string(), config.experimental.msc4354_enabled), + // // MSC4380: Invite blocking + // ("org.matrix.msc4380.stable".to_string(), true), + // // MSC4445: Sync timeline order + // ("org.matrix.msc4445.initial_sync_timeline_topological_ordering".to_string(), true), + ]), + }) +} + +/// Experimental features the server supports +pub struct UnstableFeatureMap { + msc3881: bool, + msc3575: bool, + msc4222: bool, +} + +/// Convert from [`SynapseConfig`] to the global defaults for unstable features that the +/// server supports [`UnstableFeatureMap`] +pub fn synapse_config_to_global_unstable_feature_map(config: &SynapseConfig) -> UnstableFeatureMap { + UnstableFeatureMap { + msc3881: config.experimental.msc3881_enabled, + msc3575: config.experimental.msc3575_enabled, + msc4222: config.experimental.msc4222_enabled, } } diff --git a/rust/src/storage/store.rs b/rust/src/storage/store.rs index 1b27b6036d0..ca13ff24124 100644 --- a/rust/src/storage/store.rs +++ b/rust/src/storage/store.rs @@ -32,16 +32,6 @@ pub enum PerUserExperimentalFeature { MSC4222, } -impl PerUserExperimentalFeature { - pub fn is_globally_enabled(&self, config: &SynapseConfig) -> bool { - match self { - PerUserExperimentalFeature::MSC3881 => config.experimental.msc3881_enabled, - PerUserExperimentalFeature::MSC3575 => config.experimental.msc3575_enabled, - PerUserExperimentalFeature::MSC4222 => config.experimental.msc4222_enabled, - } - } -} - impl std::fmt::Display for PerUserExperimentalFeature { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!( @@ -60,20 +50,18 @@ impl std::fmt::Display for PerUserExperimentalFeature { } pub struct Store { - pub config: SynapseConfig, pub db_pool: P, } impl Store

{ - pub async fn is_feature_enabled( + /// Checks whether a given feature is enabled/disabled for this user + /// + /// If there is no entry, returns None + pub async fn is_feature_enabled_for_user( &self, user_id: &str, feature: PerUserExperimentalFeature, - ) -> Result { - if feature.is_globally_enabled(&self.config) { - return Ok(true); - } - + ) -> Result, anyhow::Error> { // It's not enabled globally, so check whether it's enabled per-user. // // Owned copies so the callback can be `'static` (it may be moved to @@ -97,10 +85,10 @@ impl Store

{ .await?; let enabled = match &rows[..] { - // If there is no row for this user, default to disabled - [] => false, + // No row for this user + [] => None, // Otherwise, we should only find a single row for this (user, feature) - [row] => row.try_get(0)?, + [row] => Some(row.try_get(0)?), _ => { panic!("Programming error") } From 29e5830610356ec2b2c21609a5c9f4abff651a73 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 18 Jun 2026 21:05:45 -0500 Subject: [PATCH 19/55] Fix Rust warnings --- rust/src/handlers/versions.rs | 14 ++++---------- rust/src/storage/store.rs | 5 +---- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index e2d8cd1b7d1..1709b54bbe7 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -94,14 +94,11 @@ async fn build_versions_response( true } else { // Look up whether it's explicitly enabled/disabled for this user - match store + store .is_feature_enabled_for_user(user_id, PerUserExperimentalFeature::MSC3881) .await? - { - Some(enabled) => enabled, // Default to false if there is no entry for this user - None => false, - } + .unwrap_or(false) } } None => global_unstable_feature_map.msc3881, @@ -114,14 +111,11 @@ async fn build_versions_response( true } else { // Look up whether it's explicitly enabled/disabled for this user - match store + store .is_feature_enabled_for_user(user_id, PerUserExperimentalFeature::MSC3575) .await? - { - Some(enabled) => enabled, // Default to false if there is no entry for this user - None => false, - } + .unwrap_or(false) } } None => global_unstable_feature_map.msc3575, diff --git a/rust/src/storage/store.rs b/rust/src/storage/store.rs index ca13ff24124..4c8795726a2 100644 --- a/rust/src/storage/store.rs +++ b/rust/src/storage/store.rs @@ -16,10 +16,7 @@ use futures::FutureExt; use serde::Serialize; -use crate::{ - config::SynapseConfig, - storage::db::{DatabasePool, RowExt}, -}; +use crate::storage::db::{DatabasePool, RowExt}; /// Currently supported per-user features #[derive(Serialize, Debug)] From 5b1a1aef15c56d2421a6a83709f8a500d3847f5d Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 19 Jun 2026 00:07:13 -0500 Subject: [PATCH 20/55] WIP: `RoomCreationPreset` --- rust/src/config/mod.rs | 40 ++++++++++++++++++++------ rust/src/handlers/versions.rs | 46 ++++++++++++++++++------------ synapse/config/room.py | 10 +++---- tests/rest/client/test_versions.py | 4 +-- 4 files changed, 67 insertions(+), 33 deletions(-) diff --git a/rust/src/config/mod.rs b/rust/src/config/mod.rs index 763b9fcb1f1..aed0ea7762c 100644 --- a/rust/src/config/mod.rs +++ b/rust/src/config/mod.rs @@ -13,20 +13,44 @@ * */ -use pyo3::prelude::*; +use pyo3::{exceptions::PyRuntimeError, prelude::*}; +use std::collections::BTreeSet; #[derive(FromPyObject, Clone)] pub struct SynapseConfig { + pub room: RoomConfig, pub experimental: ExperimentalConfig, } -// #[derive(FromPyObject)] -// #[serde(rename_all = "snake_case")] -// pub enum RoomCreationPreset { -// PrviateChat, -// PublicChat, -// TrustedPrivateChat, -// } +#[derive(Clone, Ord)] +pub enum RoomCreationPreset { + PrviateChat, + PublicChat, + TrustedPrivateChat, +} + +impl<'a, 'py> FromPyObject<'a, 'py> for RoomCreationPreset { + type Error = PyErr; + + /// Extract from a Python `LoggingTransaction` passed as an argument. + fn extract(value_py: Borrowed<'a, 'py, PyAny>) -> PyResult { + Ok(match value_py.extract()? { + "private_chat" => RoomCreationPreset::PrviateChat, + "public_chat" => RoomCreationPreset::PublicChat, + "trusted_private_chat" => RoomCreationPreset::TrustedPrivateChat, + other => { + return Err(PyRuntimeError::new_err( + format!("Unknown variant {other:#} does not translate to `RoomCreationPreset`. This is a Synapse programming error."), + )) + } + }) + } +} + +#[derive(FromPyObject, Clone)] +pub struct RoomConfig { + pub encryption_enabled_by_default_for_room_presets: BTreeSet, +} #[derive(FromPyObject, Clone)] pub struct ExperimentalConfig { diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index 1709b54bbe7..aed3ab5f711 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -19,7 +19,7 @@ use pyo3::prelude::*; use pythonize::{pythonize, PythonizeError}; use serde::{Deserialize, Serialize}; -use crate::config::SynapseConfig; +use crate::config::{RoomCreationPreset, SynapseConfig}; use crate::deferred::create_deferred; use crate::storage::db::python_db_pool::PythonDatabasePoolWrapper; use crate::storage::store::{PerUserExperimentalFeature, Store}; @@ -121,20 +121,6 @@ async fn build_versions_response( None => global_unstable_feature_map.msc3575, }; - // TODO: Calculate these once since they shouldn't change after start-up. - // e2ee_forced_public = ( - // RoomCreationPreset.PUBLIC_CHAT - // in config.room.encryption_enabled_by_default_for_room_presets - // ); - // e2ee_forced_private = ( - // RoomCreationPreset.PRIVATE_CHAT - // in config.room.encryption_enabled_by_default_for_room_presets - // ); - // e2ee_forced_trusted_private = ( - // RoomCreationPreset.TRUSTED_PRIVATE_CHAT - // in config.room.encryption_enabled_by_default_for_room_presets - // ); - Ok(VersionsResponse { versions: Vec::from([ // XXX: at some point we need to decide whether we need to include @@ -176,9 +162,18 @@ async fn build_versions_response( // // Implements additional endpoints as described in MSC2666 // ("uk.half-shot.msc2666.query_mutual_rooms.stable".to_string(), true), // // Whether new rooms will be set to encrypted or not (based on presets). - // ("io.element.e2ee_forced.public".to_string(), e2ee_forced_public), - // ("io.element.e2ee_forced.private".to_string(), e2ee_forced_private), - // ("io.element.e2ee_forced.trusted_private".to_string(), e2ee_forced_trusted_private), + ( + "io.element.e2ee_forced.public".to_string(), + global_unstable_feature_map.e2ee_forced_public, + ), + ( + "io.element.e2ee_forced.private".to_string(), + global_unstable_feature_map.e2ee_forced_private, + ), + ( + "io.element.e2ee_forced.trusted_private".to_string(), + global_unstable_feature_map.e2ee_forced_trusted_private, + ), // // Supports the busy presence state described in MSC3026. // ("org.matrix.msc3026.busy_presence".to_string(), config.experimental.msc3026_enabled), // // Supports receiving private read receipts as per MSC2285 @@ -247,6 +242,9 @@ pub struct UnstableFeatureMap { msc3881: bool, msc3575: bool, msc4222: bool, + e2ee_forced_public: bool, + e2ee_forced_private: bool, + e2ee_forced_trusted_private: bool, } /// Convert from [`SynapseConfig`] to the global defaults for unstable features that the @@ -256,5 +254,17 @@ pub fn synapse_config_to_global_unstable_feature_map(config: &SynapseConfig) -> msc3881: config.experimental.msc3881_enabled, msc3575: config.experimental.msc3575_enabled, msc4222: config.experimental.msc4222_enabled, + e2ee_forced_public: config + .room + .encryption_enabled_by_default_for_room_presets + .contains(&RoomCreationPreset::PublicChat), + e2ee_forced_private: config + .room + .encryption_enabled_by_default_for_room_presets + .contains(&RoomCreationPreset::PrviateChat), + e2ee_forced_trusted_private: config + .room + .encryption_enabled_by_default_for_room_presets + .contains(&RoomCreationPreset::TrustedPrivateChat), } } diff --git a/synapse/config/room.py b/synapse/config/room.py index e698c7bafd6..6a1f4d1eb8b 100644 --- a/synapse/config/room.py +++ b/synapse/config/room.py @@ -48,23 +48,23 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: RoomDefaultEncryptionTypes.OFF, ) if encryption_for_room_type == RoomDefaultEncryptionTypes.ALL: - self.encryption_enabled_by_default_for_room_presets = [ + self.encryption_enabled_by_default_for_room_presets = { RoomCreationPreset.PRIVATE_CHAT, RoomCreationPreset.TRUSTED_PRIVATE_CHAT, RoomCreationPreset.PUBLIC_CHAT, - ] + } elif encryption_for_room_type == RoomDefaultEncryptionTypes.INVITE: - self.encryption_enabled_by_default_for_room_presets = [ + self.encryption_enabled_by_default_for_room_presets = { RoomCreationPreset.PRIVATE_CHAT, RoomCreationPreset.TRUSTED_PRIVATE_CHAT, - ] + } elif ( encryption_for_room_type == RoomDefaultEncryptionTypes.OFF or encryption_for_room_type is False ): # PyYAML translates "off" into False if it's unquoted, so we also need to # check for encryption_for_room_type being False. - self.encryption_enabled_by_default_for_room_presets = [] + self.encryption_enabled_by_default_for_room_presets = set() else: raise ConfigError( "Invalid value for encryption_enabled_by_default_for_room_type" diff --git a/tests/rest/client/test_versions.py b/tests/rest/client/test_versions.py index b21c5e8271e..2ef0f876b50 100644 --- a/tests/rest/client/test_versions.py +++ b/tests/rest/client/test_versions.py @@ -17,9 +17,9 @@ from synapse.rest import admin from synapse.rest.client import login, versions from synapse.server import HomeServer -from synapse.util.clock import Clock -from synapse.types import JsonDict from synapse.synapse_rust.http_client import HttpClient +from synapse.types import JsonDict +from synapse.util.clock import Clock from tests import unittest From 9688d48f33e3c7d8adaed2b136371f9bae5889ca Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 19 Jun 2026 00:13:03 -0500 Subject: [PATCH 21/55] Fix `RoomCreationPreset` --- rust/src/config/mod.rs | 31 ++++++++++++++++++++----------- rust/src/handlers/versions.rs | 2 +- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/rust/src/config/mod.rs b/rust/src/config/mod.rs index aed0ea7762c..f53014537df 100644 --- a/rust/src/config/mod.rs +++ b/rust/src/config/mod.rs @@ -15,6 +15,7 @@ use pyo3::{exceptions::PyRuntimeError, prelude::*}; use std::collections::BTreeSet; +use std::str::FromStr; #[derive(FromPyObject, Clone)] pub struct SynapseConfig { @@ -22,31 +23,39 @@ pub struct SynapseConfig { pub experimental: ExperimentalConfig, } -#[derive(Clone, Ord)] +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] pub enum RoomCreationPreset { - PrviateChat, + PrivateChat, PublicChat, TrustedPrivateChat, } -impl<'a, 'py> FromPyObject<'a, 'py> for RoomCreationPreset { - type Error = PyErr; +impl FromStr for RoomCreationPreset { + type Err = PyErr; - /// Extract from a Python `LoggingTransaction` passed as an argument. - fn extract(value_py: Borrowed<'a, 'py, PyAny>) -> PyResult { - Ok(match value_py.extract()? { - "private_chat" => RoomCreationPreset::PrviateChat, + fn from_str(s: &str) -> Result { + Ok(match s { + "private_chat" => RoomCreationPreset::PrivateChat, "public_chat" => RoomCreationPreset::PublicChat, "trusted_private_chat" => RoomCreationPreset::TrustedPrivateChat, other => { - return Err(PyRuntimeError::new_err( - format!("Unknown variant {other:#} does not translate to `RoomCreationPreset`. This is a Synapse programming error."), - )) + return Err(PyRuntimeError::new_err(format!( + "Unknown variant {other:?} does not translate to `RoomCreationPreset`. \ + This is a Synapse programming error." + ))) } }) } } +impl<'a, 'py> FromPyObject<'a, 'py> for RoomCreationPreset { + type Error = PyErr; + + fn extract(value: Borrowed<'a, 'py, PyAny>) -> PyResult { + value.extract::<&str>()?.parse() + } +} + #[derive(FromPyObject, Clone)] pub struct RoomConfig { pub encryption_enabled_by_default_for_room_presets: BTreeSet, diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index aed3ab5f711..bed37083fb3 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -261,7 +261,7 @@ pub fn synapse_config_to_global_unstable_feature_map(config: &SynapseConfig) -> e2ee_forced_private: config .room .encryption_enabled_by_default_for_room_presets - .contains(&RoomCreationPreset::PrviateChat), + .contains(&RoomCreationPreset::PrivateChat), e2ee_forced_trusted_private: config .room .encryption_enabled_by_default_for_room_presets From 941188c05ddc04b2f7c7beb3670a8597aeb8ed4c Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 19 Jun 2026 00:13:20 -0500 Subject: [PATCH 22/55] Remove unused `msc4222` --- rust/src/handlers/versions.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index bed37083fb3..1564459d57e 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -241,7 +241,6 @@ async fn build_versions_response( pub struct UnstableFeatureMap { msc3881: bool, msc3575: bool, - msc4222: bool, e2ee_forced_public: bool, e2ee_forced_private: bool, e2ee_forced_trusted_private: bool, @@ -253,7 +252,6 @@ pub fn synapse_config_to_global_unstable_feature_map(config: &SynapseConfig) -> UnstableFeatureMap { msc3881: config.experimental.msc3881_enabled, msc3575: config.experimental.msc3575_enabled, - msc4222: config.experimental.msc4222_enabled, e2ee_forced_public: config .room .encryption_enabled_by_default_for_room_presets From 7c2790a48c995d3139c3c02bece0b62db9a7dc85 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 19 Jun 2026 00:48:18 -0500 Subject: [PATCH 23/55] Serde `UnstableFeatureMap` --- rust/src/handlers/versions.rs | 101 ++++++---------------------------- 1 file changed, 16 insertions(+), 85 deletions(-) diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index 1564459d57e..cbd3b9a9573 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -121,6 +121,13 @@ async fn build_versions_response( None => global_unstable_feature_map.msc3575, }; + let unstable_feature_map = UnstableFeatureMap { + msc3575: msc3575_enabled, + msc3881: msc3881_enabled, + // The clone here isn't the best but better than manually composing things + ..global_unstable_feature_map.clone() + }; + Ok(VersionsResponse { versions: Vec::from([ // XXX: at some point we need to decide whether we need to include @@ -151,98 +158,22 @@ async fn build_versions_response( "v1.11".to_string(), "v1.12".to_string(), ]), - unstable_features: std::collections::BTreeMap::from([ - // // Implements support for label-based filtering as described in - // // MSC2326. - // ("org.matrix.label_based_filtering".to_string(), true), - // // Implements support for cross signing as described in MSC1756 - // ("org.matrix.e2e_cross_signing".to_string(), true), - // // Implements additional endpoints as described in MSC2432 - // ("org.matrix.msc2432".to_string(), true), - // // Implements additional endpoints as described in MSC2666 - // ("uk.half-shot.msc2666.query_mutual_rooms.stable".to_string(), true), - // // Whether new rooms will be set to encrypted or not (based on presets). - ( - "io.element.e2ee_forced.public".to_string(), - global_unstable_feature_map.e2ee_forced_public, - ), - ( - "io.element.e2ee_forced.private".to_string(), - global_unstable_feature_map.e2ee_forced_private, - ), - ( - "io.element.e2ee_forced.trusted_private".to_string(), - global_unstable_feature_map.e2ee_forced_trusted_private, - ), - // // Supports the busy presence state described in MSC3026. - // ("org.matrix.msc3026.busy_presence".to_string(), config.experimental.msc3026_enabled), - // // Supports receiving private read receipts as per MSC2285 - // ("org.matrix.msc2285.stable".to_string(), true), // TODO: Remove when MSC2285 becomes a part of the spec - // // Supports filtering of /publicRooms by room type as per MSC3827 - // ("org.matrix.msc3827.stable".to_string(), true), - // // Adds support for thread relations, per MSC3440. - // ("org.matrix.msc3440.stable".to_string(), true), // TODO: remove when "v1.3" is added above - // // Support for thread read receipts & notification counts. - // ("org.matrix.msc3771".to_string(), true), - // ("org.matrix.msc3773".to_string(), config.experimental.msc3773_enabled), - // // Allows moderators to fetch redacted event content as described in MSC2815 - // ("fi.mau.msc2815".to_string(), config.experimental.msc2815_enabled), - // // Adds a ping endpoint for appservices to check HS->AS connection - // ("fi.mau.msc2659.stable".to_string(), true), // TODO: remove when "v1.7" is added above - // // TODO: this is no longer needed once unstable MSC3882 does not need to be supported: - // ("org.matrix.msc3882".to_string(), config.auth.login_via_existing_enabled), - // Adds support for remotely enabling/disabling pushers, as per MSC3881 - ("org.matrix.msc3881".to_string(), msc3881_enabled), - // // Adds support for filtering /messages by event relation. - // ("org.matrix.msc3874".to_string(), config.experimental.msc3874_enabled), - // // Adds support for relation-based redactions as per MSC3912. - // ("org.matrix.msc3912".to_string(), config.experimental.msc3912_enabled), - // // Whether recursively provide relations is supported. - // // TODO This is no longer needed once unstable MSC3981 does not need to be supported. - // ("org.matrix.msc3981".to_string(), true), - // // Adds support for deleting account data. - // ("org.matrix.msc3391".to_string(), config.experimental.msc3391_enabled), - // // Allows clients to inhibit profile update propagation. - // ("org.matrix.msc4069".to_string(), config.experimental.msc4069_profile_inhibit_propagation), - // // Allows clients to handle push for encrypted events. - // ("org.matrix.msc4028".to_string(), config.experimental.msc4028_push_encrypted_events), - // // MSC4108: Mechanism to allow OIDC sign in and E2EE set up via QR code - 2024 version - // ("org.matrix.msc4108".to_string(), ( - // config.experimental.msc4108_enabled - // or ( - // config.experimental.msc4108_delegation_endpoint - // is not None - // ) - // )), - // // MSC4140: Delayed events - // ("org.matrix.msc4140".to_string(), bool(config.server.max_event_delay_ms)), - // Simplified sliding sync - ("org.matrix.simplified_msc3575".to_string(), msc3575_enabled), - // // Arbitrary key-value profile fields. - // ("uk.tcpip.msc4133".to_string(), config.experimental.msc4133_enabled), - // ("uk.tcpip.msc4133.stable".to_string(), true), - // // MSC4155: Invite filtering - // ("org.matrix.msc4155".to_string(), config.experimental.msc4155_enabled), - // // MSC4306: Support for thread subscriptions - // ("org.matrix.msc4306".to_string(), config.experimental.msc4306_enabled), - // // MSC4169: Backwards-compatible redaction sending using `/send` - // ("com.beeper.msc4169".to_string(), config.experimental.msc4169_enabled), - // // MSC4354: Sticky events - // ("org.matrix.msc4354".to_string(), config.experimental.msc4354_enabled), - // // MSC4380: Invite blocking - // ("org.matrix.msc4380.stable".to_string(), true), - // // MSC4445: Sync timeline order - // ("org.matrix.msc4445.initial_sync_timeline_topological_ordering".to_string(), true), - ]), + unstable_features: serde_json::from_value(serde_json::to_value(unstable_feature_map)?)?, }) } /// Experimental features the server supports +#[derive(Serialize, Debug, Clone)] pub struct UnstableFeatureMap { - msc3881: bool, + #[serde(rename = "org.matrix.simplified_msc3575")] msc3575: bool, + #[serde(rename = "org.matrix.msc3881")] + msc3881: bool, + #[serde(rename = "io.element.e2ee_forced.public")] e2ee_forced_public: bool, + #[serde(rename = "io.element.e2ee_forced.private")] e2ee_forced_private: bool, + #[serde(rename = "io.element.e2ee_forced.trusted_private")] e2ee_forced_trusted_private: bool, } @@ -250,8 +181,8 @@ pub struct UnstableFeatureMap { /// server supports [`UnstableFeatureMap`] pub fn synapse_config_to_global_unstable_feature_map(config: &SynapseConfig) -> UnstableFeatureMap { UnstableFeatureMap { - msc3881: config.experimental.msc3881_enabled, msc3575: config.experimental.msc3575_enabled, + msc3881: config.experimental.msc3881_enabled, e2ee_forced_public: config .room .encryption_enabled_by_default_for_room_presets From 7aec5a0786c081e56666a4eb49e1aba8624e176a Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 19 Jun 2026 00:51:54 -0500 Subject: [PATCH 24/55] Stub remaining features --- rust/src/handlers/versions.rs | 68 +++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index cbd3b9a9573..a3fc4bbd855 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -165,16 +165,84 @@ async fn build_versions_response( /// Experimental features the server supports #[derive(Serialize, Debug, Clone)] pub struct UnstableFeatureMap { + /// Simplified sliding sync #[serde(rename = "org.matrix.simplified_msc3575")] msc3575: bool, + /// Adds support for remotely enabling/disabling pushers, as per MSC3881 #[serde(rename = "org.matrix.msc3881")] msc3881: bool, + + // Whether new rooms will be set to encrypted or not (based on presets). #[serde(rename = "io.element.e2ee_forced.public")] e2ee_forced_public: bool, #[serde(rename = "io.element.e2ee_forced.private")] e2ee_forced_private: bool, #[serde(rename = "io.element.e2ee_forced.trusted_private")] e2ee_forced_trusted_private: bool, + // // Implements support for label-based filtering as described in + // // MSC2326. + // ("org.matrix.label_based_filtering".to_string(), true), + // // Implements support for cross signing as described in MSC1756 + // ("org.matrix.e2e_cross_signing".to_string(), true), + // // Implements additional endpoints as described in MSC2432 + // ("org.matrix.msc2432".to_string(), true), + // // Implements additional endpoints as described in MSC2666 + // ("uk.half-shot.msc2666.query_mutual_rooms.stable".to_string(), true), + // // Supports the busy presence state described in MSC3026. + // ("org.matrix.msc3026.busy_presence".to_string(), config.experimental.msc3026_enabled), + // // Supports receiving private read receipts as per MSC2285 + // ("org.matrix.msc2285.stable".to_string(), true), // TODO: Remove when MSC2285 becomes a part of the spec + // // Supports filtering of /publicRooms by room type as per MSC3827 + // ("org.matrix.msc3827.stable".to_string(), true), + // // Adds support for thread relations, per MSC3440. + // ("org.matrix.msc3440.stable".to_string(), true), // TODO: remove when "v1.3" is added above + // // Support for thread read receipts & notification counts. + // ("org.matrix.msc3771".to_string(), true), + // ("org.matrix.msc3773".to_string(), config.experimental.msc3773_enabled), + // // Allows moderators to fetch redacted event content as described in MSC2815 + // ("fi.mau.msc2815".to_string(), config.experimental.msc2815_enabled), + // // Adds a ping endpoint for appservices to check HS->AS connection + // ("fi.mau.msc2659.stable".to_string(), true), // TODO: remove when "v1.7" is added above + // // TODO: this is no longer needed once unstable MSC3882 does not need to be supported: + // ("org.matrix.msc3882".to_string(), config.auth.login_via_existing_enabled), + // // Adds support for filtering /messages by event relation. + // ("org.matrix.msc3874".to_string(), config.experimental.msc3874_enabled), + // // Adds support for relation-based redactions as per MSC3912. + // ("org.matrix.msc3912".to_string(), config.experimental.msc3912_enabled), + // // Whether recursively provide relations is supported. + // // TODO This is no longer needed once unstable MSC3981 does not need to be supported. + // ("org.matrix.msc3981".to_string(), true), + // // Adds support for deleting account data. + // ("org.matrix.msc3391".to_string(), config.experimental.msc3391_enabled), + // // Allows clients to inhibit profile update propagation. + // ("org.matrix.msc4069".to_string(), config.experimental.msc4069_profile_inhibit_propagation), + // // Allows clients to handle push for encrypted events. + // ("org.matrix.msc4028".to_string(), config.experimental.msc4028_push_encrypted_events), + // // MSC4108: Mechanism to allow OIDC sign in and E2EE set up via QR code - 2024 version + // ("org.matrix.msc4108".to_string(), ( + // config.experimental.msc4108_enabled + // or ( + // config.experimental.msc4108_delegation_endpoint + // is not None + // ) + // )), + // // MSC4140: Delayed events + // ("org.matrix.msc4140".to_string(), bool(config.server.max_event_delay_ms)), + // // Arbitrary key-value profile fields. + // ("uk.tcpip.msc4133".to_string(), config.experimental.msc4133_enabled), + // ("uk.tcpip.msc4133.stable".to_string(), true), + // // MSC4155: Invite filtering + // ("org.matrix.msc4155".to_string(), config.experimental.msc4155_enabled), + // // MSC4306: Support for thread subscriptions + // ("org.matrix.msc4306".to_string(), config.experimental.msc4306_enabled), + // // MSC4169: Backwards-compatible redaction sending using `/send` + // ("com.beeper.msc4169".to_string(), config.experimental.msc4169_enabled), + // // MSC4354: Sticky events + // ("org.matrix.msc4354".to_string(), config.experimental.msc4354_enabled), + // // MSC4380: Invite blocking + // ("org.matrix.msc4380.stable".to_string(), true), + // // MSC4445: Sync timeline order + // ("org.matrix.msc4445.initial_sync_timeline_topological_ordering".to_string(), true), } /// Convert from [`SynapseConfig`] to the global defaults for unstable features that the From 19c0777fd64e9a6fe0cbffa446199cac6bd73f28 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 22 Jun 2026 14:06:36 -0500 Subject: [PATCH 25/55] Add changelog --- changelog.d/19846.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/19846.misc diff --git a/changelog.d/19846.misc b/changelog.d/19846.misc new file mode 100644 index 00000000000..9ea93729c96 --- /dev/null +++ b/changelog.d/19846.misc @@ -0,0 +1 @@ +Allow Rust code to have database access via Python database connection pool. From 5f23dc6c74d193c8f27b93da30b17cb000c0bbc1 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 22 Jun 2026 14:11:56 -0500 Subject: [PATCH 26/55] Fix lint --- rust/src/config/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/src/config/mod.rs b/rust/src/config/mod.rs index f53014537df..d1788b5ab93 100644 --- a/rust/src/config/mod.rs +++ b/rust/src/config/mod.rs @@ -13,10 +13,11 @@ * */ -use pyo3::{exceptions::PyRuntimeError, prelude::*}; use std::collections::BTreeSet; use std::str::FromStr; +use pyo3::{exceptions::PyRuntimeError, prelude::*}; + #[derive(FromPyObject, Clone)] pub struct SynapseConfig { pub room: RoomConfig, From 1d6eb1e5248c9047e68431f560079fc6477db3ad Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 22 Jun 2026 15:14:08 -0500 Subject: [PATCH 27/55] Remove `await_result_with_rust` in favor of updating `await_result` with the same tricks learned in https://github.com/element-hq/synapse/pull/19871 --- tests/rest/client/test_versions.py | 10 ----- tests/server.py | 68 ++++++++++++++++++------------ 2 files changed, 41 insertions(+), 37 deletions(-) diff --git a/tests/rest/client/test_versions.py b/tests/rest/client/test_versions.py index 2ef0f876b50..429250e58e5 100644 --- a/tests/rest/client/test_versions.py +++ b/tests/rest/client/test_versions.py @@ -74,9 +74,7 @@ def test_unauthenticated(self) -> None: "GET", "/_matrix/client/versions", content={}, - await_result=False, ) - channel.await_result_with_rust() self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) @@ -89,9 +87,7 @@ def test_authenticated(self) -> None: "/_matrix/client/versions", content={}, access_token=user1_tok, - await_result=False, ) - channel.await_result_with_rust() self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) @@ -107,9 +103,7 @@ def test_authenticated_with_per_user_feature(self) -> None: "/_matrix/client/versions", content={}, access_token=user1_tok, - await_result=False, ) - channel.await_result_with_rust() self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) self.assertEqual( @@ -129,9 +123,7 @@ def test_authenticated_with_per_user_feature(self) -> None: "/_matrix/client/versions", content={}, access_token=user1_tok, - await_result=False, ) - channel.await_result_with_rust() self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) self.assertEqual( @@ -146,9 +138,7 @@ def test_authenticated_with_per_user_feature(self) -> None: "/_matrix/client/versions", content={}, access_token=user2_tok, - await_result=False, ) - channel.await_result_with_rust() self.assertEqual(channel.code, 200, channel.result) self._sanity_check_versions_response(channel.json_body) self.assertEqual( diff --git a/tests/server.py b/tests/server.py index 0963f5aa39d..9488307805c 100644 --- a/tests/server.py +++ b/tests/server.py @@ -101,6 +101,7 @@ from synapse.storage.prepare_database import prepare_database from synapse.types import ISynapseReactor, JsonDict from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.json import json_encoder from tests.utils import ( @@ -301,43 +302,56 @@ def transport(self) -> "FakeChannel": def await_result(self, timeout_ms: int = 1000) -> None: """ Wait until the request is finished. - """ - end_time = self._reactor.seconds() + timeout_ms / 1000.0 - self._reactor.run() - - while not self.is_finished(): - if self._reactor.seconds() > end_time: - raise TimedOutException("Timed out waiting for request to finish.") - self._reactor.advance(0.1) + Advances the Twisted reactor clock by 0.1s and suspending execution of the + Python thread (to allow other threads to do work) in a loop until we see a + result. We timeout when both the Twisted reactor clock has been advanced enough + AND we've waited the same amount of in real-time for the specified timeout + before giving up. - def await_result_with_rust(self, timeout_ms: int = 1000) -> None: + The loop 1) allows `clock.call_later` scheduled callbacks to run if they are + scheduled to run now and 2) will also allow other threads to make progress. This + could be things spawned on the Twisted reactor threadpool or Tokio runtime + (async Rust code). """ - Wait until the request is finished (a request that includes async Rust work). + timeout = Duration(milliseconds=timeout_ms) + start_time_seconds = self._reactor.seconds() + start_real_time_seconds = time.time() - This is separate from `await_result` because we don't want to slow down the - entire test suite with real sleeps. Prefer `await_result` if possible. - - The default `await_result` only advances the reactor's *virtual* clock in a - tight loop, never yielding real wall-clock time, so the Tokio threads never get - a chance to run and the request times out. Instead we pump the reactor while - also sleeping a little real time each iteration, the same way - `HomeserverTestCase.wait_on_thread()` does. - """ - end_time = self._reactor.seconds() + timeout_ms / 1000.0 + # TODO: Why? self._reactor.run() while not self.is_finished(): - if self._reactor.seconds() > end_time: + if ( + # Exceeded the Twisted reactor time timeout + start_time_seconds + timeout.as_secs() < self._reactor.seconds() + # And exceeded the real-time timeout + and start_real_time_seconds + timeout.as_secs() < time.time() + ): raise TimedOutException("Timed out waiting for request to finish.") - # Pump the Twisted reactor - self._reactor.advance(0.1) + # Suspend execution of this thread to allow other threads to do work. This + # could be things spawned on the Twisted reactor threadpool or Tokio thread + # pool (async Rust code). + # + # We could also use `time.sleep(0)` here but this is more precise + os.sched_yield() + # time.sleep(0) - # Give some real wall-clock time for other threads to do work. This could be - # things spawned on the Twisted reactor threadpool but this is primarily - # meant for the Tokio thread pool (async Rust code). - time.sleep(0.01) + # Advance the Twisted reactor and run any scheduled callbacks + # + # Don't advance the Twisted reactor clock further than the timeout duration + # as someone should increase the timeout if they expect things to take + # longer. + if start_time_seconds + timeout.as_secs() > self._reactor.seconds(): + self._reactor.advance(0.1) + else: + # But we want to still keep running whatever might be getting scheduled + # to run now. + # + # For example from other threads, they may have scheduled something on + # the reactor to run (like `reactor.callFromThread(...)`) + self._reactor.advance(0) def extract_cookies(self, cookies: MutableMapping[str, str]) -> None: """Process the contents of any Set-Cookie headers in the response From 4eec02ce006be2b0ac587edbc4ad689b0c5971c3 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 22 Jun 2026 16:47:31 -0500 Subject: [PATCH 28/55] Explain possible better future for async fn that need to be `Send` --- rust/src/storage/db/mod.rs | 29 +++++++++++++++++++++++++++++ rust/src/storage/store.rs | 2 +- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs index 9729bc39531..447c357c20b 100644 --- a/rust/src/storage/db/mod.rs +++ b/rust/src/storage/db/mod.rs @@ -33,8 +33,37 @@ pub trait DatabasePool: Send + Sync { /// Starts a transaction on the database and runs the given function, /// returning its result. /// + /// `name` should be a descriptive identifier for logging/metrics + /// /// `func` may be called multiple times under certain failure modes (like /// serialization and deadlock errors), so it is `Fn` rather than `FnOnce`. + /// + /// Usage: + /// ```rust + /// db_pool + /// .run_interaction(|txn| { + /// async move { + /// /* do stuff with txn */ + /// } + /// .boxed() + /// }) + /// ``` + // + // Ideally, this method signature would be slightly different to allow downstream + // usage to look like the following (simpler) but because allow the work to happen + // on other threads, the `Future` needs to be `Send`; As of 2026-06-22, the + // `AsyncFn` trait doesn't have a clean way to express that "the future this async + // closure produces is `Send`." + // ``` + // db_pool.run_interaction("description", async move |txn| { + // /* do stuff with txn */ + // }) + // ``` + // + // Refs: + // - [RFC 3668: Async closures](https://github.com/rust-lang/rfcs/pull/3668) + // - [RFC 3654: Return Type Notation](https://github.com/rust-lang/rfcs/pull/3654) + // - [Tracking Issue for return type notation](https://github.com/rust-lang/rust/issues/109417) fn run_interaction( &self, name: &'static str, diff --git a/rust/src/storage/store.rs b/rust/src/storage/store.rs index 4c8795726a2..5a86e38e5e3 100644 --- a/rust/src/storage/store.rs +++ b/rust/src/storage/store.rs @@ -63,7 +63,7 @@ impl Store

{ // // Owned copies so the callback can be `'static` (it may be moved to // another thread and called multiple times under retries). - let user_id = user_id.to_string(); + let user_id = user_id.to_owned(); let feature = feature.to_string(); let is_feature_enabled_for_user = self From dc93a239afc241f60eee81c3ffcd09c13984f511 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 22 Jun 2026 16:51:22 -0500 Subject: [PATCH 29/55] Fix grammar, add intended fix --- rust/src/storage/db/mod.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs index 447c357c20b..ff741bc43f0 100644 --- a/rust/src/storage/db/mod.rs +++ b/rust/src/storage/db/mod.rs @@ -50,10 +50,11 @@ pub trait DatabasePool: Send + Sync { /// ``` // // Ideally, this method signature would be slightly different to allow downstream - // usage to look like the following (simpler) but because allow the work to happen - // on other threads, the `Future` needs to be `Send`; As of 2026-06-22, the - // `AsyncFn` trait doesn't have a clean way to express that "the future this async - // closure produces is `Send`." + // usage to look like the following (simpler) but because we allow the work to + // happen on other threads, the `Future` needs to be `Send`; As of 2026-06-22, the + // `AsyncFn` trait has no stable way to express that "the future this async closure + // produces is `Send`". The intended fix is probably return-type-notation + // (https://github.com/rust-lang/rust/issues/109417). // ``` // db_pool.run_interaction("description", async move |txn| { // /* do stuff with txn */ From 874178afacc7f0c8099a8872c1739a18baa067ce Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 22 Jun 2026 17:29:28 -0500 Subject: [PATCH 30/55] `poll_once` instead of `futures::executor::block_on` See https://github.com/element-hq/synapse/pull/19846#discussion_r3394989664 --- rust/src/storage/db/mod.rs | 5 ++++ rust/src/storage/db/python_db_pool.rs | 37 ++++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs index ff741bc43f0..8bb0453c8c0 100644 --- a/rust/src/storage/db/mod.rs +++ b/rust/src/storage/db/mod.rs @@ -38,6 +38,10 @@ pub trait DatabasePool: Send + Sync { /// `func` may be called multiple times under certain failure modes (like /// serialization and deadlock errors), so it is `Fn` rather than `FnOnce`. /// + /// `func` is async but you should only call `.await` on [`Transaction`] methods. + /// This is a minor cosmetic flaw but seems fine, as you don't want to be doing any + /// unnecessary waiting in your transaction anyway. + /// /// Usage: /// ```rust /// db_pool @@ -82,6 +86,7 @@ pub trait DatabasePool: Send + Sync { /// interact with the database #[async_trait::async_trait] pub trait Transaction: Send { + // `async` as this is representing a round-trip between the app and database async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error>; } diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index a5f95a04ef8..d4a626ae758 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -17,7 +17,10 @@ //! - Database pool [`PythonDatabasePoolWrapper`] which allows you to start a... //! - transaction [`LoggingTransactionWrapper`] and query the database +use std::future::Future; +use std::pin::pin; use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll}; use futures::future::BoxFuture; use pyo3::{ @@ -105,12 +108,20 @@ impl DatabasePool for PythonDatabasePoolWrapper { let txn_py = args.get_item(0)?; let mut txn = txn_py.extract::()?; - match futures::executor::block_on(func(&mut txn)) { - Ok(value) => { + // Since we expect people to only call `.await` on [`Transaction`] + // related methods (mentioned in the [`Transaction`] docstring) AND + // because there is no async work to suspend on in the Python + // [`Transaction`] synchronously, we can get away with polling once + // as it should immediately resolve to [`Poll::Ready`]. Getting + // [`Poll::Pending`] would be considered a programming error. + // + // Alternatively, we could just use `futures::executor::block_on` + match poll_once(func(&mut txn)) { + Poll::Ready(Ok(value)) => { *callback_slot.lock().unwrap() = Some(Ok(value)); Ok(py.None()) } - Err(err) => { + Poll::Ready(Err(err)) => { // Re-raise into Python so `runInteraction` rolls the // transaction back (and can apply its retry logic for // serialization/deadlock errors). @@ -118,6 +129,12 @@ impl DatabasePool for PythonDatabasePoolWrapper { *callback_slot.lock().unwrap() = Some(Err(err)); Err(py_err) } + Poll::Pending => Err(PyRuntimeError::new_err( + "The `run_interaction` transaction callback future returned `Poll::Pending`, \ + but we expect Synapse Python database work to resolve synchronously. \ + This is a Synapse programming error: genuine async work is \ + not supported here.", + )), } }, )? @@ -157,6 +174,20 @@ impl DatabasePool for PythonDatabasePoolWrapper { } } +/// Poll a future exactly once. +/// +/// Returns [`Poll::Ready`] if the future resolves immediately, or +/// [`Poll::Pending`] if it would need to suspend. We use this where a future is +/// expected to never genuinely suspend (e.g. Synapse's synchronous DB query +/// path) and want to enforce that, rather than driving it to completion with a +/// blocking executor like `futures::executor::block_on`. +fn poll_once(future: F) -> Poll { + let mut future = pin!(future); + let waker = futures::task::noop_waker(); + let mut cx = Context::from_waker(&waker); + future.as_mut().poll(&mut cx) +} + /// Convert an [`anyhow::Error`] into a [`PyErr`] to re-raise into Python. /// /// If the error wraps an original Python exception (e.g. a database error From 505b5993328a7c2f30c138dd07083574d1576267 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 22 Jun 2026 17:33:39 -0500 Subject: [PATCH 31/55] Panic for programming error --- rust/src/storage/db/python_db_pool.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index d4a626ae758..13e1df2632f 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -129,12 +129,12 @@ impl DatabasePool for PythonDatabasePoolWrapper { *callback_slot.lock().unwrap() = Some(Err(err)); Err(py_err) } - Poll::Pending => Err(PyRuntimeError::new_err( + Poll::Pending => unreachable!( "The `run_interaction` transaction callback future returned `Poll::Pending`, \ but we expect Synapse Python database work to resolve synchronously. \ This is a Synapse programming error: genuine async work is \ not supported here.", - )), + ), } }, )? From d40adfad21058a8f48a182ba9085e1635460a160 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 22 Jun 2026 17:34:55 -0500 Subject: [PATCH 32/55] Explain why `poll_once` --- rust/src/storage/db/python_db_pool.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index 13e1df2632f..781b4b84732 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -116,6 +116,8 @@ impl DatabasePool for PythonDatabasePoolWrapper { // [`Poll::Pending`] would be considered a programming error. // // Alternatively, we could just use `futures::executor::block_on` + // which is probably cleaner but a single-shot poll is more + // enforcing of the concept we want to represent. match poll_once(func(&mut txn)) { Poll::Ready(Ok(value)) => { *callback_slot.lock().unwrap() = Some(Ok(value)); From 065c5cb4c88cc64d6e50595b44fd004ce74eb059 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 23 Jun 2026 19:32:29 -0500 Subject: [PATCH 33/55] Non-working: Try to save/restore logcontext --- rust/src/deferred.rs | 121 +++++++++++++++++++++----- rust/src/handlers/versions.rs | 6 ++ rust/src/storage/db/python_db_pool.rs | 8 +- 3 files changed, 111 insertions(+), 24 deletions(-) diff --git a/rust/src/deferred.rs b/rust/src/deferred.rs index 1399a9f481f..46f4fb6d7f1 100644 --- a/rust/src/deferred.rs +++ b/rust/src/deferred.rs @@ -58,10 +58,46 @@ fn defer(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { .bind(py)) } +/// A reference to the `synapse.logging.context` module. +static LOGGING_CONTEXT_MODULE: OnceCell> = OnceCell::new(); + +/// Access to the `synapse.logging.context` module. +fn logging_context_module(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + Ok(LOGGING_CONTEXT_MODULE + .get_or_try_init(|| py.import("synapse.logging.context").map(Into::into))? + .bind(py)) +} + +/// Set the Synapse logcontext active on the current (reactor) thread, returning +/// the context that was previously active so it can be restored afterwards. +fn set_current_logcontext<'py>( + py: Python<'py>, + context: &Bound<'py, PyAny>, +) -> PyResult> { + logging_context_module(py)?.call_method1(intern!(py, "set_current_context"), (context,)) +} + +tokio::task_local! { + /// The Synapse `LoggingContext` that was active on the reactor thread when + /// the Rust future was created (see [`create_deferred`]). + /// + /// Synapse attributes per-request CPU/DB usage via a thread-local + /// `LoggingContext`, but our work runs on a Tokio worker thread that is + /// detached from it. We stash the originating context here so that whenever + /// we hop back onto the reactor thread to drive Python work (see + /// [`run_awaitable`]) we can re-activate it; otherwise that work runs in + /// the sentinel context and its resource usage (e.g. database time) is lost + /// instead of being charged to the request. + static LOGGING_CONTEXT: Py; +} + /// Creates a twisted deferred from the given future, spawning the task on the /// tokio runtime. /// -/// Does not handle deferred cancellation or contextvars. +/// Captures the Synapse logcontext active on the reactor thread (so work driven +/// by the future is attributed to the originating request, see +/// [`LOGGING_CONTEXT`]), but does not handle deferred cancellation or +/// contextvars. pub fn create_deferred<'py, F, O>( py: Python<'py>, reactor: &Bound<'py, PyAny>, @@ -75,9 +111,17 @@ where let deferred_callback = deferred.getattr("callback")?.unbind(); let deferred_errback = deferred.getattr("errback")?.unbind(); + // Capture the logcontext active on the reactor thread now, while we're still + // on it, so the spawned task can re-apply it when it hops back to drive + // Python work. If there's no real context this is the sentinel, and + // re-applying it later is a no-op. + let logging_context = logging_context_module(py)? + .call_method0(intern!(py, "current_context"))? + .unbind(); + let rt = runtime(reactor)?; let handle = rt.handle()?; - let task = handle.spawn(fut); + let task = handle.spawn(LOGGING_CONTEXT.scope(logging_context, fut)); // Unbind the reactor so that we can pass it to the task let reactor = reactor.clone().unbind(); @@ -117,33 +161,53 @@ where make_deferred_yieldable(py, &deferred) } -/// Runs `make_deferred` on the Twisted reactor thread to obtain a Deferred (or -/// coroutine), then resolves once that Deferred fires. +/// Runs a Python awaitable to completion on the Twisted reactor and resolves +/// with its result. /// /// This is the inverse of [`create_deferred`]: where that turns a Rust future -/// into a Twisted Deferred, this turns a Twisted Deferred into an awaitable Rust -/// future. +/// into a Twisted `Deferred`, this turns a Python awaitable into a Rust future. +/// +/// `make_awaitable` is invoked on the reactor thread to produce the awaitable. +/// In Synapse that's typically a coroutine (e.g. from calling an `async def` +/// like `runInteraction`), though a `Deferred` works too — both are awaitable, +/// but a coroutine is not itself a `Deferred`. /// -/// We're called on a Tokio worker thread, but Twisted `Deferred`s (and the -/// coroutine that `ensureDeferred` drives) are not thread-safe and Synapse's -/// logcontext is thread-local, so the coroutine must both start and resume on -/// the reactor thread. The `callFromThread` hop is what gets us there for the -/// kickoff; the deferred's own callbacks then fire on the reactor thread too. -/// (Note this is unrelated to offloading the DB work onto a thread — that's -/// handled internally by whatever `make_deferred` calls, e.g. `runInteraction`.) +/// Despite returning a future, the awaitable is kicked off in the background and +/// runs to completion regardless of whether the returned Rust future is ever +/// polled; awaiting it only observes the result. /// -/// `make_deferred` is invoked on the reactor thread and may return either a -/// coroutine or a `Deferred`; `ensureDeferred` normalises both to a `Deferred`. -pub(crate) async fn await_deferred(reactor: Py, make_deferred: F) -> PyResult> +/// We're called on a Tokio worker thread, but Python awaitables and Synapse's +/// logcontext (thread-local) are not thread-safe, so the awaitable must be both +/// started and driven on the reactor thread. The `callFromThread` hop is what +/// gets us there for the kickoff; the resulting deferred's callbacks then fire +/// on the reactor thread too. (Note this is unrelated to offloading the DB work +/// onto a thread — that's handled internally by whatever `make_awaitable` +/// produces, e.g. `runInteraction`.) +/// +/// If a logcontext was captured when the task was created (see +/// [`create_deferred`]), it is re-activated while the awaitable is kicked off so +/// that its resource usage (e.g. database time) is attributed to the originating +/// request. We drive it with `run_coroutine_in_background` so this follows the +/// Synapse logcontext rules (see `docs/log_contexts.md`): the awaitable runs in +/// the request context but the logcontext is reset back to the sentinel once the +/// work completes, so the request context doesn't leak into the reactor. +pub(crate) async fn run_awaitable(reactor: Py, make_awaitable: F) -> PyResult> where F: for<'py> Fn(Python<'py>) -> PyResult> + Send + 'static, { - // Resolves when the deferred fires; carries the resolved value or the error. + // Resolves when the awaitable completes; carries the resolved value or error. let (tx, rx) = oneshot::channel::>>(); // Shared between the success and error callbacks (only one ever fires). let sender = Arc::new(Mutex::new(Some(tx))); + // The logcontext that is active for this thread + let calling_logging_context = LOGGING_CONTEXT + .try_with(|ctx| Python::attach(|py| ctx.clone_ref(py))) + .ok(); + Python::attach(|py| -> PyResult<()> { + // Create some deferred success/error callback functions that we will use to get + // the result from Python to Rust. let success_sender = Arc::clone(&sender); let on_success = PyCFunction::new_closure( py, @@ -174,14 +238,31 @@ where )? .unbind(); + // Create a function that we will run with the Twisted reactor that will drive + // the Python awaitable. let starter = PyCFunction::new_closure( py, None, None, move |args, _kwargs| -> PyResult> { let py = args.py(); - let deferred = defer(py)? - .call_method1(intern!(py, "ensureDeferred"), (make_deferred(py)?,))?; + + // Re-activate the logcontext while we kick the awaitable off. + let previous_context = calling_logging_context + .as_ref() + .map(|ctx| set_current_logcontext(py, ctx.bind(py))) + .transpose()?; + + // We use `run_in_background` to ensure the logcontext rules are being followed + let deferred = logging_context_module(py)? + .call_method1(intern!(py, "run_in_background"), (make_awaitable,)); + + // Put the reactor thread back to the logcontext we found it in + if let Some(previous_context) = previous_context { + set_current_logcontext(py, &previous_context)?; + } + + let deferred = deferred?; deferred.call_method1( intern!(py, "addCallbacks"), (on_success.bind(py), on_error.bind(py)), @@ -200,7 +281,7 @@ where match rx.await { Ok(result) => result, Err(_) => Err(PyRuntimeError::new_err( - "await_deferred channel closed before the deferred fired", + "run_awaitable channel closed before the awaitable completed", )), } } diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index a3fc4bbd855..aeb2efaaef9 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -121,6 +121,12 @@ async fn build_versions_response( None => global_unstable_feature_map.msc3575, }; + log::info!( + "asdf msc3881_enabled={:?} msc3575_enabled={:?}", + msc3881_enabled, + msc3575_enabled + ); + let unstable_feature_map = UnstableFeatureMap { msc3575: msc3575_enabled, msc3881: msc3881_enabled, diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index 781b4b84732..79140042e56 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -30,7 +30,7 @@ use pyo3::{ types::{PyBool, PyCFunction, PyFloat, PyInt, PyList, PyString}, }; -use crate::deferred::await_deferred; +use crate::deferred::run_awaitable; use crate::storage::db::{DatabasePool, Row, Transaction, Value}; /// The database engines we support in the Python side of Synapse @@ -151,10 +151,10 @@ impl DatabasePool for PythonDatabasePoolWrapper { .map_err(anyhow::Error::from)?; // Await `runInteraction` directly. `runInteraction` offloads the actual - // DB work onto a thread itself, so we don't; `await_deferred` only has to - // start the coroutine on the reactor thread and bridge its deferred back + // DB work onto a thread itself, so we don't; `run_awaitable` only has to + // start the coroutine on the reactor thread and bridge its result back // into our `async` world. - let outcome = await_deferred(reactor, move |py| { + let outcome = run_awaitable(reactor, move |py| { database_pool_py .bind(py) .call_method1(intern!(py, "runInteraction"), (name, callback.bind(py))) From 441e580b539f0084677a6158ecd2ec9c05f85109 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 23 Jun 2026 20:04:07 -0500 Subject: [PATCH 34/55] Fix logcontext --- rust/src/deferred.rs | 70 ++++++++++++++++++--------- rust/src/handlers/versions.rs | 1 + rust/src/storage/db/python_db_pool.rs | 9 ++-- rust/src/storage/store.rs | 6 +++ 4 files changed, 58 insertions(+), 28 deletions(-) diff --git a/rust/src/deferred.rs b/rust/src/deferred.rs index 46f4fb6d7f1..9a177fc7c00 100644 --- a/rust/src/deferred.rs +++ b/rust/src/deferred.rs @@ -70,7 +70,7 @@ fn logging_context_module(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { /// Set the Synapse logcontext active on the current (reactor) thread, returning /// the context that was previously active so it can be restored afterwards. -fn set_current_logcontext<'py>( +fn set_current_logging_context<'py>( py: Python<'py>, context: &Bound<'py, PyAny>, ) -> PyResult> { @@ -85,7 +85,7 @@ tokio::task_local! { /// `LoggingContext`, but our work runs on a Tokio worker thread that is /// detached from it. We stash the originating context here so that whenever /// we hop back onto the reactor thread to drive Python work (see - /// [`run_awaitable`]) we can re-activate it; otherwise that work runs in + /// [`run_python_awaitable`]) we can re-activate it; otherwise that work runs in /// the sentinel context and its resource usage (e.g. database time) is lost /// instead of being charged to the request. static LOGGING_CONTEXT: Py; @@ -187,11 +187,14 @@ where /// If a logcontext was captured when the task was created (see /// [`create_deferred`]), it is re-activated while the awaitable is kicked off so /// that its resource usage (e.g. database time) is attributed to the originating -/// request. We drive it with `run_coroutine_in_background` so this follows the -/// Synapse logcontext rules (see `docs/log_contexts.md`): the awaitable runs in -/// the request context but the logcontext is reset back to the sentinel once the +/// request. We drive it with `run_in_background` so this follows the Synapse +/// logcontext rules (see `docs/log_contexts.md`): the awaitable runs in the +/// request context but the logcontext is reset back to the sentinel once the /// work completes, so the request context doesn't leak into the reactor. -pub(crate) async fn run_awaitable(reactor: Py, make_awaitable: F) -> PyResult> +pub(crate) async fn run_python_awaitable( + reactor: Py, + make_awaitable: F, +) -> PyResult> where F: for<'py> Fn(Python<'py>) -> PyResult> + Send + 'static, { @@ -200,8 +203,8 @@ where // Shared between the success and error callbacks (only one ever fires). let sender = Arc::new(Mutex::new(Some(tx))); - // The logcontext that is active for this thread - let calling_logging_context = LOGGING_CONTEXT + // The logcontext that is active for this async task + let current_rust_task_logging_context = LOGGING_CONTEXT .try_with(|ctx| Python::attach(|py| ctx.clone_ref(py))) .ok(); @@ -238,6 +241,20 @@ where )? .unbind(); + // Wrap `make_awaitable` as a Python callable so we can hand it to + // `run_in_background`, which calls it (in the active logcontext) to produce + // the awaitable it then drives. + let awaitable_factory = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let py = args.py(); + Ok(make_awaitable(py)?.unbind()) + }, + )? + .unbind(); + // Create a function that we will run with the Twisted reactor that will drive // the Python awaitable. let starter = PyCFunction::new_closure( @@ -247,20 +264,29 @@ where move |args, _kwargs| -> PyResult> { let py = args.py(); - // Re-activate the logcontext while we kick the awaitable off. - let previous_context = calling_logging_context + // Activate the captured Rust task-local logging context. This way we + // properly record metrics/logging for the thing being run. + let calling_logging_context = current_rust_task_logging_context .as_ref() - .map(|ctx| set_current_logcontext(py, ctx.bind(py))) - .transpose()?; - - // We use `run_in_background` to ensure the logcontext rules are being followed - let deferred = logging_context_module(py)? - .call_method1(intern!(py, "run_in_background"), (make_awaitable,)); - - // Put the reactor thread back to the logcontext we found it in - if let Some(previous_context) = previous_context { - set_current_logcontext(py, &previous_context)?; - } + .map(|ctx| set_current_logging_context(py, ctx.bind(py))) + .transpose()? + .expect("No `LoggingContext` returned from `set_current_logging_context(...)`. This is a Synapse programming error."); + + // We fire-and-forget using `run_in_background`. Re-using + // `run_in_background` also makes sure the awaitable gets run with the + // current logcontext (the one we just activated) while following the + // logcontext rules. + let deferred = logging_context_module(py)?.call_method1( + intern!(py, "run_in_background"), + (awaitable_factory.bind(py),), + ); + + // Restore the `calling_logging_context` after we kick off the + // background task. + // + // Our goal is to have the caller logcontext unchanged after firing off + // the background task and returning. + set_current_logging_context(py, &calling_logging_context)?; let deferred = deferred?; deferred.call_method1( @@ -281,7 +307,7 @@ where match rx.await { Ok(result) => result, Err(_) => Err(PyRuntimeError::new_err( - "run_awaitable channel closed before the awaitable completed", + "run_python_awaitable channel closed before the awaitable completed", )), } } diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index aeb2efaaef9..99d28a4773c 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -121,6 +121,7 @@ async fn build_versions_response( None => global_unstable_feature_map.msc3575, }; + // TODO: Remove log::info!( "asdf msc3881_enabled={:?} msc3575_enabled={:?}", msc3881_enabled, diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index 79140042e56..70079ef402d 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -30,7 +30,7 @@ use pyo3::{ types::{PyBool, PyCFunction, PyFloat, PyInt, PyList, PyString}, }; -use crate::deferred::run_awaitable; +use crate::deferred::run_python_awaitable; use crate::storage::db::{DatabasePool, Row, Transaction, Value}; /// The database engines we support in the Python side of Synapse @@ -150,11 +150,8 @@ impl DatabasePool for PythonDatabasePoolWrapper { }) .map_err(anyhow::Error::from)?; - // Await `runInteraction` directly. `runInteraction` offloads the actual - // DB work onto a thread itself, so we don't; `run_awaitable` only has to - // start the coroutine on the reactor thread and bridge its result back - // into our `async` world. - let outcome = run_awaitable(reactor, move |py| { + // Use `runInteraction` directly + let outcome = run_python_awaitable(reactor, move |py| { database_pool_py .bind(py) .call_method1(intern!(py, "runInteraction"), (name, callback.bind(py))) diff --git a/rust/src/storage/store.rs b/rust/src/storage/store.rs index 5a86e38e5e3..be6a5774672 100644 --- a/rust/src/storage/store.rs +++ b/rust/src/storage/store.rs @@ -72,6 +72,9 @@ impl Store

{ let user_id = user_id.clone(); let feature = feature.clone(); async move { + // TODO: Remove + std::thread::sleep(std::time::Duration::from_millis(600)); + let rows = txn .query( "SELECT enabled \ @@ -91,6 +94,9 @@ impl Store

{ } }; + // TODO: Remove + log::info!("asdf feature={:?} enabled={:?}", feature, enabled); + Ok(enabled) } .boxed() From 746657498a04d443bb0dc769515308352216f9bf Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 23 Jun 2026 20:10:38 -0500 Subject: [PATCH 35/55] Less wordy --- rust/src/deferred.rs | 35 ++++++++--------------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/rust/src/deferred.rs b/rust/src/deferred.rs index 9a177fc7c00..b351938c04b 100644 --- a/rust/src/deferred.rs +++ b/rust/src/deferred.rs @@ -111,10 +111,9 @@ where let deferred_callback = deferred.getattr("callback")?.unbind(); let deferred_errback = deferred.getattr("errback")?.unbind(); - // Capture the logcontext active on the reactor thread now, while we're still - // on it, so the spawned task can re-apply it when it hops back to drive - // Python work. If there's no real context this is the sentinel, and - // re-applying it later is a no-op. + // Capture the logcontext active on the reactor thread now, while we're still on it, + // so the spawned task can keep track of what logcontext its associated with and can + // re-apply it when it hops back to drive Python work. let logging_context = logging_context_module(py)? .call_method0(intern!(py, "current_context"))? .unbind(); @@ -167,30 +166,12 @@ where /// This is the inverse of [`create_deferred`]: where that turns a Rust future /// into a Twisted `Deferred`, this turns a Python awaitable into a Rust future. /// -/// `make_awaitable` is invoked on the reactor thread to produce the awaitable. -/// In Synapse that's typically a coroutine (e.g. from calling an `async def` -/// like `runInteraction`), though a `Deferred` works too — both are awaitable, -/// but a coroutine is not itself a `Deferred`. +/// Despite returning a future, the awaitable is kicked off in the background running in +/// the Twisted reactor and runs to completion regardless of whether the returned Rust +/// future is ever polled; awaiting it only observes the result. /// -/// Despite returning a future, the awaitable is kicked off in the background and -/// runs to completion regardless of whether the returned Rust future is ever -/// polled; awaiting it only observes the result. -/// -/// We're called on a Tokio worker thread, but Python awaitables and Synapse's -/// logcontext (thread-local) are not thread-safe, so the awaitable must be both -/// started and driven on the reactor thread. The `callFromThread` hop is what -/// gets us there for the kickoff; the resulting deferred's callbacks then fire -/// on the reactor thread too. (Note this is unrelated to offloading the DB work -/// onto a thread — that's handled internally by whatever `make_awaitable` -/// produces, e.g. `runInteraction`.) -/// -/// If a logcontext was captured when the task was created (see -/// [`create_deferred`]), it is re-activated while the awaitable is kicked off so -/// that its resource usage (e.g. database time) is attributed to the originating -/// request. We drive it with `run_in_background` so this follows the Synapse -/// logcontext rules (see `docs/log_contexts.md`): the awaitable runs in the -/// request context but the logcontext is reset back to the sentinel once the -/// work completes, so the request context doesn't leak into the reactor. +/// Calls `make_awaitable` with the current logcontext associated with this async Rust +/// task. Follows the Synapse logcontext rules. pub(crate) async fn run_python_awaitable( reactor: Py, make_awaitable: F, From 37ff66099abeacbe1bf5ec30e3fbab3b84f055f3 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 23 Jun 2026 20:29:45 -0500 Subject: [PATCH 36/55] Align imports on `OnceCell` which has `get_or_try_init` --- rust/src/deferred.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/rust/src/deferred.rs b/rust/src/deferred.rs index b351938c04b..ec6e61686ee 100644 --- a/rust/src/deferred.rs +++ b/rust/src/deferred.rs @@ -15,7 +15,7 @@ use std::{ future::Future, - sync::{Arc, Mutex, OnceLock}, + sync::{Arc, Mutex}, }; use once_cell::sync::OnceCell; @@ -310,18 +310,18 @@ fn failure_to_pyerr(failure: &Bound<'_, PyAny>) -> PyErr { } } -static MAKE_DEFERRED_YIELDABLE: OnceLock> = OnceLock::new(); +static MAKE_DEFERRED_YIELDABLE: OnceCell> = OnceCell::new(); /// Given a deferred, make it follow the Synapse logcontext rules fn make_deferred_yieldable<'py>( py: Python<'py>, deferred: &Bound<'py, PyAny>, ) -> PyResult> { - let make_deferred_yieldable = MAKE_DEFERRED_YIELDABLE.get_or_init(|| { - let sys = PyModule::import(py, "synapse.logging.context").unwrap(); - let func = sys.getattr("make_deferred_yieldable").unwrap().unbind(); - func - }); + let make_deferred_yieldable = MAKE_DEFERRED_YIELDABLE.get_or_try_init(|| { + logging_context_module(py)? + .getattr("make_deferred_yieldable") + .map(Into::into) + })?; make_deferred_yieldable .call1(py, (deferred,))? @@ -333,6 +333,7 @@ fn make_deferred_yieldable<'py>( pub fn register_module(py: Python<'_>, _m: &Bound<'_, PyModule>) -> PyResult<()> { // Make sure we fail early if we can't load some modules defer(py)?; + logging_context_module(py)?; Ok(()) } From f42ba674b685560d96e756ff98c54dae8fd14451 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 23 Jun 2026 20:38:04 -0500 Subject: [PATCH 37/55] Avoid circular import issues ``` [ERROR] Traceback (most recent call last): File "/home/eric/.cache/pypoetry/virtualenvs/matrix-synapse-xCtC9ulO-py3.14/lib/python3.14/site-packages/twisted/trial/runner.py", line 711, in loadByName return self.suiteFactory([self.findByName(name, recurse=recurse)]) File "/home/eric/.cache/pypoetry/virtualenvs/matrix-synapse-xCtC9ulO-py3.14/lib/python3.14/site-packages/twisted/trial/runner.py", line 474, in findByName obj = reflect.namedModule(searchName) File "/home/eric/.cache/pypoetry/virtualenvs/matrix-synapse-xCtC9ulO-py3.14/lib/python3.14/site-packages/twisted/python/reflect.py", line 156, in namedModule topLevel = __import__(name) File "/home/eric/Documents/github/element/synapse/tests/__init__.py", line 24, in from synapse.util.patch_inline_callbacks import do_patch File "/home/eric/Documents/github/element/synapse/synapse/__init__.py", line 31, in from synapse.util.rust import check_rust_lib_up_to_date File "/home/eric/Documents/github/element/synapse/synapse/util/__init__.py", line 41, in from synapse.types import JsonDict File "/home/eric/Documents/github/element/synapse/synapse/types/__init__.py", line 63, in from synapse.api.errors import Codes, SynapseError File "/home/eric/Documents/github/element/synapse/synapse/api/errors.py", line 33, in from synapse.util.json import json_decoder File "/home/eric/Documents/github/element/synapse/synapse/util/json.py", line 23, in from synapse.synapse_rust.events import JsonObject File "/home/eric/Documents/github/element/synapse/synapse/logging/context.py", line 55, in from synapse.util.stringutils import random_string_insecure_fast File "/home/eric/Documents/github/element/synapse/synapse/util/stringutils.py", line 31, in from synapse.api.errors import Codes, SynapseError builtins.ImportError: cannot import name 'Codes' from partially initialized module 'synapse.api.errors' (most likely due to a circular import) (/home/eric/Documents/github/element/synapse/synapse/api/errors.py) ``` --- rust/src/deferred.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/src/deferred.rs b/rust/src/deferred.rs index ec6e61686ee..606eec48339 100644 --- a/rust/src/deferred.rs +++ b/rust/src/deferred.rs @@ -333,7 +333,8 @@ fn make_deferred_yieldable<'py>( pub fn register_module(py: Python<'_>, _m: &Bound<'_, PyModule>) -> PyResult<()> { // Make sure we fail early if we can't load some modules defer(py)?; - logging_context_module(py)?; + // We can't check this here because of circular import issues + // logging_context_module(py)?; Ok(()) } From c21dbbdc1f3bf375f9a20fcf6c3c62cef9eb5188 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 23 Jun 2026 21:20:45 -0500 Subject: [PATCH 38/55] Rename `Value` -> `DbValue` --- rust/src/storage/db/mod.rs | 56 +++++++++++++-------------- rust/src/storage/db/python_db_pool.rs | 18 ++++----- rust/src/storage/db/rust_db_pool.rs | 18 ++++----- 3 files changed, 46 insertions(+), 46 deletions(-) diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs index 8bb0453c8c0..bfbf081fa78 100644 --- a/rust/src/storage/db/mod.rs +++ b/rust/src/storage/db/mod.rs @@ -95,7 +95,7 @@ pub trait Transaction: Send { /// Each pool maps the values its database driver hands back into this common /// set, so callers can work with one representation regardless of engine. #[derive(Debug, Clone, PartialEq)] -pub enum Value { +pub enum DbValue { /// A SQL `NULL`. Null, Bool(bool), @@ -107,24 +107,24 @@ pub enum Value { /// A row of data returned from the database by a query. /// /// Each pool converts the cells its database driver hands back into the -/// engine-agnostic [`Value`] representation, so a row is simply a list of them. +/// engine-agnostic [`DbValue`] representation, so a row is simply a list of them. /// Values are pulled out by their numeric index with [`RowExt::try_get`]. -pub type Row = Vec; +pub type Row = Vec; /// Extension methods for reading typed values out of a [`Row`]. /// /// Modelled after [`tokio_postgres::Row`]'s `try_get`: [`try_get`](Self::try_get) -/// converts the [`Value`] at a given index into the requested type via -/// [`FromValue`] (our analogue of `tokio-postgres`'s `FromSql`). +/// converts the [`DbValue`] at a given index into the requested type via +/// [`FromDbValue`] (our analogue of `tokio-postgres`'s `FromSql`). pub trait RowExt { /// Deserializes a value from the row, specified by its numeric index, /// returning an error if the index is out of bounds or the value cannot be /// converted into `T`. - fn try_get(&self, index: usize) -> Result; + fn try_get(&self, index: usize) -> Result; } impl RowExt for Row { - fn try_get(&self, index: usize) -> Result { + fn try_get(&self, index: usize) -> Result { let value = self.get(index).cloned().ok_or_else(|| { anyhow::anyhow!( "tried to get column {index} but the row only has {} column(s)", @@ -136,56 +136,56 @@ impl RowExt for Row { } } -/// Converts a backend-agnostic [`Value`] into a concrete Rust type, analogous to +/// Converts a backend-agnostic [`DbValue`] into a concrete Rust type, analogous to /// `tokio-postgres`'s `FromSql`. -pub trait FromValue: Sized { - fn from_value(value: Value) -> Result; +pub trait FromDbValue: Sized { + fn from_value(value: DbValue) -> Result; } -impl FromValue for bool { - fn from_value(value: Value) -> Result { +impl FromDbValue for bool { + fn from_value(value: DbValue) -> Result { match value { - Value::Bool(b) => Ok(b), + DbValue::Bool(b) => Ok(b), // SQLite has no native boolean type and stores them as integers. - Value::Int(i) => Ok(i != 0), + DbValue::Int(i) => Ok(i != 0), other => anyhow::bail!("cannot read {other:?} as bool"), } } } -impl FromValue for i64 { - fn from_value(value: Value) -> Result { +impl FromDbValue for i64 { + fn from_value(value: DbValue) -> Result { match value { - Value::Int(i) => Ok(i), - Value::Bool(b) => Ok(b as i64), + DbValue::Int(i) => Ok(i), + DbValue::Bool(b) => Ok(b as i64), other => anyhow::bail!("cannot read {other:?} as i64"), } } } -impl FromValue for f64 { - fn from_value(value: Value) -> Result { +impl FromDbValue for f64 { + fn from_value(value: DbValue) -> Result { match value { - Value::Float(f) => Ok(f), - Value::Int(i) => Ok(i as f64), + DbValue::Float(f) => Ok(f), + DbValue::Int(i) => Ok(i as f64), other => anyhow::bail!("cannot read {other:?} as f64"), } } } -impl FromValue for String { - fn from_value(value: Value) -> Result { +impl FromDbValue for String { + fn from_value(value: DbValue) -> Result { match value { - Value::Text(s) => Ok(s), + DbValue::Text(s) => Ok(s), other => anyhow::bail!("cannot read {other:?} as String"), } } } -impl FromValue for Option { - fn from_value(value: Value) -> Result { +impl FromDbValue for Option { + fn from_value(value: DbValue) -> Result { match value { - Value::Null => Ok(None), + DbValue::Null => Ok(None), other => Ok(Some(T::from_value(other)?)), } } diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index 70079ef402d..b9fc142e3ff 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -31,7 +31,7 @@ use pyo3::{ }; use crate::deferred::run_python_awaitable; -use crate::storage::db::{DatabasePool, Row, Transaction, Value}; +use crate::storage::db::{DatabasePool, DbValue, Row, Transaction}; /// The database engines we support in the Python side of Synapse #[derive(Copy, Clone, Debug)] @@ -276,7 +276,7 @@ impl Transaction for LoggingTransactionWrapper { self.execute(py, sql, args.as_any())?; // Pull the rows back out, converting each cell from its Python type - // into the engine-agnostic `Value` representation as we go. + // into the engine-agnostic `DbValue` representation as we go. let rows_py = self .logging_transaction_py .bind(py) @@ -298,23 +298,23 @@ impl Transaction for LoggingTransactionWrapper { } } -/// Convert a single cell from a Python row into a backend-agnostic [`Value`] by +/// Convert a single cell from a Python row into a backend-agnostic [`DbValue`] by /// inspecting its Python type (the pyo3 equivalent of `isinstance` checks). -fn py_cell_to_value(cell: &Bound<'_, PyAny>) -> PyResult { +fn py_cell_to_value(cell: &Bound<'_, PyAny>) -> PyResult { // `None` maps to SQL `NULL`. if cell.is_none() { - return Ok(Value::Null); + return Ok(DbValue::Null); } // A `bool` *is* an `int` in SQLite, so ensure we try `bool` first. if let Ok(b) = cell.cast::() { - Ok(Value::Bool(b.extract()?)) + Ok(DbValue::Bool(b.extract()?)) } else if let Ok(i) = cell.cast::() { - Ok(Value::Int(i.extract()?)) + Ok(DbValue::Int(i.extract()?)) } else if let Ok(f) = cell.cast::() { - Ok(Value::Float(f.extract()?)) + Ok(DbValue::Float(f.extract()?)) } else if let Ok(s) = cell.cast::() { - Ok(Value::Text(s.to_string())) + Ok(DbValue::Text(s.to_string())) } else { Err(PyTypeError::new_err(format!( "unsupported column type {} returned from the database", diff --git a/rust/src/storage/db/rust_db_pool.rs b/rust/src/storage/db/rust_db_pool.rs index be280c1f9bb..94d701643f5 100644 --- a/rust/src/storage/db/rust_db_pool.rs +++ b/rust/src/storage/db/rust_db_pool.rs @@ -22,7 +22,7 @@ use bb8_postgres::PostgresConnectionManager; use futures::future::BoxFuture; use postgres_native_tls::MakeTlsConnector; -use crate::storage::db::{DatabasePool, Row, Transaction, Value}; +use crate::storage::db::{DatabasePool, DbValue, Row, Transaction}; /// Native Rust database access backed by `tokio-postgres` (for use in synapse-rust-apps) pub struct RustDatabasePool { @@ -119,25 +119,25 @@ fn tokio_row_to_row(row: &tokio_postgres::Row) -> Result { .collect() } -/// Convert a single cell of a [`tokio_postgres::Row`] into a [`Value`]. +/// Convert a single cell of a [`tokio_postgres::Row`] into a [`DbValue`]. /// /// Dispatch on the column's Postgres type, leaning on `tokio-postgres`'s own /// `FromSql` impls to read the cell. Everything is read as `Option<_>` so a SQL -/// `NULL` becomes `Value::Null`. -fn tokio_cell_to_value(row: &tokio_postgres::Row, index: usize) -> Result { +/// `NULL` becomes `DbValue::Null`. +fn tokio_cell_to_value(row: &tokio_postgres::Row, index: usize) -> Result { if let Ok(value) = row.try_get::<_, Option>(index) { - Ok(value.map_or(Value::Null, Value::Bool)) + Ok(value.map_or(DbValue::Null, DbValue::Bool)) } else if let Ok(value) = row.try_get::<_, Option>(index) { - Ok(value.map_or(Value::Null, Value::Int)) + Ok(value.map_or(DbValue::Null, DbValue::Int)) } else if let Ok(value) = row.try_get::<_, Option>(index) { - Ok(value.map_or(Value::Null, Value::Float)) + Ok(value.map_or(DbValue::Null, DbValue::Float)) } else if let Ok(value) = row.try_get::<_, Option>(index) { - Ok(value.map_or(Value::Null, Value::Text)) + Ok(value.map_or(DbValue::Null, DbValue::Text)) } else { let ty = row.columns()[index].type_(); anyhow::bail!( "Unsupported `tokio-postgres` type {} encountered when trying to convert it \ - to our generic database `Value` type. You probably just need to implement it in `tokio_cell_to_value(...)`.", + to our generic database `DbValue` type. You probably just need to implement it in `tokio_cell_to_value(...)`.", ty ) } From 2860e4ea01bc82dcff52889956fc81b976f281e7 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 24 Jun 2026 14:34:47 -0500 Subject: [PATCH 39/55] LLM attempt at switching back to `Store` with `dyn DatabasePool` --- rust/src/handlers/mod.rs | 4 +- rust/src/handlers/versions.rs | 5 +- rust/src/storage/db/mod.rs | 93 ++++++++++++- rust/src/storage/db/python_db_pool.rs | 186 +++++++++++++------------- rust/src/storage/db/rust_db_pool.rs | 85 ++++++------ rust/src/storage/store.rs | 8 +- 6 files changed, 238 insertions(+), 143 deletions(-) diff --git a/rust/src/handlers/mod.rs b/rust/src/handlers/mod.rs index 1f04b65e109..9911443721f 100644 --- a/rust/src/handlers/mod.rs +++ b/rust/src/handlers/mod.rs @@ -52,7 +52,9 @@ impl RustHandlers { let db_pool = PythonDatabasePoolWrapper::new(db_pool_py, reactor.clone_ref(py)); // Store is shared across all of the handlers so let's use an `Arc` - let store = Arc::new(Store { db_pool }); + let store = Arc::new(Store { + db_pool: Box::new(db_pool), + }); let global_unstable_feature_map = Arc::new( versions::synapse_config_to_global_unstable_feature_map(&config), diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index 99d28a4773c..029f5478b84 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -21,7 +21,6 @@ use serde::{Deserialize, Serialize}; use crate::config::{RoomCreationPreset, SynapseConfig}; use crate::deferred::create_deferred; -use crate::storage::db::python_db_pool::PythonDatabasePoolWrapper; use crate::storage::store::{PerUserExperimentalFeature, Store}; /// `GET /_matrix/client/versions` response @@ -45,7 +44,7 @@ impl<'py> IntoPyObject<'py> for VersionsResponse { #[pyclass] pub struct VersionsHandler { pub global_unstable_feature_map: Arc, - pub store: Arc>, + pub store: Arc, /// The Twisted reactor, used to bridge our `async` response back into a /// Twisted deferred that Python can `await`. pub reactor: Py, @@ -83,7 +82,7 @@ impl VersionsHandler { /// * global_unstable_feature_map: The global values before any per-user overrides /// * user_id: The user making the request async fn build_versions_response( - store: &Store, + store: &Store, global_unstable_feature_map: &UnstableFeatureMap, user_id: Option<&str>, ) -> Result { diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs index bfbf081fa78..51029161e22 100644 --- a/rust/src/storage/db/mod.rs +++ b/rust/src/storage/db/mod.rs @@ -13,23 +13,66 @@ * */ +use std::any::Any; use std::future::Future; use futures::future::BoxFuture; +use futures::FutureExt; pub mod python_db_pool; pub mod rust_db_pool; +/// A type-erased transaction callback, as accepted by +/// [`DatabasePool::run_interaction_dyn`]. +/// +/// This is the object-safe form of the `func` passed to +/// [`DatabasePoolExt::run_interaction`]: the concrete result type `R` is boxed up +/// as `Box` so the trait can stay dyn-compatible. The ergonomic +/// [`DatabasePoolExt::run_interaction`] wrapper takes care of boxing the result +/// and downcasting it back to `R`. +pub type ErasedInteraction = + Box Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, ErasedResult> + Send + Sync>; + +/// The type-erased result of an [`ErasedInteraction`]: the concrete `R` boxed up +/// as `Box` so it can pass through the object-safe +/// [`DatabasePool::run_interaction_dyn`]. +pub type ErasedResult = anyhow::Result>; + /// A database connection pool. /// /// Code is written against this trait so the same store can run against either /// the Python-backed pool (in Synapse, see [`python_db_pool`]) or a native /// `tokio-postgres` pool (in `synapse-rust-apps`, see [`rust_db_pool`]). The pool -/// type is fixed within any given binary, so callers are generic over the pool -/// (e.g. `Store`) rather than using dynamic dispatch. +/// is held behind a trait object (e.g. `Box`), so a single +/// `Store` type can wrap whichever pool the binary happens to use. +/// +/// Callers don't use this trait directly; they go through the ergonomic, +/// strongly-typed [`run_interaction`](DatabasePoolExt::run_interaction) on +/// [`DatabasePoolExt`] (blanket-implemented for every `DatabasePool`, including +/// `dyn DatabasePool`). This trait only carries the object-safe, type-erased +/// [`run_interaction_dyn`](Self::run_interaction_dyn) that the wrapper dispatches +/// to. /// /// `Send + Sync` so it can be stored in a `#[pyclass]` and shared across threads. pub trait DatabasePool: Send + Sync { + /// Object-safe, type-erased core of [`DatabasePoolExt::run_interaction`]. + /// + /// Implementors implement this; callers should prefer + /// [`DatabasePoolExt::run_interaction`], which boxes up the result and + /// downcasts it back to the concrete type for you. See that method for the + /// semantics of `name` and `func`. + fn run_interaction_dyn<'a>( + &'a self, + name: &'static str, + func: ErasedInteraction, + ) -> BoxFuture<'a, ErasedResult>; +} + +/// Ergonomic, strongly-typed access to a [`DatabasePool`]. +/// +/// Blanket-implemented for every `T: DatabasePool + ?Sized`, so it is available +/// both on concrete pools and on `dyn DatabasePool` trait objects. +pub trait DatabasePoolExt: DatabasePool { /// Starts a transaction on the database and runs the given function, /// returning its result. /// @@ -79,7 +122,51 @@ pub trait DatabasePool: Send + Sync { F: for<'txn> Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, anyhow::Result> + Send + Sync - + 'static; + + 'static, + { + async move { + // Box the concrete result `R` up as `Box` so we can + // route through the object-safe `run_interaction_dyn`, then downcast + // back to `R` once it returns. + let erased = erase_interaction(func); + + let result = self.run_interaction_dyn(name, erased).await?; + Ok(*result.downcast::().expect( + "run_interaction_dyn returned a value of an unexpected type; \ + this is a programming error in the DatabasePool implementation", + )) + } + } +} + +impl DatabasePoolExt for T {} + +/// Wrap a strongly-typed transaction callback so its result is boxed as +/// `Box`, producing the type-erased [`ErasedInteraction`] that +/// [`DatabasePool::run_interaction_dyn`] accepts. +fn erase_interaction(func: F) -> ErasedInteraction +where + R: Send + 'static, + F: for<'txn> Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, anyhow::Result> + + Send + + Sync + + 'static, +{ + // Routing the closure through a generic fn parameter that carries the + // `for<'txn>` bound pins its higher-ranked signature; assigning a closure + // that borrows its argument straight into a `Box` doesn't reliably + // infer that lifetime. + fn constrain(c: C) -> C + where + C: for<'txn> Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, ErasedResult>, + { + c + } + + Box::new(constrain(move |txn| { + let fut = func(txn); + async move { Ok(Box::new(fut.await?) as Box) }.boxed() + })) } /// A [`tokio_postgres::Transaction`] looking thing that we can use on the Rust side to diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index b9fc142e3ff..7388bc07655 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -23,6 +23,7 @@ use std::sync::{Arc, Mutex}; use std::task::{Context, Poll}; use futures::future::BoxFuture; +use futures::FutureExt; use pyo3::{ exceptions::{PyRuntimeError, PyTypeError}, intern, @@ -31,7 +32,9 @@ use pyo3::{ }; use crate::deferred::run_python_awaitable; -use crate::storage::db::{DatabasePool, DbValue, Row, Transaction}; +use crate::storage::db::{ + DatabasePool, DbValue, ErasedInteraction, ErasedResult, Row, Transaction, +}; /// The database engines we support in the Python side of Synapse #[derive(Copy, Clone, Debug)] @@ -73,103 +76,104 @@ impl PythonDatabasePoolWrapper { } impl DatabasePool for PythonDatabasePoolWrapper { - async fn run_interaction(&self, name: &'static str, func: F) -> anyhow::Result - where - R: Send + 'static, - F: for<'txn> Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, anyhow::Result> - + Send - + Sync - + 'static, - { - // `runInteraction` calls `func` with a `LoggingTransaction` on a DB - // thread and expects a synchronous return value. Since we can't - // round-trip an arbitrary Rust `R` back out through Python, the callback - // stashes the result here and we pick it up once the deferred fires. - // - // Note the callback may run more than once (`runInteraction` retries on - // serialization/deadlock errors), so we only trust this slot once the - // deferred has fired, i.e. once the transaction has finally committed or - // failed. - let result_slot: Arc>>> = Arc::new(Mutex::new(None)); + fn run_interaction_dyn<'a>( + &'a self, + name: &'static str, + func: ErasedInteraction, + ) -> BoxFuture<'a, ErasedResult> { + async move { + // `runInteraction` calls `func` with a `LoggingTransaction` on a DB + // thread and expects a synchronous return value. Since we can't + // round-trip an arbitrary Rust result back out through Python, the + // callback stashes the result here and we pick it up once the deferred + // fires. + // + // Note the callback may run more than once (`runInteraction` retries on + // serialization/deadlock errors), so we only trust this slot once the + // deferred has fired, i.e. once the transaction has finally committed or + // failed. + let result_slot: Arc>> = Arc::new(Mutex::new(None)); - // Build the callback that Python's `runInteraction` invokes on a DB - // thread with a `LoggingTransaction`, plus owned handles we can move onto - // the reactor thread. We drive `func` to completion in the callback; the - // Python query path is synchronous under the hood, so it's safe to block - // this dedicated DB thread until the future resolves. - let callback_slot = Arc::clone(&result_slot); - let (callback, database_pool_py, reactor) = Python::attach(|py| -> PyResult<_> { - let callback = PyCFunction::new_closure( - py, - None, - None, - move |args, _kwargs| -> PyResult> { - let py = args.py(); - let txn_py = args.get_item(0)?; - let mut txn = txn_py.extract::()?; + // Build the callback that Python's `runInteraction` invokes on a DB + // thread with a `LoggingTransaction`, plus owned handles we can move onto + // the reactor thread. We drive `func` to completion in the callback; the + // Python query path is synchronous under the hood, so it's safe to block + // this dedicated DB thread until the future resolves. + let callback_slot = Arc::clone(&result_slot); + let (callback, database_pool_py, reactor) = Python::attach(|py| -> PyResult<_> { + let callback = PyCFunction::new_closure( + py, + None, + None, + move |args, _kwargs| -> PyResult> { + let py = args.py(); + let txn_py = args.get_item(0)?; + let mut txn = txn_py.extract::()?; - // Since we expect people to only call `.await` on [`Transaction`] - // related methods (mentioned in the [`Transaction`] docstring) AND - // because there is no async work to suspend on in the Python - // [`Transaction`] synchronously, we can get away with polling once - // as it should immediately resolve to [`Poll::Ready`]. Getting - // [`Poll::Pending`] would be considered a programming error. - // - // Alternatively, we could just use `futures::executor::block_on` - // which is probably cleaner but a single-shot poll is more - // enforcing of the concept we want to represent. - match poll_once(func(&mut txn)) { - Poll::Ready(Ok(value)) => { - *callback_slot.lock().unwrap() = Some(Ok(value)); - Ok(py.None()) - } - Poll::Ready(Err(err)) => { - // Re-raise into Python so `runInteraction` rolls the - // transaction back (and can apply its retry logic for - // serialization/deadlock errors). - let py_err = anyhow_to_pyerr(&err); - *callback_slot.lock().unwrap() = Some(Err(err)); - Err(py_err) + // Since we expect people to only call `.await` on [`Transaction`] + // related methods (mentioned in the [`Transaction`] docstring) AND + // because there is no async work to suspend on in the Python + // [`Transaction`] synchronously, we can get away with polling once + // as it should immediately resolve to [`Poll::Ready`]. Getting + // [`Poll::Pending`] would be considered a programming error. + // + // Alternatively, we could just use `futures::executor::block_on` + // which is probably cleaner but a single-shot poll is more + // enforcing of the concept we want to represent. + match poll_once(func(&mut txn)) { + Poll::Ready(Ok(value)) => { + *callback_slot.lock().unwrap() = Some(Ok(value)); + Ok(py.None()) + } + Poll::Ready(Err(err)) => { + // Re-raise into Python so `runInteraction` rolls the + // transaction back (and can apply its retry logic for + // serialization/deadlock errors). + let py_err = anyhow_to_pyerr(&err); + *callback_slot.lock().unwrap() = Some(Err(err)); + Err(py_err) + } + Poll::Pending => unreachable!( + "The `run_interaction` transaction callback future returned `Poll::Pending`, \ + but we expect Synapse Python database work to resolve synchronously. \ + This is a Synapse programming error: genuine async work is \ + not supported here.", + ), } - Poll::Pending => unreachable!( - "The `run_interaction` transaction callback future returned `Poll::Pending`, \ - but we expect Synapse Python database work to resolve synchronously. \ - This is a Synapse programming error: genuine async work is \ - not supported here.", - ), - } - }, - )? - .unbind(); + }, + )? + .unbind(); - Ok(( - callback, - self.database_pool_py.clone_ref(py), - self.reactor.clone_ref(py), - )) - }) - .map_err(anyhow::Error::from)?; + Ok(( + callback, + self.database_pool_py.clone_ref(py), + self.reactor.clone_ref(py), + )) + }) + .map_err(anyhow::Error::from)?; - // Use `runInteraction` directly - let outcome = run_python_awaitable(reactor, move |py| { - database_pool_py - .bind(py) - .call_method1(intern!(py, "runInteraction"), (name, callback.bind(py))) - }) - .await; + // Use `runInteraction` directly + let outcome = run_python_awaitable(reactor, move |py| { + database_pool_py + .bind(py) + .call_method1(intern!(py, "runInteraction"), (name, callback.bind(py))) + }) + .await; - // Prefer the result captured by the callback (it carries the Rust - // context). If the slot is empty, `runInteraction` failed before ever - // invoking the callback (e.g. it couldn't acquire a connection), so - // surface the deferred's outcome instead. - let captured = result_slot.lock().unwrap().take(); - match captured { - Some(result) => result, - None => match outcome { - Ok(_) => Err(anyhow::anyhow!("run_interaction produced no result")), - Err(py_err) => Err(anyhow::Error::from(py_err)), - }, + // Prefer the result captured by the callback (it carries the Rust + // context). If the slot is empty, `runInteraction` failed before ever + // invoking the callback (e.g. it couldn't acquire a connection), so + // surface the deferred's outcome instead. + let captured = result_slot.lock().unwrap().take(); + match captured { + Some(result) => result, + None => match outcome { + Ok(_) => Err(anyhow::anyhow!("run_interaction produced no result")), + Err(py_err) => Err(anyhow::Error::from(py_err)), + }, + } } + .boxed() } } diff --git a/rust/src/storage/db/rust_db_pool.rs b/rust/src/storage/db/rust_db_pool.rs index 94d701643f5..8fb13b658e0 100644 --- a/rust/src/storage/db/rust_db_pool.rs +++ b/rust/src/storage/db/rust_db_pool.rs @@ -20,9 +20,12 @@ use anyhow::Context; use bb8_postgres::tokio_postgres::{self, types::ToSql, IsolationLevel}; use bb8_postgres::PostgresConnectionManager; use futures::future::BoxFuture; +use futures::FutureExt; use postgres_native_tls::MakeTlsConnector; -use crate::storage::db::{DatabasePool, DbValue, Row, Transaction}; +use crate::storage::db::{ + DatabasePool, DbValue, ErasedInteraction, ErasedResult, Row, Transaction, +}; /// Native Rust database access backed by `tokio-postgres` (for use in synapse-rust-apps) pub struct RustDatabasePool { @@ -30,52 +33,52 @@ pub struct RustDatabasePool { } impl DatabasePool for RustDatabasePool { - async fn run_interaction(&self, _name: &'static str, func: F) -> anyhow::Result - where - R: Send + 'static, - F: for<'txn> Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, anyhow::Result> - + Send - + Sync - + 'static, - { - // Like Synapse's `runInteraction`, retry the whole transaction on - // serialization/deadlock errors (which can happen under repeatable-read). - loop { - let mut conn = self - .db_pool - .get() - .await - .context("Failed to acquire database connection")?; + fn run_interaction_dyn<'a>( + &'a self, + _name: &'static str, + func: ErasedInteraction, + ) -> BoxFuture<'a, ErasedResult> { + async move { + // Like Synapse's `runInteraction`, retry the whole transaction on + // serialization/deadlock errors (which can happen under repeatable-read). + loop { + let mut conn = self + .db_pool + .get() + .await + .context("Failed to acquire database connection")?; - // Repeatable-read isolation level (like Synapse). - let txn = conn - .build_transaction() - .isolation_level(IsolationLevel::RepeatableRead) - .start() - .await - .context("Failed to start transaction")?; + // Repeatable-read isolation level (like Synapse). + let txn = conn + .build_transaction() + .isolation_level(IsolationLevel::RepeatableRead) + .start() + .await + .context("Failed to start transaction")?; - let mut wrapper = TokioPostgresTransaction { txn }; - match func(&mut wrapper).await { - Ok(value) => { - wrapper - .txn - .commit() - .await - .context("Failed to commit transaction")?; - return Ok(value); - } - Err(err) => { - // The transaction is rolled back implicitly when dropped, but - // be explicit about it before deciding whether to retry. - let _ = wrapper.txn.rollback().await; - if is_retryable(&err) { - continue; + let mut wrapper = TokioPostgresTransaction { txn }; + match func(&mut wrapper).await { + Ok(value) => { + wrapper + .txn + .commit() + .await + .context("Failed to commit transaction")?; + return Ok(value); + } + Err(err) => { + // The transaction is rolled back implicitly when dropped, but + // be explicit about it before deciding whether to retry. + let _ = wrapper.txn.rollback().await; + if is_retryable(&err) { + continue; + } + return Err(err); } - return Err(err); } } } + .boxed() } } diff --git a/rust/src/storage/store.rs b/rust/src/storage/store.rs index be6a5774672..f2bf8db3fae 100644 --- a/rust/src/storage/store.rs +++ b/rust/src/storage/store.rs @@ -16,7 +16,7 @@ use futures::FutureExt; use serde::Serialize; -use crate::storage::db::{DatabasePool, RowExt}; +use crate::storage::db::{DatabasePool, DatabasePoolExt, RowExt}; /// Currently supported per-user features #[derive(Serialize, Debug)] @@ -46,11 +46,11 @@ impl std::fmt::Display for PerUserExperimentalFeature { } } -pub struct Store { - pub db_pool: P, +pub struct Store { + pub db_pool: Box, } -impl Store

{ +impl Store { /// Checks whether a given feature is enabled/disabled for this user /// /// If there is no entry, returns None From 683f264a7f1c9c243bcb71251a214411beda5be6 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 24 Jun 2026 14:45:55 -0500 Subject: [PATCH 40/55] LLM simplify `dyn DatabasePool` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prompt: ``` Compare this with a previous approach that was swapped out in `d9a111bdfaf51cd4cadfdc071f9956eec47aea43` (look at the removals). Anything we should change/simplify or something you like better/worse about that one? Give the new approach a review and simplify and make things more clear as necessary ``` Changes made (LLM summary): 1. #[async_trait] on the object-safe method. The core trait method is now async fn run_interaction_erased(...) -> ErasedResult instead of a hand-rolled fn ... -> BoxFuture<'a, _>. This is the convention already used by Transaction in the same file, and it removed the async move { ... }.boxed() wrapper (plus the BoxFuture/FutureExt imports) from both pool impls — their bodies are now plain async fns again. 2. Dropped the constrain / erase_interaction helper. I'd over-engineered the closure erasure out of a worry that higher-ranked-lifetime inference would fail. The previous code proved a direct let erased: ErasedInteraction = Box::new(move |txn| {...}) compiles — the type annotation on the binding is enough. ~30 lines gone. 3. Dropped the unneeded + Sync on the erased callback type. ErasedInteraction is + Send only now; the boxed callback is moved to a single DB thread and called there, never shared by reference. --- rust/src/storage/db/mod.rs | 98 ++++++++++----------------- rust/src/storage/db/python_db_pool.rs | 84 +++++++++++------------ rust/src/storage/db/rust_db_pool.rs | 76 ++++++++++----------- 3 files changed, 112 insertions(+), 146 deletions(-) diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs index 51029161e22..21330dc4b6f 100644 --- a/rust/src/storage/db/mod.rs +++ b/rust/src/storage/db/mod.rs @@ -22,50 +22,51 @@ use futures::FutureExt; pub mod python_db_pool; pub mod rust_db_pool; -/// A type-erased transaction callback, as accepted by -/// [`DatabasePool::run_interaction_dyn`]. +/// A type-erased `run_interaction` callback. /// /// This is the object-safe form of the `func` passed to /// [`DatabasePoolExt::run_interaction`]: the concrete result type `R` is boxed up -/// as `Box` so the trait can stay dyn-compatible. The ergonomic -/// [`DatabasePoolExt::run_interaction`] wrapper takes care of boxing the result -/// and downcasting it back to `R`. +/// as `Box` (see [`ErasedResult`]) so [`DatabasePool`] can stay +/// object-safe (and usable as `Box`). The ergonomic +/// [`DatabasePoolExt::run_interaction`] handles the boxing and downcasts the +/// result back to `R` for the caller. +/// +/// It may be invoked multiple times under certain failure modes (serialization +/// and deadlock errors), so it is `Fn` rather than `FnOnce`. pub type ErasedInteraction = - Box Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, ErasedResult> + Send + Sync>; + Box Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, ErasedResult> + Send>; /// The type-erased result of an [`ErasedInteraction`]: the concrete `R` boxed up -/// as `Box` so it can pass through the object-safe -/// [`DatabasePool::run_interaction_dyn`]. +/// as `Box` so it can pass back through the object-safe +/// [`DatabasePool::run_interaction_erased`]. pub type ErasedResult = anyhow::Result>; /// A database connection pool. /// -/// Code is written against this trait so the same store can run against either -/// the Python-backed pool (in Synapse, see [`python_db_pool`]) or a native -/// `tokio-postgres` pool (in `synapse-rust-apps`, see [`rust_db_pool`]). The pool -/// is held behind a trait object (e.g. `Box`), so a single -/// `Store` type can wrap whichever pool the binary happens to use. +/// Held behind a trait object (e.g. `Box`) so a single `Store` +/// can run against either the Python-backed pool (in Synapse, see +/// [`python_db_pool`]) or a native `tokio-postgres` pool (in `synapse-rust-apps`, +/// see [`rust_db_pool`]). /// -/// Callers don't use this trait directly; they go through the ergonomic, -/// strongly-typed [`run_interaction`](DatabasePoolExt::run_interaction) on -/// [`DatabasePoolExt`] (blanket-implemented for every `DatabasePool`, including -/// `dyn DatabasePool`). This trait only carries the object-safe, type-erased -/// [`run_interaction_dyn`](Self::run_interaction_dyn) that the wrapper dispatches -/// to. +/// To keep the trait object-safe, the only required method is the type-erased +/// [`run_interaction_erased`](Self::run_interaction_erased); callers should +/// prefer the ergonomic, generic +/// [`run_interaction`](DatabasePoolExt::run_interaction). /// /// `Send + Sync` so it can be stored in a `#[pyclass]` and shared across threads. +#[async_trait::async_trait] pub trait DatabasePool: Send + Sync { - /// Object-safe, type-erased core of [`DatabasePoolExt::run_interaction`]. + /// Starts a transaction on the database and runs the given (type-erased) + /// `func`, returning its boxed result. /// /// Implementors implement this; callers should prefer /// [`DatabasePoolExt::run_interaction`], which boxes up the result and - /// downcasts it back to the concrete type for you. See that method for the - /// semantics of `name` and `func`. - fn run_interaction_dyn<'a>( - &'a self, + /// downcasts it back to the concrete type for you. + async fn run_interaction_erased( + &self, name: &'static str, func: ErasedInteraction, - ) -> BoxFuture<'a, ErasedResult>; + ) -> ErasedResult; } /// Ergonomic, strongly-typed access to a [`DatabasePool`]. @@ -124,16 +125,17 @@ pub trait DatabasePoolExt: DatabasePool { + Sync + 'static, { - async move { - // Box the concrete result `R` up as `Box` so we can - // route through the object-safe `run_interaction_dyn`, then downcast - // back to `R` once it returns. - let erased = erase_interaction(func); + // Erase the concrete return type `R` into `Box` so we can call + // through the object-safe `run_interaction_erased`. + let erased: ErasedInteraction = Box::new(move |txn| { + let fut = func(txn); + async move { Ok(Box::new(fut.await?) as Box) }.boxed() + }); - let result = self.run_interaction_dyn(name, erased).await?; - Ok(*result.downcast::().expect( - "run_interaction_dyn returned a value of an unexpected type; \ - this is a programming error in the DatabasePool implementation", + async move { + let boxed = self.run_interaction_erased(name, erased).await?; + Ok(*boxed.downcast::().expect( + "run_interaction return type mismatch (this is a Synapse programming error)", )) } } @@ -141,34 +143,6 @@ pub trait DatabasePoolExt: DatabasePool { impl DatabasePoolExt for T {} -/// Wrap a strongly-typed transaction callback so its result is boxed as -/// `Box`, producing the type-erased [`ErasedInteraction`] that -/// [`DatabasePool::run_interaction_dyn`] accepts. -fn erase_interaction(func: F) -> ErasedInteraction -where - R: Send + 'static, - F: for<'txn> Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, anyhow::Result> - + Send - + Sync - + 'static, -{ - // Routing the closure through a generic fn parameter that carries the - // `for<'txn>` bound pins its higher-ranked signature; assigning a closure - // that borrows its argument straight into a `Box` doesn't reliably - // infer that lifetime. - fn constrain(c: C) -> C - where - C: for<'txn> Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, ErasedResult>, - { - c - } - - Box::new(constrain(move |txn| { - let fut = func(txn); - async move { Ok(Box::new(fut.await?) as Box) }.boxed() - })) -} - /// A [`tokio_postgres::Transaction`] looking thing that we can use on the Rust side to /// interact with the database #[async_trait::async_trait] diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index 7388bc07655..53f9ac32a46 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -22,8 +22,6 @@ use std::pin::pin; use std::sync::{Arc, Mutex}; use std::task::{Context, Poll}; -use futures::future::BoxFuture; -use futures::FutureExt; use pyo3::{ exceptions::{PyRuntimeError, PyTypeError}, intern, @@ -75,32 +73,32 @@ impl PythonDatabasePoolWrapper { } } +#[async_trait::async_trait] impl DatabasePool for PythonDatabasePoolWrapper { - fn run_interaction_dyn<'a>( - &'a self, + async fn run_interaction_erased( + &self, name: &'static str, func: ErasedInteraction, - ) -> BoxFuture<'a, ErasedResult> { - async move { - // `runInteraction` calls `func` with a `LoggingTransaction` on a DB - // thread and expects a synchronous return value. Since we can't - // round-trip an arbitrary Rust result back out through Python, the - // callback stashes the result here and we pick it up once the deferred - // fires. - // - // Note the callback may run more than once (`runInteraction` retries on - // serialization/deadlock errors), so we only trust this slot once the - // deferred has fired, i.e. once the transaction has finally committed or - // failed. - let result_slot: Arc>> = Arc::new(Mutex::new(None)); + ) -> ErasedResult { + // `runInteraction` calls `func` with a `LoggingTransaction` on a DB + // thread and expects a synchronous return value. Since we can't + // round-trip an arbitrary Rust result back out through Python, the + // callback stashes the result here and we pick it up once the deferred + // fires. + // + // Note the callback may run more than once (`runInteraction` retries on + // serialization/deadlock errors), so we only trust this slot once the + // deferred has fired, i.e. once the transaction has finally committed or + // failed. + let result_slot: Arc>> = Arc::new(Mutex::new(None)); - // Build the callback that Python's `runInteraction` invokes on a DB - // thread with a `LoggingTransaction`, plus owned handles we can move onto - // the reactor thread. We drive `func` to completion in the callback; the - // Python query path is synchronous under the hood, so it's safe to block - // this dedicated DB thread until the future resolves. - let callback_slot = Arc::clone(&result_slot); - let (callback, database_pool_py, reactor) = Python::attach(|py| -> PyResult<_> { + // Build the callback that Python's `runInteraction` invokes on a DB + // thread with a `LoggingTransaction`, plus owned handles we can move onto + // the reactor thread. We drive `func` to completion in the callback; the + // Python query path is synchronous under the hood, so it's safe to block + // this dedicated DB thread until the future resolves. + let callback_slot = Arc::clone(&result_slot); + let (callback, database_pool_py, reactor) = Python::attach(|py| -> PyResult<_> { let callback = PyCFunction::new_closure( py, None, @@ -152,28 +150,26 @@ impl DatabasePool for PythonDatabasePoolWrapper { }) .map_err(anyhow::Error::from)?; - // Use `runInteraction` directly - let outcome = run_python_awaitable(reactor, move |py| { - database_pool_py - .bind(py) - .call_method1(intern!(py, "runInteraction"), (name, callback.bind(py))) - }) - .await; + // Use `runInteraction` directly + let outcome = run_python_awaitable(reactor, move |py| { + database_pool_py + .bind(py) + .call_method1(intern!(py, "runInteraction"), (name, callback.bind(py))) + }) + .await; - // Prefer the result captured by the callback (it carries the Rust - // context). If the slot is empty, `runInteraction` failed before ever - // invoking the callback (e.g. it couldn't acquire a connection), so - // surface the deferred's outcome instead. - let captured = result_slot.lock().unwrap().take(); - match captured { - Some(result) => result, - None => match outcome { - Ok(_) => Err(anyhow::anyhow!("run_interaction produced no result")), - Err(py_err) => Err(anyhow::Error::from(py_err)), - }, - } + // Prefer the result captured by the callback (it carries the Rust + // context). If the slot is empty, `runInteraction` failed before ever + // invoking the callback (e.g. it couldn't acquire a connection), so + // surface the deferred's outcome instead. + let captured = result_slot.lock().unwrap().take(); + match captured { + Some(result) => result, + None => match outcome { + Ok(_) => Err(anyhow::anyhow!("run_interaction produced no result")), + Err(py_err) => Err(anyhow::Error::from(py_err)), + }, } - .boxed() } } diff --git a/rust/src/storage/db/rust_db_pool.rs b/rust/src/storage/db/rust_db_pool.rs index 8fb13b658e0..df42be5cd2e 100644 --- a/rust/src/storage/db/rust_db_pool.rs +++ b/rust/src/storage/db/rust_db_pool.rs @@ -19,8 +19,6 @@ use anyhow::Context; use bb8_postgres::tokio_postgres::{self, types::ToSql, IsolationLevel}; use bb8_postgres::PostgresConnectionManager; -use futures::future::BoxFuture; -use futures::FutureExt; use postgres_native_tls::MakeTlsConnector; use crate::storage::db::{ @@ -32,53 +30,51 @@ pub struct RustDatabasePool { db_pool: bb8::Pool>, } +#[async_trait::async_trait] impl DatabasePool for RustDatabasePool { - fn run_interaction_dyn<'a>( - &'a self, + async fn run_interaction_erased( + &self, _name: &'static str, func: ErasedInteraction, - ) -> BoxFuture<'a, ErasedResult> { - async move { - // Like Synapse's `runInteraction`, retry the whole transaction on - // serialization/deadlock errors (which can happen under repeatable-read). - loop { - let mut conn = self - .db_pool - .get() - .await - .context("Failed to acquire database connection")?; + ) -> ErasedResult { + // Like Synapse's `runInteraction`, retry the whole transaction on + // serialization/deadlock errors (which can happen under repeatable-read). + loop { + let mut conn = self + .db_pool + .get() + .await + .context("Failed to acquire database connection")?; - // Repeatable-read isolation level (like Synapse). - let txn = conn - .build_transaction() - .isolation_level(IsolationLevel::RepeatableRead) - .start() - .await - .context("Failed to start transaction")?; + // Repeatable-read isolation level (like Synapse). + let txn = conn + .build_transaction() + .isolation_level(IsolationLevel::RepeatableRead) + .start() + .await + .context("Failed to start transaction")?; - let mut wrapper = TokioPostgresTransaction { txn }; - match func(&mut wrapper).await { - Ok(value) => { - wrapper - .txn - .commit() - .await - .context("Failed to commit transaction")?; - return Ok(value); - } - Err(err) => { - // The transaction is rolled back implicitly when dropped, but - // be explicit about it before deciding whether to retry. - let _ = wrapper.txn.rollback().await; - if is_retryable(&err) { - continue; - } - return Err(err); + let mut wrapper = TokioPostgresTransaction { txn }; + match func(&mut wrapper).await { + Ok(value) => { + wrapper + .txn + .commit() + .await + .context("Failed to commit transaction")?; + return Ok(value); + } + Err(err) => { + // The transaction is rolled back implicitly when dropped, but + // be explicit about it before deciding whether to retry. + let _ = wrapper.txn.rollback().await; + if is_retryable(&err) { + continue; } + return Err(err); } } } - .boxed() } } From adeb03960cc0094964a47386fbe2fc55c10ce93c Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 24 Jun 2026 14:51:16 -0500 Subject: [PATCH 41/55] Sync trait not needed on `run_interaction` LLM summary: func only ever needs to be moved to a thread and called there sequentially, never shared by & across threads: - In the erasure, func is captured by-move into the Box callback (only Send is required to box it that way). - The Python pool moves that callback into the runInteraction DB-thread closure and calls it there; the Rust pool calls it within a single async task. Neither sends &func to multiple threads concurrently. --- rust/src/storage/db/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs index 21330dc4b6f..ed705b8306e 100644 --- a/rust/src/storage/db/mod.rs +++ b/rust/src/storage/db/mod.rs @@ -122,7 +122,6 @@ pub trait DatabasePoolExt: DatabasePool { R: Send + 'static, F: for<'txn> Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, anyhow::Result> + Send - + Sync + 'static, { // Erase the concrete return type `R` into `Box` so we can call From a4e4af7fa88735be0b41abdc19d7c721d827a473 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 24 Jun 2026 14:55:47 -0500 Subject: [PATCH 42/55] "Dyn-compatibility" is the new name for "object safety" > This concept was formerly known as object safety. > > *-- https://doc.rust-lang.org/reference/items/traits.html#dyn-compatibility* --- rust/src/storage/db/mod.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs index ed705b8306e..c34ea2899eb 100644 --- a/rust/src/storage/db/mod.rs +++ b/rust/src/storage/db/mod.rs @@ -24,10 +24,10 @@ pub mod rust_db_pool; /// A type-erased `run_interaction` callback. /// -/// This is the object-safe form of the `func` passed to +/// This is the dyn-compatible form of the `func` passed to /// [`DatabasePoolExt::run_interaction`]: the concrete result type `R` is boxed up /// as `Box` (see [`ErasedResult`]) so [`DatabasePool`] can stay -/// object-safe (and usable as `Box`). The ergonomic +/// dyn-compatible (and usable as `Box`). The ergonomic /// [`DatabasePoolExt::run_interaction`] handles the boxing and downcasts the /// result back to `R` for the caller. /// @@ -37,7 +37,7 @@ pub type ErasedInteraction = Box Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, ErasedResult> + Send>; /// The type-erased result of an [`ErasedInteraction`]: the concrete `R` boxed up -/// as `Box` so it can pass back through the object-safe +/// as `Box` so it can pass back through the dyn-compatible /// [`DatabasePool::run_interaction_erased`]. pub type ErasedResult = anyhow::Result>; @@ -48,7 +48,7 @@ pub type ErasedResult = anyhow::Result>; /// [`python_db_pool`]) or a native `tokio-postgres` pool (in `synapse-rust-apps`, /// see [`rust_db_pool`]). /// -/// To keep the trait object-safe, the only required method is the type-erased +/// To keep the trait dyn-compatible, the only required method is the type-erased /// [`run_interaction_erased`](Self::run_interaction_erased); callers should /// prefer the ergonomic, generic /// [`run_interaction`](DatabasePoolExt::run_interaction). @@ -125,7 +125,7 @@ pub trait DatabasePoolExt: DatabasePool { + 'static, { // Erase the concrete return type `R` into `Box` so we can call - // through the object-safe `run_interaction_erased`. + // through the dyn-compatible `run_interaction_erased`. let erased: ErasedInteraction = Box::new(move |txn| { let fut = func(txn); async move { Ok(Box::new(fut.await?) as Box) }.boxed() From 87ca67d4b63ed361290629b99d0302adfc21e88c Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 24 Jun 2026 15:03:51 -0500 Subject: [PATCH 43/55] Update comments --- rust/src/storage/db/mod.rs | 9 +++++---- rust/src/storage/db/python_db_pool.rs | 3 +++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs index c34ea2899eb..88ed6dc21ff 100644 --- a/rust/src/storage/db/mod.rs +++ b/rust/src/storage/db/mod.rs @@ -48,10 +48,9 @@ pub type ErasedResult = anyhow::Result>; /// [`python_db_pool`]) or a native `tokio-postgres` pool (in `synapse-rust-apps`, /// see [`rust_db_pool`]). /// -/// To keep the trait dyn-compatible, the only required method is the type-erased -/// [`run_interaction_erased`](Self::run_interaction_erased); callers should -/// prefer the ergonomic, generic -/// [`run_interaction`](DatabasePoolExt::run_interaction). +/// To keep the trait dyn-compatible, we have to specify a type-erased +/// [`run_interaction_erased`](Self::run_interaction_erased) version; callers should +/// prefer the ergonomic, generic [`run_interaction`](DatabasePoolExt::run_interaction). /// /// `Send + Sync` so it can be stored in a `#[pyclass]` and shared across threads. #[async_trait::async_trait] @@ -140,6 +139,8 @@ pub trait DatabasePoolExt: DatabasePool { } } +// Make [`run_interaction`](DatabasePoolExt::run_interaction) available on all +// `DatabasePool` impl DatabasePoolExt for T {} /// A [`tokio_postgres::Transaction`] looking thing that we can use on the Rust side to diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index 53f9ac32a46..b2fbb7b5119 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -231,6 +231,9 @@ pub struct LoggingTransactionWrapper { logging_transaction_py: Py, /// Disambiguate which underlying database engine we're working with + /// + /// Some features are only available on Postgres vs SQLite and the queries need to + /// be differentiated (for compatibility or performance reasons). pub database_engine: DatabaseEngine, } From 590fd2b95bb24b782d4b577b2160c97a2a80f955 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 24 Jun 2026 15:27:53 -0500 Subject: [PATCH 44/55] Fill in the rest of `rust/src/handlers/versions.rs` --- rust/src/handlers/versions.rs | 196 ++++++++++++++++++++++------------ 1 file changed, 128 insertions(+), 68 deletions(-) diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index 029f5478b84..f74a56ff64e 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -171,12 +171,104 @@ async fn build_versions_response( /// Experimental features the server supports #[derive(Serialize, Debug, Clone)] pub struct UnstableFeatureMap { - /// Simplified sliding sync - #[serde(rename = "org.matrix.simplified_msc3575")] - msc3575: bool, + /// Implements support for label-based filtering as described in + /// MSC2326. + #[serde(rename = "org.matrix.label_based_filtering")] + msc2326: bool, + /// Implements support for cross signing as described in MSC1756 + #[serde(rename = "org.matrix.e2e_cross_signing")] + msc1756: bool, + /// Implements additional endpoints as described in MSC2432 + #[serde(rename = "org.matrix.msc2432")] + msc2432: bool, + /// Implements additional endpoints as described in MSC2666 + #[serde(rename = "uk.half-shot.msc2666.query_mutual_rooms.stable")] + msc2666: bool, + // Supports the busy presence state described in MSC3026. + #[serde(rename = "org.matrix.msc3026.busy_presence")] + msc3026: bool, + /// Supports receiving private read receipts as per MSC2285 + // TODO: Remove when MSC2285 becomes a part of the spec + #[serde(rename = "org.matrix.msc2285.stable")] + msc2285: bool, + /// Supports filtering of /publicRooms by room type as per MSC3827 + #[serde(rename = "org.matrix.msc3827.stable")] + msc3827: bool, + /// Adds support for thread relations, per MSC3440. + // TODO: remove when "v1.3" is added above + #[serde(rename = "org.matrix.msc3440.stable")] + msc3440: bool, + /// Support for thread read receipts & notification counts. + #[serde(rename = "org.matrix.msc3771")] + msc3771: bool, + #[serde(rename = "org.matrix.msc3773")] + msc3773: bool, + /// Allows moderators to fetch redacted event content as described in MSC2815 + #[serde(rename = "fi.mau.msc2815")] + msc2815: bool, + /// Adds a ping endpoint for appservices to check HS->AS connection + // TODO: remove when "v1.7" is added above + #[serde(rename = "fi.mau.msc2659.stable")] + msc2659: bool, + // TODO: this is no longer needed once unstable MSC3882 does not need to be supported: + #[serde(rename = "org.matrix.msc3882")] + msc3882: bool, /// Adds support for remotely enabling/disabling pushers, as per MSC3881 #[serde(rename = "org.matrix.msc3881")] msc3881: bool, + /// Adds support for filtering /messages by event relation. + #[serde(rename = "org.matrix.msc3874")] + msc3874: bool, + // Adds support for relation-based redactions as per MSC3912. + #[serde(rename = "org.matrix.msc3912")] + msc3912: bool, + /// Whether recursively provide relations is supported. + // TODO This is no longer needed once unstable MSC3981 does not need to be supported. + #[serde(rename = "org.matrix.msc3981")] + msc3981: bool, + /// Adds support for deleting account data. + #[serde(rename = "org.matrix.msc3391")] + msc3391: bool, + /// Allows clients to inhibit profile update propagation. + #[serde(rename = "org.matrix.msc4069")] + msc4069: bool, + // Allows clients to handle push for encrypted events. + #[serde(rename = "org.matrix.msc4028")] + msc4028: bool, + /// MSC4108: Mechanism to allow OIDC sign in and E2EE set up via QR code - 2024 version + #[serde(rename = "org.matrix.msc4108")] + msc4108: bool, + /// MSC4140: Delayed events + #[serde(rename = "org.matrix.msc4140")] + msc4140: bool, + /// Simplified sliding sync + #[serde(rename = "org.matrix.simplified_msc3575")] + msc3575: bool, + /// Arbitrary key-value profile fields. + #[serde(rename = "uk.tcpip.msc4133")] + msc4133: bool, + /// Arbitrary key-value profile fields (stable identifier) + #[serde(rename = "uk.tcpip.msc4133.stable")] + msc4133_stable: bool, + /// MSC4155: Invite filtering + #[serde(rename = "org.matrix.msc4155")] + msc4155: bool, + /// MSC4306: Support for thread subscriptions + #[serde(rename = "org.matrix.msc4306")] + msc4306: bool, + /// MSC4169: Backwards-compatible redaction sending using `/send` + #[serde(rename = "com.beeper.msc4169")] + msc4169: bool, + /// MSC4354: Sticky events + #[serde(rename = "org.matrix.msc4354")] + msc4354: bool, + /// MSC4380: Invite blocking + #[serde(rename = "org.matrix.msc4380.stable")] + msc4380: bool, + // MSC4445: Sync timeline order + #[serde(rename = "org.matrix.msc4445.initial_sync_timeline_topological_ordering")] + msc4445_initial_sync_timeline_topological_ordering: bool, + // Whether new rooms will be set to encrypted or not (based on presets). #[serde(rename = "io.element.e2ee_forced.public")] @@ -185,78 +277,46 @@ pub struct UnstableFeatureMap { e2ee_forced_private: bool, #[serde(rename = "io.element.e2ee_forced.trusted_private")] e2ee_forced_trusted_private: bool, - // // Implements support for label-based filtering as described in - // // MSC2326. - // ("org.matrix.label_based_filtering".to_string(), true), - // // Implements support for cross signing as described in MSC1756 - // ("org.matrix.e2e_cross_signing".to_string(), true), - // // Implements additional endpoints as described in MSC2432 - // ("org.matrix.msc2432".to_string(), true), - // // Implements additional endpoints as described in MSC2666 - // ("uk.half-shot.msc2666.query_mutual_rooms.stable".to_string(), true), - // // Supports the busy presence state described in MSC3026. - // ("org.matrix.msc3026.busy_presence".to_string(), config.experimental.msc3026_enabled), - // // Supports receiving private read receipts as per MSC2285 - // ("org.matrix.msc2285.stable".to_string(), true), // TODO: Remove when MSC2285 becomes a part of the spec - // // Supports filtering of /publicRooms by room type as per MSC3827 - // ("org.matrix.msc3827.stable".to_string(), true), - // // Adds support for thread relations, per MSC3440. - // ("org.matrix.msc3440.stable".to_string(), true), // TODO: remove when "v1.3" is added above - // // Support for thread read receipts & notification counts. - // ("org.matrix.msc3771".to_string(), true), - // ("org.matrix.msc3773".to_string(), config.experimental.msc3773_enabled), - // // Allows moderators to fetch redacted event content as described in MSC2815 - // ("fi.mau.msc2815".to_string(), config.experimental.msc2815_enabled), - // // Adds a ping endpoint for appservices to check HS->AS connection - // ("fi.mau.msc2659.stable".to_string(), true), // TODO: remove when "v1.7" is added above - // // TODO: this is no longer needed once unstable MSC3882 does not need to be supported: - // ("org.matrix.msc3882".to_string(), config.auth.login_via_existing_enabled), - // // Adds support for filtering /messages by event relation. - // ("org.matrix.msc3874".to_string(), config.experimental.msc3874_enabled), - // // Adds support for relation-based redactions as per MSC3912. - // ("org.matrix.msc3912".to_string(), config.experimental.msc3912_enabled), - // // Whether recursively provide relations is supported. - // // TODO This is no longer needed once unstable MSC3981 does not need to be supported. - // ("org.matrix.msc3981".to_string(), true), - // // Adds support for deleting account data. - // ("org.matrix.msc3391".to_string(), config.experimental.msc3391_enabled), - // // Allows clients to inhibit profile update propagation. - // ("org.matrix.msc4069".to_string(), config.experimental.msc4069_profile_inhibit_propagation), - // // Allows clients to handle push for encrypted events. - // ("org.matrix.msc4028".to_string(), config.experimental.msc4028_push_encrypted_events), - // // MSC4108: Mechanism to allow OIDC sign in and E2EE set up via QR code - 2024 version - // ("org.matrix.msc4108".to_string(), ( - // config.experimental.msc4108_enabled - // or ( - // config.experimental.msc4108_delegation_endpoint - // is not None - // ) - // )), - // // MSC4140: Delayed events - // ("org.matrix.msc4140".to_string(), bool(config.server.max_event_delay_ms)), - // // Arbitrary key-value profile fields. - // ("uk.tcpip.msc4133".to_string(), config.experimental.msc4133_enabled), - // ("uk.tcpip.msc4133.stable".to_string(), true), - // // MSC4155: Invite filtering - // ("org.matrix.msc4155".to_string(), config.experimental.msc4155_enabled), - // // MSC4306: Support for thread subscriptions - // ("org.matrix.msc4306".to_string(), config.experimental.msc4306_enabled), - // // MSC4169: Backwards-compatible redaction sending using `/send` - // ("com.beeper.msc4169".to_string(), config.experimental.msc4169_enabled), - // // MSC4354: Sticky events - // ("org.matrix.msc4354".to_string(), config.experimental.msc4354_enabled), - // // MSC4380: Invite blocking - // ("org.matrix.msc4380.stable".to_string(), true), - // // MSC4445: Sync timeline order - // ("org.matrix.msc4445.initial_sync_timeline_topological_ordering".to_string(), true), } /// Convert from [`SynapseConfig`] to the global defaults for unstable features that the /// server supports [`UnstableFeatureMap`] pub fn synapse_config_to_global_unstable_feature_map(config: &SynapseConfig) -> UnstableFeatureMap { UnstableFeatureMap { - msc3575: config.experimental.msc3575_enabled, + msc2326: true, + msc1756: true, + msc2432: true, + msc2666: true, + msc3026: config.experimental.msc3026_enabled, + msc2285: true, + msc3827: true, + msc3440: true, + msc3771: true, + msc3773: config.experimental.msc3773_enabled, + msc2815: config.experimental.msc2815_enabled, + msc2659: true, + msc3882: config.auth.login_via_existing_enabled, msc3881: config.experimental.msc3881_enabled, + msc3874: config.experimental.msc3874_enabled, + msc3912: config.experimental.msc3912_enabled, + msc3981: true, + msc3391: config.experimental.msc3391_enabled, + msc4069: config.experimental.msc4069_profile_inhibit_propagation, + msc4028: config.experimental.msc4028_push_encrypted_events, + msc4108: config.experimental.msc4108_enabled || ( + config.experimental.msc4108_delegation_endpoint + is not None + ), + msc4140: bool(config.server.max_event_delay_ms), + msc3575: config.experimental.msc3575_enabled, + msc4133: config.experimental.msc4133_enabled, + msc4133_stable: true, + msc4155: config.experimental.msc4155_enabled, + msc4306: config.experimental.msc4306_enabled, + msc4169: config.experimental.msc4169_enabled, + msc4354: config.experimental.msc4354_enabled, + msc4380: true, + msc4445_initial_sync_timeline_topological_ordering: true, e2ee_forced_public: config .room .encryption_enabled_by_default_for_room_presets From 2a609a5cdf8c81f89ab2bbc26f575df92dc3b038 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 24 Jun 2026 15:32:38 -0500 Subject: [PATCH 45/55] Fill in missing experimental config --- rust/src/config/mod.rs | 15 +++++++++++++++ rust/src/handlers/versions.rs | 11 ++++------- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/rust/src/config/mod.rs b/rust/src/config/mod.rs index d1788b5ab93..6dd4ae220a0 100644 --- a/rust/src/config/mod.rs +++ b/rust/src/config/mod.rs @@ -64,7 +64,22 @@ pub struct RoomConfig { #[derive(FromPyObject, Clone)] pub struct ExperimentalConfig { + pub msc3026_enabled: bool, + pub msc3773_enabled: bool, + pub msc2815_enabled: bool, pub msc3881_enabled: bool, + pub msc3874_enabled: bool, + pub msc3912_enabled: bool, + pub msc3391_enabled: bool, + pub msc4069_profile_inhibit_propagation: bool, + pub msc4028_push_encrypted_events: bool, + pub msc4108_enabled: bool, + pub msc4108_delegation_endpoint: Option, pub msc3575_enabled: bool, + pub msc4133_enabled: bool, + pub msc4155_enabled: bool, + pub msc4306_enabled: bool, + pub msc4169_enabled: bool, + pub msc4354_enabled: bool, pub msc4222_enabled: bool, } diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index f74a56ff64e..d2627b9e798 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -236,10 +236,10 @@ pub struct UnstableFeatureMap { #[serde(rename = "org.matrix.msc4028")] msc4028: bool, /// MSC4108: Mechanism to allow OIDC sign in and E2EE set up via QR code - 2024 version - #[serde(rename = "org.matrix.msc4108")] + #[serde(rename = "org.matrix.msc4108")] msc4108: bool, /// MSC4140: Delayed events - #[serde(rename = "org.matrix.msc4140")] + #[serde(rename = "org.matrix.msc4140")] msc4140: bool, /// Simplified sliding sync #[serde(rename = "org.matrix.simplified_msc3575")] @@ -269,7 +269,6 @@ pub struct UnstableFeatureMap { #[serde(rename = "org.matrix.msc4445.initial_sync_timeline_topological_ordering")] msc4445_initial_sync_timeline_topological_ordering: bool, - // Whether new rooms will be set to encrypted or not (based on presets). #[serde(rename = "io.element.e2ee_forced.public")] e2ee_forced_public: bool, @@ -303,10 +302,8 @@ pub fn synapse_config_to_global_unstable_feature_map(config: &SynapseConfig) -> msc3391: config.experimental.msc3391_enabled, msc4069: config.experimental.msc4069_profile_inhibit_propagation, msc4028: config.experimental.msc4028_push_encrypted_events, - msc4108: config.experimental.msc4108_enabled || ( - config.experimental.msc4108_delegation_endpoint - is not None - ), + msc4108: config.experimental.msc4108_enabled + || (config.experimental.msc4108_delegation_endpoint.is_some()), msc4140: bool(config.server.max_event_delay_ms), msc3575: config.experimental.msc3575_enabled, msc4133: config.experimental.msc4133_enabled, From 59228e5f58a14f4375ebc2e09449baa0cd91a23d Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 24 Jun 2026 15:37:42 -0500 Subject: [PATCH 46/55] Fill in the rest of the config --- rust/src/config/mod.rs | 11 +++++++++++ rust/src/handlers/versions.rs | 5 ++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/rust/src/config/mod.rs b/rust/src/config/mod.rs index 6dd4ae220a0..562beebfc92 100644 --- a/rust/src/config/mod.rs +++ b/rust/src/config/mod.rs @@ -21,6 +21,8 @@ use pyo3::{exceptions::PyRuntimeError, prelude::*}; #[derive(FromPyObject, Clone)] pub struct SynapseConfig { pub room: RoomConfig, + pub auth: AuthConfig, + pub server: ServerConfig, pub experimental: ExperimentalConfig, } @@ -62,6 +64,15 @@ pub struct RoomConfig { pub encryption_enabled_by_default_for_room_presets: BTreeSet, } +#[derive(FromPyObject, Clone)] +pub struct AuthConfig { + pub login_via_existing_enabled: bool, +} +#[derive(FromPyObject, Clone)] +pub struct ServerConfig { + pub max_event_delay_ms: Option, +} + #[derive(FromPyObject, Clone)] pub struct ExperimentalConfig { pub msc3026_enabled: bool, diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index d2627b9e798..fde47f4a33d 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -304,7 +304,10 @@ pub fn synapse_config_to_global_unstable_feature_map(config: &SynapseConfig) -> msc4028: config.experimental.msc4028_push_encrypted_events, msc4108: config.experimental.msc4108_enabled || (config.experimental.msc4108_delegation_endpoint.is_some()), - msc4140: bool(config.server.max_event_delay_ms), + msc4140: config + .server + .max_event_delay_ms + .is_some_and(|max_event_delay_ms| max_event_delay_ms > 0), msc3575: config.experimental.msc3575_enabled, msc4133: config.experimental.msc4133_enabled, msc4133_stable: true, From 55d89f972bcf11732b73c4cbc84b5b6812043dda Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 24 Jun 2026 15:54:15 -0500 Subject: [PATCH 47/55] Use `time.sleep(0)` to align with decision in #19871 See https://github.com/element-hq/synapse/pull/19871#discussion_r3462533989 --- tests/server.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/server.py b/tests/server.py index 9488307805c..52b7dba7491 100644 --- a/tests/server.py +++ b/tests/server.py @@ -334,9 +334,12 @@ def await_result(self, timeout_ms: int = 1000) -> None: # could be things spawned on the Twisted reactor threadpool or Tokio thread # pool (async Rust code). # - # We could also use `time.sleep(0)` here but this is more precise - os.sched_yield() - # time.sleep(0) + # Note: Since we're waiting real-time (`timeout` duration), the tests also + # pass with `time.sleep(0)` commented out because Python has a default + # thread switch interval (5ms for cpython) (see + # `sys.setswitchinterval(interval)`). We still want this here as we're able + # to preempt and cause the thread context swtich to happen faster. + time.sleep(0) # Advance the Twisted reactor and run any scheduled callbacks # From 7ed525ddd11700cc0de09278e201e9b6ee5ae7c6 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 24 Jun 2026 16:02:19 -0500 Subject: [PATCH 48/55] Rename `SynapseConfig` -> `SynapseHomeServerConfig` --- rust/src/config/mod.rs | 9 ++++++++- rust/src/handlers/mod.rs | 4 ++-- rust/src/handlers/versions.rs | 8 +++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/rust/src/config/mod.rs b/rust/src/config/mod.rs index 562beebfc92..709b4428952 100644 --- a/rust/src/config/mod.rs +++ b/rust/src/config/mod.rs @@ -18,8 +18,15 @@ use std::str::FromStr; use pyo3::{exceptions::PyRuntimeError, prelude::*}; +/// A Rust-side view of Synapse's Python `HomeServerConfig`. +/// +/// This only mirrors the subset of config that the Rust handlers need, rather +/// than the whole thing. Thanks to `#[derive(FromPyObject)]`, each field is +/// pulled directly off the corresponding attribute of the Python `config` +/// object, so you can populate it in one shot with +/// `homeserver.getattr("config")?.extract()?`. #[derive(FromPyObject, Clone)] -pub struct SynapseConfig { +pub struct SynapseHomeServerConfig { pub room: RoomConfig, pub auth: AuthConfig, pub server: ServerConfig, diff --git a/rust/src/handlers/mod.rs b/rust/src/handlers/mod.rs index 9911443721f..d5604231669 100644 --- a/rust/src/handlers/mod.rs +++ b/rust/src/handlers/mod.rs @@ -21,7 +21,7 @@ use pyo3::{ Bound, PyResult, Python, }; -use crate::config::SynapseConfig; +use crate::config::SynapseHomeServerConfig; use crate::storage::db::python_db_pool::PythonDatabasePoolWrapper; use crate::storage::store::Store; @@ -37,7 +37,7 @@ impl RustHandlers { #[new] #[pyo3(signature = (homeserver))] pub fn py_new(py: Python<'_>, homeserver: &Bound<'_, PyAny>) -> PyResult { - let config: SynapseConfig = homeserver.getattr("config")?.extract()?; + let config: SynapseHomeServerConfig = homeserver.getattr("config")?.extract()?; // The Twisted reactor, used both to drive our Tokio runtime and to // marshal database work back onto the reactor thread. diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index fde47f4a33d..e46a13a5365 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -19,7 +19,7 @@ use pyo3::prelude::*; use pythonize::{pythonize, PythonizeError}; use serde::{Deserialize, Serialize}; -use crate::config::{RoomCreationPreset, SynapseConfig}; +use crate::config::{RoomCreationPreset, SynapseHomeServerConfig}; use crate::deferred::create_deferred; use crate::storage::store::{PerUserExperimentalFeature, Store}; @@ -278,9 +278,11 @@ pub struct UnstableFeatureMap { e2ee_forced_trusted_private: bool, } -/// Convert from [`SynapseConfig`] to the global defaults for unstable features that the +/// Convert from [`SynapseHomeServerConfig`] to the global defaults for unstable features that the /// server supports [`UnstableFeatureMap`] -pub fn synapse_config_to_global_unstable_feature_map(config: &SynapseConfig) -> UnstableFeatureMap { +pub fn synapse_config_to_global_unstable_feature_map( + config: &SynapseHomeServerConfig, +) -> UnstableFeatureMap { UnstableFeatureMap { msc2326: true, msc1756: true, From 8ddab4f23eac5a79951308089bcc7bbc7178de2d Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 24 Jun 2026 16:09:29 -0500 Subject: [PATCH 49/55] Iterate on Database trait comments --- rust/src/storage/db/mod.rs | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs index 88ed6dc21ff..42e9ef39ce8 100644 --- a/rust/src/storage/db/mod.rs +++ b/rust/src/storage/db/mod.rs @@ -25,28 +25,25 @@ pub mod rust_db_pool; /// A type-erased `run_interaction` callback. /// /// This is the dyn-compatible form of the `func` passed to -/// [`DatabasePoolExt::run_interaction`]: the concrete result type `R` is boxed up -/// as `Box` (see [`ErasedResult`]) so [`DatabasePool`] can stay -/// dyn-compatible (and usable as `Box`). The ergonomic -/// [`DatabasePoolExt::run_interaction`] handles the boxing and downcasts the -/// result back to `R` for the caller. +/// [`DatabasePoolExt::run_interaction`]. +/// +/// The ergonomic [`DatabasePoolExt::run_interaction`] handles the boxing and downcasts +/// the result back to `R` for the caller. /// /// It may be invoked multiple times under certain failure modes (serialization /// and deadlock errors), so it is `Fn` rather than `FnOnce`. pub type ErasedInteraction = Box Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, ErasedResult> + Send>; -/// The type-erased result of an [`ErasedInteraction`]: the concrete `R` boxed up -/// as `Box` so it can pass back through the dyn-compatible +/// The type-erased *result* of an [`ErasedInteraction`] /// [`DatabasePool::run_interaction_erased`]. pub type ErasedResult = anyhow::Result>; /// A database connection pool. /// -/// Held behind a trait object (e.g. `Box`) so a single `Store` -/// can run against either the Python-backed pool (in Synapse, see -/// [`python_db_pool`]) or a native `tokio-postgres` pool (in `synapse-rust-apps`, -/// see [`rust_db_pool`]). +/// Held behind a trait object (e.g. `Box`) as it can be backed by +/// either the Python-backed pool (in Synapse, see [`python_db_pool`]) or a Rust native +/// `tokio-postgres` pool (expected to be used in `synapse-rust-apps`). /// /// To keep the trait dyn-compatible, we have to specify a type-erased /// [`run_interaction_erased`](Self::run_interaction_erased) version; callers should @@ -69,9 +66,6 @@ pub trait DatabasePool: Send + Sync { } /// Ergonomic, strongly-typed access to a [`DatabasePool`]. -/// -/// Blanket-implemented for every `T: DatabasePool + ?Sized`, so it is available -/// both on concrete pools and on `dyn DatabasePool` trait objects. pub trait DatabasePoolExt: DatabasePool { /// Starts a transaction on the database and runs the given function, /// returning its result. @@ -139,12 +133,13 @@ pub trait DatabasePoolExt: DatabasePool { } } -// Make [`run_interaction`](DatabasePoolExt::run_interaction) available on all -// `DatabasePool` +/// Blanket-implemented for every [`DatabasePool`] so +/// [`run_interaction`](DatabasePoolExt::run_interaction) is always available impl DatabasePoolExt for T {} -/// A [`tokio_postgres::Transaction`] looking thing that we can use on the Rust side to -/// interact with the database +/// A transaction to interact with the database +/// +/// Based on the ergonomics of [`tokio_postgres::Transaction`] #[async_trait::async_trait] pub trait Transaction: Send { // `async` as this is representing a round-trip between the app and database From 79a8cc1f71399385ae1f59ab761b21fc4e75bdf3 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 24 Jun 2026 16:12:43 -0500 Subject: [PATCH 50/55] Rename `Row` -> `DbRow` --- rust/src/storage/db/mod.rs | 16 ++++++++-------- rust/src/storage/db/python_db_pool.rs | 10 +++++----- rust/src/storage/db/rust_db_pool.rs | 6 +++--- rust/src/storage/store.rs | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs index 42e9ef39ce8..78fd0b4efb8 100644 --- a/rust/src/storage/db/mod.rs +++ b/rust/src/storage/db/mod.rs @@ -143,10 +143,10 @@ impl DatabasePoolExt for T {} #[async_trait::async_trait] pub trait Transaction: Send { // `async` as this is representing a round-trip between the app and database - async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error>; + async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error>; } -/// A single backend-agnostic value within a [`Row`]. +/// A single backend-agnostic value within a [`DbRow`]. /// /// Each pool maps the values its database driver hands back into this common /// set, so callers can work with one representation regardless of engine. @@ -164,22 +164,22 @@ pub enum DbValue { /// /// Each pool converts the cells its database driver hands back into the /// engine-agnostic [`DbValue`] representation, so a row is simply a list of them. -/// Values are pulled out by their numeric index with [`RowExt::try_get`]. -pub type Row = Vec; +/// Values are pulled out by their numeric index with [`DbRowExt::try_get`]. +pub type DbRow = Vec; -/// Extension methods for reading typed values out of a [`Row`]. +/// Extension methods for reading typed values out of a [`DbRow`]. /// -/// Modelled after [`tokio_postgres::Row`]'s `try_get`: [`try_get`](Self::try_get) +/// Based on [`tokio_postgres::Row`]'s `try_get`: [`try_get`](Self::try_get) /// converts the [`DbValue`] at a given index into the requested type via /// [`FromDbValue`] (our analogue of `tokio-postgres`'s `FromSql`). -pub trait RowExt { +pub trait DbRowExt { /// Deserializes a value from the row, specified by its numeric index, /// returning an error if the index is out of bounds or the value cannot be /// converted into `T`. fn try_get(&self, index: usize) -> Result; } -impl RowExt for Row { +impl DbRowExt for DbRow { fn try_get(&self, index: usize) -> Result { let value = self.get(index).cloned().ok_or_else(|| { anyhow::anyhow!( diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index b2fbb7b5119..e8452c44ae8 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -31,7 +31,7 @@ use pyo3::{ use crate::deferred::run_python_awaitable; use crate::storage::db::{ - DatabasePool, DbValue, ErasedInteraction, ErasedResult, Row, Transaction, + DatabasePool, DbRow, DbValue, ErasedInteraction, ErasedResult, Transaction, }; /// The database engines we support in the Python side of Synapse @@ -268,8 +268,8 @@ impl LoggingTransactionWrapper { #[async_trait::async_trait] impl Transaction for LoggingTransactionWrapper { - async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error> { - Python::attach(|py| -> PyResult> { + async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error> { + Python::attach(|py| -> PyResult> { // Convert the Rust `&[&str]` of SQL parameters into a Python sequence // so it can be passed through to the Python-side `execute`. Note that // `LoggingTransaction.execute` converts `?` placeholders into the @@ -285,10 +285,10 @@ impl Transaction for LoggingTransactionWrapper { .bind(py) .call_method0(intern!(py, "fetchall"))?; - let mut rows: Vec = Vec::new(); + let mut rows: Vec = Vec::new(); for row_py in rows_py.try_iter()? { let row_py = row_py?; - let mut row: Row = Vec::new(); + let mut row: DbRow = Vec::new(); for cell in row_py.try_iter()? { row.push(py_cell_to_value(&cell?)?); } diff --git a/rust/src/storage/db/rust_db_pool.rs b/rust/src/storage/db/rust_db_pool.rs index df42be5cd2e..96b50b78fee 100644 --- a/rust/src/storage/db/rust_db_pool.rs +++ b/rust/src/storage/db/rust_db_pool.rs @@ -22,7 +22,7 @@ use bb8_postgres::PostgresConnectionManager; use postgres_native_tls::MakeTlsConnector; use crate::storage::db::{ - DatabasePool, DbValue, ErasedInteraction, ErasedResult, Row, Transaction, + DatabasePool, DbRow, DbValue, ErasedInteraction, ErasedResult, Transaction, }; /// Native Rust database access backed by `tokio-postgres` (for use in synapse-rust-apps) @@ -95,7 +95,7 @@ struct TokioPostgresTransaction<'a> { #[async_trait::async_trait] impl Transaction for TokioPostgresTransaction<'_> { - async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error> { + async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error> { // Synapse SQL uses `?` placeholders; `tokio-postgres` uses `$1`, `$2`, ... let sql = convert_param_style(sql); @@ -112,7 +112,7 @@ impl Transaction for TokioPostgresTransaction<'_> { } /// Convert a native [`tokio_postgres::Row`] into our engine-agnostic [`Row`]. -fn tokio_row_to_row(row: &tokio_postgres::Row) -> Result { +fn tokio_row_to_row(row: &tokio_postgres::Row) -> Result { (0..row.len()) .map(|index| tokio_cell_to_value(row, index)) .collect() diff --git a/rust/src/storage/store.rs b/rust/src/storage/store.rs index f2bf8db3fae..4e8f63549c0 100644 --- a/rust/src/storage/store.rs +++ b/rust/src/storage/store.rs @@ -16,7 +16,7 @@ use futures::FutureExt; use serde::Serialize; -use crate::storage::db::{DatabasePool, DatabasePoolExt, RowExt}; +use crate::storage::db::{DatabasePool, DatabasePoolExt, DbRowExt}; /// Currently supported per-user features #[derive(Serialize, Debug)] From a2e68adb1c961e967760e00146691f73085fb3a1 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 24 Jun 2026 16:17:00 -0500 Subject: [PATCH 51/55] Explain more about `python_db_pool` --- rust/src/storage/db/python_db_pool.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index e8452c44ae8..6d164ae779e 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -13,9 +13,17 @@ * */ +//! A database pool that calls into Python to re-use the same database pool used on the +//! Python side. This is desirable because we want to avoid having two separate database +//! pools (one for Rust, one for Python) to avoid database connection exhaustion +//! problems. This is a stepping stone until all of our database interactions are in +//! Rust. +//! //! We have these main classes: -//! - Database pool [`PythonDatabasePoolWrapper`] which allows you to start a... -//! - transaction [`LoggingTransactionWrapper`] and query the database +//! - Database pool [`PythonDatabasePoolWrapper`] (implements [`DatabasePool`]) which +//! allows you to start a... +//! - transaction [`LoggingTransactionWrapper`] (implements [`Transaction`]) and query +//! the database. use std::future::Future; use std::pin::pin; From 0090330585e53bcb48adab756b637bd7e1dd4bb2 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 24 Jun 2026 16:40:28 -0500 Subject: [PATCH 52/55] Iterate on `python_db_pool` comments --- rust/src/storage/db/mod.rs | 5 +++++ rust/src/storage/db/python_db_pool.rs | 27 ++++++++++++++------------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs index 78fd0b4efb8..ce159a6429d 100644 --- a/rust/src/storage/db/mod.rs +++ b/rust/src/storage/db/mod.rs @@ -142,6 +142,11 @@ impl DatabasePoolExt for T {} /// Based on the ergonomics of [`tokio_postgres::Transaction`] #[async_trait::async_trait] pub trait Transaction: Send { + /// Run a database query, returning a list of resulting rows. + /// + /// We expect the `sql` query should use `?` placeholders for the `args`. Downstream + /// implementations should string-replace `?` as necessary. + // // `async` as this is representing a round-trip between the app and database async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error>; } diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index 6d164ae779e..12a879a0074 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -116,11 +116,12 @@ impl DatabasePool for PythonDatabasePoolWrapper { let txn_py = args.get_item(0)?; let mut txn = txn_py.extract::()?; - // Since we expect people to only call `.await` on [`Transaction`] - // related methods (mentioned in the [`Transaction`] docstring) AND - // because there is no async work to suspend on in the Python - // [`Transaction`] synchronously, we can get away with polling once - // as it should immediately resolve to [`Poll::Ready`]. Getting + // Since we expect people to only call `.await` on + // [`Transaction`] related methods (mentioned in the + // [`Transaction`] docstring) AND because there is no actual + // async work to suspend on in the Python [`Transaction`] + // (resolves synchonously), we can get away with polling once as + // it should immediately resolve to [`Poll::Ready`]. Getting // [`Poll::Pending`] would be considered a programming error. // // Alternatively, we could just use `futures::executor::block_on` @@ -183,10 +184,9 @@ impl DatabasePool for PythonDatabasePoolWrapper { /// Poll a future exactly once. /// -/// Returns [`Poll::Ready`] if the future resolves immediately, or -/// [`Poll::Pending`] if it would need to suspend. We use this where a future is -/// expected to never genuinely suspend (e.g. Synapse's synchronous DB query -/// path) and want to enforce that, rather than driving it to completion with a +/// Returns [`Poll::Ready`] if the future resolves immediately, or [`Poll::Pending`] if +/// it would need to suspend. We use this where a future is expected to never genuinely +/// suspend and want to enforce/check that, rather than driving it to completion with a /// blocking executor like `futures::executor::block_on`. fn poll_once(future: F) -> Poll { let mut future = pin!(future); @@ -208,6 +208,7 @@ fn anyhow_to_pyerr(err: &anyhow::Error) -> PyErr { PyRuntimeError::new_err(format!("{err:#}")) } +/// Given a Python `LoggingTransaction`, figures out the database engine that backs it fn detect_engine(txn_py: &Bound<'_, PyAny>) -> PyResult { let name = txn_py .getattr("database_engine") @@ -279,10 +280,10 @@ impl Transaction for LoggingTransactionWrapper { async fn query(&mut self, sql: &str, args: &[&str]) -> Result, anyhow::Error> { Python::attach(|py| -> PyResult> { // Convert the Rust `&[&str]` of SQL parameters into a Python sequence - // so it can be passed through to the Python-side `execute`. Note that - // `LoggingTransaction.execute` converts `?` placeholders into the - // appropriate param style for the underlying engine, so we pass - // `?`-style SQL. + // so it can be passed through to the Python-side `execute`. + // + // We don't need to do anything to the SQL as the `?`-style arg placeholders + // already align with what `LoggingTransaction` expects. let args = PyList::new(py, args)?; self.execute(py, sql, args.as_any())?; From 619507482afe8eb1cbf805ea59a66a3c10827794 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 24 Jun 2026 17:28:57 -0500 Subject: [PATCH 53/55] Only return runInteraction result if it succeeded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previous logic (LLM summary): > The current logic always prefers captured over outcome, even when outcome is an Err. Consider: callback runs successfully → slot = Some(Ok(value)) → returns None → but then runInteraction's commit fails and the deferred errbacks. Now captured = Some(Ok(value)) and outcome = Err(...), and we'd return Ok(value) — silently swallowing the commit failure. > > Whether that's reachable depends on Synapse's retry behavior (it may retry the whole interaction on commit failure, re-running the callback). It's likely a narrow edge case, but the comment ("we only trust this slot once the deferred has fired") implies the slot is authoritative once fired, which isn't quite true if the deferred fires with an error after a successful callback run. --- rust/src/storage/db/python_db_pool.rs | 43 ++++++++++++++++----------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index 12a879a0074..28d23e57800 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -28,8 +28,9 @@ use std::future::Future; use std::pin::pin; use std::sync::{Arc, Mutex}; -use std::task::{Context, Poll}; +use std::task::Poll; +use anyhow::Context; use pyo3::{ exceptions::{PyRuntimeError, PyTypeError}, intern, @@ -88,11 +89,11 @@ impl DatabasePool for PythonDatabasePoolWrapper { name: &'static str, func: ErasedInteraction, ) -> ErasedResult { - // `runInteraction` calls `func` with a `LoggingTransaction` on a DB - // thread and expects a synchronous return value. Since we can't - // round-trip an arbitrary Rust result back out through Python, the - // callback stashes the result here and we pick it up once the deferred - // fires. + // `runInteraction` calls `func` with a `LoggingTransaction` on a DB thread and + // expects a synchronous return value. Since we can't round-trip an arbitrary + // Rust result back out through Python (remember, `func` returns an + // `ErasedResult`, not a `PyAny`), the callback stashes the result here and we + // pick it up once the deferred fires. // // Note the callback may run more than once (`runInteraction` retries on // serialization/deadlock errors), so we only trust this slot once the @@ -160,24 +161,30 @@ impl DatabasePool for PythonDatabasePoolWrapper { .map_err(anyhow::Error::from)?; // Use `runInteraction` directly - let outcome = run_python_awaitable(reactor, move |py| { + let run_interaction_outcome = run_python_awaitable(reactor, move |py| { database_pool_py .bind(py) .call_method1(intern!(py, "runInteraction"), (name, callback.bind(py))) }) .await; - // Prefer the result captured by the callback (it carries the Rust - // context). If the slot is empty, `runInteraction` failed before ever - // invoking the callback (e.g. it couldn't acquire a connection), so - // surface the deferred's outcome instead. - let captured = result_slot.lock().unwrap().take(); - match captured { - Some(result) => result, - None => match outcome { - Ok(_) => Err(anyhow::anyhow!("run_interaction produced no result")), - Err(py_err) => Err(anyhow::Error::from(py_err)), + // Return the result we captured based on if `runInteraction` was successful + let captured_result = result_slot.lock().unwrap().take(); + match run_interaction_outcome { + // Only return the `captured_result` if `runInteraction` succeeded. We don't + // want to accidentally return a successful result when the transaction + // actually failed to commit. + Ok(_) => match captured_result { + Some(result) => result, + // This is unexpected as we either expect `runInteraction` to have + // completed successfully and run the provided `callback` which runs the + // `func` and we capture a result or it fails. + None => Err(anyhow::anyhow!( + "Expected to capture result after running `runInteraction` and seeing it succeed (but saw nothing). \ + This is a Synapse programming error." + )), }, + Err(py_err) => Err(anyhow::Error::from(py_err)).with_context(|| format!("run_interaction(name={}) failed", name)), } } } @@ -191,7 +198,7 @@ impl DatabasePool for PythonDatabasePoolWrapper { fn poll_once(future: F) -> Poll { let mut future = pin!(future); let waker = futures::task::noop_waker(); - let mut cx = Context::from_waker(&waker); + let mut cx = std::task::Context::from_waker(&waker); future.as_mut().poll(&mut cx) } From b5ea2f7154472b1a1890538b0e0366d5060dd4ac Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 24 Jun 2026 17:40:59 -0500 Subject: [PATCH 54/55] Avoid `String` allocation --- rust/src/storage/store.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/rust/src/storage/store.rs b/rust/src/storage/store.rs index 4e8f63549c0..36cb775c1b5 100644 --- a/rust/src/storage/store.rs +++ b/rust/src/storage/store.rs @@ -13,6 +13,8 @@ * */ +use std::sync::Arc; + use futures::FutureExt; use serde::Serialize; @@ -61,10 +63,12 @@ impl Store { ) -> Result, anyhow::Error> { // It's not enabled globally, so check whether it's enabled per-user. // - // Owned copies so the callback can be `'static` (it may be moved to - // another thread and called multiple times under retries). - let user_id = user_id.to_owned(); - let feature = feature.to_string(); + // We need owned copies to move into the callback because it is `'static` (it + // may be moved to another thread). We use `Arc` rather than `String` so + // the per-call clone is just a cheap refcount bump rather than a fresh + // allocation. + let user_id: Arc = user_id.into(); + let feature: Arc = feature.to_string().into(); let is_feature_enabled_for_user = self .db_pool @@ -80,7 +84,7 @@ impl Store { "SELECT enabled \ FROM per_user_experimental_features \ WHERE user_id = ? AND feature = ?", - &[user_id.as_str(), feature.as_str()], + &[user_id.as_ref(), feature.as_ref()], ) .await?; From 5ce6b8a307ab9bb3eebeb0384665ecd0f8234679 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 24 Jun 2026 17:53:28 -0500 Subject: [PATCH 55/55] Better `failure_to_pyerr` --- rust/src/deferred.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/rust/src/deferred.rs b/rust/src/deferred.rs index 606eec48339..4347474985d 100644 --- a/rust/src/deferred.rs +++ b/rust/src/deferred.rs @@ -293,20 +293,23 @@ where } } -/// Convert a Twisted `Failure` (as passed to an errback) into a [`PyErr`]. +/// Convert a Twisted `Failure` (as passed to an Deferred errback) into a [`PyErr`]. /// -/// A `Failure` carries the original exception instance in its `.value` -/// attribute, which we re-raise so callers see the real error. If that can't be -/// reached, fall back to the `Failure`'s textual representation. +/// A Twisted `Failure` carries the original exception instance in its `.value` +/// attribute, which we re-raise so callers see the real error. If the `Failure` is +/// mangled, we fallback to raising a generic [`PyRuntimeError`] explaining what we saw +/// instead. fn failure_to_pyerr(failure: &Bound<'_, PyAny>) -> PyErr { match failure.getattr(intern!(failure.py(), "value")) { Ok(value) => PyErr::from_value(value), - Err(_) => PyRuntimeError::new_err( + Err(_) => PyRuntimeError::new_err(format!( + "Expected Python object passed here to be a Twisted `Failure` with a `value` attribute \ + but saw something else: {}", failure .str() .map(|s| s.to_string_lossy().into_owned()) - .unwrap_or_else(|_| "".to_owned()), - ), + .unwrap_or_else(|_| "".to_owned()), + )), } }