Compare commits

...

7 commits

Author SHA1 Message Date
Jonas Zohren
dd14270a6e Fix formatting.
Signed-off-by: Jonas Zohren <git-pbkyr@jzohren.de>
2021-08-26 15:47:10 +02:00
Timo Kösters
e8140cda33 chore: bump dependencies 2021-08-26 15:25:49 +02:00
Timo Kösters
0ac7ba4c24 improvement: persist cached auth chains in db 2021-08-26 15:25:49 +02:00
Timo Kösters
a15ea4c156 fix: better migration, force e2ee device key updates 2021-08-26 15:25:49 +02:00
Timo Kösters
33923ad6a7 fix: wrong soft fail check, too many events in /sync state response 2021-08-26 15:25:49 +02:00
Timo Kösters
6da3f62747 fix: e2ee over federation
to device events were not being sent
2021-08-26 15:25:49 +02:00
Jonas Zohren
e534199195 Add show_cache_usage admin room command
Signed-off-by: Jonas Zohren <git-pbkyr@jzohren.de>
2021-08-26 15:17:17 +02:00
9 changed files with 529 additions and 287 deletions

333
Cargo.lock generated
View file

@ -146,9 +146,9 @@ dependencies = [
[[package]]
name = "bitflags"
version = "1.2.1"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "blake2b_simd"
@ -190,9 +190,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]]
name = "bytes"
version = "1.0.1"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040"
checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
[[package]]
name = "cc"
@ -295,12 +295,6 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
[[package]]
name = "convert_case"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e"
[[package]]
name = "cookie"
version = "0.15.1"
@ -372,9 +366,9 @@ dependencies = [
[[package]]
name = "crossbeam-deque"
version = "0.8.0"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-epoch",
@ -435,9 +429,9 @@ dependencies = [
[[package]]
name = "curve25519-dalek"
version = "3.1.0"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "639891fde0dbea823fc3d798a0fdf9d2f9440a42d64a78ab3488b0ca025117b3"
checksum = "0b9fdf9972b2bd6af2d913799d9ebc165ea4d2e65878e329d9c6b372c4491b61"
dependencies = [
"byteorder",
"digest",
@ -464,26 +458,13 @@ dependencies = [
[[package]]
name = "der"
version = "0.4.0"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49f215f706081a44cb702c71c39a52c05da637822e9c1645a50b7202689e982d"
checksum = "31e21d2d0f22cde6e88694108429775c0219760a07779bf96503b434a03d7412"
dependencies = [
"const-oid",
]
[[package]]
name = "derive_more"
version = "0.99.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40eebddd2156ce1bb37b20bbe5151340a31828b1f2d22ba4141f3531710e38df"
dependencies = [
"convert_case",
"proc-macro2",
"quote",
"rustc_version 0.3.3",
"syn",
]
[[package]]
name = "devise"
version = "0.3.1"
@ -839,9 +820,9 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
[[package]]
name = "h2"
version = "0.3.3"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "825343c4eef0b63f541f8903f395dc5beb362a979b5799a84062527ef1e37726"
checksum = "d7f3675cfef6a30c8031cf9e6493ebdc3bb3272a3fea3923c4210d1830e6a472"
dependencies = [
"bytes",
"fnv",
@ -952,9 +933,9 @@ dependencies = [
[[package]]
name = "http-body"
version = "0.4.2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9"
checksum = "399c583b2979440c60be0821a6199eca73bc3c8dcd9d070d75ac726e2c6186e5"
dependencies = [
"bytes",
"http",
@ -963,9 +944,9 @@ dependencies = [
[[package]]
name = "httparse"
version = "1.4.1"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68"
checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503"
[[package]]
name = "httpdate"
@ -984,9 +965,9 @@ dependencies = [
[[package]]
name = "hyper"
version = "0.14.11"
version = "0.14.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b61cf2d1aebcf6e6352c97b81dc2244ca29194be1b276f5d8ad5c6330fffb11"
checksum = "13f67199e765030fa08fe0bd581af683f0d5bc04ea09c2b1102012c5fb90e7fd"
dependencies = [
"bytes",
"futures-channel",
@ -1044,7 +1025,7 @@ dependencies = [
"gif",
"jpeg-decoder",
"num-iter",
"num-rational",
"num-rational 0.3.2",
"num-traits",
"png",
]
@ -1128,15 +1109,15 @@ dependencies = [
[[package]]
name = "itoa"
version = "0.4.7"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"
checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
[[package]]
name = "jobserver"
version = "0.1.22"
version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "972f5ae5d1cb9c6ae417789196c803205313edde988685da5e3aae0827b9e7fd"
checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa"
dependencies = [
"libc",
]
@ -1149,9 +1130,9 @@ checksum = "229d53d58899083193af11e15917b5640cd40b29ff475a1fe4ef725deb02d0f2"
[[package]]
name = "js-sys"
version = "0.3.51"
version = "0.3.53"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83bdfbace3a0e81a4253f73b49e960b053e396a11012cbd49b9b74d6a2b67062"
checksum = "e4bf49d50e2961077d9c99f4b7997d770a1114f087c3c2e0069b36c13fc2979d"
dependencies = [
"wasm-bindgen",
]
@ -1187,9 +1168,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.98"
version = "0.2.101"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320cfe77175da3a483efed4bc0adc1968ca050b098ce4f2f1c13a56626128790"
checksum = "3cb00336871be5ed2c8ed44b60ae9959dc5b9f08539422ed43f09e34ecaeba21"
[[package]]
name = "libsqlite3-sys"
@ -1282,15 +1263,15 @@ dependencies = [
[[package]]
name = "matches"
version = "0.1.8"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"
[[package]]
name = "memchr"
version = "2.4.0"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
[[package]]
name = "memoffset"
@ -1340,12 +1321,11 @@ dependencies = [
[[package]]
name = "multer"
version = "2.0.0"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fdd568fea4758b30d6423f013f7171e193c34aa97828d1bd9f924fb3af30a8c"
checksum = "408327e2999b839cd1af003fc01b2019a6c10a1361769542203f6fedc5179680"
dependencies = [
"bytes",
"derive_more",
"encoding_rs",
"futures-util",
"http",
@ -1368,6 +1348,20 @@ dependencies = [
"winapi",
]
[[package]]
name = "num"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43db66d1170d347f9a065114077f7dccb00c1b9478c89384490a3425279a4606"
dependencies = [
"num-bigint 0.4.0",
"num-complex",
"num-integer",
"num-iter",
"num-rational 0.4.0",
"num-traits",
]
[[package]]
name = "num-bigint"
version = "0.2.6"
@ -1379,6 +1373,26 @@ dependencies = [
"num-traits",
]
[[package]]
name = "num-bigint"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e0d047c1062aa51e256408c560894e5251f08925980e53cf1aa5bd00eec6512"
dependencies = [
"autocfg",
"num-integer",
"num-traits",
]
[[package]]
name = "num-complex"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26873667bbbb7c5182d4a37c1add32cdf09f841af72da53318fdb81543c15085"
dependencies = [
"num-traits",
]
[[package]]
name = "num-integer"
version = "0.1.44"
@ -1411,6 +1425,18 @@ dependencies = [
"num-traits",
]
[[package]]
name = "num-rational"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d41702bd167c2df5520b384281bc111a4b5efcf7fbc4c9c222c815b07e0a6a6a"
dependencies = [
"autocfg",
"num-bigint 0.4.0",
"num-integer",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.14"
@ -1598,15 +1624,6 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]]
name = "pest"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53"
dependencies = [
"ucd-trie",
]
[[package]]
name = "pin-project"
version = "1.0.8"
@ -1826,9 +1843,9 @@ dependencies = [
[[package]]
name = "redox_syscall"
version = "0.2.9"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ab49abadf3f9e1c4bc499e8845e152ad87d2ad2d30371841171169e9d75feee"
checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff"
dependencies = [
"bitflags",
]
@ -2044,8 +2061,9 @@ dependencies = [
[[package]]
name = "ruma"
version = "0.3.0"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "668031e3108d6a2cfbe6eca271d8698f4593440e71a44afdadcf67ce3cb93c1f"
dependencies = [
"assign",
"js_int",
@ -2066,7 +2084,8 @@ dependencies = [
[[package]]
name = "ruma-api"
version = "0.18.3"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5f1843792b6749ec1ece62595cf99ad30bf9589c96bb237515235e71da396ea"
dependencies = [
"bytes",
"http",
@ -2082,7 +2101,8 @@ dependencies = [
[[package]]
name = "ruma-api-macros"
version = "0.18.3"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b18abda5cca94178d08b622bca042e1cbb5eb7d4ebf3a2a81590a3bb3c57008"
dependencies = [
"proc-macro-crate",
"proc-macro2",
@ -2093,7 +2113,8 @@ dependencies = [
[[package]]
name = "ruma-appservice-api"
version = "0.4.0"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49369332a5f299e832e19661f92d49e08c345c3c6c4ab16e09cb31c5ff6da878"
dependencies = [
"ruma-api",
"ruma-common",
@ -2107,7 +2128,8 @@ dependencies = [
[[package]]
name = "ruma-client-api"
version = "0.12.2"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9568a222c12cf6220e751484ab78feec28071f85965113a5bb802936a2920ff0"
dependencies = [
"assign",
"bytes",
@ -2127,7 +2149,8 @@ dependencies = [
[[package]]
name = "ruma-common"
version = "0.6.0"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41d5b7605f58dc0d9cf1848cc7f1af2bae4e4bcd1d2b7a87bbb9864c8a785b91"
dependencies = [
"indexmap",
"js_int",
@ -2141,8 +2164,9 @@ dependencies = [
[[package]]
name = "ruma-events"
version = "0.24.4"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
version = "0.24.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87801e1207cfebdee02e7997ebf181a1c9837260b78c1b8ce96b896a2bcb3763"
dependencies = [
"indoc",
"js_int",
@ -2157,8 +2181,9 @@ dependencies = [
[[package]]
name = "ruma-events-macros"
version = "0.24.4"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
version = "0.24.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5da4498845347de88adf1b7da4578e2ca7355ad4ce47b0976f6594bacf958660"
dependencies = [
"proc-macro-crate",
"proc-macro2",
@ -2169,7 +2194,8 @@ dependencies = [
[[package]]
name = "ruma-federation-api"
version = "0.3.0"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c61c9adbe1a29c301ae627604406d60102c89fc833b110cd35bbf29ae205ea6c"
dependencies = [
"js_int",
"ruma-api",
@ -2184,7 +2210,8 @@ dependencies = [
[[package]]
name = "ruma-identifiers"
version = "0.20.0"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb417d091e8dd5a633e4e5998231a156049d7fcc221045cfdc0642eb72067732"
dependencies = [
"paste",
"rand 0.8.4",
@ -2198,7 +2225,8 @@ dependencies = [
[[package]]
name = "ruma-identifiers-macros"
version = "0.20.0"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c708edad7f605638f26c951cbad7501fbf28ab01009e5ca65ea5a2db74a882b1"
dependencies = [
"quote",
"ruma-identifiers-validation",
@ -2208,15 +2236,14 @@ dependencies = [
[[package]]
name = "ruma-identifiers-validation"
version = "0.5.0"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
dependencies = [
"thiserror",
]
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42285e7fb5d5f2d5268e45bb683e36d5c6fd9fc1e11a4559ba3c3521f3bbb2cb"
[[package]]
name = "ruma-identity-service-api"
version = "0.3.0"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e76e66e24f2d5a31511fbf6c79e79f67a7a6a98ebf48d72381b7d5bb6c09f035"
dependencies = [
"js_int",
"ruma-api",
@ -2229,7 +2256,8 @@ dependencies = [
[[package]]
name = "ruma-push-gateway-api"
version = "0.3.0"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ef5b29da7065efc5b1e1a8f61add7543c9ab4ecce5ee0dd1c1c5ecec83fbeec"
dependencies = [
"js_int",
"ruma-api",
@ -2244,7 +2272,8 @@ dependencies = [
[[package]]
name = "ruma-serde"
version = "0.5.0"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b2b22aae842e7ecda695e42b7b39d4558959d9d9a27acc2a16acf4f4f7f00c3"
dependencies = [
"bytes",
"form_urlencoded",
@ -2258,7 +2287,8 @@ dependencies = [
[[package]]
name = "ruma-serde-macros"
version = "0.5.0"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "243e9bef188b08f94c79bc2f8fd1eb307a9e636b2b8e4571acf8c7be16381d28"
dependencies = [
"proc-macro-crate",
"proc-macro2",
@ -2269,7 +2299,8 @@ dependencies = [
[[package]]
name = "ruma-signatures"
version = "0.9.0"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a4f64027165b59500162d10d435b1253898bf3ad4f5002cb0d56913fe7f76d7"
dependencies = [
"base64 0.13.0",
"ed25519-dalek",
@ -2285,8 +2316,9 @@ dependencies = [
[[package]]
name = "ruma-state-res"
version = "0.3.0"
source = "git+https://github.com/timokoesters/ruma?rev=2215049b60a1c3358f5a52215adf1e7bb88619a1#2215049b60a1c3358f5a52215adf1e7bb88619a1"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "796427aaa2d266238c5c1b1a6ca4640a4d282ec2cb2e844c69a8f3a262d3db15"
dependencies = [
"itertools 0.10.1",
"js_int",
@ -2334,16 +2366,7 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
dependencies = [
"semver 0.9.0",
]
[[package]]
name = "rustc_version"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee"
dependencies = [
"semver 0.11.0",
"semver",
]
[[package]]
@ -2417,22 +2440,23 @@ dependencies = [
[[package]]
name = "security-framework"
version = "2.3.1"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467"
checksum = "5b9bd29cdffb8875b04f71c51058f940cf4e390bbfd2ce669c4f22cd70b492a5"
dependencies = [
"bitflags",
"core-foundation",
"core-foundation-sys",
"libc",
"num",
"security-framework-sys",
]
[[package]]
name = "security-framework-sys"
version = "2.3.0"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284"
checksum = "19133a286e494cc3311c165c4676ccb1fd47bed45b55f9d71fbd784ad4cea6f8"
dependencies = [
"core-foundation-sys",
"libc",
@ -2444,16 +2468,7 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
dependencies = [
"semver-parser 0.7.0",
]
[[package]]
name = "semver"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6"
dependencies = [
"semver-parser 0.10.2",
"semver-parser",
]
[[package]]
@ -2462,29 +2477,20 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
[[package]]
name = "semver-parser"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7"
dependencies = [
"pest",
]
[[package]]
name = "serde"
version = "1.0.126"
version = "1.0.129"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03"
checksum = "d1f72836d2aa753853178eda473a3b9d8e4eefdaf20523b919677e6de489f8f1"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.126"
version = "1.0.129"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43"
checksum = "e57ae87ad533d9a56427558b516d0adac283614e347abf85b0dc0cbbf0a249f3"
dependencies = [
"proc-macro2",
"quote",
@ -2516,12 +2522,12 @@ dependencies = [
[[package]]
name = "serde_yaml"
version = "0.8.17"
version = "0.8.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15654ed4ab61726bf918a39cb8d98a2e2995b002387807fa6ba58fdf7f59bb23"
checksum = "6375dbd828ed6964c3748e4ef6d18e7a175d408ffe184bca01698d0c73f915a9"
dependencies = [
"dtoa",
"linked-hash-map",
"indexmap",
"serde",
"yaml-rust",
]
@ -2547,9 +2553,9 @@ dependencies = [
[[package]]
name = "sharded-slab"
version = "0.1.1"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79c719719ee05df97490f80a45acfc99e5a30ce98a1e4fb67aee422745ae14e3"
checksum = "740223c51853f3145fe7c90360d2d4232f2b62e3449489c207eccde818979982"
dependencies = [
"lazy_static",
]
@ -2576,15 +2582,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "692ca13de57ce0613a363c8c2f1de925adebc81b04c923ac60c5488bb44abe4b"
dependencies = [
"chrono",
"num-bigint",
"num-bigint 0.2.6",
"num-traits",
]
[[package]]
name = "slab"
version = "0.4.3"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527"
checksum = "c307a32c1c5c437f38c7fd45d753050587732ba8628319fbdf12a7e289ccc590"
[[package]]
name = "sled"
@ -2685,7 +2691,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5"
dependencies = [
"discard",
"rustc_version 0.2.3",
"rustc_version",
"stdweb-derive",
"stdweb-internal-macros",
"stdweb-internal-runtime",
@ -2735,9 +2741,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
[[package]]
name = "syn"
version = "1.0.74"
version = "1.0.75"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1873d832550d4588c3dbc20f01361ab00bfe741048f71e3fecf145a7cc18b29c"
checksum = "b7f58f7e8eaa0009c5fec437aabf511bd9933e4b2d7407bd05273c01a8906ea7"
dependencies = [
"proc-macro2",
"quote",
@ -2904,9 +2910,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]]
name = "tokio"
version = "1.9.0"
version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b7b349f11a7047e6d1276853e612d152f5e8a352c61917887cc2169e2366b4c"
checksum = "92036be488bb6594459f2e03b60e42df6f937fe6ca5c5ffdcb539c6b84dc40f5"
dependencies = [
"autocfg",
"bytes",
@ -3020,9 +3026,9 @@ dependencies = [
[[package]]
name = "tracing-core"
version = "0.1.18"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052"
checksum = "2ca517f43f0fb96e0c3072ed5c275fe5eece87e8cb52f4a77b69226d3b1c9df8"
dependencies = [
"lazy_static",
]
@ -3074,9 +3080,9 @@ dependencies = [
[[package]]
name = "tracing-subscriber"
version = "0.2.19"
version = "0.2.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab69019741fca4d98be3c62d2b75254528b5432233fd8a4d2739fec20278de48"
checksum = "b9cbe87a2fa7e35900ce5de20220a582a9483a7063811defce79d7cbd59d4cfe"
dependencies = [
"ansi_term",
"chrono",
@ -3170,12 +3176,6 @@ dependencies = [
"serde",
]
[[package]]
name = "ucd-trie"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c"
[[package]]
name = "uncased"
version = "0.9.6"
@ -3194,12 +3194,9 @@ checksum = "eeba86d422ce181a719445e51872fa30f1f7413b62becb52e95ec91aa262d85c"
[[package]]
name = "unicode-bidi"
version = "0.3.5"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0"
dependencies = [
"matches",
]
checksum = "246f4c42e67e7a4e3c6106ff716a5d067d4132a642840b242e357e468a2a0085"
[[package]]
name = "unicode-normalization"
@ -3282,9 +3279,9 @@ checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]]
name = "wasm-bindgen"
version = "0.2.74"
version = "0.2.76"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd"
checksum = "8ce9b1b516211d33767048e5d47fa2a381ed8b76fc48d2ce4aa39877f9f183e0"
dependencies = [
"cfg-if 1.0.0",
"serde",
@ -3294,9 +3291,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.74"
version = "0.2.76"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b33f6a0694ccfea53d94db8b2ed1c3a8a4c86dd936b13b9f0a15ec4a451b900"
checksum = "cfe8dc78e2326ba5f845f4b5bf548401604fa20b1dd1d365fb73b6c1d6364041"
dependencies = [
"bumpalo",
"lazy_static",
@ -3309,9 +3306,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.24"
version = "0.4.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fba7978c679d53ce2d0ac80c8c175840feb849a161664365d1287b41f2e67f1"
checksum = "95fded345a6559c2cfee778d562300c581f7d4ff3edb9b0d230d69800d213972"
dependencies = [
"cfg-if 1.0.0",
"js-sys",
@ -3321,9 +3318,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.74"
version = "0.2.76"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "088169ca61430fe1e58b8096c24975251700e7b1f6fd91cc9d59b04fb9b18bd4"
checksum = "44468aa53335841d9d6b6c023eaab07c0cd4bddbcfdee3e2bb1e8d2cb8069fef"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@ -3331,9 +3328,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.74"
version = "0.2.76"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be2241542ff3d9f241f5e2cb6dd09b37efe786df8851c54957683a49f0987a97"
checksum = "0195807922713af1e67dc66132c7328206ed9766af3858164fb583eedc25fbad"
dependencies = [
"proc-macro2",
"quote",
@ -3344,15 +3341,15 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.74"
version = "0.2.76"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7cff876b8f18eed75a66cf49b65e7f967cb354a7aa16003fb55dbfd25b44b4f"
checksum = "acdb075a845574a1fa5f09fd77e43f7747599301ea3417a9fbffdeedfc1f4a29"
[[package]]
name = "web-sys"
version = "0.3.51"
version = "0.3.53"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e828417b379f3df7111d3a2a9e5753706cae29c41f7c4029ee9fd77f3e09e582"
checksum = "224b2f6b67919060055ef1a67807367c2066ed520c3862cc013d26cf893a783c"
dependencies = [
"js-sys",
"wasm-bindgen",

View file

@ -18,8 +18,9 @@ edition = "2018"
rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle requests
# Used for matrix spec type definitions and helpers
ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
#ruma = { git = "https://github.com/ruma/ruma", rev = "f5ab038e22421ed338396ece977b6b2844772ced", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
ruma = { git = "https://github.com/timokoesters/ruma", rev = "2215049b60a1c3358f5a52215adf1e7bb88619a1", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
#ruma = { git = "https://github.com/timokoesters/ruma", rev = "2215049b60a1c3358f5a52215adf1e7bb88619a1", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
#ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
# Used for long polling and federation sender, should be the same as rocket::tokio

View file

@ -12,7 +12,7 @@ use std::{
time::Duration,
};
use tokio::sync::watch::Sender;
use tracing::{error, warn};
use tracing::error;
#[cfg(feature = "conduit_bin")]
use rocket::{get, tokio};
@ -246,31 +246,13 @@ async fn sync_helper(
.current_shortstatehash(&room_id)?
.expect("All rooms have state");
let first_pdu_before_since = db
.rooms
.pdus_until(&sender_user, &room_id, since)?
.next()
.transpose()?;
let pdus_after_since = db
.rooms
.pdus_after(&sender_user, &room_id, since)?
.next()
.is_some();
let since_shortstatehash = first_pdu_before_since
.as_ref()
.map(|pdu| {
db.rooms
.pdu_shortstatehash(&pdu.1.event_id)
.transpose()
.ok_or_else(|| {
warn!("PDU without state: {}", pdu.1.event_id);
Error::bad_database("Found PDU without state")
})
})
.transpose()?
.transpose()?;
let since_shortstatehash = db.rooms.get_token_shortstatehash(&room_id, since)?;
// Calculates joined_member_count, invited_member_count and heroes
let calculate_counts = || {
@ -359,7 +341,7 @@ async fn sync_helper(
true,
state_events,
)
} else if !pdus_after_since || since_shortstatehash == Some(current_shortstatehash) {
} else if !pdus_after_since && since_shortstatehash == Some(current_shortstatehash) {
// No state changes
(Vec::new(), None, None, false, Vec::new())
} else {
@ -400,11 +382,6 @@ async fn sync_helper(
current_state_ids
.iter()
.filter(|(key, id)| since_state_ids.get(key) != Some(id))
.filter(|(_, id)| {
!timeline_pdus
.iter()
.any(|(_, timeline_pdu)| timeline_pdu.event_id == **id)
})
.map(|(_, id)| db.rooms.get_pdu(id))
.filter_map(|r| r.ok().flatten())
.collect()
@ -585,6 +562,10 @@ async fn sync_helper(
);
}
// Save the state after this sync so we can send the correct state diff next sync
db.rooms
.associate_token_shortstatehash(&room_id, next_batch, current_shortstatehash)?;
let joined_room = sync_events::JoinedRoom {
account_data: sync_events::RoomAccountData {
events: db

View file

@ -56,6 +56,7 @@ pub async fn send_event_to_device_route(
},
))
.expect("DirectToDevice EDU can be serialized"),
db.globals.next_count()?,
)?;
continue;

View file

@ -1,3 +1,34 @@
use std::hash::Hash;
use std::{
collections::{BTreeMap, HashMap, HashSet},
convert::{TryFrom, TryInto},
fs::{self, remove_dir_all},
io::Write,
mem::size_of,
ops::Deref,
path::Path,
sync::{Arc, Mutex, RwLock},
};
use directories::ProjectDirs;
use lru_cache::LruCache;
use rocket::{
futures::{channel::mpsc, stream::FuturesUnordered, StreamExt},
outcome::{try_outcome, IntoOutcome},
request::{FromRequest, Request},
Shutdown, State,
};
use ruma::{DeviceId, EventId, RoomId, ServerName, UserId};
use serde::{de::IgnoredAny, Deserialize};
use tokio::sync::{OwnedRwLockReadGuard, RwLock as TokioRwLock, Semaphore};
use tracing::{debug, error, warn};
use abstraction::DatabaseEngine;
use crate::{utils, Error, Result};
use self::proxy::ProxyConfig;
pub mod abstraction;
pub mod account_data;
@ -14,33 +45,6 @@ pub mod transaction_ids;
pub mod uiaa;
pub mod users;
use crate::{utils, Error, Result};
use abstraction::DatabaseEngine;
use directories::ProjectDirs;
use lru_cache::LruCache;
use rocket::{
futures::{channel::mpsc, stream::FuturesUnordered, StreamExt},
outcome::{try_outcome, IntoOutcome},
request::{FromRequest, Request},
Shutdown, State,
};
use ruma::{DeviceId, EventId, RoomId, ServerName, UserId};
use serde::{de::IgnoredAny, Deserialize};
use std::{
collections::{BTreeMap, HashMap, HashSet},
convert::{TryFrom, TryInto},
fs::{self, remove_dir_all},
io::Write,
mem::size_of,
ops::Deref,
path::Path,
sync::{Arc, Mutex, RwLock},
};
use tokio::sync::{OwnedRwLockReadGuard, RwLock as TokioRwLock, Semaphore};
use tracing::{debug, error, warn};
use self::proxy::ProxyConfig;
#[derive(Clone, Debug, Deserialize)]
pub struct Config {
server_name: Box<ServerName>,
@ -132,6 +136,17 @@ pub type Engine = abstraction::sqlite::Engine;
#[cfg(feature = "heed")]
pub type Engine = abstraction::heed::Engine;
// for each key: (memory_usage in bytes, items in cache, capacity)
pub struct CacheUsageStatistics {
pdu_cache: (usize, usize, usize),
auth_chain_cache: (usize, usize, usize),
shorteventid_cache: (usize, usize, usize),
eventidshort_cache: (usize, usize, usize),
statekeyshort_cache: (usize, usize, usize),
shortstatekey_cache: (usize, usize, usize),
stateinfo_cache: (usize, usize, usize),
}
pub struct Database {
_db: Arc<Engine>,
pub globals: globals::Globals,
@ -264,6 +279,8 @@ impl Database {
statekey_shortstatekey: builder.open_tree("statekey_shortstatekey")?,
shortstatekey_statekey: builder.open_tree("shortstatekey_statekey")?,
shorteventid_authchain: builder.open_tree("shorteventid_authchain")?,
roomid_shortroomid: builder.open_tree("roomid_shortroomid")?,
shortstatehash_statediff: builder.open_tree("shortstatehash_statediff")?,
@ -271,12 +288,13 @@ impl Database {
shorteventid_eventid: builder.open_tree("shorteventid_eventid")?,
shorteventid_shortstatehash: builder.open_tree("shorteventid_shortstatehash")?,
roomid_shortstatehash: builder.open_tree("roomid_shortstatehash")?,
roomsynctoken_shortstatehash: builder.open_tree("roomsynctoken_shortstatehash")?,
statehash_shortstatehash: builder.open_tree("statehash_shortstatehash")?,
eventid_outlierpdu: builder.open_tree("eventid_outlierpdu")?,
referencedevents: builder.open_tree("referencedevents")?,
pdu_cache: Mutex::new(LruCache::new(100_000)),
auth_chain_cache: Mutex::new(LruCache::new(100_000)),
auth_chain_cache: Mutex::new(LruCache::new(1_000_000)),
shorteventid_cache: Mutex::new(LruCache::new(1_000_000)),
eventidshort_cache: Mutex::new(LruCache::new(1_000_000)),
shortstatekey_cache: Mutex::new(LruCache::new(1_000_000)),
@ -709,6 +727,12 @@ impl Database {
.insert(&shortstatekey, &statekey)?;
}
// Force E2EE device list updates so we can send them over federation
for user_id in db.users.iter().filter_map(|r| r.ok()) {
db.users
.mark_device_key_update(&user_id, &db.rooms, &db.globals)?;
}
db.globals.bump_database_version(10)?;
println!("Migration: 9 -> 10 finished");
@ -908,6 +932,66 @@ impl Database {
}
});
}
/// Measures memory usage in bytes and how full the caches are in percent for all caches in the Database struct.
pub fn get_cache_usage(&mut self) -> Result<CacheUsageStatistics> {
fn memory_usage_of_locked_cache<K: Eq + Hash, V>(
cache: &mut Mutex<LruCache<K, V>>,
) -> usize {
let raw_cache = cache.lock().unwrap();
let mut cache_items_size_sum: usize = 0;
for cache_item in raw_cache.iter() {
cache_items_size_sum += std::mem::size_of_val(&cache_item);
}
cache_items_size_sum += std::mem::size_of_val(&cache);
cache_items_size_sum
}
fn items_in_locked_cache<K: Eq + Hash, V>(cache: &mut Mutex<LruCache<K, V>>) -> usize {
cache.lock().unwrap().len()
}
fn capacity_of_locked_cache<K: Eq + Hash, V>(cache: &mut Mutex<LruCache<K, V>>) -> usize {
cache.lock().unwrap().capacity()
}
return Ok(CacheUsageStatistics {
pdu_cache: (
memory_usage_of_locked_cache(&mut self.rooms.pdu_cache),
items_in_locked_cache(&mut self.rooms.pdu_cache),
capacity_of_locked_cache(&mut self.rooms.pdu_cache),
),
auth_chain_cache: (
memory_usage_of_locked_cache(&mut self.rooms.auth_chain_cache),
items_in_locked_cache(&mut self.rooms.auth_chain_cache),
capacity_of_locked_cache(&mut self.rooms.auth_chain_cache),
),
shorteventid_cache: (
memory_usage_of_locked_cache(&mut self.rooms.shorteventid_cache),
items_in_locked_cache(&mut self.rooms.shorteventid_cache),
capacity_of_locked_cache(&mut self.rooms.shorteventid_cache),
),
eventidshort_cache: (
memory_usage_of_locked_cache(&mut self.rooms.eventidshort_cache),
items_in_locked_cache(&mut self.rooms.eventidshort_cache),
capacity_of_locked_cache(&mut self.rooms.eventidshort_cache),
),
statekeyshort_cache: (
memory_usage_of_locked_cache(&mut self.rooms.statekeyshort_cache),
items_in_locked_cache(&mut self.rooms.statekeyshort_cache),
capacity_of_locked_cache(&mut self.rooms.statekeyshort_cache),
),
shortstatekey_cache: (
memory_usage_of_locked_cache(&mut self.rooms.shortstatekey_cache),
items_in_locked_cache(&mut self.rooms.shortstatekey_cache),
capacity_of_locked_cache(&mut self.rooms.shortstatekey_cache),
),
stateinfo_cache: (
memory_usage_of_locked_cache(&mut self.rooms.stateinfo_cache),
items_in_locked_cache(&mut self.rooms.stateinfo_cache),
capacity_of_locked_cache(&mut self.rooms.stateinfo_cache),
),
});
}
}
pub struct DatabaseGuard(OwnedRwLockReadGuard<Database>);

View file

@ -3,19 +3,21 @@ use std::{
sync::Arc,
};
use crate::{pdu::PduBuilder, Database};
use rocket::futures::{channel::mpsc, stream::StreamExt};
use ruma::{
events::{room::message, EventType},
UserId,
};
use tokio::sync::{MutexGuard, RwLock, RwLockReadGuard};
use tokio::sync::{MutexGuard, RwLock, RwLockWriteGuard};
use tracing::warn;
use crate::{pdu::PduBuilder, Database};
pub enum AdminCommand {
RegisterAppservice(serde_yaml::Value),
ListAppservices,
SendMessage(message::MessageEventContent),
ShowCacheUsage,
}
#[derive(Clone)]
@ -59,7 +61,7 @@ impl Admin {
drop(guard);
let send_message = |message: message::MessageEventContent,
guard: RwLockReadGuard<'_, Database>,
guard: RwLockWriteGuard<'_, Database>,
mutex_lock: &MutexGuard<'_, ()>| {
guard
.rooms
@ -83,7 +85,7 @@ impl Admin {
loop {
tokio::select! {
Some(event) = receiver.next() => {
let guard = db.read().await;
let mut guard = db.write().await;
let mutex_state = Arc::clone(
guard.globals
.roomid_mutex_state
@ -92,6 +94,7 @@ impl Admin {
.entry(conduit_room.clone())
.or_default(),
);
let state_lock = mutex_state.lock().await;
match event {
@ -114,6 +117,37 @@ impl Admin {
AdminCommand::SendMessage(message) => {
send_message(message, guard, &state_lock);
}
AdminCommand::ShowCacheUsage => {
fn format_cache_statistics_triple(name: String, triple: (usize, usize, usize)) -> String {
let (memory_usage, item_count, capacity) = triple;
format!(
"{0} is using {1} MB ({2} bytes) of RAM at {3:.2}% utilization.",
name,
memory_usage / 100_00,
memory_usage ,
((item_count as f32 / capacity as f32) * 100.0)
)
}
if let Ok(cache_usage_statistics) = guard.get_cache_usage() {
let mut statistics_lines = Vec::with_capacity(7);
statistics_lines.push(format_cache_statistics_triple("pdu_cache".to_string(), cache_usage_statistics.pdu_cache));
statistics_lines.push(format_cache_statistics_triple("auth_chain_cache".to_string(), cache_usage_statistics.auth_chain_cache));
statistics_lines.push(format_cache_statistics_triple("shorteventid_cache".to_string(), cache_usage_statistics.shorteventid_cache));
statistics_lines.push(format_cache_statistics_triple("eventidshort_cache".to_string(), cache_usage_statistics.eventidshort_cache));
statistics_lines.push(format_cache_statistics_triple("statekeyshort_cache".to_string(), cache_usage_statistics.statekeyshort_cache));
statistics_lines.push(format_cache_statistics_triple("shortstatekey_cache".to_string(), cache_usage_statistics.shortstatekey_cache));
statistics_lines.push(format_cache_statistics_triple("stateinfo_cache".to_string(), cache_usage_statistics.stateinfo_cache));
send_message(message::MessageEventContent::text_plain(statistics_lines.join("\n")), guard, &state_lock);
} else {
let result_text = "Could not calculate database cache size";
send_message(message::MessageEventContent::text_plain(result_text), guard, &state_lock);
}
}
}
drop(state_lock);

View file

@ -1,10 +1,12 @@
mod edus;
use std::{
collections::{BTreeMap, HashMap, HashSet},
convert::{TryFrom, TryInto},
mem::size_of,
sync::{Arc, Mutex},
};
pub use edus::RoomEdus;
use member::MembershipState;
use crate::{pdu::PduBuilder, utils, Database, Error, PduEvent, Result};
use lru_cache::LruCache;
use member::MembershipState;
use regex::Regex;
use ring::digest;
use rocket::http::RawStr;
@ -22,17 +24,17 @@ use ruma::{
state_res::{self, RoomVersion, StateMap},
uint, EventId, RoomAliasId, RoomId, RoomVersionId, ServerName, UserId,
};
use std::{
collections::{BTreeMap, HashMap, HashSet},
convert::{TryFrom, TryInto},
mem::size_of,
sync::{Arc, Mutex},
};
use tokio::sync::MutexGuard;
use tracing::{error, warn};
pub use edus::RoomEdus;
use crate::{pdu::PduBuilder, utils, Database, Error, PduEvent, Result};
use super::{abstraction::Tree, admin::AdminCommand, pusher};
mod edus;
/// The unique identifier of each state group.
///
/// This is created when a state group is added to the database by
@ -69,6 +71,7 @@ pub struct Rooms {
/// Remember the current state hash of a room.
pub(super) roomid_shortstatehash: Arc<dyn Tree>,
pub(super) roomsynctoken_shortstatehash: Arc<dyn Tree>,
/// Remember the state hash at events in the past.
pub(super) shorteventid_shortstatehash: Arc<dyn Tree>,
/// StateKey = EventType + StateKey, ShortStateKey = Count
@ -83,6 +86,8 @@ pub struct Rooms {
pub(super) statehash_shortstatehash: Arc<dyn Tree>,
pub(super) shortstatehash_statediff: Arc<dyn Tree>, // StateDiff = parent (or 0) + (shortstatekey+shorteventid++) + 0_u64 + (shortstatekey+shorteventid--)
pub(super) shorteventid_authchain: Arc<dyn Tree>,
/// RoomId + EventId -> outlier PDU.
/// Any pdu that has passed the steps 1-8 in the incoming event /federation/send/txn.
pub(super) eventid_outlierpdu: Arc<dyn Tree>,
@ -91,7 +96,7 @@ pub struct Rooms {
pub(super) referencedevents: Arc<dyn Tree>,
pub(super) pdu_cache: Mutex<LruCache<EventId, Arc<PduEvent>>>,
pub(super) auth_chain_cache: Mutex<LruCache<Vec<u64>, HashSet<u64>>>,
pub(super) auth_chain_cache: Mutex<LruCache<Vec<u64>, Arc<HashSet<u64>>>>,
pub(super) shorteventid_cache: Mutex<LruCache<u64, EventId>>,
pub(super) eventidshort_cache: Mutex<LruCache<EventId, u64>>,
pub(super) statekeyshort_cache: Mutex<LruCache<(EventType, String), u64>>,
@ -1563,10 +1568,13 @@ impl Rooms {
));
}
}
"show_cache_usage" => {
db.admin.send(AdminCommand::ShowCacheUsage);
}
_ => {
db.admin.send(AdminCommand::SendMessage(
message::MessageEventContent::text_plain(format!(
"Unrecognized command: {}",
"Unrecognized command: {}\nAvailable commands:\n- register_appservice\n- list_appservices\n- get_pdu\n- show_cache_usage",
command
)),
));
@ -1800,6 +1808,38 @@ impl Rooms {
Ok(())
}
pub fn associate_token_shortstatehash(
&self,
room_id: &RoomId,
token: u64,
shortstatehash: u64,
) -> Result<()> {
let shortroomid = self.get_shortroomid(room_id)?.expect("room exists");
let mut key = shortroomid.to_be_bytes().to_vec();
key.extend_from_slice(&token.to_be_bytes());
self.roomsynctoken_shortstatehash
.insert(&key, &shortstatehash.to_be_bytes())
}
pub fn get_token_shortstatehash(&self, room_id: &RoomId, token: u64) -> Result<Option<u64>> {
let shortroomid = self.get_shortroomid(room_id)?.expect("room exists");
let mut key = shortroomid.to_be_bytes().to_vec();
key.extend_from_slice(&token.to_be_bytes());
Ok(self
.roomsynctoken_shortstatehash
.get(&key)?
.map(|bytes| {
utils::u64_from_bytes(&bytes).map_err(|_| {
Error::bad_database("Invalid shortstatehash in roomsynctoken_shortstatehash")
})
})
.transpose()?)
}
/// Creates a new persisted data unit and adds it to a room.
#[tracing::instrument(skip(self, db, _mutex_lock))]
pub fn build_and_append_pdu(
@ -3166,7 +3206,64 @@ impl Rooms {
}
#[tracing::instrument(skip(self))]
pub fn auth_chain_cache(&self) -> std::sync::MutexGuard<'_, LruCache<Vec<u64>, HashSet<u64>>> {
self.auth_chain_cache.lock().unwrap()
pub fn get_auth_chain_from_cache<'a>(
&'a self,
key: &[u64],
) -> Result<Option<Arc<HashSet<u64>>>> {
// Check RAM cache
if let Some(result) = self.auth_chain_cache.lock().unwrap().get_mut(key) {
return Ok(Some(Arc::clone(result)));
}
// Check DB cache
if key.len() == 1 {
if let Some(chain) =
self.shorteventid_authchain
.get(&key[0].to_be_bytes())?
.map(|chain| {
chain
.chunks_exact(size_of::<u64>())
.map(|chunk| {
utils::u64_from_bytes(chunk).expect("byte length is correct")
})
.collect()
})
{
let chain = Arc::new(chain);
// Cache in RAM
self.auth_chain_cache
.lock()
.unwrap()
.insert(vec![key[0]], Arc::clone(&chain));
return Ok(Some(chain));
}
}
Ok(None)
}
#[tracing::instrument(skip(self))]
pub fn cache_auth_chain(&self, key: Vec<u64>, chain: Arc<HashSet<u64>>) -> Result<()> {
// Persist in db
if key.len() == 1 {
self.shorteventid_authchain.insert(
&key[0].to_be_bytes(),
&chain
.iter()
.map(|s| s.to_be_bytes().to_vec())
.flatten()
.collect::<Vec<u8>>(),
)?;
}
// Cache in RAM
self.auth_chain_cache
.lock()
.unwrap()
.insert(key.clone(), chain);
Ok(())
}
}

View file

@ -84,8 +84,8 @@ pub enum SendingEventType {
pub struct Sending {
/// The state for a given state hash.
pub(super) servername_educount: Arc<dyn Tree>, // EduCount: Count of last EDU sync
pub(super) servernameevent_data: Arc<dyn Tree>, // ServernamEvent = (+ / $)SenderKey / ServerName / UserId + PduId / * (for edus), Data = EDU content
pub(super) servercurrentevent_data: Arc<dyn Tree>, // ServerCurrentEvents = (+ / $)ServerName / UserId + PduId / * (for edus), Data = EDU content
pub(super) servernameevent_data: Arc<dyn Tree>, // ServernamEvent = (+ / $)SenderKey / ServerName / UserId + PduId / Id (for edus), Data = EDU content
pub(super) servercurrentevent_data: Arc<dyn Tree>, // ServerCurrentEvents = (+ / $)ServerName / UserId + PduId / Id (for edus), Data = EDU content
pub(super) maximum_requests: Arc<Semaphore>,
pub sender: mpsc::UnboundedSender<(Vec<u8>, Vec<u8>)>,
}
@ -435,10 +435,15 @@ impl Sending {
}
#[tracing::instrument(skip(self, server, serialized))]
pub fn send_reliable_edu(&self, server: &ServerName, serialized: Vec<u8>) -> Result<()> {
pub fn send_reliable_edu(
&self,
server: &ServerName,
serialized: Vec<u8>,
id: u64,
) -> Result<()> {
let mut key = server.as_bytes().to_vec();
key.push(0xff);
key.push(b'*');
key.extend_from_slice(&id.to_be_bytes());
self.servernameevent_data.insert(&key, &serialized)?;
self.sender.unbounded_send((key, serialized)).unwrap();
@ -714,10 +719,10 @@ impl Sending {
OutgoingKind::Appservice(Box::<ServerName>::try_from(server).map_err(|_| {
Error::bad_database("Invalid server string in server_currenttransaction")
})?),
if event.starts_with(b"*") {
SendingEventType::Edu(value)
} else {
if value.is_empty() {
SendingEventType::Pdu(event.to_vec())
} else {
SendingEventType::Edu(value)
},
)
} else if key.starts_with(b"$") {
@ -732,10 +737,10 @@ impl Sending {
.ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?;
(
OutgoingKind::Push(user.to_vec(), pushkey.to_vec()),
if event.starts_with(b"*") {
SendingEventType::Edu(value)
} else {
if value.is_empty() {
SendingEventType::Pdu(event.to_vec())
} else {
SendingEventType::Edu(value)
},
)
} else {
@ -753,10 +758,10 @@ impl Sending {
OutgoingKind::Normal(Box::<ServerName>::try_from(server).map_err(|_| {
Error::bad_database("Invalid server string in server_currenttransaction")
})?),
if event.starts_with(b"*") {
SendingEventType::Edu(event[1..].to_vec())
} else {
if value.is_empty() {
SendingEventType::Pdu(event.to_vec())
} else {
SendingEventType::Edu(value)
},
)
})

View file

@ -51,7 +51,7 @@ use ruma::{
ServerSigningKeyId, UserId,
};
use std::{
collections::{btree_map, hash_map, BTreeMap, HashMap, HashSet},
collections::{btree_map, hash_map, BTreeMap, BTreeSet, HashMap, HashSet},
convert::{TryFrom, TryInto},
fmt::Debug,
future::Future,
@ -1387,6 +1387,17 @@ async fn upgrade_outlier_to_timeline_pdu(
.state_full_ids(current_sstatehash)
.map_err(|_| "Failed to load room state.")?;
let auth_events = db
.rooms
.get_auth_events(
&room_id,
&incoming_pdu.kind,
&incoming_pdu.sender,
incoming_pdu.state_key.as_deref(),
&incoming_pdu.content,
)
.map_err(|_| "Failed to get_auth_events.".to_owned())?;
if incoming_pdu.state_key.is_some() {
let mut extremity_sstatehashes = HashMap::new();
@ -1541,18 +1552,8 @@ async fn upgrade_outlier_to_timeline_pdu(
extremities.insert(incoming_pdu.event_id.clone());
debug!("starting soft fail auth check");
// 13. Check if the event passes auth based on the "current state" of the room, if not "soft fail" it
let auth_events = db
.rooms
.get_auth_events(
&room_id,
&incoming_pdu.kind,
&incoming_pdu.sender,
incoming_pdu.state_key.as_deref(),
&incoming_pdu.content,
)
.map_err(|_| "Failed to get_auth_events.".to_owned())?;
debug!("starting soft fail auth check");
let soft_fail = !state_res::event_auth::auth_check(
&room_version,
@ -1974,44 +1975,85 @@ fn get_auth_chain(
starting_events: Vec<EventId>,
db: &Database,
) -> Result<impl Iterator<Item = EventId> + '_> {
let mut full_auth_chain = HashSet::new();
const NUM_BUCKETS: usize = 50;
const NUM_BUCKETS: usize = 100;
let mut buckets = vec![HashSet::new(); NUM_BUCKETS];
let mut buckets = vec![BTreeSet::new(); NUM_BUCKETS];
for id in starting_events {
let short = db.rooms.get_or_create_shorteventid(&id, &db.globals)?;
let bucket_id = (short % NUM_BUCKETS as u64) as usize;
buckets[bucket_id].insert((short, id));
if let Some(pdu) = db.rooms.get_pdu(&id)? {
for auth_event in &pdu.auth_events {
let short = db
.rooms
.get_or_create_shorteventid(&auth_event, &db.globals)?;
let bucket_id = (short % NUM_BUCKETS as u64) as usize;
buckets[bucket_id].insert((short, auth_event.clone()));
}
}
}
let mut cache = db.rooms.auth_chain_cache();
let mut full_auth_chain = HashSet::new();
let mut hits = 0;
let mut misses = 0;
for chunk in buckets {
let chunk_key = chunk.iter().map(|(short, _)| short).copied().collect();
if let Some(cached) = cache.get_mut(&chunk_key) {
full_auth_chain.extend(cached.iter().cloned());
if chunk.is_empty() {
continue;
}
// The code below will only get the auth chains, not the events in the chunk. So let's add
// them first
full_auth_chain.extend(chunk.iter().map(|(id, _)| id));
let chunk_key = chunk
.iter()
.map(|(short, _)| short)
.copied()
.collect::<Vec<u64>>();
if let Some(cached) = db.rooms.get_auth_chain_from_cache(&chunk_key)? {
hits += 1;
full_auth_chain.extend(cached.iter().cloned());
continue;
}
misses += 1;
let mut chunk_cache = HashSet::new();
let mut hits2 = 0;
let mut misses2 = 0;
for (sevent_id, event_id) in chunk {
if let Some(cached) = cache.get_mut(&[sevent_id][..]) {
if let Some(cached) = db.rooms.get_auth_chain_from_cache(&[sevent_id])? {
hits2 += 1;
chunk_cache.extend(cached.iter().cloned());
} else {
drop(cache);
let auth_chain = get_auth_chain_inner(&event_id, db)?;
cache = db.rooms.auth_chain_cache();
cache.insert(vec![sevent_id], auth_chain.clone());
chunk_cache.extend(auth_chain);
misses2 += 1;
let auth_chain = Arc::new(get_auth_chain_inner(&event_id, db)?);
db.rooms
.cache_auth_chain(vec![sevent_id], Arc::clone(&auth_chain))?;
println!(
"cache missed event {} with auth chain len {}",
event_id,
auth_chain.len()
);
chunk_cache.extend(auth_chain.iter());
};
}
cache.insert(chunk_key, chunk_cache.clone());
full_auth_chain.extend(chunk_cache);
println!(
"chunk missed with len {}, event hits2: {}, misses2: {}",
chunk_cache.len(),
hits2,
misses2
);
let chunk_cache = Arc::new(chunk_cache);
db.rooms
.cache_auth_chain(chunk_key, Arc::clone(&chunk_cache))?;
full_auth_chain.extend(chunk_cache.iter());
}
drop(cache);
println!(
"total: {}, chunk hits: {}, misses: {}",
full_auth_chain.len(),
hits,
misses
);
Ok(full_auth_chain
.into_iter()