TudbuT/conduit
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

improvement: memory usage for caches

Browse source
This commit is contained in:
Timo Kösters 2022-01-10 15:53:28 +01:00
parent 4cd18f573d
commit 9bd3147763
No known key found for this signature in database
GPG key ID: 356E705610F626D5
4 changed files with 26 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Cargo.lock generated
View file

@ -2074,9 +2074,9 @@ dependencies = [
[[package]]
name = "rocksdb"
version = "0.16.0"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c749134fda8bfc90d0de643d59bfc841dcb3ac8a1062e12b6754bd60235c48b3"
checksum = "7a62eca5cacf2c8261128631bed9f045598d40bfbe4b29f5163f0f802f8f44a7"
dependencies = [
"libc",
"librocksdb-sys",

2
Cargo.toml
View file

@ -78,7 +78,7 @@ crossbeam = { version = "0.8.1", optional = true }
num_cpus = "1.13.0"
threadpool = "1.8.1"
heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c758867e05ad973ef800a6fe1d5d", optional = true }
rocksdb = { version = "0.16.0", features = ["multi-threaded-cf"], optional = true }
rocksdb = { version = "0.17.0", features = ["multi-threaded-cf"], optional = true }
thread_local = "1.1.3"
# used for TURN server authentication
hmac = "0.11.0"

40
src/database/abstraction/rocksdb.rs
View file

@ -10,6 +10,7 @@ use tokio::sync::watch;
pub struct Engine {
rocks: rocksdb::DBWithThreadMode<rocksdb::MultiThreaded>,
cache: rocksdb::Cache,
old_cfs: Vec<String>,
}
@ -62,6 +63,7 @@ impl DatabaseEngine for Arc<Engine> {
Ok(Arc::new(Engine {
rocks: db,
cache: rocksdb_cache,
old_cfs: cfs,
}))
}
@ -90,34 +92,36 @@ impl DatabaseEngine for Arc<Engine> {
}
fn memory_usage(&self) -> Result<String> {
let stats = rocksdb::perf::get_memory_usage_stats(Some(&[&self.rocks]), None)?;
Ok(format!("Approximate memory usage of all the mem-tables: {:.3} MB\n\
let stats =
rocksdb::perf::get_memory_usage_stats(Some(&[&self.rocks]), Some(&[&self.cache]))?;
Ok(format!(
"Approximate memory usage of all the mem-tables: {:.3} MB\n\
Approximate memory usage of un-flushed mem-tables: {:.3} MB\n\
Approximate memory usage of all the table readers: {:.3} MB\n\
Approximate memory usage by cache: {:.3} MB",
stats.mem_table_total as f64 / 1024.0 / 1024.0,
stats.mem_table_unflushed as f64 / 1024.0 / 1024.0,
stats.mem_table_readers_total as f64 / 1024.0 / 1024.0,
stats.cache_total as f64 / 1024.0 / 1024.0
stats.mem_table_total as f64 / 1024.0 / 1024.0,
stats.mem_table_unflushed as f64 / 1024.0 / 1024.0,
stats.mem_table_readers_total as f64 / 1024.0 / 1024.0,
stats.cache_total as f64 / 1024.0 / 1024.0
))
}
}
impl RocksDbEngineTree<'_> {
fn cf(&self) -> rocksdb::BoundColumnFamily<'_> {
fn cf(&self) -> Arc<rocksdb::BoundColumnFamily<'_>> {
self.db.rocks.cf_handle(self.name).unwrap()
}
}
impl Tree for RocksDbEngineTree<'_> {
fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
Ok(self.db.rocks.get_cf(self.cf(), key)?)
Ok(self.db.rocks.get_cf(&self.cf(), key)?)
}
fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> {
let lock = self.write_lock.read().unwrap();
let result = self.db.rocks.put_cf(self.cf(), key, value)?;
let result = self.db.rocks.put_cf(&self.cf(), key, value)?;
drop(lock);
@ -146,21 +150,21 @@ impl Tree for RocksDbEngineTree<'_> {
fn insert_batch<'a>(&self, iter: &mut dyn Iterator<Item = (Vec<u8>, Vec<u8>)>) -> Result<()> {
for (key, value) in iter {
self.db.rocks.put_cf(self.cf(), key, value)?;
self.db.rocks.put_cf(&self.cf(), key, value)?;
}
Ok(())
}
fn remove(&self, key: &[u8]) -> Result<()> {
Ok(self.db.rocks.delete_cf(self.cf(), key)?)
Ok(self.db.rocks.delete_cf(&self.cf(), key)?)
}
fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a> {
Box::new(
self.db
.rocks
.iterator_cf(self.cf(), rocksdb::IteratorMode::Start)
.iterator_cf(&self.cf(), rocksdb::IteratorMode::Start)
.map(|(k, v)| (Vec::from(k), Vec::from(v))),
)
}
@ -174,7 +178,7 @@ impl Tree for RocksDbEngineTree<'_> {
self.db
.rocks
.iterator_cf(
self.cf(),
&self.cf(),
rocksdb::IteratorMode::From(
from,
if backwards {
@ -191,9 +195,9 @@ impl Tree for RocksDbEngineTree<'_> {
fn increment(&self, key: &[u8]) -> Result<Vec<u8>> {
let lock = self.write_lock.write().unwrap();
let old = self.db.rocks.get_cf(self.cf(), &key)?;
let old = self.db.rocks.get_cf(&self.cf(), &key)?;
let new = utils::increment(old.as_deref()).unwrap();
self.db.rocks.put_cf(self.cf(), key, &new)?;
self.db.rocks.put_cf(&self.cf(), key, &new)?;
drop(lock);
Ok(new)
@ -203,9 +207,9 @@ impl Tree for RocksDbEngineTree<'_> {
let lock = self.write_lock.write().unwrap();
for key in iter {
let old = self.db.rocks.get_cf(self.cf(), &key)?;
let old = self.db.rocks.get_cf(&self.cf(), &key)?;
let new = utils::increment(old.as_deref()).unwrap();
self.db.rocks.put_cf(self.cf(), key, new)?;
self.db.rocks.put_cf(&self.cf(), key, new)?;
}
drop(lock);
@ -221,7 +225,7 @@ impl Tree for RocksDbEngineTree<'_> {
self.db
.rocks
.iterator_cf(
self.cf(),
&self.cf(),
rocksdb::IteratorMode::From(&prefix, rocksdb::Direction::Forward),
)
.map(|(k, v)| (Vec::from(k), Vec::from(v)))

2
src/database/admin.rs
View file

@ -114,7 +114,7 @@ impl Admin {
if let Ok(response) = guard._db.memory_usage() {
send_message(RoomMessageEventContent::text_plain(response), guard, &state_lock);
} else {
send_message(RoomMessageEventContent::text_plain("Failed to get database memory usage".to_string()), guard, &state_lock);
send_message(RoomMessageEventContent::text_plain("Failed to get database memory usage.".to_string()), guard, &state_lock);
}
}
AdminCommand::SendMessage(message) => {