fix refering uninit static, use upstream rocksdb again, don't compile debug info

even rust-analyzer themselves don't compile with debug info because it
makes cargo checks and builds in general a lot longer. helps speed
things up and we mainly care about panics/stacktraces and compiler
errors.

https://github.com/rust-lang/rust-analyzer/blob/master/Cargo.toml#L12-L15

upstream rust-rocksdb is active again finally

Signed-off-by: strawberry <strawberry@puppygock.gay>
This commit is contained in:
strawberry 2023-12-03 01:00:23 -05:00
parent 4d52dd2f44
commit d9d1ce3cb5
3 changed files with 19 additions and 18 deletions

4
Cargo.lock generated
View file

@ -1353,7 +1353,7 @@ dependencies = [
[[package]] [[package]]
name = "librocksdb-sys" name = "librocksdb-sys"
version = "0.14.0+8.8.1" version = "0.14.0+8.8.1"
source = "git+https://github.com/girlbossceo/rust-rocksdb?rev=eaa2beedb1f36466a52ca01ffbf3a2118b2cb41b#eaa2beedb1f36466a52ca01ffbf3a2118b2cb41b" source = "git+https://github.com/rust-rocksdb/rust-rocksdb?rev=30ffe0ad78a037694eb3e834ac0afc436eea4ebf#30ffe0ad78a037694eb3e834ac0afc436eea4ebf"
dependencies = [ dependencies = [
"bindgen", "bindgen",
"bzip2-sys", "bzip2-sys",
@ -2136,7 +2136,7 @@ dependencies = [
[[package]] [[package]]
name = "rocksdb" name = "rocksdb"
version = "0.21.0" version = "0.21.0"
source = "git+https://github.com/girlbossceo/rust-rocksdb?rev=eaa2beedb1f36466a52ca01ffbf3a2118b2cb41b#eaa2beedb1f36466a52ca01ffbf3a2118b2cb41b" source = "git+https://github.com/rust-rocksdb/rust-rocksdb?rev=30ffe0ad78a037694eb3e834ac0afc436eea4ebf#30ffe0ad78a037694eb3e834ac0afc436eea4ebf"
dependencies = [ dependencies = [
"libc", "libc",
"librocksdb-sys", "librocksdb-sys",

View file

@ -88,7 +88,7 @@ heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c7
# Used for ruma wrapper # Used for ruma wrapper
serde_html_form = "0.2.2" serde_html_form = "0.2.2"
rocksdb = { git = "https://github.com/girlbossceo/rust-rocksdb", rev = "eaa2beedb1f36466a52ca01ffbf3a2118b2cb41b", default-features = false, features = ["multi-threaded-cf", "snappy", "lz4", "zstd"], optional = true } rocksdb = { git = "https://github.com/rust-rocksdb/rust-rocksdb", rev = "30ffe0ad78a037694eb3e834ac0afc436eea4ebf", default-features = false, features = ["multi-threaded-cf", "snappy", "lz4", "zstd"], optional = true }
thread_local = "1.1.7" thread_local = "1.1.7"
# used for TURN server authentication # used for TURN server authentication
@ -164,6 +164,7 @@ maintainer-scripts = "debian/"
systemd-units = { unit-name = "matrix-conduit" } systemd-units = { unit-name = "matrix-conduit" }
[profile.dev] [profile.dev]
debug = 0
lto = 'off' lto = 'off'
incremental = true incremental = true
@ -175,13 +176,16 @@ opt-level = 3
overflow-checks = true overflow-checks = true
strip = "symbols" strip = "symbols"
panic = "abort" panic = "abort"
debug = 0
# If you want to make flamegraphs, enable debug info: # If you want to make flamegraphs, enable debug info:
# debug = true # debug = true
# For releases also try to max optimizations for dependencies: # For releases also try to max optimizations for dependencies:
[profile.release.build-override] [profile.release.build-override]
debug = 0
opt-level = 3 opt-level = 3
codegen-units=1 codegen-units=1
[profile.release.package."*"] [profile.release.package."*"]
debug = 0
opt-level = 3 opt-level = 3
codegen-units=1 codegen-units=1

View file

@ -1,5 +1,5 @@
use super::{super::Config, watchers::Watchers, KeyValueDatabaseEngine, KvTree}; use super::{super::Config, watchers::Watchers, KeyValueDatabaseEngine, KvTree};
use crate::{services, utils, Result}; use crate::{utils, Result};
use std::{ use std::{
future::Future, future::Future,
pin::Pin, pin::Pin,
@ -10,9 +10,9 @@ use rocksdb::LogLevel::{Debug, Error, Fatal, Info, Warn};
pub struct Engine { pub struct Engine {
rocks: rocksdb::DBWithThreadMode<rocksdb::MultiThreaded>, rocks: rocksdb::DBWithThreadMode<rocksdb::MultiThreaded>,
max_open_files: i32,
cache: rocksdb::Cache, cache: rocksdb::Cache,
old_cfs: Vec<String>, old_cfs: Vec<String>,
config: Config,
} }
pub struct RocksDbEngineTree<'a> { pub struct RocksDbEngineTree<'a> {
@ -22,7 +22,7 @@ pub struct RocksDbEngineTree<'a> {
write_lock: RwLock<()>, write_lock: RwLock<()>,
} }
fn db_options(max_open_files: i32, rocksdb_cache: &rocksdb::Cache) -> rocksdb::Options { fn db_options(rocksdb_cache: &rocksdb::Cache, config: &Config) -> rocksdb::Options {
// block-based options: https://docs.rs/rocksdb/latest/rocksdb/struct.BlockBasedOptions.html# // block-based options: https://docs.rs/rocksdb/latest/rocksdb/struct.BlockBasedOptions.html#
let mut block_based_options = rocksdb::BlockBasedOptions::default(); let mut block_based_options = rocksdb::BlockBasedOptions::default();
@ -36,7 +36,7 @@ fn db_options(max_open_files: i32, rocksdb_cache: &rocksdb::Cache) -> rocksdb::O
// database options: https://docs.rs/rocksdb/latest/rocksdb/struct.Options.html# // database options: https://docs.rs/rocksdb/latest/rocksdb/struct.Options.html#
let mut db_opts = rocksdb::Options::default(); let mut db_opts = rocksdb::Options::default();
let rocksdb_log_level = match services().globals.rocksdb_log_level().as_str() { let rocksdb_log_level = match config.rocksdb_log_level.as_ref() {
"debug" => Debug, "debug" => Debug,
"info" => Info, "info" => Info,
"warn" => Warn, "warn" => Warn,
@ -46,10 +46,10 @@ fn db_options(max_open_files: i32, rocksdb_cache: &rocksdb::Cache) -> rocksdb::O
}; };
db_opts.set_log_level(rocksdb_log_level); db_opts.set_log_level(rocksdb_log_level);
db_opts.set_max_log_file_size(services().globals.rocksdb_max_log_file_size()); db_opts.set_max_log_file_size(config.rocksdb_max_log_file_size);
db_opts.set_log_file_time_to_roll(services().globals.rocksdb_log_time_to_roll()); db_opts.set_log_file_time_to_roll(config.rocksdb_log_time_to_roll);
if services().globals.rocksdb_optimize_for_spinning_disks() { if config.rocksdb_optimize_for_spinning_disks {
// useful for hard drives but on literally any half-decent SSD this is not useful // useful for hard drives but on literally any half-decent SSD this is not useful
// and the benefits of improved compaction based on up to date stats are good. // and the benefits of improved compaction based on up to date stats are good.
// current conduwut users have NVMe/SSDs. // current conduwut users have NVMe/SSDs.
@ -69,7 +69,7 @@ fn db_options(max_open_files: i32, rocksdb_cache: &rocksdb::Cache) -> rocksdb::O
db_opts.set_level_compaction_dynamic_level_bytes(true); db_opts.set_level_compaction_dynamic_level_bytes(true);
db_opts.create_if_missing(true); db_opts.create_if_missing(true);
db_opts.increase_parallelism(num_cpus::get() as i32); db_opts.increase_parallelism(num_cpus::get() as i32);
db_opts.set_max_open_files(max_open_files); db_opts.set_max_open_files(config.rocksdb_max_open_files);
db_opts.set_compression_type(rocksdb::DBCompressionType::Zstd); db_opts.set_compression_type(rocksdb::DBCompressionType::Zstd);
db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level); db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level);
db_opts.optimize_level_style_compaction(10 * 1024 * 1024); db_opts.optimize_level_style_compaction(10 * 1024 * 1024);
@ -96,7 +96,7 @@ impl KeyValueDatabaseEngine for Arc<Engine> {
let cache_capacity_bytes = (config.db_cache_capacity_mb * 1024.0 * 1024.0) as usize; let cache_capacity_bytes = (config.db_cache_capacity_mb * 1024.0 * 1024.0) as usize;
let rocksdb_cache = rocksdb::Cache::new_lru_cache(cache_capacity_bytes); let rocksdb_cache = rocksdb::Cache::new_lru_cache(cache_capacity_bytes);
let db_opts = db_options(config.rocksdb_max_open_files, &rocksdb_cache); let db_opts = db_options(&rocksdb_cache, config);
let cfs = rocksdb::DBWithThreadMode::<rocksdb::MultiThreaded>::list_cf( let cfs = rocksdb::DBWithThreadMode::<rocksdb::MultiThreaded>::list_cf(
&db_opts, &db_opts,
@ -108,18 +108,15 @@ impl KeyValueDatabaseEngine for Arc<Engine> {
&db_opts, &db_opts,
&config.database_path, &config.database_path,
cfs.iter().map(|name| { cfs.iter().map(|name| {
rocksdb::ColumnFamilyDescriptor::new( rocksdb::ColumnFamilyDescriptor::new(name, db_options(&rocksdb_cache, config))
name,
db_options(config.rocksdb_max_open_files, &rocksdb_cache),
)
}), }),
)?; )?;
Ok(Arc::new(Engine { Ok(Arc::new(Engine {
rocks: db, rocks: db,
max_open_files: config.rocksdb_max_open_files,
cache: rocksdb_cache, cache: rocksdb_cache,
old_cfs: cfs, old_cfs: cfs,
config: config.clone(),
})) }))
} }
@ -128,7 +125,7 @@ impl KeyValueDatabaseEngine for Arc<Engine> {
// Create if it didn't exist // Create if it didn't exist
let _ = self let _ = self
.rocks .rocks
.create_cf(name, &db_options(self.max_open_files, &self.cache)); .create_cf(name, &db_options(&self.cache, &self.config));
} }
Ok(Arc::new(RocksDbEngineTree { Ok(Arc::new(RocksDbEngineTree {