dont build sha2, opentelemetry, or zstd code if unused

reduces unnecessary crates being compiled. splits them
into features.

i have yet to see anyone use conduit's opentelemetry
stuff, and realistically those people who do
performance benchmarking and measurements will be
building stuff anyways so they can just enable this
feature.

Signed-off-by: strawberry <strawberry@puppygock.gay>
This commit is contained in:
strawberry 2024-03-09 15:53:22 -05:00 committed by June
parent 958b738e5a
commit 39aef8d1b9
5 changed files with 149 additions and 94 deletions

View file

@ -338,6 +338,7 @@ impl fmt::Display for Config {
} }
&lst.join(", ") &lst.join(", ")
}), }),
#[cfg(feature = "compression-zstd")]
("zstd Response Body Compression", &self.zstd_compression.to_string()), ("zstd Response Body Compression", &self.zstd_compression.to_string()),
("RocksDB database log level", &self.rocksdb_log_level), ("RocksDB database log level", &self.rocksdb_log_level),
("RocksDB database log time-to-roll", &self.rocksdb_log_time_to_roll.to_string()), ("RocksDB database log time-to-roll", &self.rocksdb_log_time_to_roll.to_string()),

View file

@ -894,26 +894,28 @@ impl KeyValueDatabase {
warn!("Migration: 12 -> 13 finished"); warn!("Migration: 12 -> 13 finished");
} }
if services().globals.database_version()? < 14 && cfg!(feature = "sha256_media") { #[cfg(feature = "sha256_media")]
warn!("sha256_media feature flag is enabled, migrating legacy base64 file names to sha256 file names"); {
// Move old media files to new names if services().globals.database_version()? < 14 && cfg!(feature = "sha256_media") {
for (key, _) in db.mediaid_file.iter() { warn!(
// we know that this method is deprecated, but we need to use it to migrate the "sha256_media feature flag is enabled, migrating legacy base64 file names to sha256 file names"
// old files to the new location );
// // Move old media files to new names
// TODO: remove this once we're sure that all users have migrated for (key, _) in db.mediaid_file.iter() {
#[allow(deprecated)] let old_path = services().globals.get_media_file(&key);
let old_path = services().globals.get_media_file(&key); debug!("Old file path: {old_path:?}");
let path = services().globals.get_media_file_new(&key); let path = services().globals.get_media_file_new(&key);
// move the file to the new location debug!("New file path: {path:?}");
if old_path.exists() { // move the file to the new location
tokio::fs::rename(&old_path, &path).await?; if old_path.exists() {
tokio::fs::rename(&old_path, &path).await?;
}
} }
services().globals.bump_database_version(14)?;
warn!("Migration: 13 -> 14 finished");
} }
services().globals.bump_database_version(14)?;
warn!("Migration: 13 -> 14 finished");
} }
assert_eq!( assert_eq!(
@ -981,7 +983,7 @@ impl KeyValueDatabase {
); );
} }
// Inserting registraions into cache // Inserting registrations into cache
for appservice in services().appservice.all()? { for appservice in services().appservice.all()? {
services().appservice.registration_info.write().await.insert( services().appservice.registration_info.write().await.insert(
appservice.0, appservice.0,

View file

@ -85,33 +85,39 @@ async fn main() {
}; };
if config.allow_jaeger { if config.allow_jaeger {
opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); #[cfg(feature = "perf_measurements")]
let tracer = opentelemetry_jaeger::new_agent_pipeline() {
.with_auto_split_batch(true) opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new());
.with_service_name("conduit") let tracer = opentelemetry_jaeger::new_agent_pipeline()
.install_batch(opentelemetry_sdk::runtime::Tokio) .with_auto_split_batch(true)
.unwrap(); .with_service_name("conduit")
let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); .install_batch(opentelemetry_sdk::runtime::Tokio)
.unwrap();
let telemetry = tracing_opentelemetry::layer().with_tracer(tracer);
let filter_layer = match EnvFilter::try_new(&config.log) { let filter_layer = match EnvFilter::try_new(&config.log) {
Ok(s) => s, Ok(s) => s,
Err(e) => { Err(e) => {
eprintln!("It looks like your log config is invalid. The following error occurred: {e}"); eprintln!("It looks like your log config is invalid. The following error occurred: {e}");
EnvFilter::try_new("warn").unwrap() EnvFilter::try_new("warn").unwrap()
}, },
}; };
let subscriber = tracing_subscriber::Registry::default().with(filter_layer).with(telemetry); let subscriber = tracing_subscriber::Registry::default().with(filter_layer).with(telemetry);
tracing::subscriber::set_global_default(subscriber).unwrap(); tracing::subscriber::set_global_default(subscriber).unwrap();
}
} else if config.tracing_flame { } else if config.tracing_flame {
let registry = tracing_subscriber::Registry::default(); #[cfg(feature = "perf_measurements")]
let (flame_layer, _guard) = tracing_flame::FlameLayer::with_file("./tracing.folded").unwrap(); {
let flame_layer = flame_layer.with_empty_samples(false); let registry = tracing_subscriber::Registry::default();
let (flame_layer, _guard) = tracing_flame::FlameLayer::with_file("./tracing.folded").unwrap();
let flame_layer = flame_layer.with_empty_samples(false);
let filter_layer = EnvFilter::new("trace,h2=off"); let filter_layer = EnvFilter::new("trace,h2=off");
let subscriber = registry.with(filter_layer).with(flame_layer); let subscriber = registry.with(filter_layer).with(flame_layer);
tracing::subscriber::set_global_default(subscriber).unwrap(); tracing::subscriber::set_global_default(subscriber).unwrap();
}
} else { } else {
let registry = tracing_subscriber::Registry::default(); let registry = tracing_subscriber::Registry::default();
let fmt_layer = tracing_subscriber::fmt::Layer::new(); let fmt_layer = tracing_subscriber::fmt::Layer::new();
@ -296,6 +302,7 @@ async fn main() {
// if server runs into critical error and shuts down, shut down the tracer // if server runs into critical error and shuts down, shut down the tracer
// provider if jaegar is used. awaiting run_server() is a blocking call so // provider if jaegar is used. awaiting run_server() is a blocking call so
// putting this after is fine, but not the other options above. // putting this after is fine, but not the other options above.
#[cfg(feature = "perf_measurements")]
if config.allow_jaeger { if config.allow_jaeger {
opentelemetry::global::shutdown_tracer_provider(); opentelemetry::global::shutdown_tracer_provider();
} }
@ -359,11 +366,21 @@ async fn run_server() -> io::Result<()> {
config.max_request_size.try_into().expect("failed to convert max request size"), config.max_request_size.try_into().expect("failed to convert max request size"),
)); ));
let app = if cfg!(feature = "zstd_compression") && config.zstd_compression { let app;
debug!("zstd body compression is enabled");
routes().layer(middlewares.compression()).into_make_service() #[cfg(feature = "zstd_compression")]
} else { {
routes().layer(middlewares).into_make_service() app = if cfg!(feature = "zstd_compression") && config.zstd_compression {
debug!("zstd body compression is enabled");
routes().layer(middlewares.compression()).into_make_service()
} else {
routes().layer(middlewares).into_make_service()
}
};
#[cfg(not(feature = "zstd_compression"))]
{
app = routes().layer(middlewares).into_make_service()
}; };
let handle = ServerHandle::new(); let handle = ServerHandle::new();

View file

@ -32,7 +32,6 @@ use ruma::{
DeviceId, OwnedDeviceId, OwnedEventId, OwnedRoomId, OwnedServerName, OwnedServerSigningKeyId, OwnedUserId, DeviceId, OwnedDeviceId, OwnedEventId, OwnedRoomId, OwnedServerName, OwnedServerSigningKeyId, OwnedUserId,
RoomVersionId, ServerName, UserId, RoomVersionId, ServerName, UserId,
}; };
use sha2::Digest;
use tokio::sync::{broadcast, watch::Receiver, Mutex, RwLock, Semaphore}; use tokio::sync::{broadcast, watch::Receiver, Mutex, RwLock, Semaphore};
use tracing::{error, info}; use tracing::{error, info};
use trust_dns_resolver::TokioAsyncResolver; use trust_dns_resolver::TokioAsyncResolver;
@ -430,6 +429,7 @@ impl Service<'_> {
/// new SHA256 file name media function, requires "sha256_media" feature /// new SHA256 file name media function, requires "sha256_media" feature
/// flag enabled and database migrated uses SHA256 hash of the base64 key as /// flag enabled and database migrated uses SHA256 hash of the base64 key as
/// the file name /// the file name
#[cfg(feature = "sha256_media")]
pub fn get_media_file_new(&self, key: &[u8]) -> PathBuf { pub fn get_media_file_new(&self, key: &[u8]) -> PathBuf {
let mut r = PathBuf::new(); let mut r = PathBuf::new();
r.push(self.config.database_path.clone()); r.push(self.config.database_path.clone());
@ -437,17 +437,13 @@ impl Service<'_> {
// Using the hash of the base64 key as the filename // Using the hash of the base64 key as the filename
// This is to prevent the total length of the path from exceeding the maximum // This is to prevent the total length of the path from exceeding the maximum
// length in most filesystems // length in most filesystems
r.push(general_purpose::URL_SAFE_NO_PAD.encode(sha2::Sha256::digest(key))); r.push(general_purpose::URL_SAFE_NO_PAD.encode(<sha2::Sha256 as sha2::Digest>::digest(key)));
r r
} }
/// old base64 file name media function /// old base64 file name media function
/// This is the old version of `get_media_file` that uses the full base64 /// This is the old version of `get_media_file` that uses the full base64
/// key as the filename. /// key as the filename.
///
/// This is deprecated and will be removed in a future release.
/// Please use `get_media_file_new` instead.
#[deprecated(note = "Use get_media_file_new instead")]
pub fn get_media_file(&self, key: &[u8]) -> PathBuf { pub fn get_media_file(&self, key: &[u8]) -> PathBuf {
let mut r = PathBuf::new(); let mut r = PathBuf::new();
r.push(self.config.database_path.clone()); r.push(self.config.database_path.clone());

View file

@ -50,11 +50,16 @@ impl Service {
// Width, Height = 0 if it's not a thumbnail // Width, Height = 0 if it's not a thumbnail
let key = self.db.create_file_metadata(mxc, 0, 0, content_disposition, content_type)?; let key = self.db.create_file_metadata(mxc, 0, 0, content_disposition, content_type)?;
let path = if cfg!(feature = "sha256_media") { let path;
services().globals.get_media_file_new(&key)
} else { #[cfg(feature = "sha256_media")]
#[allow(deprecated)] {
services().globals.get_media_file(&key) path = services().globals.get_media_file_new(&key)
};
#[cfg(not(feature = "sha256_media"))]
{
path = services().globals.get_media_file(&key)
}; };
let mut f = File::create(path).await?; let mut f = File::create(path).await?;
@ -66,12 +71,18 @@ impl Service {
pub async fn delete(&self, mxc: String) -> Result<()> { pub async fn delete(&self, mxc: String) -> Result<()> {
if let Ok(keys) = self.db.search_mxc_metadata_prefix(mxc.clone()) { if let Ok(keys) = self.db.search_mxc_metadata_prefix(mxc.clone()) {
for key in keys { for key in keys {
let file_path = if cfg!(feature = "sha256_media") { let file_path;
services().globals.get_media_file_new(&key)
} else { #[cfg(feature = "sha256_media")]
#[allow(deprecated)] {
services().globals.get_media_file(&key) file_path = services().globals.get_media_file_new(&key)
}; };
#[cfg(not(feature = "sha256_media"))]
{
file_path = services().globals.get_media_file(&key)
};
debug!("Got local file path: {:?}", file_path); debug!("Got local file path: {:?}", file_path);
debug!("Deleting local file {:?} from filesystem, original MXC: {}", file_path, mxc); debug!("Deleting local file {:?} from filesystem, original MXC: {}", file_path, mxc);
@ -96,13 +107,17 @@ impl Service {
file: &[u8], file: &[u8],
) -> Result<()> { ) -> Result<()> {
let key = self.db.create_file_metadata(mxc, width, height, content_disposition, content_type)?; let key = self.db.create_file_metadata(mxc, width, height, content_disposition, content_type)?;
let path;
let path = if cfg!(feature = "sha256_media") { #[cfg(feature = "sha256_media")]
services().globals.get_media_file_new(&key) {
} else { path = services().globals.get_media_file_new(&key);
#[allow(deprecated)] }
services().globals.get_media_file(&key)
}; #[cfg(not(feature = "sha256_media"))]
{
path = services().globals.get_media_file(&key);
}
let mut f = File::create(path).await?; let mut f = File::create(path).await?;
f.write_all(file).await?; f.write_all(file).await?;
@ -113,11 +128,16 @@ impl Service {
/// Downloads a file. /// Downloads a file.
pub async fn get(&self, mxc: String) -> Result<Option<FileMeta>> { pub async fn get(&self, mxc: String) -> Result<Option<FileMeta>> {
if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc, 0, 0) { if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc, 0, 0) {
let path = if cfg!(feature = "sha256_media") { let path;
services().globals.get_media_file_new(&key)
} else { #[cfg(feature = "sha256_media")]
#[allow(deprecated)] {
services().globals.get_media_file(&key) path = services().globals.get_media_file_new(&key)
};
#[cfg(not(feature = "sha256_media"))]
{
path = services().globals.get_media_file(&key)
}; };
let mut file = Vec::new(); let mut file = Vec::new();
@ -186,11 +206,16 @@ impl Service {
continue; continue;
} }
let path = if cfg!(feature = "sha256_media") { let path;
services().globals.get_media_file_new(&key)
} else { #[cfg(feature = "sha256_media")]
#[allow(deprecated)] {
services().globals.get_media_file(&key) path = services().globals.get_media_file_new(&key)
};
#[cfg(not(feature = "sha256_media"))]
{
path = services().globals.get_media_file(&key)
}; };
debug!("MXC path: {:?}", path); debug!("MXC path: {:?}", path);
@ -265,11 +290,16 @@ impl Service {
if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc.clone(), width, height) { if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc.clone(), width, height) {
// Using saved thumbnail // Using saved thumbnail
let path = if cfg!(feature = "sha256_media") { let path;
services().globals.get_media_file_new(&key)
} else { #[cfg(feature = "sha256_media")]
#[allow(deprecated)] {
services().globals.get_media_file(&key) path = services().globals.get_media_file_new(&key)
};
#[cfg(not(feature = "sha256_media"))]
{
path = services().globals.get_media_file(&key)
}; };
let mut file = Vec::new(); let mut file = Vec::new();
@ -282,11 +312,16 @@ impl Service {
})) }))
} else if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc.clone(), 0, 0) { } else if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc.clone(), 0, 0) {
// Generate a thumbnail // Generate a thumbnail
let path = if cfg!(feature = "sha256_media") { let path;
services().globals.get_media_file_new(&key)
} else { #[cfg(feature = "sha256_media")]
#[allow(deprecated)] {
services().globals.get_media_file(&key) path = services().globals.get_media_file_new(&key)
};
#[cfg(not(feature = "sha256_media"))]
{
path = services().globals.get_media_file(&key)
}; };
let mut file = Vec::new(); let mut file = Vec::new();
@ -351,11 +386,16 @@ impl Service {
content_type.as_deref(), content_type.as_deref(),
)?; )?;
let path = if cfg!(feature = "sha256_media") { let path;
services().globals.get_media_file_new(&thumbnail_key)
} else { #[cfg(feature = "sha256_media")]
#[allow(deprecated)] {
services().globals.get_media_file(&thumbnail_key) path = services().globals.get_media_file_new(&thumbnail_key)
};
#[cfg(not(feature = "sha256_media"))]
{
path = services().globals.get_media_file(&thumbnail_key)
}; };
let mut f = File::create(path).await?; let mut f = File::create(path).await?;
@ -397,7 +437,6 @@ mod tests {
use std::path::PathBuf; use std::path::PathBuf;
use base64::{engine::general_purpose, Engine as _}; use base64::{engine::general_purpose, Engine as _};
use sha2::Digest;
use super::*; use super::*;