TudbuT/conduit
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

dont build sha2, opentelemetry, or zstd code if unused

Browse source 
reduces unnecessary crates being compiled. splits them
into features.

i have yet to see anyone use conduit's opentelemetry
stuff, and realistically those people who do
performance benchmarking and measurements will be
building stuff anyways so they can just enable this
feature.

Signed-off-by: strawberry <strawberry@puppygock.gay>
This commit is contained in:
strawberry 2024-03-09 15:53:22 -05:00 • committed by June
parent 958b738e5a
commit 39aef8d1b9
5 changed files with 149 additions and 94 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/config/mod.rs
View file

@ -338,6 +338,7 @@ impl fmt::Display for Config {
}
&lst.join(", ")
}),
#[cfg(feature = "compression-zstd")]
("zstd Response Body Compression", &self.zstd_compression.to_string()),
("RocksDB database log level", &self.rocksdb_log_level),
("RocksDB database log time-to-roll", &self.rocksdb_log_time_to_roll.to_string()),

40
src/database/mod.rs
View file

@ -894,26 +894,28 @@ impl KeyValueDatabase {
warn!("Migration: 12 -> 13 finished");
}
if services().globals.database_version()? < 14 && cfg!(feature = "sha256_media") {
warn!("sha256_media feature flag is enabled, migrating legacy base64 file names to sha256 file names");
// Move old media files to new names
for (key, _) in db.mediaid_file.iter() {
// we know that this method is deprecated, but we need to use it to migrate the
// old files to the new location
//
// TODO: remove this once we're sure that all users have migrated
#[allow(deprecated)]
let old_path = services().globals.get_media_file(&key);
let path = services().globals.get_media_file_new(&key);
// move the file to the new location
if old_path.exists() {
tokio::fs::rename(&old_path, &path).await?;
#[cfg(feature = "sha256_media")]
{
if services().globals.database_version()? < 14 && cfg!(feature = "sha256_media") {
warn!(
"sha256_media feature flag is enabled, migrating legacy base64 file names to sha256 file names"
);
// Move old media files to new names
for (key, _) in db.mediaid_file.iter() {
let old_path = services().globals.get_media_file(&key);
debug!("Old file path: {old_path:?}");
let path = services().globals.get_media_file_new(&key);
debug!("New file path: {path:?}");
// move the file to the new location
if old_path.exists() {
tokio::fs::rename(&old_path, &path).await?;
}
}
services().globals.bump_database_version(14)?;
warn!("Migration: 13 -> 14 finished");
}
services().globals.bump_database_version(14)?;
warn!("Migration: 13 -> 14 finished");
}
assert_eq!(
@ -981,7 +983,7 @@ impl KeyValueDatabase {
);
}
// Inserting registraions into cache
// Inserting registrations into cache
for appservice in services().appservice.all()? {
services().appservice.registration_info.write().await.insert(
appservice.0,

71
src/main.rs
View file

@ -85,33 +85,39 @@ async fn main() {
};
if config.allow_jaeger {
opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new());
let tracer = opentelemetry_jaeger::new_agent_pipeline()
.with_auto_split_batch(true)
.with_service_name("conduit")
.install_batch(opentelemetry_sdk::runtime::Tokio)
.unwrap();
let telemetry = tracing_opentelemetry::layer().with_tracer(tracer);
#[cfg(feature = "perf_measurements")]
{
opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new());
let tracer = opentelemetry_jaeger::new_agent_pipeline()
.with_auto_split_batch(true)
.with_service_name("conduit")
.install_batch(opentelemetry_sdk::runtime::Tokio)
.unwrap();
let telemetry = tracing_opentelemetry::layer().with_tracer(tracer);
let filter_layer = match EnvFilter::try_new(&config.log) {
Ok(s) => s,
Err(e) => {
eprintln!("It looks like your log config is invalid. The following error occurred: {e}");
EnvFilter::try_new("warn").unwrap()
},
};
let filter_layer = match EnvFilter::try_new(&config.log) {
Ok(s) => s,
Err(e) => {
eprintln!("It looks like your log config is invalid. The following error occurred: {e}");
EnvFilter::try_new("warn").unwrap()
},
};
let subscriber = tracing_subscriber::Registry::default().with(filter_layer).with(telemetry);
tracing::subscriber::set_global_default(subscriber).unwrap();
let subscriber = tracing_subscriber::Registry::default().with(filter_layer).with(telemetry);
tracing::subscriber::set_global_default(subscriber).unwrap();
}
} else if config.tracing_flame {
let registry = tracing_subscriber::Registry::default();
let (flame_layer, _guard) = tracing_flame::FlameLayer::with_file("./tracing.folded").unwrap();
let flame_layer = flame_layer.with_empty_samples(false);
#[cfg(feature = "perf_measurements")]
{
let registry = tracing_subscriber::Registry::default();
let (flame_layer, _guard) = tracing_flame::FlameLayer::with_file("./tracing.folded").unwrap();
let flame_layer = flame_layer.with_empty_samples(false);
let filter_layer = EnvFilter::new("trace,h2=off");
let filter_layer = EnvFilter::new("trace,h2=off");
let subscriber = registry.with(filter_layer).with(flame_layer);
tracing::subscriber::set_global_default(subscriber).unwrap();
let subscriber = registry.with(filter_layer).with(flame_layer);
tracing::subscriber::set_global_default(subscriber).unwrap();
}
} else {
let registry = tracing_subscriber::Registry::default();
let fmt_layer = tracing_subscriber::fmt::Layer::new();
@ -296,6 +302,7 @@ async fn main() {
// if server runs into critical error and shuts down, shut down the tracer
// provider if jaegar is used. awaiting run_server() is a blocking call so
// putting this after is fine, but not the other options above.
#[cfg(feature = "perf_measurements")]
if config.allow_jaeger {
opentelemetry::global::shutdown_tracer_provider();
}
@ -359,11 +366,21 @@ async fn run_server() -> io::Result<()> {
config.max_request_size.try_into().expect("failed to convert max request size"),
));
let app = if cfg!(feature = "zstd_compression") && config.zstd_compression {
debug!("zstd body compression is enabled");
routes().layer(middlewares.compression()).into_make_service()
} else {
routes().layer(middlewares).into_make_service()
let app;
#[cfg(feature = "zstd_compression")]
{
app = if cfg!(feature = "zstd_compression") && config.zstd_compression {
debug!("zstd body compression is enabled");
routes().layer(middlewares.compression()).into_make_service()
} else {
routes().layer(middlewares).into_make_service()
}
};
#[cfg(not(feature = "zstd_compression"))]
{
app = routes().layer(middlewares).into_make_service()
};
let handle = ServerHandle::new();

8
src/service/globals/mod.rs
View file

@ -32,7 +32,6 @@ use ruma::{
DeviceId, OwnedDeviceId, OwnedEventId, OwnedRoomId, OwnedServerName, OwnedServerSigningKeyId, OwnedUserId,
RoomVersionId, ServerName, UserId,
};
use sha2::Digest;
use tokio::sync::{broadcast, watch::Receiver, Mutex, RwLock, Semaphore};
use tracing::{error, info};
use trust_dns_resolver::TokioAsyncResolver;
@ -430,6 +429,7 @@ impl Service<'_> {
/// new SHA256 file name media function, requires "sha256_media" feature
/// flag enabled and database migrated uses SHA256 hash of the base64 key as
/// the file name
#[cfg(feature = "sha256_media")]
pub fn get_media_file_new(&self, key: &[u8]) -> PathBuf {
let mut r = PathBuf::new();
r.push(self.config.database_path.clone());
@ -437,17 +437,13 @@ impl Service<'_> {
// Using the hash of the base64 key as the filename
// This is to prevent the total length of the path from exceeding the maximum
// length in most filesystems
r.push(general_purpose::URL_SAFE_NO_PAD.encode(sha2::Sha256::digest(key)));
r.push(general_purpose::URL_SAFE_NO_PAD.encode(<sha2::Sha256 as sha2::Digest>::digest(key)));
r
}
/// old base64 file name media function
/// This is the old version of `get_media_file` that uses the full base64
/// key as the filename.
///
/// This is deprecated and will be removed in a future release.
/// Please use `get_media_file_new` instead.
#[deprecated(note = "Use get_media_file_new instead")]
pub fn get_media_file(&self, key: &[u8]) -> PathBuf {
let mut r = PathBuf::new();
r.push(self.config.database_path.clone());

123
src/service/media/mod.rs
View file

@ -50,11 +50,16 @@ impl Service {
// Width, Height = 0 if it's not a thumbnail
let key = self.db.create_file_metadata(mxc, 0, 0, content_disposition, content_type)?;
let path = if cfg!(feature = "sha256_media") {
services().globals.get_media_file_new(&key)
} else {
#[allow(deprecated)]
services().globals.get_media_file(&key)
let path;
#[cfg(feature = "sha256_media")]
{
path = services().globals.get_media_file_new(&key)
};
#[cfg(not(feature = "sha256_media"))]
{
path = services().globals.get_media_file(&key)
};
let mut f = File::create(path).await?;
@ -66,12 +71,18 @@ impl Service {
pub async fn delete(&self, mxc: String) -> Result<()> {
if let Ok(keys) = self.db.search_mxc_metadata_prefix(mxc.clone()) {
for key in keys {
let file_path = if cfg!(feature = "sha256_media") {
services().globals.get_media_file_new(&key)
} else {
#[allow(deprecated)]
services().globals.get_media_file(&key)
let file_path;
#[cfg(feature = "sha256_media")]
{
file_path = services().globals.get_media_file_new(&key)
};
#[cfg(not(feature = "sha256_media"))]
{
file_path = services().globals.get_media_file(&key)
};
debug!("Got local file path: {:?}", file_path);
debug!("Deleting local file {:?} from filesystem, original MXC: {}", file_path, mxc);
@ -96,13 +107,17 @@ impl Service {
file: &[u8],
) -> Result<()> {
let key = self.db.create_file_metadata(mxc, width, height, content_disposition, content_type)?;
let path;
let path = if cfg!(feature = "sha256_media") {
services().globals.get_media_file_new(&key)
} else {
#[allow(deprecated)]
services().globals.get_media_file(&key)
};
#[cfg(feature = "sha256_media")]
{
path = services().globals.get_media_file_new(&key);
}
#[cfg(not(feature = "sha256_media"))]
{
path = services().globals.get_media_file(&key);
}
let mut f = File::create(path).await?;
f.write_all(file).await?;
@ -113,11 +128,16 @@ impl Service {
/// Downloads a file.
pub async fn get(&self, mxc: String) -> Result<Option<FileMeta>> {
if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc, 0, 0) {
let path = if cfg!(feature = "sha256_media") {
services().globals.get_media_file_new(&key)
} else {
#[allow(deprecated)]
services().globals.get_media_file(&key)
let path;
#[cfg(feature = "sha256_media")]
{
path = services().globals.get_media_file_new(&key)
};
#[cfg(not(feature = "sha256_media"))]
{
path = services().globals.get_media_file(&key)
};
let mut file = Vec::new();
@ -186,11 +206,16 @@ impl Service {
continue;
}
let path = if cfg!(feature = "sha256_media") {
services().globals.get_media_file_new(&key)
} else {
#[allow(deprecated)]
services().globals.get_media_file(&key)
let path;
#[cfg(feature = "sha256_media")]
{
path = services().globals.get_media_file_new(&key)
};
#[cfg(not(feature = "sha256_media"))]
{
path = services().globals.get_media_file(&key)
};
debug!("MXC path: {:?}", path);
@ -265,11 +290,16 @@ impl Service {
if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc.clone(), width, height) {
// Using saved thumbnail
let path = if cfg!(feature = "sha256_media") {
services().globals.get_media_file_new(&key)
} else {
#[allow(deprecated)]
services().globals.get_media_file(&key)
let path;
#[cfg(feature = "sha256_media")]
{
path = services().globals.get_media_file_new(&key)
};
#[cfg(not(feature = "sha256_media"))]
{
path = services().globals.get_media_file(&key)
};
let mut file = Vec::new();
@ -282,11 +312,16 @@ impl Service {
}))
} else if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc.clone(), 0, 0) {
// Generate a thumbnail
let path = if cfg!(feature = "sha256_media") {
services().globals.get_media_file_new(&key)
} else {
#[allow(deprecated)]
services().globals.get_media_file(&key)
let path;
#[cfg(feature = "sha256_media")]
{
path = services().globals.get_media_file_new(&key)
};
#[cfg(not(feature = "sha256_media"))]
{
path = services().globals.get_media_file(&key)
};
let mut file = Vec::new();
@ -351,11 +386,16 @@ impl Service {
content_type.as_deref(),
)?;
let path = if cfg!(feature = "sha256_media") {
services().globals.get_media_file_new(&thumbnail_key)
} else {
#[allow(deprecated)]
services().globals.get_media_file(&thumbnail_key)
let path;
#[cfg(feature = "sha256_media")]
{
path = services().globals.get_media_file_new(&thumbnail_key)
};
#[cfg(not(feature = "sha256_media"))]
{
path = services().globals.get_media_file(&thumbnail_key)
};
let mut f = File::create(path).await?;
@ -397,7 +437,6 @@ mod tests {
use std::path::PathBuf;
use base64::{engine::general_purpose, Engine as _};
use sha2::Digest;
use super::*;