From cc578d9a67a01ffb04dea415ce0028d947fa0eac Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Fri, 26 Apr 2024 18:10:20 -0700 Subject: [PATCH] keep stdout logs when tracing-flame/jaeger is enabled Previously, enabling the `tracing_flame` or `allow_jaeger` options would prevent any logs from being written to stdout. In addition, enabling the `allow_jaeger` option would inhibit the `tracing_flame` option. Now that we have a way to use separate tracing filters with different layers, we can enable all three at the same time without issues. This commit also prevents the `debug log_level` command from modifying the `tracing-flame` filter. This was supported previously, but I don't think it's something that you would ever want to do intentionally. Now that we have both the normal log filter and the `tracing-flame` filter enabled at the same time, we want to `debug log_level` to only modify the normal filter. --- src/main.rs | 106 ++++++++++++++++++++-------------------------------- 1 file changed, 41 insertions(+), 65 deletions(-) diff --git a/src/main.rs b/src/main.rs index bf2f40db..c7de0516 100644 --- a/src/main.rs +++ b/src/main.rs @@ -476,24 +476,7 @@ fn init(args: clap::Args) -> Result { None }; - let tracing_reload_handle; - - #[cfg(feature = "perf_measurements")] - { - tracing_reload_handle = if config.allow_jaeger { - init_tracing_jaeger(&config) - } else if config.tracing_flame { - #[cfg(feature = "perf_measurements")] - init_tracing_flame(&config) - } else { - init_tracing_sub(&config) - }; - }; - - #[cfg(not(feature = "perf_measurements"))] - { - tracing_reload_handle = init_tracing_sub(&config); - }; + let tracing_reload_handle = init_tracing(&config); config.check()?; @@ -596,7 +579,10 @@ impl LogLevelReloadHandles { } } -fn init_tracing_sub(config: &Config) -> LogLevelReloadHandles { +// clippy thinks the filter_layer clones are redundant if the next usage is +// behind a disabled feature. +#[allow(clippy::redundant_clone)] +fn init_tracing(config: &Config) -> LogLevelReloadHandles { let registry = Registry::default(); let fmt_layer = tracing_subscriber::fmt::Layer::new(); let filter_layer = match EnvFilter::try_new(&config.log) { @@ -623,11 +609,46 @@ fn init_tracing_sub(config: &Config) -> LogLevelReloadHandles { #[cfg(feature = "sentry_telemetry")] let subscriber = { let sentry_layer = sentry_tracing::layer(); - let (sentry_reload_filter, sentry_reload_handle) = reload::Layer::new(filter_layer); + let (sentry_reload_filter, sentry_reload_handle) = reload::Layer::new(filter_layer.clone()); reload_handles.push(Box::new(sentry_reload_handle)); subscriber.with(sentry_layer.with_filter(sentry_reload_filter)) }; + #[cfg(feature = "perf_measurements")] + let subscriber = { + let flame_layer = if config.tracing_flame { + let flame_filter = EnvFilter::new("trace,h2=off"); + + // TODO: actually preserve this guard until exit: https://docs.rs/tracing-flame/latest/tracing_flame/struct.FlameLayer.html#dropping-and-flushing + let (flame_layer, _guard) = tracing_flame::FlameLayer::with_file("./tracing.folded").unwrap(); + Some( + flame_layer + .with_empty_samples(false) + .with_filter(flame_filter), + ) + } else { + None + }; + + let jaeger_layer = if config.allow_jaeger { + opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); + let tracer = opentelemetry_jaeger::new_agent_pipeline() + .with_auto_split_batch(true) + .with_service_name("conduwuit") + .install_batch(opentelemetry_sdk::runtime::Tokio) + .unwrap(); + let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); + + let (jaeger_reload_filter, jaeger_reload_handle) = reload::Layer::new(filter_layer); + reload_handles.push(Box::new(jaeger_reload_handle)); + Some(telemetry.with_filter(jaeger_reload_filter)) + } else { + None + }; + + subscriber.with(flame_layer).with(jaeger_layer) + }; + tracing::subscriber::set_global_default(subscriber).unwrap(); #[cfg(all(feature = "tokio_console", feature = "release_max_log_level"))] @@ -639,51 +660,6 @@ fn init_tracing_sub(config: &Config) -> LogLevelReloadHandles { LogLevelReloadHandles::new(reload_handles) } -#[cfg(feature = "perf_measurements")] -fn init_tracing_jaeger(config: &Config) -> LogLevelReloadHandles { - opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); - let tracer = opentelemetry_jaeger::new_agent_pipeline() - .with_auto_split_batch(true) - .with_service_name("conduwuit") - .install_batch(opentelemetry_sdk::runtime::Tokio) - .unwrap(); - let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); - - let filter_layer = match EnvFilter::try_new(&config.log) { - Ok(s) => s, - Err(e) => { - eprintln!("It looks like your log config is invalid. The following error occurred: {e}"); - EnvFilter::try_new("warn").unwrap() - }, - }; - - let (reload_filter, reload_handle) = reload::Layer::new(filter_layer); - - let subscriber = Registry::default().with(reload_filter).with(telemetry); - - tracing::subscriber::set_global_default(subscriber).unwrap(); - - LogLevelReloadHandles::new(vec![Box::new(reload_handle)]) -} - -// TODO: tokio-console here? -#[cfg(feature = "perf_measurements")] -fn init_tracing_flame(_config: &Config) -> LogLevelReloadHandles { - let registry = Registry::default(); - let (flame_layer, _guard) = tracing_flame::FlameLayer::with_file("./tracing.folded").unwrap(); - let flame_layer = flame_layer.with_empty_samples(false); - - let filter_layer = EnvFilter::new("trace,h2=off"); - - let (reload_filter, reload_handle) = reload::Layer::new(filter_layer); - - let subscriber = registry.with(reload_filter).with(flame_layer); - - tracing::subscriber::set_global_default(subscriber).unwrap(); - - LogLevelReloadHandles::new(vec![Box::new(reload_handle)]) -} - // This is needed for opening lots of file descriptors, which tends to // happen more often when using RocksDB and making lots of federation // connections at startup. The soft limit is usually 1024, and the hard