From 39aef8d1b9f7c1d35725503bd38754c231f795fd Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 9 Mar 2024 15:53:22 -0500 Subject: [PATCH] dont build sha2, opentelemetry, or zstd code if unused reduces unnecessary crates being compiled. splits them into features. i have yet to see anyone use conduit's opentelemetry stuff, and realistically those people who do performance benchmarking and measurements will be building stuff anyways so they can just enable this feature. Signed-off-by: strawberry --- src/config/mod.rs | 1 + src/database/mod.rs | 40 ++++++------ src/main.rs | 71 +++++++++++++-------- src/service/globals/mod.rs | 8 +-- src/service/media/mod.rs | 123 ++++++++++++++++++++++++------------- 5 files changed, 149 insertions(+), 94 deletions(-) diff --git a/src/config/mod.rs b/src/config/mod.rs index 5b4cc2c8..c3e309c2 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -338,6 +338,7 @@ impl fmt::Display for Config { } &lst.join(", ") }), + #[cfg(feature = "compression-zstd")] ("zstd Response Body Compression", &self.zstd_compression.to_string()), ("RocksDB database log level", &self.rocksdb_log_level), ("RocksDB database log time-to-roll", &self.rocksdb_log_time_to_roll.to_string()), diff --git a/src/database/mod.rs b/src/database/mod.rs index 3f52f845..0e207dfd 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -894,26 +894,28 @@ impl KeyValueDatabase { warn!("Migration: 12 -> 13 finished"); } - if services().globals.database_version()? < 14 && cfg!(feature = "sha256_media") { - warn!("sha256_media feature flag is enabled, migrating legacy base64 file names to sha256 file names"); - // Move old media files to new names - for (key, _) in db.mediaid_file.iter() { - // we know that this method is deprecated, but we need to use it to migrate the - // old files to the new location - // - // TODO: remove this once we're sure that all users have migrated - #[allow(deprecated)] - let old_path = services().globals.get_media_file(&key); - let path = services().globals.get_media_file_new(&key); - // move the file to the new location - if old_path.exists() { - tokio::fs::rename(&old_path, &path).await?; + #[cfg(feature = "sha256_media")] + { + if services().globals.database_version()? < 14 && cfg!(feature = "sha256_media") { + warn!( + "sha256_media feature flag is enabled, migrating legacy base64 file names to sha256 file names" + ); + // Move old media files to new names + for (key, _) in db.mediaid_file.iter() { + let old_path = services().globals.get_media_file(&key); + debug!("Old file path: {old_path:?}"); + let path = services().globals.get_media_file_new(&key); + debug!("New file path: {path:?}"); + // move the file to the new location + if old_path.exists() { + tokio::fs::rename(&old_path, &path).await?; + } } + + services().globals.bump_database_version(14)?; + + warn!("Migration: 13 -> 14 finished"); } - - services().globals.bump_database_version(14)?; - - warn!("Migration: 13 -> 14 finished"); } assert_eq!( @@ -981,7 +983,7 @@ impl KeyValueDatabase { ); } - // Inserting registraions into cache + // Inserting registrations into cache for appservice in services().appservice.all()? { services().appservice.registration_info.write().await.insert( appservice.0, diff --git a/src/main.rs b/src/main.rs index c0825b89..65bc37f3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -85,33 +85,39 @@ async fn main() { }; if config.allow_jaeger { - opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); - let tracer = opentelemetry_jaeger::new_agent_pipeline() - .with_auto_split_batch(true) - .with_service_name("conduit") - .install_batch(opentelemetry_sdk::runtime::Tokio) - .unwrap(); - let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); + #[cfg(feature = "perf_measurements")] + { + opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); + let tracer = opentelemetry_jaeger::new_agent_pipeline() + .with_auto_split_batch(true) + .with_service_name("conduit") + .install_batch(opentelemetry_sdk::runtime::Tokio) + .unwrap(); + let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); - let filter_layer = match EnvFilter::try_new(&config.log) { - Ok(s) => s, - Err(e) => { - eprintln!("It looks like your log config is invalid. The following error occurred: {e}"); - EnvFilter::try_new("warn").unwrap() - }, - }; + let filter_layer = match EnvFilter::try_new(&config.log) { + Ok(s) => s, + Err(e) => { + eprintln!("It looks like your log config is invalid. The following error occurred: {e}"); + EnvFilter::try_new("warn").unwrap() + }, + }; - let subscriber = tracing_subscriber::Registry::default().with(filter_layer).with(telemetry); - tracing::subscriber::set_global_default(subscriber).unwrap(); + let subscriber = tracing_subscriber::Registry::default().with(filter_layer).with(telemetry); + tracing::subscriber::set_global_default(subscriber).unwrap(); + } } else if config.tracing_flame { - let registry = tracing_subscriber::Registry::default(); - let (flame_layer, _guard) = tracing_flame::FlameLayer::with_file("./tracing.folded").unwrap(); - let flame_layer = flame_layer.with_empty_samples(false); + #[cfg(feature = "perf_measurements")] + { + let registry = tracing_subscriber::Registry::default(); + let (flame_layer, _guard) = tracing_flame::FlameLayer::with_file("./tracing.folded").unwrap(); + let flame_layer = flame_layer.with_empty_samples(false); - let filter_layer = EnvFilter::new("trace,h2=off"); + let filter_layer = EnvFilter::new("trace,h2=off"); - let subscriber = registry.with(filter_layer).with(flame_layer); - tracing::subscriber::set_global_default(subscriber).unwrap(); + let subscriber = registry.with(filter_layer).with(flame_layer); + tracing::subscriber::set_global_default(subscriber).unwrap(); + } } else { let registry = tracing_subscriber::Registry::default(); let fmt_layer = tracing_subscriber::fmt::Layer::new(); @@ -296,6 +302,7 @@ async fn main() { // if server runs into critical error and shuts down, shut down the tracer // provider if jaegar is used. awaiting run_server() is a blocking call so // putting this after is fine, but not the other options above. + #[cfg(feature = "perf_measurements")] if config.allow_jaeger { opentelemetry::global::shutdown_tracer_provider(); } @@ -359,11 +366,21 @@ async fn run_server() -> io::Result<()> { config.max_request_size.try_into().expect("failed to convert max request size"), )); - let app = if cfg!(feature = "zstd_compression") && config.zstd_compression { - debug!("zstd body compression is enabled"); - routes().layer(middlewares.compression()).into_make_service() - } else { - routes().layer(middlewares).into_make_service() + let app; + + #[cfg(feature = "zstd_compression")] + { + app = if cfg!(feature = "zstd_compression") && config.zstd_compression { + debug!("zstd body compression is enabled"); + routes().layer(middlewares.compression()).into_make_service() + } else { + routes().layer(middlewares).into_make_service() + } + }; + + #[cfg(not(feature = "zstd_compression"))] + { + app = routes().layer(middlewares).into_make_service() }; let handle = ServerHandle::new(); diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index a40514f4..93e9bca4 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -32,7 +32,6 @@ use ruma::{ DeviceId, OwnedDeviceId, OwnedEventId, OwnedRoomId, OwnedServerName, OwnedServerSigningKeyId, OwnedUserId, RoomVersionId, ServerName, UserId, }; -use sha2::Digest; use tokio::sync::{broadcast, watch::Receiver, Mutex, RwLock, Semaphore}; use tracing::{error, info}; use trust_dns_resolver::TokioAsyncResolver; @@ -430,6 +429,7 @@ impl Service<'_> { /// new SHA256 file name media function, requires "sha256_media" feature /// flag enabled and database migrated uses SHA256 hash of the base64 key as /// the file name + #[cfg(feature = "sha256_media")] pub fn get_media_file_new(&self, key: &[u8]) -> PathBuf { let mut r = PathBuf::new(); r.push(self.config.database_path.clone()); @@ -437,17 +437,13 @@ impl Service<'_> { // Using the hash of the base64 key as the filename // This is to prevent the total length of the path from exceeding the maximum // length in most filesystems - r.push(general_purpose::URL_SAFE_NO_PAD.encode(sha2::Sha256::digest(key))); + r.push(general_purpose::URL_SAFE_NO_PAD.encode(::digest(key))); r } /// old base64 file name media function /// This is the old version of `get_media_file` that uses the full base64 /// key as the filename. - /// - /// This is deprecated and will be removed in a future release. - /// Please use `get_media_file_new` instead. - #[deprecated(note = "Use get_media_file_new instead")] pub fn get_media_file(&self, key: &[u8]) -> PathBuf { let mut r = PathBuf::new(); r.push(self.config.database_path.clone()); diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 3677fe52..688c1abf 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -50,11 +50,16 @@ impl Service { // Width, Height = 0 if it's not a thumbnail let key = self.db.create_file_metadata(mxc, 0, 0, content_disposition, content_type)?; - let path = if cfg!(feature = "sha256_media") { - services().globals.get_media_file_new(&key) - } else { - #[allow(deprecated)] - services().globals.get_media_file(&key) + let path; + + #[cfg(feature = "sha256_media")] + { + path = services().globals.get_media_file_new(&key) + }; + + #[cfg(not(feature = "sha256_media"))] + { + path = services().globals.get_media_file(&key) }; let mut f = File::create(path).await?; @@ -66,12 +71,18 @@ impl Service { pub async fn delete(&self, mxc: String) -> Result<()> { if let Ok(keys) = self.db.search_mxc_metadata_prefix(mxc.clone()) { for key in keys { - let file_path = if cfg!(feature = "sha256_media") { - services().globals.get_media_file_new(&key) - } else { - #[allow(deprecated)] - services().globals.get_media_file(&key) + let file_path; + + #[cfg(feature = "sha256_media")] + { + file_path = services().globals.get_media_file_new(&key) }; + + #[cfg(not(feature = "sha256_media"))] + { + file_path = services().globals.get_media_file(&key) + }; + debug!("Got local file path: {:?}", file_path); debug!("Deleting local file {:?} from filesystem, original MXC: {}", file_path, mxc); @@ -96,13 +107,17 @@ impl Service { file: &[u8], ) -> Result<()> { let key = self.db.create_file_metadata(mxc, width, height, content_disposition, content_type)?; + let path; - let path = if cfg!(feature = "sha256_media") { - services().globals.get_media_file_new(&key) - } else { - #[allow(deprecated)] - services().globals.get_media_file(&key) - }; + #[cfg(feature = "sha256_media")] + { + path = services().globals.get_media_file_new(&key); + } + + #[cfg(not(feature = "sha256_media"))] + { + path = services().globals.get_media_file(&key); + } let mut f = File::create(path).await?; f.write_all(file).await?; @@ -113,11 +128,16 @@ impl Service { /// Downloads a file. pub async fn get(&self, mxc: String) -> Result> { if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc, 0, 0) { - let path = if cfg!(feature = "sha256_media") { - services().globals.get_media_file_new(&key) - } else { - #[allow(deprecated)] - services().globals.get_media_file(&key) + let path; + + #[cfg(feature = "sha256_media")] + { + path = services().globals.get_media_file_new(&key) + }; + + #[cfg(not(feature = "sha256_media"))] + { + path = services().globals.get_media_file(&key) }; let mut file = Vec::new(); @@ -186,11 +206,16 @@ impl Service { continue; } - let path = if cfg!(feature = "sha256_media") { - services().globals.get_media_file_new(&key) - } else { - #[allow(deprecated)] - services().globals.get_media_file(&key) + let path; + + #[cfg(feature = "sha256_media")] + { + path = services().globals.get_media_file_new(&key) + }; + + #[cfg(not(feature = "sha256_media"))] + { + path = services().globals.get_media_file(&key) }; debug!("MXC path: {:?}", path); @@ -265,11 +290,16 @@ impl Service { if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc.clone(), width, height) { // Using saved thumbnail - let path = if cfg!(feature = "sha256_media") { - services().globals.get_media_file_new(&key) - } else { - #[allow(deprecated)] - services().globals.get_media_file(&key) + let path; + + #[cfg(feature = "sha256_media")] + { + path = services().globals.get_media_file_new(&key) + }; + + #[cfg(not(feature = "sha256_media"))] + { + path = services().globals.get_media_file(&key) }; let mut file = Vec::new(); @@ -282,11 +312,16 @@ impl Service { })) } else if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc.clone(), 0, 0) { // Generate a thumbnail - let path = if cfg!(feature = "sha256_media") { - services().globals.get_media_file_new(&key) - } else { - #[allow(deprecated)] - services().globals.get_media_file(&key) + let path; + + #[cfg(feature = "sha256_media")] + { + path = services().globals.get_media_file_new(&key) + }; + + #[cfg(not(feature = "sha256_media"))] + { + path = services().globals.get_media_file(&key) }; let mut file = Vec::new(); @@ -351,11 +386,16 @@ impl Service { content_type.as_deref(), )?; - let path = if cfg!(feature = "sha256_media") { - services().globals.get_media_file_new(&thumbnail_key) - } else { - #[allow(deprecated)] - services().globals.get_media_file(&thumbnail_key) + let path; + + #[cfg(feature = "sha256_media")] + { + path = services().globals.get_media_file_new(&thumbnail_key) + }; + + #[cfg(not(feature = "sha256_media"))] + { + path = services().globals.get_media_file(&thumbnail_key) }; let mut f = File::create(path).await?; @@ -397,7 +437,6 @@ mod tests { use std::path::PathBuf; use base64::{engine::general_purpose, Engine as _}; - use sha2::Digest; use super::*;