diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 8bc59108..4d186f67 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -289,8 +289,8 @@ url_preview_check_root_domain = false # Defaults to false #rocksdb_optimize_for_spinning_disks = false -# RocksDB log level. This is not the same as conduwuit's log level. This is the log level for RocksDB itself -# which show up in your database folder/path as `LOG` files. Defaults to warn. conduwuit will typically log RocksDB errors. +# RocksDB log level. This is not the same as conduwuit's log level. This is the log level for the RocksDB engine/library +# which show up in your database folder/path as `LOG` files. Defaults to error. conduwuit will typically log RocksDB errors as normal. #rocksdb_log_level = "error" # Max RocksDB `LOG` file size before rotating in bytes. Defaults to 4MB. @@ -311,9 +311,28 @@ url_preview_check_root_domain = false # Type of RocksDB database compression to use. # Available options are "zstd", "zlib", "bz2" and "lz4" +# It is best to use ZSTD as an overall good balance between speed/performance, storage, IO amplification, and CPU usage. +# For more performance but less compression (more storage used) and less CPU usage, use LZ4. +# See https://github.com/facebook/rocksdb/wiki/Compression for more details. +# # Defaults to "zstd" #rocksdb_compression_algo = "zstd" +# Level of compression the specified compression algorithm for RocksDB to use. +# Default is 32767, which is internally read by RocksDB as the default magic number and +# translated to the library's default compression level as they all differ. +# See their `kDefaultCompressionLevel`. +# +#rocksdb_compression_level = 32767 + +# Whether to enable RocksDB "bottommost_compression". +# At the expense of more CPU usage, this will further compress the database to reduce more storage. +# It is recommended to use ZSTD compression with this for best compression results. +# See https://github.com/facebook/rocksdb/wiki/Compression for more details. +# +# Defaults to false as this uses more CPU when compressing. +#rocksdb_bottommost_compression = false + ### Presence diff --git a/src/config/mod.rs b/src/config/mod.rs index 6a80eadb..02ff745c 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -121,6 +121,10 @@ pub struct Config { pub rocksdb_max_log_files: usize, #[serde(default = "default_rocksdb_compression_algo")] pub rocksdb_compression_algo: String, + #[serde(default = "default_rocksdb_compression_level")] + pub rocksdb_compression_level: i32, + #[serde(default)] + pub rocksdb_bottommost_compression: bool, pub emergency_password: Option, @@ -342,19 +346,33 @@ impl fmt::Display for Config { }), #[cfg(feature = "compression-zstd")] ("zstd Response Body Compression", &self.zstd_compression.to_string()), + #[cfg(feature = "rocksdb")] ("RocksDB database log level", &self.rocksdb_log_level), + #[cfg(feature = "rocksdb")] ("RocksDB database log time-to-roll", &self.rocksdb_log_time_to_roll.to_string()), + #[cfg(feature = "rocksdb")] ("RocksDB Max LOG Files", &self.rocksdb_max_log_files.to_string()), + #[cfg(feature = "rocksdb")] ( "RocksDB database max log file size", &self.rocksdb_max_log_file_size.to_string(), ), + #[cfg(feature = "rocksdb")] ( "RocksDB database optimize for spinning disks", &self.rocksdb_optimize_for_spinning_disks.to_string(), ), + #[cfg(feature = "rocksdb")] ("RocksDB Parallelism Threads", &self.rocksdb_parallelism_threads.to_string()), + #[cfg(feature = "rocksdb")] ("RocksDB Compression Algorithm", &self.rocksdb_compression_algo), + #[cfg(feature = "rocksdb")] + ("RocksDB Compression Level", &self.rocksdb_compression_level.to_string()), + #[cfg(feature = "rocksdb")] + ( + "RocksDB Bottommost Level Compression", + &self.rocksdb_bottommost_compression.to_string(), + ), ("Prevent Media Downloads From", { let mut lst = vec![]; for domain in &self.prevent_media_downloads_from { @@ -456,6 +474,11 @@ fn default_rocksdb_parallelism_threads() -> usize { num_cpus::get_physical() / 2 fn default_rocksdb_compression_algo() -> String { "zstd".to_owned() } +/// Default RocksDB compression level is 32767, which is internally read by +/// RocksDB as the default magic number and translated to the library's default +/// compression level as they all differ. See their `kDefaultCompressionLevel`. +fn default_rocksdb_compression_level() -> i32 { 32767 } + // I know, it's a great name pub(crate) fn default_default_room_version() -> RoomVersionId { RoomVersionId::V10 } diff --git a/src/database/abstraction/rocksdb.rs b/src/database/abstraction/rocksdb.rs index 6faec3c1..fbeb29f3 100644 --- a/src/database/abstraction/rocksdb.rs +++ b/src/database/abstraction/rocksdb.rs @@ -84,6 +84,17 @@ fn db_options(rocksdb_cache: &rust_rocksdb::Cache, config: &Config) -> rust_rock db_opts.set_use_direct_io_for_flush_and_compaction(true); } + if config.rocksdb_bottommost_compression { + db_opts.set_bottommost_compression_type(rocksdb_compression_algo); + db_opts.set_bottommost_zstd_max_train_bytes(0, true); + + // -14 w_bits is only read by zlib. + db_opts.set_bottommost_compression_options(-14, config.rocksdb_compression_level, 0, 0, true); + } + + // -14 w_bits is only read by zlib. + db_opts.set_compression_options(-14, config.rocksdb_compression_level, 0, 0); + db_opts.set_block_based_table_factory(&block_based_options); db_opts.create_if_missing(true); db_opts.increase_parallelism( diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index ce45ca5b..bb4e0f6c 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -371,6 +371,8 @@ impl Service<'_> { pub fn rocksdb_compression_algo(&self) -> &String { &self.config.rocksdb_compression_algo } + pub fn rocksdb_compression_level(&self) -> i32 { self.config.rocksdb_compression_level } + pub fn prevent_media_downloads_from(&self) -> &[OwnedServerName] { &self.config.prevent_media_downloads_from } pub fn ip_range_denylist(&self) -> &[String] { &self.config.ip_range_denylist }