From 277b4951e8e7f28e6319a17e91229e77c9db090d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 20 Jan 2025 11:50:17 +0000 Subject: [PATCH] add compression-shaping; tweak default compression levels Signed-off-by: Jason Volk --- conduwuit-example.toml | 8 +++++++- src/core/config/mod.rs | 8 +++++++- src/database/engine/cf_opts.rs | 32 +++++++++++++++++++++++++++---- src/database/engine/descriptor.rs | 19 ++++++++++++++++-- 4 files changed, 59 insertions(+), 8 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 54143ced..79efbd14 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -818,6 +818,9 @@ # magic number and translated to the library's default compression level # as they all differ. See their `kDefaultCompressionLevel`. # +# Note when using the default value we may override it with a setting +# tailored specifically conduwuit. +# #rocksdb_compression_level = 32767 # Level of compression the specified compression algorithm for the @@ -831,6 +834,9 @@ # less likely for this data to be used. Research your chosen compression # algorithm. # +# Note when using the default value we may override it with a setting +# tailored specifically conduwuit. +# #rocksdb_bottommost_compression_level = 32767 # Whether to enable RocksDB's "bottommost_compression". @@ -842,7 +848,7 @@ # # See https://github.com/facebook/rocksdb/wiki/Compression for more details. # -#rocksdb_bottommost_compression = false +#rocksdb_bottommost_compression = true # Database recovery mode (for RocksDB WAL corruption). # diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index cb42940b..5cfed0b9 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -958,6 +958,9 @@ pub struct Config { /// magic number and translated to the library's default compression level /// as they all differ. See their `kDefaultCompressionLevel`. /// + /// Note when using the default value we may override it with a setting + /// tailored specifically conduwuit. + /// /// default: 32767 #[serde(default = "default_rocksdb_compression_level")] pub rocksdb_compression_level: i32, @@ -973,6 +976,9 @@ pub struct Config { /// less likely for this data to be used. Research your chosen compression /// algorithm. /// + /// Note when using the default value we may override it with a setting + /// tailored specifically conduwuit. + /// /// default: 32767 #[serde(default = "default_rocksdb_bottommost_compression_level")] pub rocksdb_bottommost_compression_level: i32, @@ -985,7 +991,7 @@ pub struct Config { /// if you're trying to reduce storage usage from the database. /// /// See https://github.com/facebook/rocksdb/wiki/Compression for more details. - #[serde(default)] + #[serde(default = "true_fn")] pub rocksdb_bottommost_compression: bool, /// Database recovery mode (for RocksDB WAL corruption). diff --git a/src/database/engine/cf_opts.rs b/src/database/engine/cf_opts.rs index 7b3a1d49..da636718 100644 --- a/src/database/engine/cf_opts.rs +++ b/src/database/engine/cf_opts.rs @@ -8,6 +8,8 @@ use rocksdb::{ use super::descriptor::{CacheDisp, Descriptor}; use crate::{util::map_err, Context}; +pub(super) const SENTINEL_COMPRESSION_LEVEL: i32 = 32767; + /// Adjust options for the specific column by name. Provide the result of /// db_options() as the argument to this function and use the return value in /// the arguments to open the specific column. @@ -45,7 +47,15 @@ fn descriptor_cf_options( opts.set_compaction_pri(desc.compaction_pri); opts.set_universal_compaction_options(&uc_options(&desc)); + let compression_shape: Vec<_> = desc + .compression_shape + .into_iter() + .map(|val| (val > 0).then_some(desc.compression)) + .map(|val| val.unwrap_or(CompressionType::None)) + .collect(); + opts.set_compression_type(desc.compression); + opts.set_compression_per_level(compression_shape.as_slice()); opts.set_compression_options(-14, desc.compression_level, 0, 0); // -14 w_bits used by zlib. if let Some(&bottommost_level) = desc.bottommost_level.as_ref() { opts.set_bottommost_compression_type(desc.compression); @@ -95,10 +105,24 @@ fn set_compression(desc: &mut Descriptor, config: &Config) { | _ => CompressionType::Zstd, }; - desc.compression_level = config.rocksdb_compression_level; - desc.bottommost_level = config - .rocksdb_bottommost_compression - .then_some(config.rocksdb_bottommost_compression_level); + let can_override_level = config.rocksdb_compression_level == SENTINEL_COMPRESSION_LEVEL + && desc.compression == CompressionType::Zstd; + + if !can_override_level { + desc.compression_level = config.rocksdb_compression_level; + } + + let can_override_bottom = config.rocksdb_bottommost_compression_level + == SENTINEL_COMPRESSION_LEVEL + && desc.compression == CompressionType::Zstd; + + if !can_override_bottom { + desc.bottommost_level = Some(config.rocksdb_bottommost_compression_level); + } + + if !config.rocksdb_bottommost_compression { + desc.bottommost_level = None; + } } fn uc_options(desc: &Descriptor) -> UniversalCompactOptions { diff --git a/src/database/engine/descriptor.rs b/src/database/engine/descriptor.rs index 234ca2bf..2c84ac53 100644 --- a/src/database/engine/descriptor.rs +++ b/src/database/engine/descriptor.rs @@ -4,6 +4,8 @@ use rocksdb::{ DBCompressionType as CompressionType, }; +use super::cf_opts::SENTINEL_COMPRESSION_LEVEL; + #[derive(Debug, Clone, Copy)] pub(crate) enum CacheDisp { Unique, @@ -32,6 +34,7 @@ pub(crate) struct Descriptor { pub(crate) compaction: CompactionStyle, pub(crate) compaction_pri: CompactionPri, pub(crate) compression: CompressionType, + pub(crate) compression_shape: [i32; 7], pub(crate) compression_level: i32, pub(crate) bottommost_level: Option, pub(crate) block_index_hashing: Option, @@ -58,8 +61,9 @@ pub(crate) static BASE: Descriptor = Descriptor { compaction: CompactionStyle::Level, compaction_pri: CompactionPri::MinOverlappingRatio, compression: CompressionType::Zstd, - compression_level: 32767, - bottommost_level: Some(32767), + compression_shape: [0, 0, 0, 1, 1, 1, 1], + compression_level: SENTINEL_COMPRESSION_LEVEL, + bottommost_level: Some(SENTINEL_COMPRESSION_LEVEL), block_index_hashing: None, cache_shards: 64, }; @@ -68,6 +72,8 @@ pub(crate) static RANDOM: Descriptor = Descriptor { compaction_pri: CompactionPri::OldestSmallestSeqFirst, write_size: 1024 * 1024 * 32, cache_shards: 128, + compression_level: -3, + bottommost_level: Some(4), ..BASE }; @@ -77,6 +83,9 @@ pub(crate) static SEQUENTIAL: Descriptor = Descriptor { level_size: 1024 * 1024 * 32, file_size: 1024 * 1024 * 2, cache_shards: 128, + compression_level: -1, + bottommost_level: Some(6), + compression_shape: [0, 0, 1, 1, 1, 1, 1], ..BASE }; @@ -88,6 +97,9 @@ pub(crate) static RANDOM_SMALL: Descriptor = Descriptor { index_size: 512, block_size: 512, cache_shards: 64, + compression_level: -4, + bottommost_level: Some(1), + compression_shape: [0, 0, 0, 0, 0, 1, 1], ..RANDOM }; @@ -99,5 +111,8 @@ pub(crate) static SEQUENTIAL_SMALL: Descriptor = Descriptor { block_size: 512, cache_shards: 64, block_index_hashing: Some(false), + compression_level: -2, + bottommost_level: Some(4), + compression_shape: [0, 0, 0, 0, 1, 1, 1], ..SEQUENTIAL };