From 98f95705478dbe60a56206dd2ae9057f602040ea Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 26 Jan 2025 07:05:00 +0000 Subject: [PATCH] add option to disable rocksdb checksums reference runtime state for default option initialization Signed-off-by: Jason Volk --- conduwuit-example.toml | 7 ++++ src/core/config/mod.rs | 9 ++++++ src/database/engine.rs | 1 + src/database/engine/open.rs | 1 + src/database/map.rs | 6 ++-- src/database/map/keys.rs | 2 +- src/database/map/keys_from.rs | 2 +- src/database/map/options.rs | 50 +++++++++++++++++------------ src/database/map/rev_keys.rs | 2 +- src/database/map/rev_keys_from.rs | 2 +- src/database/map/rev_stream.rs | 4 +-- src/database/map/rev_stream_from.rs | 4 +-- src/database/map/stream.rs | 4 +-- src/database/map/stream_from.rs | 4 +-- 14 files changed, 62 insertions(+), 36 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 3ecc1628..51d948e8 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -897,6 +897,13 @@ # #rocksdb_paranoid_file_checks = false +# Enables or disables checksum verification in rocksdb at runtime. +# Checksums are usually hardware accelerated with low overhead; they are +# enabled in rocksdb by default. Older or slower platforms may see gains +# from disabling. +# +#rocksdb_checksums = true + # Database repair mode (for RocksDB SST corruption). # # Use this option when the server reports corruption while running or diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 133f0887..94788fa4 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1049,6 +1049,15 @@ pub struct Config { #[serde(default)] pub rocksdb_paranoid_file_checks: bool, + /// Enables or disables checksum verification in rocksdb at runtime. + /// Checksums are usually hardware accelerated with low overhead; they are + /// enabled in rocksdb by default. Older or slower platforms may see gains + /// from disabling. + /// + /// default: true + #[serde(default = "true_fn")] + pub rocksdb_checksums: bool, + /// Database repair mode (for RocksDB SST corruption). /// /// Use this option when the server reports corruption while running or diff --git a/src/database/engine.rs b/src/database/engine.rs index 76b2889b..be3d62cf 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -32,6 +32,7 @@ use crate::{ pub struct Engine { pub(super) read_only: bool, pub(super) secondary: bool, + pub(crate) checksums: bool, corks: AtomicU32, pub(crate) db: Db, pub(crate) pool: Arc, diff --git a/src/database/engine/open.rs b/src/database/engine/open.rs index 6a801878..ad724765 100644 --- a/src/database/engine/open.rs +++ b/src/database/engine/open.rs @@ -58,6 +58,7 @@ pub(crate) async fn open(ctx: Arc, desc: &[Descriptor]) -> Result) -> impl Stream>> + Send { use crate::pool::Seek; - let opts = super::iter_options_default(); + let opts = super::iter_options_default(&self.db); let state = stream::State::new(self, opts); if is_cached(self) { let state = state.init_fwd(None); diff --git a/src/database/map/keys_from.rs b/src/database/map/keys_from.rs index b83775ac..76c76325 100644 --- a/src/database/map/keys_from.rs +++ b/src/database/map/keys_from.rs @@ -53,7 +53,7 @@ where { use crate::pool::Seek; - let opts = super::iter_options_default(); + let opts = super::iter_options_default(&self.db); let state = stream::State::new(self, opts); if is_cached(self, from) { return stream::Keys::<'_>::from(state.init_fwd(from.as_ref().into())).boxed(); diff --git a/src/database/map/options.rs b/src/database/map/options.rs index f726036d..9e2ad898 100644 --- a/src/database/map/options.rs +++ b/src/database/map/options.rs @@ -1,35 +1,43 @@ +use std::sync::Arc; + use rocksdb::{ReadOptions, ReadTier, WriteOptions}; -#[inline] -pub(crate) fn iter_options_default() -> ReadOptions { - let mut options = read_options_default(); - options.set_background_purge_on_iterator_cleanup(true); - //options.set_pin_data(true); - options -} +use crate::Engine; #[inline] -pub(crate) fn cache_iter_options_default() -> ReadOptions { - let mut options = cache_read_options_default(); - options.set_background_purge_on_iterator_cleanup(true); - //options.set_pin_data(true); - options -} - -#[inline] -pub(crate) fn cache_read_options_default() -> ReadOptions { - let mut options = read_options_default(); +pub(crate) fn cache_iter_options_default(db: &Arc) -> ReadOptions { + let mut options = iter_options_default(db); options.set_read_tier(ReadTier::BlockCache); options.fill_cache(false); options } #[inline] -pub(crate) fn read_options_default() -> ReadOptions { - let mut options = ReadOptions::default(); - options.set_total_order_seek(true); +pub(crate) fn iter_options_default(db: &Arc) -> ReadOptions { + let mut options = read_options_default(db); + options.set_background_purge_on_iterator_cleanup(true); options } #[inline] -pub(crate) fn write_options_default() -> WriteOptions { WriteOptions::default() } +pub(crate) fn cache_read_options_default(db: &Arc) -> ReadOptions { + let mut options = read_options_default(db); + options.set_read_tier(ReadTier::BlockCache); + options.fill_cache(false); + options +} + +#[inline] +pub(crate) fn read_options_default(db: &Arc) -> ReadOptions { + let mut options = ReadOptions::default(); + options.set_total_order_seek(true); + + if !db.checksums { + options.set_verify_checksums(false); + } + + options +} + +#[inline] +pub(crate) fn write_options_default(_db: &Arc) -> WriteOptions { WriteOptions::default() } diff --git a/src/database/map/rev_keys.rs b/src/database/map/rev_keys.rs index a559d04b..21558a17 100644 --- a/src/database/map/rev_keys.rs +++ b/src/database/map/rev_keys.rs @@ -22,7 +22,7 @@ where pub fn rev_raw_keys(self: &Arc) -> impl Stream>> + Send { use crate::pool::Seek; - let opts = super::iter_options_default(); + let opts = super::iter_options_default(&self.db); let state = stream::State::new(self, opts); if is_cached(self) { let state = state.init_rev(None); diff --git a/src/database/map/rev_keys_from.rs b/src/database/map/rev_keys_from.rs index 5b159195..65072337 100644 --- a/src/database/map/rev_keys_from.rs +++ b/src/database/map/rev_keys_from.rs @@ -61,7 +61,7 @@ where { use crate::pool::Seek; - let opts = super::iter_options_default(); + let opts = super::iter_options_default(&self.db); let state = stream::State::new(self, opts); if is_cached(self, from) { return stream::KeysRev::<'_>::from(state.init_rev(from.as_ref().into())).boxed(); diff --git a/src/database/map/rev_stream.rs b/src/database/map/rev_stream.rs index 56b20b9b..f55053be 100644 --- a/src/database/map/rev_stream.rs +++ b/src/database/map/rev_stream.rs @@ -31,7 +31,7 @@ where pub fn rev_raw_stream(self: &Arc) -> impl Stream>> + Send { use crate::pool::Seek; - let opts = super::iter_options_default(); + let opts = super::iter_options_default(&self.db); let state = stream::State::new(self, opts); if is_cached(self) { let state = state.init_rev(None); @@ -66,7 +66,7 @@ pub fn rev_raw_stream(self: &Arc) -> impl Stream> fields(%map), )] pub(super) fn is_cached(map: &Arc) -> bool { - let opts = super::cache_iter_options_default(); + let opts = super::cache_iter_options_default(&map.db); let state = stream::State::new(map, opts).init_rev(None); !state.is_incomplete() diff --git a/src/database/map/rev_stream_from.rs b/src/database/map/rev_stream_from.rs index 83832bdd..ddc98607 100644 --- a/src/database/map/rev_stream_from.rs +++ b/src/database/map/rev_stream_from.rs @@ -80,7 +80,7 @@ where { use crate::pool::Seek; - let opts = super::iter_options_default(); + let opts = super::iter_options_default(&self.db); let state = stream::State::new(self, opts); if is_cached(self, from) { let state = state.init_rev(from.as_ref().into()); @@ -118,7 +118,7 @@ pub(super) fn is_cached

(map: &Arc, from: &P) -> bool where P: AsRef<[u8]> + ?Sized, { - let cache_opts = super::cache_iter_options_default(); + let cache_opts = super::cache_iter_options_default(&map.db); let cache_status = stream::State::new(map, cache_opts) .init_rev(from.as_ref().into()) .status(); diff --git a/src/database/map/stream.rs b/src/database/map/stream.rs index f1b5fdc3..bfc8ba04 100644 --- a/src/database/map/stream.rs +++ b/src/database/map/stream.rs @@ -30,7 +30,7 @@ where pub fn raw_stream(self: &Arc) -> impl Stream>> + Send { use crate::pool::Seek; - let opts = super::iter_options_default(); + let opts = super::iter_options_default(&self.db); let state = stream::State::new(self, opts); if is_cached(self) { let state = state.init_fwd(None); @@ -65,7 +65,7 @@ pub fn raw_stream(self: &Arc) -> impl Stream>> + fields(%map), )] pub(super) fn is_cached(map: &Arc) -> bool { - let opts = super::cache_iter_options_default(); + let opts = super::cache_iter_options_default(&map.db); let state = stream::State::new(map, opts).init_fwd(None); !state.is_incomplete() diff --git a/src/database/map/stream_from.rs b/src/database/map/stream_from.rs index 562ab6b1..74140a65 100644 --- a/src/database/map/stream_from.rs +++ b/src/database/map/stream_from.rs @@ -77,7 +77,7 @@ where { use crate::pool::Seek; - let opts = super::iter_options_default(); + let opts = super::iter_options_default(&self.db); let state = stream::State::new(self, opts); if is_cached(self, from) { let state = state.init_fwd(from.as_ref().into()); @@ -115,7 +115,7 @@ pub(super) fn is_cached

(map: &Arc, from: &P) -> bool where P: AsRef<[u8]> + ?Sized, { - let opts = super::cache_iter_options_default(); + let opts = super::cache_iter_options_default(&map.db); let state = stream::State::new(map, opts).init_fwd(from.as_ref().into()); !state.is_incomplete()