magmaus3/continuwuity
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add conf item for write buffer size

Browse source 
Signed-off-by: Jason Volk <jason@zemos.net>
This commit is contained in:
Jason Volk 2025-01-09 19:55:25 +00:00
parent 6a0f9add0c
commit 16fa2eca87
4 changed files with 92 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

26
src/database/engine/cf_opts.rs
View file

@ -10,7 +10,7 @@ use rocksdb::{
};
use super::descriptor::{CacheDisp, Descriptor};
use crate::Context;
use crate::{util::map_err, Context};
/// Adjust options for the specific column by name. Provide the result of
/// db_options() as the argument to this function and use the return value in
@ -28,7 +28,7 @@ fn descriptor_cf_options(
cache: Option<&Cache>,
) -> Result<Options> {
set_compression(&mut desc, config);
set_table_options(&mut opts, &desc, cache);
set_table_options(&mut opts, &desc, cache)?;
opts.set_min_write_buffer_number(1);
opts.set_max_write_buffer_number(2);
@ -65,10 +65,13 @@ fn descriptor_cf_options(
);
}
opts.set_options_from_string("{{arena_block_size=2097152;}}")
.map_err(map_err)?;
Ok(opts)
}
fn set_table_options(opts: &mut Options, desc: &Descriptor, cache: Option<&Cache>) {
fn set_table_options(opts: &mut Options, desc: &Descriptor, cache: Option<&Cache>) -> Result {
let mut table = table_options(desc);
if let Some(cache) = cache {
table.set_block_cache(cache);
@ -76,7 +79,15 @@ fn set_table_options(opts: &mut Options, desc: &Descriptor, cache: Option<&Cache
table.disable_cache();
}
opts.set_options_from_string(
"{{block_based_table_factory={num_file_reads_for_auto_readahead=0;\
max_auto_readahead_size=524288;initial_auto_readahead_size=16384}}}",
)
.map_err(map_err)?;
opts.set_block_based_table_factory(&table);
Ok(())
}
fn set_compression(desc: &mut Descriptor, config: &Config) {
@ -121,6 +132,7 @@ fn table_options(desc: &Descriptor) -> BlockBasedOptions {
opts.set_unpartitioned_pinning_tier(BlockBasedPinningTier::None);
opts.set_top_level_index_pinning_tier(BlockBasedPinningTier::None);
opts.set_partition_filters(true);
opts.set_use_delta_encoding(false);
opts.set_index_type(BlockBasedIndexType::TwoLevelIndexSearch);
opts.set_data_block_index_type(
@ -203,9 +215,13 @@ fn get_cache(ctx: &Context, desc: &Descriptor) -> Option<Cache> {
}
}
#[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)]
pub(crate) fn cache_size(config: &Config, base_size: u32, entity_size: usize) -> usize {
let ents = f64::from(base_size) * config.cache_capacity_modifier;
cache_size_f64(config, f64::from(base_size), entity_size)
}
#[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)]
pub(crate) fn cache_size_f64(config: &Config, base_size: f64, entity_size: usize) -> usize {
let ents = base_size * config.cache_capacity_modifier;
(ents as usize)
.checked_mul(entity_size)

11
src/database/engine/db_opts.rs
View file

@ -3,7 +3,7 @@ use std::{cmp, convert::TryFrom};
use conduwuit::{utils, Config, Result};
use rocksdb::{statistics::StatsLevel, Cache, DBRecoveryMode, Env, LogLevel, Options};
use super::{cf_opts::cache_size, logger::handle as handle_log};
use super::{cf_opts::cache_size_f64, logger::handle as handle_log};
/// Create database-wide options suitable for opening the database. This also
/// sets our default column options in case of opening a column with the same
@ -41,16 +41,23 @@ pub(crate) fn db_options(config: &Config, env: &Env, row_cache: &Cache) -> Resul
opts.set_skip_checking_sst_file_sizes_on_db_open(true);
opts.set_skip_stats_update_on_db_open(true);
//opts.set_max_file_opening_threads(threads.try_into().unwrap());
} else {
opts.set_compaction_readahead_size(1024 * 512);
}
// Blocks
opts.set_row_cache(row_cache);
opts.set_db_write_buffer_size(cache_size_f64(
config,
config.db_write_buffer_capacity_mb,
1_048_576,
));
// Files
opts.set_table_cache_num_shard_bits(7);
opts.set_wal_size_limit_mb(1024 * 1024 * 1024);
opts.set_max_total_wal_size(1024 * 1024 * 512);
opts.set_db_write_buffer_size(cache_size(config, 1024 * 1024 * 32, 1));
opts.set_writable_file_max_buffer_size(1024 * 1024 * 2);
// Misc
opts.set_disable_auto_compactions(!config.rocksdb_compaction);