magmaus3/continuwuity
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

misc database options tweaks

Browse source 
Signed-off-by: Jason Volk <jason@zemos.net>
This commit is contained in:
Jason Volk 2024-12-08 03:03:34 +00:00
parent 139a1ac504
commit 6aa35260e6
3 changed files with 18 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

1
deps/rust-rocksdb/lib.rs vendored
View file

@ -5,6 +5,7 @@ pub use rust_rocksdb::*;
extern "C" {
pub fn rocksdb_list_column_families();
pub fn rocksdb_logger_create_stderr_logger();
pub fn rocksdb_logger_create_callback_logger();
pub fn rocksdb_options_set_info_log();
pub fn rocksdb_get_options_from_string();
pub fn rocksdb_writebatch_create();

2
src/database/map.rs
View file

@ -118,6 +118,8 @@ fn open(db: &Arc<Engine>, name: &str) -> Result<Arc<ColumnFamily>> {
#[inline]
pub(crate) fn iter_options_default() -> ReadOptions {
let mut read_options = read_options_default();
read_options.set_background_purge_on_iterator_cleanup(true);
//read_options.set_pin_data(true);
read_options
}

20
src/database/opts.rs
View file

@ -3,7 +3,7 @@ use std::{cmp, collections::HashMap, convert::TryFrom};
use conduit::{err, utils, Config, Result};
use rocksdb::{
statistics::StatsLevel, BlockBasedOptions, Cache, DBCompactionStyle, DBCompressionType, DBRecoveryMode, Env,
LogLevel, Options, UniversalCompactOptions, UniversalCompactionStopStyle,
LogLevel, LruCacheOptions, Options, UniversalCompactOptions, UniversalCompactionStopStyle,
};
/// Create database-wide options suitable for opening the database. This also
@ -24,9 +24,9 @@ pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_
set_logging_defaults(&mut opts, config);
// Processing
opts.set_enable_pipelined_write(true);
opts.set_max_background_jobs(num_threads::<i32>(config)?);
opts.set_max_subcompactions(num_threads::<u32>(config)?);
opts.set_avoid_unnecessary_blocking_io(true);
opts.set_max_file_opening_threads(0);
if config.rocksdb_compaction_prio_idle {
env.lower_thread_pool_cpu_priority();
@ -34,6 +34,7 @@ pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_
// IO
opts.set_manual_wal_flush(true);
opts.set_enable_pipelined_write(true);
if config.rocksdb_direct_io {
opts.set_use_direct_reads(true);
opts.set_use_direct_io_for_flush_and_compaction(true);
@ -60,6 +61,7 @@ pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_
opts.set_min_write_buffer_number(1);
// Files
opts.set_table_cache_num_shard_bits(7);
opts.set_max_total_wal_size(96 * 1024 * 1024);
set_level_defaults(&mut opts, config);
@ -328,10 +330,16 @@ fn uc_options(_config: &Config) -> UniversalCompactOptions {
}
fn set_table_with_new_cache(
opts: &mut Options, config: &Config, cache: &mut HashMap<String, Cache>, name: &str, size: usize,
opts: &mut Options, config: &Config, caches: &mut HashMap<String, Cache>, name: &str, size: usize,
) {
cache.insert(name.to_owned(), Cache::new_lru_cache(size));
set_table_with_shared_cache(opts, config, cache, name, name);
let mut cache_opts = LruCacheOptions::default();
cache_opts.set_capacity(size);
cache_opts.set_num_shard_bits(7);
let cache = Cache::new_lru_cache_opts(&cache_opts);
caches.insert(name.into(), cache);
set_table_with_shared_cache(opts, config, caches, name, name);
}
fn set_table_with_shared_cache(
@ -343,6 +351,7 @@ fn set_table_with_shared_cache(
.get(cache_name)
.expect("existing cache to share with this column"),
);
opts.set_block_based_table_factory(&table);
}
@ -361,6 +370,7 @@ fn table_options(_config: &Config) -> BlockBasedOptions {
opts.set_block_size(4 * 1024);
opts.set_metadata_block_size(4 * 1024);
opts.set_use_delta_encoding(false);
opts.set_optimize_filters_for_memory(true);
opts.set_cache_index_and_filter_blocks(true);
opts.set_pin_top_level_index_and_filter(true);