diff --git a/deps/rust-rocksdb/lib.rs b/deps/rust-rocksdb/lib.rs index 2d53d4b1..2e699efb 100644 --- a/deps/rust-rocksdb/lib.rs +++ b/deps/rust-rocksdb/lib.rs @@ -5,6 +5,7 @@ pub use rust_rocksdb::*; extern "C" { pub fn rocksdb_list_column_families(); pub fn rocksdb_logger_create_stderr_logger(); + pub fn rocksdb_logger_create_callback_logger(); pub fn rocksdb_options_set_info_log(); pub fn rocksdb_get_options_from_string(); pub fn rocksdb_writebatch_create(); diff --git a/src/database/map.rs b/src/database/map.rs index d9601599..0f4d740a 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -118,6 +118,8 @@ fn open(db: &Arc, name: &str) -> Result> { #[inline] pub(crate) fn iter_options_default() -> ReadOptions { let mut read_options = read_options_default(); + read_options.set_background_purge_on_iterator_cleanup(true); + //read_options.set_pin_data(true); read_options } diff --git a/src/database/opts.rs b/src/database/opts.rs index 28a39cca..f1b4d3a9 100644 --- a/src/database/opts.rs +++ b/src/database/opts.rs @@ -3,7 +3,7 @@ use std::{cmp, collections::HashMap, convert::TryFrom}; use conduit::{err, utils, Config, Result}; use rocksdb::{ statistics::StatsLevel, BlockBasedOptions, Cache, DBCompactionStyle, DBCompressionType, DBRecoveryMode, Env, - LogLevel, Options, UniversalCompactOptions, UniversalCompactionStopStyle, + LogLevel, LruCacheOptions, Options, UniversalCompactOptions, UniversalCompactionStopStyle, }; /// Create database-wide options suitable for opening the database. This also @@ -24,9 +24,9 @@ pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_ set_logging_defaults(&mut opts, config); // Processing - opts.set_enable_pipelined_write(true); opts.set_max_background_jobs(num_threads::(config)?); opts.set_max_subcompactions(num_threads::(config)?); + opts.set_avoid_unnecessary_blocking_io(true); opts.set_max_file_opening_threads(0); if config.rocksdb_compaction_prio_idle { env.lower_thread_pool_cpu_priority(); @@ -34,6 +34,7 @@ pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_ // IO opts.set_manual_wal_flush(true); + opts.set_enable_pipelined_write(true); if config.rocksdb_direct_io { opts.set_use_direct_reads(true); opts.set_use_direct_io_for_flush_and_compaction(true); @@ -60,6 +61,7 @@ pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_ opts.set_min_write_buffer_number(1); // Files + opts.set_table_cache_num_shard_bits(7); opts.set_max_total_wal_size(96 * 1024 * 1024); set_level_defaults(&mut opts, config); @@ -328,10 +330,16 @@ fn uc_options(_config: &Config) -> UniversalCompactOptions { } fn set_table_with_new_cache( - opts: &mut Options, config: &Config, cache: &mut HashMap, name: &str, size: usize, + opts: &mut Options, config: &Config, caches: &mut HashMap, name: &str, size: usize, ) { - cache.insert(name.to_owned(), Cache::new_lru_cache(size)); - set_table_with_shared_cache(opts, config, cache, name, name); + let mut cache_opts = LruCacheOptions::default(); + cache_opts.set_capacity(size); + cache_opts.set_num_shard_bits(7); + + let cache = Cache::new_lru_cache_opts(&cache_opts); + caches.insert(name.into(), cache); + + set_table_with_shared_cache(opts, config, caches, name, name); } fn set_table_with_shared_cache( @@ -343,6 +351,7 @@ fn set_table_with_shared_cache( .get(cache_name) .expect("existing cache to share with this column"), ); + opts.set_block_based_table_factory(&table); } @@ -361,6 +370,7 @@ fn table_options(_config: &Config) -> BlockBasedOptions { opts.set_block_size(4 * 1024); opts.set_metadata_block_size(4 * 1024); + opts.set_use_delta_encoding(false); opts.set_optimize_filters_for_memory(true); opts.set_cache_index_and_filter_blocks(true); opts.set_pin_top_level_index_and_filter(true);