From 3dae02b886a3428c58d0a11e1cc19271722b4b47 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 18 Jan 2025 01:30:41 +0000 Subject: [PATCH] add preferred jemalloc config add muzzy/dirty configuration mallctl interface add program argument for --gc-muzzy=false Signed-off-by: Jason Volk --- Cargo.lock | 57 +++++----- Cargo.toml | 16 ++- deps/rust-rocksdb/Cargo.toml | 2 +- src/admin/debug/commands.rs | 2 +- src/core/Cargo.toml | 1 + src/core/alloc/default.rs | 2 +- src/core/alloc/hardened.rs | 2 +- src/core/alloc/je.rs | 194 +++++++++++++++++++++++++++++------ src/database/pool.rs | 16 ++- src/main/Cargo.toml | 4 + src/main/clap.rs | 16 +++ src/main/runtime.rs | 59 ++++++++--- 12 files changed, 289 insertions(+), 82 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 18bd7aab..8de3abf4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -175,9 +175,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-lc-rs" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f409eb70b561706bf8abba8ca9c112729c481595893fd06a2dd9af8ed8441148" +checksum = "1ea835662a0af02443aa1396d39be523bbf8f11ee6fad20329607c480bea48c3" dependencies = [ "aws-lc-sys", "paste", @@ -186,9 +186,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "923ded50f602b3007e5e63e3f094c479d9c8a9b42d7f4034e4afe456aa48bfd2" +checksum = "71b2ddd3ada61a305e1d8bb6c005d1eaa7d14d903681edfc400406d523a9b491" dependencies = [ "bindgen", "cc", @@ -368,7 +368,7 @@ version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", "cexpr", "clang-sys", "itertools 0.12.1", @@ -393,9 +393,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.7.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1be3f42a67d6d345ecd59f675f3f012d6974981560836e938c22b424b85ce1be" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" [[package]] name = "blake2" @@ -495,9 +495,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.9" +version = "1.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8293772165d9345bdaaa39b45b2109591e63fe5e6fbc23c6ff930a048aa310b" +checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" dependencies = [ "jobserver", "libc", @@ -1047,7 +1047,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", "crossterm_winapi", "futures-core", "mio", @@ -1122,9 +1122,9 @@ checksum = "817fa642fb0ee7fe42e95783e00e0969927b96091bdd4b9b1af082acd943913b" [[package]] name = "data-encoding" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" +checksum = "0e60eed09d8c01d3cee5b7d30acb059b76614c918fa0f992e0dd6eeb10daad6f" [[package]] name = "date_header" @@ -2378,7 +2378,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", "cfg-if", "cfg_aliases", "libc", @@ -2911,7 +2911,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f86ba2052aebccc42cbbb3ed234b8b13ce76f75c3551a303cb2bcffcff12bb14" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", "memchr", "pulldown-cmark-escape", "unicase", @@ -3032,7 +3032,7 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", ] [[package]] @@ -3377,7 +3377,7 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" version = "0.31.0+9.9.3" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=123d6302fed23fc706344becb2f19623265a83f8#123d6302fed23fc706344becb2f19623265a83f8" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=2d31cf323df7c6d95396ef0213e28936c2218bd6#2d31cf323df7c6d95396ef0213e28936c2218bd6" dependencies = [ "bindgen", "bzip2-sys", @@ -3394,7 +3394,7 @@ dependencies = [ [[package]] name = "rust-rocksdb" version = "0.35.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=123d6302fed23fc706344becb2f19623265a83f8#123d6302fed23fc706344becb2f19623265a83f8" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=2d31cf323df7c6d95396ef0213e28936c2218bd6#2d31cf323df7c6d95396ef0213e28936c2218bd6" dependencies = [ "libc", "rust-librocksdb-sys", @@ -3441,7 +3441,7 @@ version = "0.38.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", "errno", "libc", "linux-raw-sys", @@ -3559,9 +3559,12 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sd-notify" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1be20c5f7f393ee700f8b2f28ea35812e4e212f40774b550cd2a93ea91684451" +checksum = "561e6b346a5e59e0b8a07894004897d7160567e3352d2ebd6c3741d4e086b6f5" +dependencies = [ + "libc", +] [[package]] name = "security-framework" @@ -3569,7 +3572,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", "core-foundation", "core-foundation-sys", "libc", @@ -4171,7 +4174,7 @@ dependencies = [ [[package]] name = "tikv-jemalloc-ctl" version = "0.6.0" -source = "git+https://github.com/girlbossceo/jemallocator?rev=d87938bfddc26377dd7fdf14bbcd345f3ab19442#d87938bfddc26377dd7fdf14bbcd345f3ab19442" +source = "git+https://github.com/girlbossceo/jemallocator?rev=82af58d6a13ddd5dcdc7d4e91eae3b63292995b8#82af58d6a13ddd5dcdc7d4e91eae3b63292995b8" dependencies = [ "libc", "paste", @@ -4181,7 +4184,7 @@ dependencies = [ [[package]] name = "tikv-jemalloc-sys" version = "0.6.0+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" -source = "git+https://github.com/girlbossceo/jemallocator?rev=d87938bfddc26377dd7fdf14bbcd345f3ab19442#d87938bfddc26377dd7fdf14bbcd345f3ab19442" +source = "git+https://github.com/girlbossceo/jemallocator?rev=82af58d6a13ddd5dcdc7d4e91eae3b63292995b8#82af58d6a13ddd5dcdc7d4e91eae3b63292995b8" dependencies = [ "cc", "libc", @@ -4190,7 +4193,7 @@ dependencies = [ [[package]] name = "tikv-jemallocator" version = "0.6.0" -source = "git+https://github.com/girlbossceo/jemallocator?rev=d87938bfddc26377dd7fdf14bbcd345f3ab19442#d87938bfddc26377dd7fdf14bbcd345f3ab19442" +source = "git+https://github.com/girlbossceo/jemallocator?rev=82af58d6a13ddd5dcdc7d4e91eae3b63292995b8#82af58d6a13ddd5dcdc7d4e91eae3b63292995b8" dependencies = [ "libc", "tikv-jemalloc-sys", @@ -4445,7 +4448,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" dependencies = [ "async-compression", - "bitflags 2.7.0", + "bitflags 2.8.0", "bytes", "futures-core", "futures-util", @@ -4709,9 +4712,9 @@ dependencies = [ [[package]] name = "valuable" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "vcpkg" diff --git a/Cargo.toml b/Cargo.toml index c0b31a69..4d738a11 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -431,17 +431,23 @@ version = "0.35.0" # jemalloc usage [workspace.dependencies.tikv-jemalloc-sys] git = "https://github.com/girlbossceo/jemallocator" -rev = "d87938bfddc26377dd7fdf14bbcd345f3ab19442" +rev = "82af58d6a13ddd5dcdc7d4e91eae3b63292995b8" default-features = false -features = ["unprefixed_malloc_on_supported_platforms"] +features = [ + "background_threads_runtime_support", + "unprefixed_malloc_on_supported_platforms", +] [workspace.dependencies.tikv-jemallocator] git = "https://github.com/girlbossceo/jemallocator" -rev = "d87938bfddc26377dd7fdf14bbcd345f3ab19442" +rev = "82af58d6a13ddd5dcdc7d4e91eae3b63292995b8" default-features = false -features = ["unprefixed_malloc_on_supported_platforms"] +features = [ + "background_threads_runtime_support", + "unprefixed_malloc_on_supported_platforms", +] [workspace.dependencies.tikv-jemalloc-ctl] git = "https://github.com/girlbossceo/jemallocator" -rev = "d87938bfddc26377dd7fdf14bbcd345f3ab19442" +rev = "82af58d6a13ddd5dcdc7d4e91eae3b63292995b8" default-features = false features = ["use_std"] diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index f06c44e8..96554aed 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "123d6302fed23fc706344becb2f19623265a83f8" +rev = "2d31cf323df7c6d95396ef0213e28936c2218bd6" #branch = "master" default-features = false diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index ad61440c..a77587b0 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -967,7 +967,7 @@ pub(super) async fn database_stats( #[admin_command] pub(super) async fn trim_memory(&self) -> Result { - conduwuit::alloc::trim()?; + conduwuit::alloc::trim(None)?; writeln!(self, "done").await?; diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index c716e9c2..ef2df4ff 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -36,6 +36,7 @@ jemalloc_stats = [ "tikv-jemalloc-ctl/stats", "tikv-jemallocator/stats", ] +jemalloc_conf = [] hardened_malloc = [ "dep:hardened_malloc-rs" ] diff --git a/src/core/alloc/default.rs b/src/core/alloc/default.rs index 56e8c407..65354b7d 100644 --- a/src/core/alloc/default.rs +++ b/src/core/alloc/default.rs @@ -1,7 +1,7 @@ //! Default allocator with no special features /// Always returns Ok -pub fn trim() -> crate::Result { Ok(()) } +pub fn trim>>(_: I) -> crate::Result { Ok(()) } /// Always returns None #[must_use] diff --git a/src/core/alloc/hardened.rs b/src/core/alloc/hardened.rs index ff10cf2b..5f850673 100644 --- a/src/core/alloc/hardened.rs +++ b/src/core/alloc/hardened.rs @@ -3,7 +3,7 @@ #[global_allocator] static HMALLOC: hardened_malloc_rs::HardenedMalloc = hardened_malloc_rs::HardenedMalloc; -pub fn trim() -> crate::Result { Ok(()) } +pub fn trim>>(_: I) -> crate::Result { Ok(()) } #[must_use] //TODO: get usage diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index ccb213c9..119ff45e 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -2,8 +2,9 @@ use std::{ cell::OnceCell, - ffi::{c_char, c_void}, + ffi::{c_char, c_void, CStr}, fmt::Debug, + sync::RwLock, }; use arrayvec::ArrayVec; @@ -11,10 +12,14 @@ use tikv_jemalloc_ctl as mallctl; use tikv_jemalloc_sys as ffi; use tikv_jemallocator as jemalloc; -use crate::{err, is_equal_to, utils::math::Tried, Result}; +use crate::{ + err, is_equal_to, is_nonzero, + utils::{math, math::Tried}, + Result, +}; #[cfg(feature = "jemalloc_conf")] -#[no_mangle] +#[unsafe(no_mangle)] pub static malloc_conf: &[u8] = b"\ metadata_thp:always\ ,percpu_arena:percpu\ @@ -22,19 +27,26 @@ metadata_thp:always\ ,max_background_threads:-1\ ,lg_extent_max_active_fit:4\ ,oversize_threshold:33554432\ -,tcache_max:2097152\ +,tcache_max:1048576\ ,dirty_decay_ms:16000\ ,muzzy_decay_ms:144000\ \0"; #[global_allocator] static JEMALLOC: jemalloc::Jemalloc = jemalloc::Jemalloc; +static CONTROL: RwLock<()> = RwLock::new(()); -type Key = ArrayVec; type Name = ArrayVec; +type Key = ArrayVec; -const KEY_SEGS: usize = 8; const NAME_MAX: usize = 128; +const KEY_SEGS: usize = 8; + +#[crate::ctor] +fn _static_initialization() { + acq_epoch().expect("pre-initialization of jemalloc failed"); + acq_epoch().expect("pre-initialization of jemalloc failed"); +} #[must_use] #[cfg(feature = "jemalloc_stats")] @@ -49,6 +61,9 @@ pub fn memory_usage() -> Option { kibs / 1024.0 }; + // Acquire the epoch; ensure latest stats are pulled in + acq_epoch().ok()?; + let allocated = mibs(stats::allocated::read()); let active = mibs(stats::active::read()); let mapped = mibs(stats::mapped::read()); @@ -76,6 +91,9 @@ pub fn memory_stats(opts: &str) -> Option { .into_raw() .cast_const(); + // Acquire the epoch; ensure latest stats are pulled in + acq_epoch().ok()?; + // SAFETY: calls malloc_stats_print() with our string instance which must remain // in this frame. https://docs.rs/tikv-jemalloc-sys/latest/tikv_jemalloc_sys/fn.malloc_stats_print.html unsafe { ffi::malloc_stats_print(Some(malloc_stats_cb), opaque, opts_p) }; @@ -95,7 +113,7 @@ unsafe extern "C" fn malloc_stats_cb(opaque: *mut c_void, msg: *const c_char) { }; // SAFETY: we have to trust the string is null terminated. - let msg = unsafe { std::ffi::CStr::from_ptr(msg) }; + let msg = unsafe { CStr::from_ptr(msg) }; let msg = String::from_utf8_lossy(msg.to_bytes()); res.push_str(msg.as_ref()); @@ -114,58 +132,168 @@ macro_rules! mallctl { }}; } -pub fn trim() -> Result { set(&mallctl!("arena.4096.purge"), ()) } - -pub fn decay() -> Result { set(&mallctl!("arena.4096.purge"), ()) } - -pub fn set_by_name(name: &str, val: T) -> Result { set(&key(name)?, val) } - -pub fn get_by_name(name: &str) -> Result { get(&key(name)?) } - pub mod this_thread { - use super::{get, key, set, Key, OnceCell, Result}; + use super::{is_nonzero, key, math, Debug, Key, OnceCell, Result}; - pub fn trim() -> Result { - let mut key = mallctl!("arena.0.purge"); - key[1] = arena_id()?.try_into()?; - set(&key, ()) + pub fn trim() -> Result { notify(mallctl!("arena.0.purge")) } + + pub fn decay() -> Result { notify(mallctl!("arena.0.decay")) } + + pub fn flush() -> Result { super::notify(&mallctl!("thread.tcache.flush")) } + + pub fn set_muzzy_decay(decay_ms: isize) -> Result { + set(mallctl!("arena.0.muzzy_decay_ms"), decay_ms) } - pub fn decay() -> Result { - let mut key = mallctl!("arena.0.decay"); - key[1] = arena_id()?.try_into()?; - set(&key, ()) + pub fn get_muzzy_decay() -> Result { get(mallctl!("arena.0.muzzy_decay_ms")) } + + pub fn set_dirty_decay(decay_ms: isize) -> Result { + set(mallctl!("arena.0.dirty_decay_ms"), decay_ms) } - pub fn cache(enable: bool) -> Result { - set(&mallctl!("thread.tcache.enabled"), u8::from(enable)) + pub fn get_dirty_decay() -> Result { get(mallctl!("arena.0.dirty_decay_ms")) } + + pub fn enable_cache(enable: bool) -> Result { + super::set::(&mallctl!("thread.tcache.enabled"), enable.into()).map(is_nonzero!()) } - pub fn flush() -> Result { set(&mallctl!("thread.tcache.flush"), ()) } + pub fn is_cache_enabled() -> Result { + super::get::(&mallctl!("thread.tcache.enabled")).map(is_nonzero!()) + } - pub fn allocated() -> Result { get::(&mallctl!("thread.allocated")) } + pub fn set_arena(id: usize) -> Result { + super::set::(&mallctl!("thread.arena"), id.try_into()?).and_then(math::try_into) + } - pub fn deallocated() -> Result { get::(&mallctl!("thread.deallocated")) } + pub fn arena_id() -> Result { + super::get::(&mallctl!("thread.arena")).and_then(math::try_into) + } - pub fn arena_id() -> Result { get::(&mallctl!("thread.arena")) } + pub fn allocated() -> Result { super::get(&mallctl!("thread.allocated")) } + + pub fn deallocated() -> Result { super::get(&mallctl!("thread.deallocated")) } + + fn notify(key: Key) -> Result { super::notify_by_arena(Some(arena_id()?), key) } + + fn set(key: Key, val: T) -> Result + where + T: Copy + Debug, + { + super::set_by_arena(Some(arena_id()?), key, val) + } + + fn get(key: Key) -> Result + where + T: Copy + Debug, + { + super::get_by_arena(Some(arena_id()?), key) + } } -fn set(key: &Key, val: T) -> Result +pub fn trim>>(arena: I) -> Result { + notify_by_arena(arena.into(), mallctl!("arena.4096.purge")) +} + +pub fn decay>>(arena: I) -> Result { + notify_by_arena(arena.into(), mallctl!("arena.4096.decay")) +} + +pub fn set_muzzy_decay>>(arena: I, decay_ms: isize) -> Result { + if let Some(arena) = arena.into() { + set_by_arena(Some(arena), mallctl!("arena.4096.muzzy_decay_ms"), decay_ms) + } else { + set(&mallctl!("arenas.muzzy_decay_ms"), decay_ms) + } +} + +pub fn set_dirty_decay>>(arena: I, decay_ms: isize) -> Result { + if let Some(arena) = arena.into() { + set_by_arena(Some(arena), mallctl!("arena.4096.dirty_decay_ms"), decay_ms) + } else { + set(&mallctl!("arenas.dirty_decay_ms"), decay_ms) + } +} + +#[inline] +#[must_use] +pub fn is_affine_arena() -> bool { is_percpu_arena() || is_phycpu_arena() } + +#[inline] +#[must_use] +pub fn is_percpu_arena() -> bool { percpu_arenas().is_ok_and(is_equal_to!("percpu")) } + +#[inline] +#[must_use] +pub fn is_phycpu_arena() -> bool { percpu_arenas().is_ok_and(is_equal_to!("phycpu")) } + +pub fn percpu_arenas() -> Result<&'static str> { + let ptr = get::<*const c_char>(&mallctl!("opt.percpu_arena"))?; + //SAFETY: ptr points to a null-terminated string returned for opt.percpu_arena. + let cstr = unsafe { CStr::from_ptr(ptr) }; + cstr.to_str().map_err(Into::into) +} + +pub fn arenas() -> Result { + get::(&mallctl!("arenas.narenas")).and_then(math::try_into) +} + +pub fn inc_epoch() -> Result { xchg(&mallctl!("epoch"), 1_u64) } + +pub fn acq_epoch() -> Result { xchg(&mallctl!("epoch"), 0_u64) } + +fn notify_by_arena(id: Option, mut key: Key) -> Result { + key[1] = id.unwrap_or(4096); + notify(&key) +} + +fn set_by_arena(id: Option, mut key: Key, val: T) -> Result where T: Copy + Debug, { - // SAFETY: T must be the exact expected type. - unsafe { mallctl::raw::write_mib(key.as_slice(), val) }.map_err(map_err) + key[1] = id.unwrap_or(4096); + set(&key, val) +} + +fn get_by_arena(id: Option, mut key: Key) -> Result +where + T: Copy + Debug, +{ + key[1] = id.unwrap_or(4096); + get(&key) +} + +fn notify(key: &Key) -> Result { xchg(key, ()) } + +fn set(key: &Key, val: T) -> Result +where + T: Copy + Debug, +{ + let _lock = CONTROL.write()?; + let res = xchg(key, val)?; + inc_epoch()?; + + Ok(res) } fn get(key: &Key) -> Result where T: Copy + Debug, { + acq_epoch()?; + acq_epoch()?; + // SAFETY: T must be perfectly valid to receive value. unsafe { mallctl::raw::read_mib(key.as_slice()) }.map_err(map_err) } +fn xchg(key: &Key, val: T) -> Result +where + T: Copy + Debug, +{ + // SAFETY: T must be the exact expected type. + unsafe { mallctl::raw::update_mib(key.as_slice(), val) }.map_err(map_err) +} + fn key(name: &str) -> Result { // tikv asserts the output buffer length is tight to the number of required mibs // so we slice that down here. diff --git a/src/database/pool.rs b/src/database/pool.rs index f5600c36..86516c31 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -13,7 +13,7 @@ use std::{ use async_channel::{QueueStrategy, Receiver, RecvError, Sender}; use conduwuit::{ debug, debug_warn, err, error, implement, - result::DebugInspect, + result::{DebugInspect, LogDebugErr}, trace, utils::sys::compute::{get_affinity, nth_core_available, set_affinity}, Error, Result, Server, @@ -289,6 +289,20 @@ fn worker_init(&self, id: usize) { // affinity is empty (no-op) if there's only one queue set_affinity(affinity.clone()); + + #[cfg(feature = "jemalloc")] + if affinity.clone().count() == 1 && conduwuit::alloc::je::is_affine_arena() { + use conduwuit::alloc::je::this_thread::{arena_id, set_arena}; + + let id = affinity.clone().next().expect("at least one id"); + + if let Ok(arena) = arena_id() { + if arena != id { + set_arena(id).log_debug_err().ok(); + } + } + } + debug!( ?group, affinity = ?affinity.collect::>(), diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index baf5336f..f774c37a 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -41,6 +41,7 @@ default = [ "gzip_compression", "io_uring", "jemalloc", + "jemalloc_conf", "media_thumbnail", "release_max_log_level", "systemd", @@ -85,6 +86,9 @@ jemalloc_prof = [ jemalloc_stats = [ "conduwuit-core/jemalloc_stats", ] +jemalloc_conf = [ + "conduwuit-core/jemalloc_conf", +] media_thumbnail = [ "conduwuit-service/media_thumbnail", ] diff --git a/src/main/clap.rs b/src/main/clap.rs index d3d40491..2bb6f3f2 100644 --- a/src/main/clap.rs +++ b/src/main/clap.rs @@ -92,6 +92,22 @@ pub(crate) struct Args { require_equals(false), )] pub(crate) gc_on_park: Option, + + /// Toggles muzzy decay for jemalloc arenas associated with a tokio + /// worker (when worker-affinity is enabled). Setting to false releases + /// memory to the operating system using MADV_FREE without MADV_DONTNEED. + /// Setting to false increases performance by reducing pagefaults, but + /// resident memory usage appears high until there is memory pressure. The + /// default is true unless the system has four or more cores. + #[arg( + long, + hide(true), + env = "CONDUWUIT_RUNTIME_GC_MUZZY", + action = ArgAction::Set, + num_args = 0..=1, + require_equals(false), + )] + pub(crate) gc_muzzy: Option, } /// Parse commandline arguments into structured data diff --git a/src/main/runtime.rs b/src/main/runtime.rs index 315336b0..9f4f60f8 100644 --- a/src/main/runtime.rs +++ b/src/main/runtime.rs @@ -9,8 +9,12 @@ use std::{ }; use conduwuit::{ - result::LogErr, - utils::sys::compute::{nth_core_available, set_affinity}, + is_true, + result::LogDebugErr, + utils::{ + available_parallelism, + sys::compute::{nth_core_available, set_affinity}, + }, Result, }; use tokio::runtime::Builder; @@ -21,9 +25,11 @@ const WORKER_NAME: &str = "conduwuit:worker"; const WORKER_MIN: usize = 2; const WORKER_KEEPALIVE: u64 = 36; const MAX_BLOCKING_THREADS: usize = 1024; +const DISABLE_MUZZY_THRESHOLD: usize = 4; static WORKER_AFFINITY: OnceLock = OnceLock::new(); static GC_ON_PARK: OnceLock> = OnceLock::new(); +static GC_MUZZY: OnceLock> = OnceLock::new(); pub(super) fn new(args: &Args) -> Result { WORKER_AFFINITY @@ -34,6 +40,10 @@ pub(super) fn new(args: &Args) -> Result { .set(args.gc_on_park) .expect("set GC_ON_PARK from program argument"); + GC_MUZZY + .set(args.gc_muzzy) + .expect("set GC_MUZZY from program argument"); + let mut builder = Builder::new_multi_thread(); builder .enable_io() @@ -83,11 +93,13 @@ fn enable_histogram(builder: &mut Builder, args: &Args) { ), )] fn thread_start() { - if WORKER_AFFINITY - .get() - .copied() - .expect("WORKER_AFFINITY initialized by runtime::new()") - { + debug_assert_eq!( + Some(WORKER_NAME), + thread::current().name(), + "tokio worker name mismatch at thread start" + ); + + if WORKER_AFFINITY.get().is_some_and(is_true!()) { set_worker_affinity(); } } @@ -95,10 +107,6 @@ fn thread_start() { fn set_worker_affinity() { static CORES_OCCUPIED: AtomicUsize = AtomicUsize::new(0); - if thread::current().name() != Some(WORKER_NAME) { - return; - } - let handle = tokio::runtime::Handle::current(); let num_workers = handle.metrics().num_workers(); let i = CORES_OCCUPIED.fetch_add(1, Ordering::Relaxed); @@ -111,8 +119,33 @@ fn set_worker_affinity() { }; set_affinity(once(id)); + set_worker_mallctl(id); } +#[cfg(feature = "jemalloc")] +fn set_worker_mallctl(id: usize) { + use conduwuit::alloc::je::{ + is_affine_arena, + this_thread::{set_arena, set_muzzy_decay}, + }; + + if is_affine_arena() { + set_arena(id).log_debug_err().ok(); + } + + let muzzy_option = GC_MUZZY + .get() + .expect("GC_MUZZY initialized by runtime::new()"); + + let muzzy_auto_disable = available_parallelism() >= DISABLE_MUZZY_THRESHOLD; + if matches!(muzzy_option, Some(false) | None if muzzy_auto_disable) { + set_muzzy_decay(-1).log_debug_err().ok(); + } +} + +#[cfg(not(feature = "jemalloc"))] +fn set_worker_mallctl(_: usize) {} + #[tracing::instrument( name = "join", level = "debug", @@ -157,7 +190,9 @@ fn thread_park() { fn gc_on_park() { #[cfg(feature = "jemalloc")] - conduwuit::alloc::je::this_thread::decay().log_err().ok(); + conduwuit::alloc::je::this_thread::decay() + .log_debug_err() + .ok(); } #[cfg(tokio_unstable)]