From 8a5599adf9eafe9111f3d1597f8fb333b8b76849 Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Thu, 25 Apr 2024 09:07:59 -0700 Subject: [PATCH] add optional support for tokio-console This turned out to be quite hairy, mostly because we need to apply the config's log level filter to the actual logs (stdout and, optionally sentry), but do not want to filter out the tokio tracing events needed by the console_subscriber. I hit several edge cases in tracing getting this to work, and we now depend on a git version of tracing with a backported patch :( --- Cargo.lock | 420 +++++++++++++++++++--- Cargo.toml | 18 + docs/development.md | 18 + src/database/mod.rs | 12 +- src/main.rs | 105 ++++-- src/service/admin/debug/debug_commands.rs | 4 +- src/service/globals/mod.rs | 8 +- src/service/mod.rs | 8 +- 8 files changed, 488 insertions(+), 105 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 38cf12b7..d9930e9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,6 +65,12 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" +[[package]] +name = "anyhow" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" + [[package]] name = "arc-swap" version = "1.7.1" @@ -137,6 +143,34 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core 0.3.4", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper 0.1.2", + "tower", + "tower-layer", + "tower-service", +] + [[package]] name = "axum" version = "0.7.5" @@ -144,13 +178,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" dependencies = [ "async-trait", - "axum-core", + "axum-core 0.4.3", "bytes", "futures-util", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.0", "http-body-util", - "hyper", + "hyper 1.3.1", "hyper-util", "itoa", "matchit", @@ -169,6 +203,23 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + [[package]] name = "axum-core" version = "0.4.3" @@ -178,8 +229,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.0", "http-body-util", "mime", "pin-project-lite", @@ -195,13 +246,13 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0be6ea09c9b96cb5076af0de2e383bd2bc0c18f827cf1967bdd353e0b910d733" dependencies = [ - "axum", - "axum-core", + "axum 0.7.5", + "axum-core 0.4.3", "bytes", "futures-util", "headers", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.0", "http-body-util", "mime", "pin-project-lite", @@ -220,10 +271,10 @@ dependencies = [ "arc-swap", "bytes", "futures-util", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.0", "http-body-util", - "hyper", + "hyper 1.3.1", "hyper-util", "pin-project-lite", "rustls 0.21.11", @@ -242,7 +293,7 @@ checksum = "7ea4cd08ae2a5f075d28fa31190163c8106a1d2d3189442494bae22b39040a0d" dependencies = [ "axum-server", "bytes", - "http", + "http 1.1.0", "http-body-util", "pin-project", "tokio", @@ -505,7 +556,7 @@ version = "0.3.0" dependencies = [ "argon2", "async-trait", - "axum", + "axum 0.7.5", "axum-extra", "axum-server", "axum-server-dual-protocol", @@ -513,6 +564,7 @@ dependencies = [ "bytes", "chrono", "clap", + "console-subscriber", "cyborgtime", "either", "figment", @@ -521,9 +573,9 @@ dependencies = [ "hickory-resolver", "hmac", "hot-lib-reloader", - "http", + "http 1.1.0", "http-body-util", - "hyper", + "hyper 1.3.1", "hyper-util", "image", "ipaddress", @@ -573,6 +625,42 @@ dependencies = [ "webpage", ] +[[package]] +name = "console-api" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2895653b4d9f1538a83970077cb01dfc77a4810524e51a110944688e916b18e" +dependencies = [ + "prost", + "prost-types", + "tonic", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4cf42660ac07fcebed809cfe561dd8730bcd35b075215e6479c516bcd0d11cb" +dependencies = [ + "console-api", + "crossbeam-channel", + "crossbeam-utils", + "futures", + "hdrhistogram", + "humantime", + "prost-types", + "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -917,6 +1005,20 @@ dependencies = [ "new_debug_unreachable", ] +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.30" @@ -1035,6 +1137,25 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.2.6", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.4" @@ -1046,8 +1167,8 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", - "indexmap", + "http 1.1.0", + "indexmap 2.2.6", "slab", "tokio", "tokio-util", @@ -1060,6 +1181,12 @@ version = "0.1.2+12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "647deb1583b14d160f85f3ff626f20b6edd366e3852c9843b06077388f794cb6" +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.3" @@ -1076,7 +1203,20 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "692eaaf7f7607518dd3cef090f1474b61edc5301d8012f09579920df68b725ee" dependencies = [ - "hashbrown", + "hashbrown 0.14.3", +] + +[[package]] +name = "hdrhistogram" +version = "7.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" +dependencies = [ + "base64 0.21.7", + "byteorder", + "flate2", + "nom", + "num-traits", ] [[package]] @@ -1088,7 +1228,7 @@ dependencies = [ "base64 0.21.7", "bytes", "headers-core", - "http", + "http 1.1.0", "httpdate", "mime", "sha1", @@ -1100,7 +1240,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" dependencies = [ - "http", + "http 1.1.0", ] [[package]] @@ -1242,6 +1382,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.1.0" @@ -1253,6 +1404,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.0" @@ -1260,7 +1422,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ "bytes", - "http", + "http 1.1.0", ] [[package]] @@ -1271,8 +1433,8 @@ checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -1288,6 +1450,36 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "0.14.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.3.1" @@ -1297,9 +1489,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2", - "http", - "http-body", + "h2 0.4.4", + "http 1.1.0", + "http-body 1.0.0", "httparse", "httpdate", "itoa", @@ -1316,8 +1508,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" dependencies = [ "futures-util", - "http", - "hyper", + "http 1.1.0", + "hyper 1.3.1", "hyper-util", "rustls 0.22.4", "rustls-pki-types", @@ -1326,6 +1518,18 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper 0.14.28", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "hyper-util" version = "0.1.3" @@ -1335,9 +1539,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", - "hyper", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.3.1", "pin-project-lite", "socket2", "tokio", @@ -1393,6 +1597,16 @@ dependencies = [ "thiserror", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.2.6" @@ -1400,7 +1614,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.14.3", "serde", ] @@ -1477,6 +1691,15 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.11.0" @@ -2016,7 +2239,7 @@ checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" dependencies = [ "futures-core", "futures-sink", - "indexmap", + "indexmap 2.2.6", "js-sys", "once_cell", "pin-project-lite", @@ -2340,6 +2563,38 @@ dependencies = [ "yansi", ] +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools 0.10.5", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "prost-types" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -2451,10 +2706,10 @@ dependencies = [ "futures-core", "futures-util", "hickory-resolver", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.0", "http-body-util", - "hyper", + "hyper 1.3.1", "hyper-rustls", "hyper-util", "ipnet", @@ -2551,7 +2806,7 @@ dependencies = [ "assign", "bytes", "date_header", - "http", + "http 1.1.0", "js_int", "js_option", "maplit", @@ -2573,8 +2828,8 @@ dependencies = [ "base64 0.21.7", "bytes", "form_urlencoded", - "http", - "indexmap", + "http 1.1.0", + "indexmap 2.2.6", "js_int", "konst", "percent-encoding", @@ -2600,7 +2855,7 @@ version = "0.27.11" source = "git+https://github.com/girlbossceo/ruma?branch=conduwuit-changes#7136799881d0fcb18aad26c3dac18a34a1f892fc" dependencies = [ "as_variant", - "indexmap", + "indexmap 2.2.6", "js_int", "js_option", "percent-encoding", @@ -3020,7 +3275,7 @@ version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df141464944fdf8e2a6f2184eb1d973a20456466f788346b6e3a51791cdaa370" dependencies = [ - "http", + "http 1.1.0", "pin-project", "sentry-core", "tower-layer", @@ -3084,7 +3339,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de514ef58196f1fc96dcaef80fe6170a1ce6215df9687a93fe8300e773fefc5" dependencies = [ "form_urlencoded", - "indexmap", + "indexmap 2.2.6", "itoa", "ryu", "serde", @@ -3148,7 +3403,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap", + "indexmap 2.2.6", "itoa", "ryu", "serde", @@ -3516,9 +3771,20 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", + "tracing", "windows-sys 0.48.0", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "2.2.0" @@ -3615,13 +3881,41 @@ version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338" dependencies = [ - "indexmap", + "indexmap 2.2.6", "serde", "serde_spanned", "toml_datetime", "winnow", ] +[[package]] +name = "tonic" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" +dependencies = [ + "async-trait", + "axum 0.6.20", + "base64 0.21.7", + "bytes", + "futures-core", + "futures-util", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.4.13" @@ -3630,9 +3924,13 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", + "indexmap 1.9.3", "pin-project", "pin-project-lite", + "rand", + "slab", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -3649,8 +3947,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.0", "http-body-util", "pin-project-lite", "tokio", @@ -3676,8 +3974,7 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" version = "0.1.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +source = "git+https://github.com/Benjamin-L/tracing?branch=tracing-subscriber/env-filter-clone-0.1.x-backport#b348dca742af641c47bc390261f60711c2af573c" dependencies = [ "log", "pin-project-lite", @@ -3688,8 +3985,7 @@ dependencies = [ [[package]] name = "tracing-attributes" version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +source = "git+https://github.com/Benjamin-L/tracing?branch=tracing-subscriber/env-filter-clone-0.1.x-backport#b348dca742af641c47bc390261f60711c2af573c" dependencies = [ "proc-macro2", "quote", @@ -3699,8 +3995,7 @@ dependencies = [ [[package]] name = "tracing-core" version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +source = "git+https://github.com/Benjamin-L/tracing?branch=tracing-subscriber/env-filter-clone-0.1.x-backport#b348dca742af641c47bc390261f60711c2af573c" dependencies = [ "once_cell", "valuable", @@ -3728,6 +4023,16 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "git+https://github.com/Benjamin-L/tracing?branch=tracing-subscriber/env-filter-clone-0.1.x-backport#b348dca742af641c47bc390261f60711c2af573c" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + [[package]] name = "tracing-opentelemetry" version = "0.22.0" @@ -3741,7 +4046,7 @@ dependencies = [ "smallvec", "tracing", "tracing-core", - "tracing-log", + "tracing-log 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "tracing-subscriber", "web-time 0.2.4", ] @@ -3749,8 +4054,7 @@ dependencies = [ [[package]] name = "tracing-subscriber" version = "0.3.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +source = "git+https://github.com/Benjamin-L/tracing?branch=tracing-subscriber/env-filter-clone-0.1.x-backport#b348dca742af641c47bc390261f60711c2af573c" dependencies = [ "matchers", "nu-ansi-term", @@ -3761,7 +4065,7 @@ dependencies = [ "thread_local", "tracing", "tracing-core", - "tracing-log", + "tracing-log 0.2.0 (git+https://github.com/Benjamin-L/tracing?branch=tracing-subscriber/env-filter-clone-0.1.x-backport)", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d240ebee..9bdf9aa7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,8 @@ rust-version = "1.75.0" [dependencies] +console-subscriber = { version = "0.1", optional = true } + hot-lib-reloader = { version = "^0.6", optional = true } # Used for secure identifiers @@ -338,6 +340,18 @@ hardened_malloc-rs = { version = "0.1.2", optional = true, features = [ #hardened_malloc-rs = { optional = true, features = ["static","clang","light"], path = "../hardened_malloc-rs", default-features = false } +# backport of [https://github.com/tokio-rs/tracing/pull/2956] to the 0.1.x branch of tracing. +# we can switch back to upstream if #2956 is merged and backported in the upstream repo. +[patch.crates-io.tracing-subscriber] +git = "https://github.com/Benjamin-L/tracing" +branch = "tracing-subscriber/env-filter-clone-0.1.x-backport" +[patch.crates-io.tracing] +git = "https://github.com/Benjamin-L/tracing" +branch = "tracing-subscriber/env-filter-clone-0.1.x-backport" +[patch.crates-io.tracing-core] +git = "https://github.com/Benjamin-L/tracing" +branch = "tracing-subscriber/env-filter-clone-0.1.x-backport" + [features] default = [ "backend_rocksdb", @@ -373,6 +387,10 @@ perf_measurements = [ "opentelemetry-jaeger", ] +# enable the tokio_console server +# incompatible with release_max_log_level +tokio_console = ["console-subscriber", "tokio/tracing"] + hot_reload = ["dep:hot-lib-reloader"] hardened_malloc = ["hardened_malloc-rs"] diff --git a/docs/development.md b/docs/development.md index d6ee27eb..49e055e1 100644 --- a/docs/development.md +++ b/docs/development.md @@ -2,3 +2,21 @@ Information about developing the project. If you are only interested in using it, you can safely ignore this section. + +## Debugging with `tokio-console` + +[`tokio-console`][1] can be a useful tool for debugging and profiling. To make +a `tokio-console`-enabled build of Conduwuit, enable the `tokio_console` feature, +disable the default `release_max_log_level` feature, and set the +`--cfg tokio_unstable` flag to enable experimental tokio APIs. A build might +look like this: + +```bash +RUSTFLAGS="--cfg tokio_unstable" cargo build \ + --release \ + --no-default-features \ + --features + backend_rocksdb,systemd,element_hacks,sentry_telemetry,gzip_compression,brotli_compression,zstd_compression,tokio_console +``` + +[1]: https://docs.rs/tokio-console/latest/tokio_console/ diff --git a/src/database/mod.rs b/src/database/mod.rs index c8e1a17b..e31e0a78 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -40,8 +40,8 @@ use tokio::time::{interval, Instant}; use tracing::{debug, error, warn}; use crate::{ - database::migrations::migrations, service::rooms::timeline::PduCount, services, Config, Error, Result, Services, - SERVICES, + database::migrations::migrations, service::rooms::timeline::PduCount, services, Config, Error, + LogLevelReloadHandles, Result, Services, SERVICES, }; pub(crate) struct KeyValueDatabase { @@ -203,13 +203,7 @@ struct CheckForUpdatesResponse { impl KeyValueDatabase { /// Load an existing database or create a new one. #[allow(clippy::too_many_lines)] - pub(crate) async fn load_or_create( - config: Config, - tracing_reload_handler: tracing_subscriber::reload::Handle< - tracing_subscriber::EnvFilter, - tracing_subscriber::Registry, - >, - ) -> Result<()> { + pub(crate) async fn load_or_create(config: Config, tracing_reload_handler: LogLevelReloadHandles) -> Result<()> { Self::check_db_setup(&config)?; if !Path::new(&config.database_path).exists() { diff --git a/src/main.rs b/src/main.rs index be133055..af235146 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,7 +6,7 @@ use std::os::unix::fs::PermissionsExt as _; /* not unix specific, just only for // Not async due to services() being used in many closures, and async closures // are not stable as of writing This is the case for every other occurence of // sync Mutex/RwLock, except for database related ones -use std::sync::RwLock; +use std::sync::{Arc, RwLock}; use std::{any::Any, io, net::SocketAddr, sync::atomic, time::Duration}; use api::ruma_wrapper::{Ruma, RumaResponse}; @@ -79,7 +79,7 @@ struct Server { runtime: tokio::runtime::Runtime, - tracing_reload_handle: reload::Handle, + tracing_reload_handle: LogLevelReloadHandles, #[cfg(feature = "sentry_telemetry")] _sentry_guard: Option, @@ -547,7 +547,56 @@ fn init_sentry(config: &Config) -> sentry::ClientInitGuard { )) } -fn init_tracing_sub(config: &Config) -> reload::Handle { +// We need to store a reload::Handle value, but can't name it's type explicitly +// because the S type parameter depends on the subscriber's previous layers. In +// our case, this includes unnameable 'impl Trait' types. +// +// This is fixed[1] in the unreleased tracing-subscriber from the master branch, +// which removes the S parameter. Unfortunately can't use it without pulling in +// a version of tracing that's incompatible with the rest of our deps. +// +// To work around this, we define an trait without the S paramter that forwards +// to the reload::Handle::reload method, and then store the handle as a trait +// object. +// +// [1]: https://github.com/tokio-rs/tracing/pull/1035/commits/8a87ea52425098d3ef8f56d92358c2f6c144a28f +trait ReloadHandle { + fn reload(&self, new_value: L) -> Result<(), reload::Error>; +} + +impl ReloadHandle for reload::Handle { + fn reload(&self, new_value: L) -> Result<(), reload::Error> { reload::Handle::reload(self, new_value) } +} + +struct LogLevelReloadHandlesInner { + handles: Vec + Send + Sync>>, +} + +/// Wrapper to allow reloading the filter on several several +/// [`tracing_subscriber::reload::Handle`]s at once, with the same value. +#[derive(Clone)] +struct LogLevelReloadHandles { + inner: Arc, +} + +impl LogLevelReloadHandles { + fn new(handles: Vec + Send + Sync>>) -> LogLevelReloadHandles { + LogLevelReloadHandles { + inner: Arc::new(LogLevelReloadHandlesInner { + handles, + }), + } + } + + fn reload(&self, new_value: &EnvFilter) -> Result<(), reload::Error> { + for handle in &self.inner.handles { + handle.reload(new_value.clone())?; + } + Ok(()) + } +} + +fn init_tracing_sub(config: &Config) -> LogLevelReloadHandles { let registry = Registry::default(); let fmt_layer = tracing_subscriber::fmt::Layer::new(); let filter_layer = match EnvFilter::try_new(&config.log) { @@ -558,35 +607,40 @@ fn init_tracing_sub(config: &Config) -> reload::Handle { }, }; - let (reload_filter, reload_handle) = reload::Layer::new(filter_layer); + let mut reload_handles = Vec:: + Send + Sync>>::new(); + let subscriber = registry; - #[cfg(feature = "sentry_telemetry")] - let sentry_layer = sentry_tracing::layer(); - - let subscriber; - - #[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental - #[cfg(feature = "sentry_telemetry")] - { - subscriber = registry - .with(reload_filter) - .with(fmt_layer) - .with(sentry_layer); + #[cfg(feature = "tokio_console")] + let subscriber = { + let console_layer = console_subscriber::spawn(); + subscriber.with(console_layer) }; - #[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental - #[cfg(not(feature = "sentry_telemetry"))] - { - subscriber = registry.with(reload_filter).with(fmt_layer); + let (fmt_reload_filter, fmt_reload_handle) = reload::Layer::new(filter_layer.clone()); + reload_handles.push(Box::new(fmt_reload_handle)); + let subscriber = subscriber.with(fmt_layer.with_filter(fmt_reload_filter)); + + #[cfg(feature = "sentry_telemetry")] + let subscriber = { + let sentry_layer = sentry_tracing::layer(); + let (sentry_reload_filter, sentry_reload_handle) = reload::Layer::new(filter_layer); + reload_handles.push(Box::new(sentry_reload_handle)); + subscriber.with(sentry_layer.with_filter(sentry_reload_filter)) }; tracing::subscriber::set_global_default(subscriber).unwrap(); - reload_handle + #[cfg(all(feature = "tokio_console", feature = "release_max_log_level"))] + error!( + "'tokio_console' feature and 'release_max_log_level' feature are incompatible, because console-subscriber \ + needs access to trace-level events. 'release_max_log_level' must be disabled to use tokio-console." + ); + + LogLevelReloadHandles::new(reload_handles) } #[cfg(feature = "perf_measurements")] -fn init_tracing_jaeger(config: &Config) -> reload::Handle { +fn init_tracing_jaeger(config: &Config) -> LogLevelReloadHandles { opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); let tracer = opentelemetry_jaeger::new_agent_pipeline() .with_auto_split_batch(true) @@ -609,11 +663,12 @@ fn init_tracing_jaeger(config: &Config) -> reload::Handle { tracing::subscriber::set_global_default(subscriber).unwrap(); - reload_handle + LogLevelReloadHandles::new(vec![Box::new(reload_handle)]) } +// TODO: tokio-console here? #[cfg(feature = "perf_measurements")] -fn init_tracing_flame(_config: &Config) -> reload::Handle { +fn init_tracing_flame(_config: &Config) -> LogLevelReloadHandles { let registry = Registry::default(); let (flame_layer, _guard) = tracing_flame::FlameLayer::with_file("./tracing.folded").unwrap(); let flame_layer = flame_layer.with_empty_samples(false); @@ -626,7 +681,7 @@ fn init_tracing_flame(_config: &Config) -> reload::Handle { tracing::subscriber::set_global_default(subscriber).unwrap(); - reload_handle + LogLevelReloadHandles::new(vec![Box::new(reload_handle)]) } // This is needed for opening lots of file descriptors, which tends to diff --git a/src/service/admin/debug/debug_commands.rs b/src/service/admin/debug/debug_commands.rs index b3c84e40..c91c8f4f 100644 --- a/src/service/admin/debug/debug_commands.rs +++ b/src/service/admin/debug/debug_commands.rs @@ -331,7 +331,7 @@ pub(crate) async fn change_log_level( match services() .globals .tracing_reload_handle - .modify(|filter| *filter = old_filter_layer) + .reload(&old_filter_layer) { Ok(()) => { return Ok(RoomMessageEventContent::text_plain(format!( @@ -360,7 +360,7 @@ pub(crate) async fn change_log_level( match services() .globals .tracing_reload_handle - .modify(|filter| *filter = new_filter_layer) + .reload(&new_filter_layer) { Ok(()) => { return Ok(RoomMessageEventContent::text_plain("Successfully changed log level")); diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 31d81141..b58ebbee 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -27,10 +27,9 @@ use ruma::{ }; use tokio::sync::{broadcast, watch::Receiver, Mutex, RwLock}; use tracing::{error, info, trace}; -use tracing_subscriber::{EnvFilter, Registry}; use url::Url; -use crate::{services, Config, Result}; +use crate::{services, Config, LogLevelReloadHandles, Result}; mod client; mod data; @@ -45,7 +44,7 @@ type SyncHandle = ( pub(crate) struct Service<'a> { pub(crate) db: &'static dyn Data, - pub(crate) tracing_reload_handle: tracing_subscriber::reload::Handle, + pub(crate) tracing_reload_handle: LogLevelReloadHandles, pub(crate) config: Config, pub(crate) cidr_range_denylist: Vec, keypair: Arc, @@ -99,8 +98,7 @@ impl Default for RotationHandler { impl Service<'_> { pub(crate) fn load( - db: &'static dyn Data, config: &Config, - tracing_reload_handle: tracing_subscriber::reload::Handle, + db: &'static dyn Data, config: &Config, tracing_reload_handle: LogLevelReloadHandles, ) -> Result { let keypair = db.load_keypair(); diff --git a/src/service/mod.rs b/src/service/mod.rs index 6dd7d447..86fd103a 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -6,7 +6,7 @@ use std::{ use lru_cache::LruCache; use tokio::sync::{broadcast, Mutex, RwLock}; -use crate::{Config, Result}; +use crate::{Config, LogLevelReloadHandles, Result}; pub(crate) mod account_data; pub(crate) mod admin; @@ -55,11 +55,7 @@ impl Services<'_> { + sending::Data + 'static, >( - db: &'static D, config: &Config, - tracing_reload_handle: tracing_subscriber::reload::Handle< - tracing_subscriber::EnvFilter, - tracing_subscriber::Registry, - >, + db: &'static D, config: &Config, tracing_reload_handle: LogLevelReloadHandles, ) -> Result { Ok(Self { appservice: appservice::Service::build(db)?,