From 71341ea05ad6bbb8dad01d023024b287f9debd2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Fri, 3 Sep 2021 11:26:15 +0200 Subject: [PATCH 001/201] fix: make sure old events don't sneek into the timeline --- src/server_server.rs | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/server_server.rs b/src/server_server.rs index bac7203c..a6557cc1 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -977,6 +977,12 @@ pub(crate) async fn handle_incoming_pdu<'a>( .map_err(|_| "Failed to ask database for event.".to_owned())? .ok_or_else(|| "Failed to find create event in db.".to_owned())?; + let first_pdu_in_room = db + .rooms + .first_pdu_in_room(&room_id) + .map_err(|_| "Error loading first room event.".to_owned())? + .expect("Room exists"); + let (incoming_pdu, val) = handle_outlier_pdu( origin, &create_event, @@ -993,13 +999,7 @@ pub(crate) async fn handle_incoming_pdu<'a>( return Ok(None); } - if incoming_pdu.origin_server_ts - < db.rooms - .first_pdu_in_room(&room_id) - .map_err(|_| "Error loading first room event.".to_owned())? - .expect("Room exists") - .origin_server_ts - { + if incoming_pdu.origin_server_ts < first_pdu_in_room.origin_server_ts { return Ok(None); } @@ -1037,13 +1037,7 @@ pub(crate) async fn handle_incoming_pdu<'a>( if let Some(json) = json_opt.or_else(|| db.rooms.get_outlier_pdu_json(&prev_event_id).ok().flatten()) { - if pdu.origin_server_ts - > db.rooms - .first_pdu_in_room(&room_id) - .map_err(|_| "Error loading first room event.".to_owned())? - .expect("Room exists") - .origin_server_ts - { + if pdu.origin_server_ts > first_pdu_in_room.origin_server_ts { amount += 1; for prev_prev in &pdu.prev_events { if !graph.contains_key(prev_prev) { @@ -1095,6 +1089,10 @@ pub(crate) async fn handle_incoming_pdu<'a>( break; } if let Some((pdu, json)) = eventid_info.remove(&prev_id) { + if pdu.origin_server_ts < first_pdu_in_room.origin_server_ts { + continue; + } + let start_time = Instant::now(); let event_id = pdu.event_id.clone(); if let Err(e) = upgrade_outlier_to_timeline_pdu( From bbe36810ec0be944769a34f7d3927eb6dd07a8f7 Mon Sep 17 00:00:00 2001 From: Daniel Wiesenberg Date: Fri, 3 Sep 2021 21:51:17 +0200 Subject: [PATCH 002/201] Fix deprecated/removed Traefik label --- docker/docker-compose.override.traefik.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/docker-compose.override.traefik.yml b/docker/docker-compose.override.traefik.yml index 56333483..9525078d 100644 --- a/docker/docker-compose.override.traefik.yml +++ b/docker/docker-compose.override.traefik.yml @@ -12,7 +12,7 @@ services: - "traefik.http.routers.to-conduit.tls.certresolver=letsencrypt" - "traefik.http.routers.to-conduit.middlewares=cors-headers@docker" - - "traefik.http.middlewares.cors-headers.headers.accessControlAllowOrigin=*" + - "traefik.http.middlewares.cors-headers.headers.accessControlAllowOriginList=*" - "traefik.http.middlewares.cors-headers.headers.accessControlAllowHeaders=Origin, X-Requested-With, Content-Type, Accept, Authorization" - "traefik.http.middlewares.cors-headers.headers.accessControlAllowMethods=GET, POST, PUT, DELETE, OPTIONS" @@ -29,7 +29,7 @@ services: - "traefik.http.routers.to-matrix-wellknown.tls.certresolver=letsencrypt" - "traefik.http.routers.to-matrix-wellknown.middlewares=cors-headers@docker" - - "traefik.http.middlewares.cors-headers.headers.accessControlAllowOrigin=*" + - "traefik.http.middlewares.cors-headers.headers.accessControlAllowOriginList=*" - "traefik.http.middlewares.cors-headers.headers.accessControlAllowHeaders=Origin, X-Requested-With, Content-Type, Accept, Authorization" - "traefik.http.middlewares.cors-headers.headers.accessControlAllowMethods=GET, POST, PUT, DELETE, OPTIONS" From 487046571c5561c554558db9fdb17f27b0ca1215 Mon Sep 17 00:00:00 2001 From: charludo Date: Mon, 6 Sep 2021 18:35:35 +0000 Subject: [PATCH 003/201] These lines get *generated* by certbot. Having them in the file before running certbot results in an apache2 error, and putting them in afterwards is not necessary, since certbot places them there on its own. --- DEPLOY.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/DEPLOY.md b/DEPLOY.md index 3a81eb01..84dd2beb 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -162,9 +162,6 @@ AllowEncodedSlashes NoDecode ProxyPass /_matrix/ http://127.0.0.1:6167/_matrix/ nocanon ProxyPassReverse /_matrix/ http://127.0.0.1:6167/_matrix/ -Include /etc/letsencrypt/options-ssl-apache.conf -SSLCertificateFile /etc/letsencrypt/live/your.server.name/fullchain.pem # EDIT THIS -SSLCertificateKeyFile /etc/letsencrypt/live/your.server.name/privkey.pem # EDIT THIS ``` From 5821b8e705ea3b0fcb23ebac6d4d61f6400f760e Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Tue, 7 Sep 2021 15:41:01 +0200 Subject: [PATCH 004/201] Remove unused dependencies --- Cargo.lock | 67 ++---------------------------------------------------- Cargo.toml | 2 -- 2 files changed, 2 insertions(+), 67 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a60ca29f..86ef48c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "adler32" version = "1.2.0" @@ -251,7 +253,6 @@ dependencies = [ "opentelemetry", "opentelemetry-jaeger", "parking_lot", - "pretty_env_logger", "rand 0.8.4", "regex", "reqwest", @@ -272,7 +273,6 @@ dependencies = [ "tokio", "tracing", "tracing-flame", - "tracing-opentelemetry", "tracing-subscriber", "trust-dns-resolver", "webpki 0.22.0", @@ -590,19 +590,6 @@ dependencies = [ "syn", ] -[[package]] -name = "env_logger" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" -dependencies = [ - "atty", - "humantime", - "log", - "regex", - "termcolor", -] - [[package]] name = "fallible-iterator" version = "0.2.0" @@ -955,15 +942,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440" -[[package]] -name = "humantime" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" -dependencies = [ - "quick-error", -] - [[package]] name = "hyper" version = "0.14.12" @@ -1629,16 +1607,6 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" -[[package]] -name = "pretty_env_logger" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d" -dependencies = [ - "env_logger", - "log", -] - [[package]] name = "proc-macro-crate" version = "1.0.0" @@ -2721,15 +2689,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "termcolor" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" -dependencies = [ - "winapi-util", -] - [[package]] name = "thiserror" version = "1.0.28" @@ -2991,19 +2950,6 @@ dependencies = [ "tracing-core", ] -[[package]] -name = "tracing-opentelemetry" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "599f388ecb26b28d9c1b2e4437ae019a7b336018b45ed911458cd9ebf91129f6" -dependencies = [ - "opentelemetry", - "tracing", - "tracing-core", - "tracing-log", - "tracing-subscriber", -] - [[package]] name = "tracing-serde" version = "0.1.2" @@ -3345,15 +3291,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -[[package]] -name = "winapi-util" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" -dependencies = [ - "winapi", -] - [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index a78307a1..04887d0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,11 +68,9 @@ jsonwebtoken = "7.2.0" # Performance measurements tracing = { version = "0.1.26", features = ["release_max_level_warn"] } tracing-subscriber = "0.2.20" -tracing-opentelemetry = "0.15.0" tracing-flame = "0.1.0" opentelemetry = { version = "0.16.0", features = ["rt-tokio"] } opentelemetry-jaeger = { version = "0.15.0", features = ["rt-tokio"] } -pretty_env_logger = "0.4.0" lru-cache = "0.1.2" rusqlite = { version = "0.25.3", optional = true, features = ["bundled"] } parking_lot = { version = "0.11.2", optional = true } From 51245d34f172f9025699e8d9653be62625872a49 Mon Sep 17 00:00:00 2001 From: Rasmus Thomsen Date: Tue, 7 Sep 2021 19:41:14 +0100 Subject: [PATCH 005/201] fix(database): handle errors in config parsin or database creation Showing the user a backtrace can be pretty confusing, so just printing a nice error message makes errors easier to understand for end users. fixes #121 --- src/database.rs | 5 +++++ src/main.rs | 23 +++++++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/database.rs b/src/database.rs index 6ea0abdc..5fb6de46 100644 --- a/src/database.rs +++ b/src/database.rs @@ -198,6 +198,11 @@ impl Database { pub async fn load_or_create(config: &Config) -> Result>> { Self::check_sled_or_sqlite_db(&config)?; + if !Path::new(&config.database_path).exists() { + std::fs::create_dir_all(&config.database_path) + .map_err(|_| Error::BadConfig("Database folder doesn't exists and couldn't be created (e.g. due to missing permissions). Please create the database folder yourself."))?; + } + let builder = Engine::open(&config)?; if config.max_request_size < 1024 { diff --git a/src/main.rs b/src/main.rs index 2ca49e25..06409eeb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -199,16 +199,27 @@ async fn main() { std::env::set_var("RUST_LOG", "warn"); - let config = raw_config - .extract::() - .expect("It looks like your config is invalid. Please take a look at the error"); + let config = match raw_config.extract::() { + Ok(s) => s, + Err(e) => { + eprintln!("It looks like your config is invalid. The following error occured while parsing it: {}", e); + std::process::exit(1); + } + }; let start = async { config.warn_deprecated(); - let db = Database::load_or_create(&config) - .await - .expect("config is valid"); + let db = match Database::load_or_create(&config).await { + Ok(db) => db, + Err(e) => { + eprintln!( + "The database couldn't be loaded or created. The following error occured: {}", + e + ); + std::process::exit(1); + } + }; let rocket = setup_rocket(raw_config, Arc::clone(&db)) .ignite() From 23c5ec8099829a6d8115d09ee9a9cf6a0380f848 Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Wed, 8 Sep 2021 14:50:44 +0200 Subject: [PATCH 006/201] fix sync not firing on new events in room --- src/database.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/database.rs b/src/database.rs index 5fb6de46..dcba2abf 100644 --- a/src/database.rs +++ b/src/database.rs @@ -815,12 +815,21 @@ impl Database { // Events for rooms we are in for room_id in self.rooms.rooms_joined(user_id).filter_map(|r| r.ok()) { + let short_roomid = self + .rooms + .get_shortroomid(&room_id) + .ok() + .flatten() + .expect("room exists") + .to_be_bytes() + .to_vec(); + let roomid_bytes = room_id.as_bytes().to_vec(); let mut roomid_prefix = roomid_bytes.clone(); roomid_prefix.push(0xff); // PDUs - futures.push(self.rooms.pduid_pdu.watch_prefix(&roomid_prefix)); + futures.push(self.rooms.pduid_pdu.watch_prefix(&short_roomid)); // EDUs futures.push( From 00927a7ce367df88f1230e2ba60492fda0901750 Mon Sep 17 00:00:00 2001 From: Daniel Wiesenberg Date: Thu, 9 Sep 2021 10:02:11 +0200 Subject: [PATCH 007/201] Add mautrix-signal appservice instructions --- APPSERVICES.md | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/APPSERVICES.md b/APPSERVICES.md index a84f1d26..ba9ae89a 100644 --- a/APPSERVICES.md +++ b/APPSERVICES.md @@ -7,9 +7,32 @@ If you run into any problems while setting up an Appservice, write an email to ` ## Tested appservices Here are some appservices we tested and that work with Conduit: -- matrix-appservice-discord -- mautrix-hangouts -- mautrix-telegram +- [matrix-appservice-discord](https://github.com/Half-Shot/matrix-appservice-discord) +- [mautrix-hangouts](https://github.com/mautrix/hangouts/) +- [mautrix-telegram](https://github.com/mautrix/telegram/) +- [mautrix-signal](https://github.com/mautrix/signal) + - There are a few things you need to do, in order for the bridge (at least up to version `0.2.0`) to work. Before following the bridge installation guide, you need to map apply a patch to bridges `portal.py`. Go to [portal.py](https://github.com/mautrix/signal/blob/master/mautrix_signal/portal.py) at [mautrix-signal](https://github.com/mautrix/signal) (don't forget to change to the correct commit/version of the file) and copy its content, create a `portal.py` on your host system and paste it in. Now you need to change two lines: + [Line 1020](https://github.com/mautrix/signal/blob/4ea831536f154aba6419d13292479eb383ea3308/mautrix_signal/portal.py#L1020) + + ```diff + --- levels.users[self.main_intent.mxid] = 9001 if is_initial else 100 + +++ levels.users[self.main_intent.mxid] = 100 if is_initial else 100 + ``` + + and add a new line between [Lines 1041 and 1042](https://github.com/mautrix/signal/blob/4ea831536f154aba6419d13292479eb383ea3308/mautrix_signal/portal.py#L1041-L1042) + + ```diff + "type": str(EventType.ROOM_POWER_LEVELS), + +++ "state_key": "", + "content": power_levels.serialize(), + ``` + + Now you just need to map the patched `portal.py` into the `mautrix-signal` container + ```yml + volumes: + - ./////portal.py:/usr/lib/python3.9/site-packages/mautrix_signal/portal.py + ``` + and then read below and start following the bridge [installation instructions](https://docs.mau.fi/bridges/index.html). ## Set up the appservice From 2c8412fe58efa91a467fad34cfa93dca57204b92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 13 Sep 2021 20:11:50 +0200 Subject: [PATCH 008/201] improvement: more efficient sqlite --- src/database/abstraction/sqlite.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/database/abstraction/sqlite.rs b/src/database/abstraction/sqlite.rs index 06e371e0..feac690e 100644 --- a/src/database/abstraction/sqlite.rs +++ b/src/database/abstraction/sqlite.rs @@ -56,7 +56,7 @@ impl Engine { conn.pragma_update(Some(Main), "journal_mode", &"WAL")?; conn.pragma_update(Some(Main), "synchronous", &"NORMAL")?; conn.pragma_update(Some(Main), "cache_size", &(-i64::from(cache_size_kb)))?; - conn.pragma_update(Some(Main), "wal_autocheckpoint", &2000)?; + conn.pragma_update(Some(Main), "wal_autocheckpoint", &0)?; Ok(conn) } @@ -77,7 +77,7 @@ impl Engine { pub fn flush_wal(self: &Arc) -> Result<()> { self.write_lock() - .pragma_update(Some(Main), "wal_checkpoint", &"TRUNCATE")?; + .pragma_update(Some(Main), "wal_checkpoint", &"RESTART")?; Ok(()) } } From 910ad7fed1dad053f81b721d2d021d320d793bd5 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Sun, 5 Sep 2021 00:05:59 +0200 Subject: [PATCH 009/201] Get rid of more unnecessary intermediate collections --- src/server_server.rs | 69 +++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 36 deletions(-) diff --git a/src/server_server.rs b/src/server_server.rs index a6557cc1..c27ea228 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1396,49 +1396,46 @@ async fn upgrade_outlier_to_timeline_pdu( } if okay { - let fork_states: Vec<_> = extremity_sstatehashes - .into_iter() - .map(|(sstatehash, prev_event)| { - let mut leaf_state = db + let mut fork_states = Vec::with_capacity(extremity_sstatehashes.len()); + let mut auth_chain_sets = Vec::with_capacity(extremity_sstatehashes.len()); + + for (sstatehash, prev_event) in extremity_sstatehashes { + let mut leaf_state: BTreeMap<_, _> = db + .rooms + .state_full_ids(sstatehash) + .map_err(|_| "Failed to ask db for room state.".to_owned())?; + + if let Some(state_key) = &prev_event.state_key { + let shortstatekey = db .rooms - .state_full_ids(sstatehash) - .map_err(|_| "Failed to ask db for room state.".to_owned())?; + .get_or_create_shortstatekey(&prev_event.kind, state_key, &db.globals) + .map_err(|_| "Failed to create shortstatekey.".to_owned())?; + leaf_state.insert(shortstatekey, Arc::new(prev_event.event_id.clone())); + // Now it's the state after the pdu + } - if let Some(state_key) = &prev_event.state_key { - let shortstatekey = db - .rooms - .get_or_create_shortstatekey(&prev_event.kind, state_key, &db.globals) - .map_err(|_| "Failed to create shortstatekey.".to_owned())?; - leaf_state.insert(shortstatekey, Arc::new(prev_event.event_id.clone())); - // Now it's the state after the pdu - } + let mut state = StateMap::with_capacity(leaf_state.len()); + let mut starting_events = Vec::with_capacity(leaf_state.len()); - leaf_state - .into_iter() - .map(|(k, id)| (db.rooms.get_statekey_from_short(k).map(|k| (k, id)))) - .collect::>>() - .map_err(|_| "Failed to get_statekey_from_short.".to_owned()) - }) - .collect::>()?; + for (k, id) in leaf_state { + let k = db + .rooms + .get_statekey_from_short(k) + .map_err(|_| "Failed to get_statekey_from_short.".to_owned())?; + + state.insert(k, (*id).clone()); + starting_events.push(id); + } - let mut auth_chain_sets = Vec::new(); - for state in &fork_states { auth_chain_sets.push( - get_auth_chain( - &room_id, - state.iter().map(|(_, id)| id.clone()).collect(), - db, - ) - .map_err(|_| "Failed to load auth chain.".to_owned())? - .map(|event_id| (*event_id).clone()) - .collect(), + get_auth_chain(&room_id, starting_events, db) + .map_err(|_| "Failed to load auth chain.".to_owned())? + .map(|event_id| (*event_id).clone()) + .collect(), ); - } - let fork_states = &fork_states - .into_iter() - .map(|map| map.into_iter().map(|(k, id)| (k, (*id).clone())).collect()) - .collect::>(); + fork_states.push(state); + } state_at_incoming_event = match state_res::StateResolution::resolve( &room_id, From 979ec6b4fa5cd5b415f56ebc8c59ba742f835eb6 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Wed, 1 Sep 2021 15:28:02 +0200 Subject: [PATCH 010/201] Upgrade ruma --- Cargo.lock | 54 ++++++++++------------------- Cargo.toml | 4 +-- src/client_server/membership.rs | 6 ++-- src/database/rooms.rs | 13 +++---- src/pdu.rs | 30 ++++++++++------ src/server_server.rs | 61 ++++++++++++++++----------------- 6 files changed, 76 insertions(+), 92 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 86ef48c1..70d7f4b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1968,8 +1968,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668031e3108d6a2cfbe6eca271d8698f4593440e71a44afdadcf67ce3cb93c1f" +source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" dependencies = [ "assign", "js_int", @@ -1990,8 +1989,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.18.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5f1843792b6749ec1ece62595cf99ad30bf9589c96bb237515235e71da396ea" +source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" dependencies = [ "bytes", "http", @@ -2007,8 +2005,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.18.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b18abda5cca94178d08b622bca042e1cbb5eb7d4ebf3a2a81590a3bb3c57008" +source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2019,8 +2016,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49369332a5f299e832e19661f92d49e08c345c3c6c4ab16e09cb31c5ff6da878" +source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" dependencies = [ "ruma-api", "ruma-common", @@ -2034,8 +2030,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9568a222c12cf6220e751484ab78feec28071f85965113a5bb802936a2920ff0" +source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" dependencies = [ "assign", "bytes", @@ -2055,8 +2050,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d5b7605f58dc0d9cf1848cc7f1af2bae4e4bcd1d2b7a87bbb9864c8a785b91" +source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" dependencies = [ "indexmap", "js_int", @@ -2071,8 +2065,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.24.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87801e1207cfebdee02e7997ebf181a1c9837260b78c1b8ce96b896a2bcb3763" +source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" dependencies = [ "indoc", "js_int", @@ -2088,8 +2081,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.24.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5da4498845347de88adf1b7da4578e2ca7355ad4ce47b0976f6594bacf958660" +source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2100,8 +2092,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa3d1db1a064ab26484df6ef5d96c384fc053022004f34d96c3b4939e13dc204" +source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" dependencies = [ "js_int", "ruma-api", @@ -2116,8 +2107,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb417d091e8dd5a633e4e5998231a156049d7fcc221045cfdc0642eb72067732" +source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" dependencies = [ "paste", "rand 0.8.4", @@ -2131,8 +2121,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c708edad7f605638f26c951cbad7501fbf28ab01009e5ca65ea5a2db74a882b1" +source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" dependencies = [ "quote", "ruma-identifiers-validation", @@ -2142,14 +2131,12 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42285e7fb5d5f2d5268e45bb683e36d5c6fd9fc1e11a4559ba3c3521f3bbb2cb" +source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" [[package]] name = "ruma-identity-service-api" version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e76e66e24f2d5a31511fbf6c79e79f67a7a6a98ebf48d72381b7d5bb6c09f035" +source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" dependencies = [ "js_int", "ruma-api", @@ -2162,8 +2149,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef5b29da7065efc5b1e1a8f61add7543c9ab4ecce5ee0dd1c1c5ecec83fbeec" +source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" dependencies = [ "js_int", "ruma-api", @@ -2178,8 +2164,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2b22aae842e7ecda695e42b7b39d4558959d9d9a27acc2a16acf4f4f7f00c3" +source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" dependencies = [ "bytes", "form_urlencoded", @@ -2193,8 +2178,7 @@ dependencies = [ [[package]] name = "ruma-serde-macros" version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243e9bef188b08f94c79bc2f8fd1eb307a9e636b2b8e4571acf8c7be16381d28" +source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2205,8 +2189,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a4f64027165b59500162d10d435b1253898bf3ad4f5002cb0d56913fe7f76d7" +source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" dependencies = [ "base64 0.13.0", "ed25519-dalek", @@ -2223,8 +2206,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "518c1afbddfcc5ffac8818a5cf0902709e6eca11aca8f24f6479df6f0601f1ba" +source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" dependencies = [ "itertools 0.10.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 04887d0f..593a1fd3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,8 +18,8 @@ edition = "2018" rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle requests # Used for matrix spec type definitions and helpers -ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } -#ruma = { git = "https://github.com/ruma/ruma", rev = "f5ab038e22421ed338396ece977b6b2844772ced", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +#ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +ruma = { git = "https://github.com/ruma/ruma", rev = "a6a1224652912a957b09f136ec5da2686be6e0e2", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { git = "https://github.com/timokoesters/ruma", rev = "50c1db7e0a3a21fc794b0cce3b64285a4c750c71", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 52e074c2..01c19c21 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -976,10 +976,10 @@ pub(crate) async fn invite_helper<'a>( let auth_check = state_res::auth_check( &room_version, - &Arc::new(pdu.clone()), + &pdu, create_prev_event, - None, // TODO: third_party_invite - |k, s| auth_events.get(&(k.clone(), s.to_owned())).map(Arc::clone), + None::, // TODO: third_party_invite + |k, s| auth_events.get(&(k.clone(), s.to_owned())), ) .map_err(|e| { error!("{:?}", e); diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 42454833..51023ba9 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -252,12 +252,7 @@ impl Rooms { return Ok(HashMap::new()); }; - let auth_events = state_res::auth_types_for_event( - kind, - sender, - state_key.map(|s| s.to_string()), - content.clone(), - ); + let auth_events = state_res::auth_types_for_event(kind, sender, state_key, &content); let mut sauthevents = auth_events .into_iter() @@ -2046,10 +2041,10 @@ impl Rooms { let auth_check = state_res::auth_check( &room_version, - &Arc::new(pdu.clone()), + &pdu, create_prev_event, - None, // TODO: third_party_invite - |k, s| auth_events.get(&(k.clone(), s.to_owned())).map(Arc::clone), + None::, // TODO: third_party_invite + |k, s| auth_events.get(&(k.clone(), s.to_owned())), ) .map_err(|e| { error!("{:?}", e); diff --git a/src/pdu.rs b/src/pdu.rs index 1016fe66..8623b1a0 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -260,37 +260,47 @@ impl state_res::Event for PduEvent { fn sender(&self) -> &UserId { &self.sender } - fn kind(&self) -> EventType { - self.kind.clone() + + fn event_type(&self) -> &EventType { + &self.kind } - fn content(&self) -> serde_json::Value { - self.content.clone() + fn content(&self) -> &serde_json::Value { + &self.content } + fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch { MilliSecondsSinceUnixEpoch(self.origin_server_ts) } - fn state_key(&self) -> Option { - self.state_key.clone() + + fn state_key(&self) -> Option<&str> { + self.state_key.as_deref() } - fn prev_events(&self) -> Vec { - self.prev_events.to_vec() + + fn prev_events(&self) -> Box + '_> { + Box::new(self.prev_events.iter()) } + fn depth(&self) -> &UInt { &self.depth } - fn auth_events(&self) -> Vec { - self.auth_events.to_vec() + + fn auth_events(&self) -> Box + '_> { + Box::new(self.auth_events.iter()) } + fn redacts(&self) -> Option<&EventId> { self.redacts.as_ref() } + fn hashes(&self) -> &EventHash { &self.hashes } + fn signatures(&self) -> BTreeMap, BTreeMap> { self.signatures.clone() } + fn unsigned(&self) -> &BTreeMap { &self.unsigned } diff --git a/src/server_server.rs b/src/server_server.rs index c27ea228..89b9c5cf 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1065,23 +1065,22 @@ pub(crate) async fn handle_incoming_pdu<'a>( } } - let sorted = - state_res::StateResolution::lexicographical_topological_sort(dbg!(&graph), |event_id| { - // This return value is the key used for sorting events, - // events are then sorted by power level, time, - // and lexically by event_id. - println!("{}", event_id); - Ok(( - 0, - MilliSecondsSinceUnixEpoch( - eventid_info - .get(event_id) - .map_or_else(|| uint!(0), |info| info.0.origin_server_ts), - ), - ruma::event_id!("$notimportant"), - )) - }) - .map_err(|_| "Error sorting prev events".to_owned())?; + let sorted = state_res::lexicographical_topological_sort(dbg!(&graph), |event_id| { + // This return value is the key used for sorting events, + // events are then sorted by power level, time, + // and lexically by event_id. + println!("{}", event_id); + Ok(( + 0, + MilliSecondsSinceUnixEpoch( + eventid_info + .get(event_id) + .map_or_else(|| uint!(0), |info| info.0.origin_server_ts), + ), + ruma::event_id!("$notimportant"), + )) + }) + .map_err(|_| "Error sorting prev events".to_owned())?; let mut errors = 0; for prev_id in dbg!(sorted) { @@ -1280,8 +1279,8 @@ fn handle_outlier_pdu<'a>( &room_version, &incoming_pdu, previous_create, - None, // TODO: third party invite - |k, s| auth_events.get(&(k.clone(), s.to_owned())).map(Arc::clone), + None::, // TODO: third party invite + |k, s| auth_events.get(&(k.clone(), s.to_owned())), ) .map_err(|_e| "Auth check failed".to_string())? { @@ -1437,8 +1436,7 @@ async fn upgrade_outlier_to_timeline_pdu( fork_states.push(state); } - state_at_incoming_event = match state_res::StateResolution::resolve( - &room_id, + state_at_incoming_event = match state_res::resolve( room_version_id, &fork_states, auth_chain_sets, @@ -1566,8 +1564,8 @@ async fn upgrade_outlier_to_timeline_pdu( let check_result = state_res::event_auth::auth_check( &room_version, &incoming_pdu, - previous_create.clone(), - None, // TODO: third party invite + previous_create.as_deref(), + None::, // TODO: third party invite |k, s| { db.rooms .get_shortstatekey(&k, &s) @@ -1650,9 +1648,9 @@ async fn upgrade_outlier_to_timeline_pdu( let soft_fail = !state_res::event_auth::auth_check( &room_version, &incoming_pdu, - previous_create, - None, - |k, s| auth_events.get(&(k.clone(), s.to_owned())).map(Arc::clone), + previous_create.as_deref(), + None::, + |k, s| auth_events.get(&(k.clone(), s.to_owned())), ) .map_err(|_e| "Auth check failed.".to_owned())?; @@ -1795,8 +1793,7 @@ async fn upgrade_outlier_to_timeline_pdu( .collect::>>() .map_err(|_| "Failed to get_statekey_from_short.".to_owned())?; - let state = match state_res::StateResolution::resolve( - &room_id, + let state = match state_res::resolve( room_version_id, fork_states, auth_chain_sets, @@ -2773,10 +2770,10 @@ pub fn create_join_event_template_route( let auth_check = state_res::auth_check( &room_version, - &Arc::new(pdu.clone()), - create_prev_event, - None, // TODO: third_party_invite - |k, s| auth_events.get(&(k.clone(), s.to_owned())).map(Arc::clone), + &pdu, + create_prev_event.as_deref(), + None::, // TODO: third_party_invite + |k, s| auth_events.get(&(k.clone(), s.to_owned())), ) .map_err(|e| { error!("{:?}", e); From d68c93b5fa0a7fcbf749fc15cbddd82da29b285d Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Mon, 13 Sep 2021 19:45:56 +0200 Subject: [PATCH 011/201] Clean up (mostly automated with cargo clippy --fix) --- src/appservice_server.rs | 2 +- src/client_server/account.rs | 24 ++--- src/client_server/alias.rs | 4 +- src/client_server/backup.rs | 39 ++++---- src/client_server/capabilities.rs | 3 +- src/client_server/context.rs | 4 +- src/client_server/device.rs | 22 ++--- src/client_server/keys.rs | 18 ++-- src/client_server/media.rs | 3 +- src/client_server/membership.rs | 66 +++++++------- src/client_server/message.rs | 6 +- src/client_server/presence.rs | 10 +-- src/client_server/profile.rs | 34 +++---- src/client_server/push.rs | 52 ++++------- src/client_server/read_marker.rs | 14 +-- src/client_server/redact.rs | 2 +- src/client_server/room.rs | 32 +++---- src/client_server/search.rs | 4 +- src/client_server/session.rs | 8 +- src/client_server/state.rs | 6 +- src/client_server/to_device.rs | 8 +- src/client_server/typing.rs | 4 +- src/database.rs | 14 +-- src/database/abstraction/sqlite.rs | 4 +- src/database/account_data.rs | 8 +- src/database/globals.rs | 2 +- src/database/key_backups.rs | 28 +++--- src/database/media.rs | 5 +- src/database/pusher.rs | 6 +- src/database/rooms.rs | 137 ++++++++++++++--------------- src/database/rooms/edus.rs | 26 +++--- src/database/sending.rs | 20 ++--- src/database/uiaa.rs | 2 +- src/database/users.rs | 22 ++--- src/ruma_wrapper.rs | 4 +- src/server_server.rs | 114 ++++++++++++------------ 36 files changed, 364 insertions(+), 393 deletions(-) diff --git a/src/appservice_server.rs b/src/appservice_server.rs index 8be524c2..ed886d6c 100644 --- a/src/appservice_server.rs +++ b/src/appservice_server.rs @@ -21,7 +21,7 @@ where let hs_token = registration.get("hs_token").unwrap().as_str().unwrap(); let mut http_request = request - .try_into_http_request::(&destination, SendAccessToken::IfRequired("")) + .try_into_http_request::(destination, SendAccessToken::IfRequired("")) .unwrap() .map(|body| body.freeze()); diff --git a/src/client_server/account.rs b/src/client_server/account.rs index e9300b50..fb338422 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -572,7 +572,7 @@ pub async fn change_password_route( if let Some(auth) = &body.auth { let (worked, uiaainfo) = db.uiaa.try_auth( - &sender_user, + sender_user, sender_device, auth, &uiaainfo, @@ -586,24 +586,24 @@ pub async fn change_password_route( } else if let Some(json) = body.json_body { uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); db.uiaa - .create(&sender_user, &sender_device, &uiaainfo, &json)?; + .create(sender_user, sender_device, &uiaainfo, &json)?; return Err(Error::Uiaa(uiaainfo)); } else { return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); } db.users - .set_password(&sender_user, Some(&body.new_password))?; + .set_password(sender_user, Some(&body.new_password))?; if body.logout_devices { // Logout all devices except the current one for id in db .users - .all_device_ids(&sender_user) + .all_device_ids(sender_user) .filter_map(|id| id.ok()) .filter(|id| id != sender_device) { - db.users.remove_device(&sender_user, &id)?; + db.users.remove_device(sender_user, &id)?; } } @@ -664,8 +664,8 @@ pub async fn deactivate_route( if let Some(auth) = &body.auth { let (worked, uiaainfo) = db.uiaa.try_auth( - &sender_user, - &sender_device, + sender_user, + sender_device, auth, &uiaainfo, &db.users, @@ -678,7 +678,7 @@ pub async fn deactivate_route( } else if let Some(json) = body.json_body { uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); db.uiaa - .create(&sender_user, &sender_device, &uiaainfo, &json)?; + .create(sender_user, sender_device, &uiaainfo, &json)?; return Err(Error::Uiaa(uiaainfo)); } else { return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); @@ -688,10 +688,10 @@ pub async fn deactivate_route( // TODO: work over federation invites let all_rooms = db .rooms - .rooms_joined(&sender_user) + .rooms_joined(sender_user) .chain( db.rooms - .rooms_invited(&sender_user) + .rooms_invited(sender_user) .map(|t| t.map(|(r, _)| r)), ) .collect::>(); @@ -726,7 +726,7 @@ pub async fn deactivate_route( state_key: Some(sender_user.to_string()), redacts: None, }, - &sender_user, + sender_user, &room_id, &db, &state_lock, @@ -734,7 +734,7 @@ pub async fn deactivate_route( } // Remove devices and mark account as deactivated - db.users.deactivate_account(&sender_user)?; + db.users.deactivate_account(sender_user)?; info!("{} deactivated their account", sender_user); diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index c806a9ce..129ac166 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -112,7 +112,7 @@ pub(crate) async fn get_alias_helper( } let mut room_id = None; - match db.rooms.id_from_alias(&room_alias)? { + match db.rooms.id_from_alias(room_alias)? { Some(r) => room_id = Some(r), None => { for (_id, registration) in db.appservice.all()? { @@ -140,7 +140,7 @@ pub(crate) async fn get_alias_helper( .await .is_ok() { - room_id = Some(db.rooms.id_from_alias(&room_alias)?.ok_or_else(|| { + room_id = Some(db.rooms.id_from_alias(room_alias)?.ok_or_else(|| { Error::bad_config("Appservice lied to us. Room does not exist.") })?); break; diff --git a/src/client_server/backup.rs b/src/client_server/backup.rs index 259f1a93..bbb86726 100644 --- a/src/client_server/backup.rs +++ b/src/client_server/backup.rs @@ -27,7 +27,7 @@ pub async fn create_backup_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let version = db .key_backups - .create_backup(&sender_user, &body.algorithm, &db.globals)?; + .create_backup(sender_user, &body.algorithm, &db.globals)?; db.flush()?; @@ -48,7 +48,7 @@ pub async fn update_backup_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); db.key_backups - .update_backup(&sender_user, &body.version, &body.algorithm, &db.globals)?; + .update_backup(sender_user, &body.version, &body.algorithm, &db.globals)?; db.flush()?; @@ -71,7 +71,7 @@ pub async fn get_latest_backup_route( let (version, algorithm) = db.key_backups - .get_latest_backup(&sender_user)? + .get_latest_backup(sender_user)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "Key backup does not exist.", @@ -101,7 +101,7 @@ pub async fn get_backup_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let algorithm = db .key_backups - .get_backup(&sender_user, &body.version)? + .get_backup(sender_user, &body.version)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "Key backup does not exist.", @@ -132,7 +132,7 @@ pub async fn delete_backup_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - db.key_backups.delete_backup(&sender_user, &body.version)?; + db.key_backups.delete_backup(sender_user, &body.version)?; db.flush()?; @@ -172,11 +172,11 @@ pub async fn add_backup_keys_route( for (room_id, room) in &body.rooms { for (session_id, key_data) in &room.sessions { db.key_backups.add_key( - &sender_user, + sender_user, &body.version, - &room_id, - &session_id, - &key_data, + room_id, + session_id, + key_data, &db.globals, )? } @@ -223,11 +223,11 @@ pub async fn add_backup_key_sessions_route( for (session_id, key_data) in &body.sessions { db.key_backups.add_key( - &sender_user, + sender_user, &body.version, &body.room_id, - &session_id, - &key_data, + session_id, + key_data, &db.globals, )? } @@ -272,7 +272,7 @@ pub async fn add_backup_key_session_route( } db.key_backups.add_key( - &sender_user, + sender_user, &body.version, &body.room_id, &body.session_id, @@ -303,7 +303,7 @@ pub async fn get_backup_keys_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let rooms = db.key_backups.get_all(&sender_user, &body.version)?; + let rooms = db.key_backups.get_all(sender_user, &body.version)?; Ok(get_backup_keys::Response { rooms }.into()) } @@ -324,7 +324,7 @@ pub async fn get_backup_key_sessions_route( let sessions = db .key_backups - .get_room(&sender_user, &body.version, &body.room_id)?; + .get_room(sender_user, &body.version, &body.room_id)?; Ok(get_backup_key_sessions::Response { sessions }.into()) } @@ -345,7 +345,7 @@ pub async fn get_backup_key_session_route( let key_data = db .key_backups - .get_session(&sender_user, &body.version, &body.room_id, &body.session_id)? + .get_session(sender_user, &body.version, &body.room_id, &body.session_id)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "Backup key not found for this user's session.", @@ -368,8 +368,7 @@ pub async fn delete_backup_keys_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - db.key_backups - .delete_all_keys(&sender_user, &body.version)?; + db.key_backups.delete_all_keys(sender_user, &body.version)?; db.flush()?; @@ -395,7 +394,7 @@ pub async fn delete_backup_key_sessions_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); db.key_backups - .delete_room_keys(&sender_user, &body.version, &body.room_id)?; + .delete_room_keys(sender_user, &body.version, &body.room_id)?; db.flush()?; @@ -421,7 +420,7 @@ pub async fn delete_backup_key_session_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); db.key_backups - .delete_room_key(&sender_user, &body.version, &body.room_id, &body.session_id)?; + .delete_room_key(sender_user, &body.version, &body.room_id, &body.session_id)?; db.flush()?; diff --git a/src/client_server/capabilities.rs b/src/client_server/capabilities.rs index 2eacd8f2..f86b23b5 100644 --- a/src/client_server/capabilities.rs +++ b/src/client_server/capabilities.rs @@ -1,5 +1,4 @@ -use crate::ConduitResult; -use crate::Ruma; +use crate::{ConduitResult, Ruma}; use ruma::{ api::client::r0::capabilities::{ get_capabilities, Capabilities, RoomVersionStability, RoomVersionsCapability, diff --git a/src/client_server/context.rs b/src/client_server/context.rs index aaae8d6b..b2346f5e 100644 --- a/src/client_server/context.rs +++ b/src/client_server/context.rs @@ -50,7 +50,7 @@ pub async fn get_context_route( let events_before = db .rooms - .pdus_until(&sender_user, &body.room_id, base_token)? + .pdus_until(sender_user, &body.room_id, base_token)? .take( u32::try_from(body.limit).map_err(|_| { Error::BadRequest(ErrorKind::InvalidParam, "Limit value is invalid.") @@ -72,7 +72,7 @@ pub async fn get_context_route( let events_after = db .rooms - .pdus_after(&sender_user, &body.room_id, base_token)? + .pdus_after(sender_user, &body.room_id, base_token)? .take( u32::try_from(body.limit).map_err(|_| { Error::BadRequest(ErrorKind::InvalidParam, "Limit value is invalid.") diff --git a/src/client_server/device.rs b/src/client_server/device.rs index 4aa3047e..100b591f 100644 --- a/src/client_server/device.rs +++ b/src/client_server/device.rs @@ -50,7 +50,7 @@ pub async fn get_device_route( let device = db .users - .get_device_metadata(&sender_user, &body.body.device_id)? + .get_device_metadata(sender_user, &body.body.device_id)? .ok_or(Error::BadRequest(ErrorKind::NotFound, "Device not found."))?; Ok(get_device::Response { device }.into()) @@ -72,13 +72,13 @@ pub async fn update_device_route( let mut device = db .users - .get_device_metadata(&sender_user, &body.device_id)? + .get_device_metadata(sender_user, &body.device_id)? .ok_or(Error::BadRequest(ErrorKind::NotFound, "Device not found."))?; device.display_name = body.display_name.clone(); db.users - .update_device_metadata(&sender_user, &body.device_id, &device)?; + .update_device_metadata(sender_user, &body.device_id, &device)?; db.flush()?; @@ -119,8 +119,8 @@ pub async fn delete_device_route( if let Some(auth) = &body.auth { let (worked, uiaainfo) = db.uiaa.try_auth( - &sender_user, - &sender_device, + sender_user, + sender_device, auth, &uiaainfo, &db.users, @@ -133,13 +133,13 @@ pub async fn delete_device_route( } else if let Some(json) = body.json_body { uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); db.uiaa - .create(&sender_user, &sender_device, &uiaainfo, &json)?; + .create(sender_user, sender_device, &uiaainfo, &json)?; return Err(Error::Uiaa(uiaainfo)); } else { return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); } - db.users.remove_device(&sender_user, &body.device_id)?; + db.users.remove_device(sender_user, &body.device_id)?; db.flush()?; @@ -182,8 +182,8 @@ pub async fn delete_devices_route( if let Some(auth) = &body.auth { let (worked, uiaainfo) = db.uiaa.try_auth( - &sender_user, - &sender_device, + sender_user, + sender_device, auth, &uiaainfo, &db.users, @@ -196,14 +196,14 @@ pub async fn delete_devices_route( } else if let Some(json) = body.json_body { uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); db.uiaa - .create(&sender_user, &sender_device, &uiaainfo, &json)?; + .create(sender_user, sender_device, &uiaainfo, &json)?; return Err(Error::Uiaa(uiaainfo)); } else { return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); } for device_id in &body.devices { - db.users.remove_device(&sender_user, &device_id)? + db.users.remove_device(sender_user, device_id)? } db.flush()?; diff --git a/src/client_server/keys.rs b/src/client_server/keys.rs index 3295e16b..a74c4097 100644 --- a/src/client_server/keys.rs +++ b/src/client_server/keys.rs @@ -158,8 +158,8 @@ pub async fn upload_signing_keys_route( if let Some(auth) = &body.auth { let (worked, uiaainfo) = db.uiaa.try_auth( - &sender_user, - &sender_device, + sender_user, + sender_device, auth, &uiaainfo, &db.users, @@ -172,7 +172,7 @@ pub async fn upload_signing_keys_route( } else if let Some(json) = body.json_body { uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); db.uiaa - .create(&sender_user, &sender_device, &uiaainfo, &json)?; + .create(sender_user, sender_device, &uiaainfo, &json)?; return Err(Error::Uiaa(uiaainfo)); } else { return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); @@ -181,7 +181,7 @@ pub async fn upload_signing_keys_route( if let Some(master_key) = &body.master_key { db.users.add_cross_signing_keys( sender_user, - &master_key, + master_key, &body.self_signing_key, &body.user_signing_key, &db.rooms, @@ -242,10 +242,10 @@ pub async fn upload_signatures_route( .to_owned(), ); db.users.sign_key( - &user_id, - &key_id, + user_id, + key_id, signature, - &sender_user, + sender_user, &db.rooms, &db.globals, )?; @@ -359,8 +359,8 @@ pub(crate) async fn get_keys_helper bool>( } else { for device_id in device_ids { let mut container = BTreeMap::new(); - if let Some(mut keys) = db.users.get_device_keys(&user_id.clone(), &device_id)? { - let metadata = db.users.get_device_metadata(user_id, &device_id)?.ok_or( + if let Some(mut keys) = db.users.get_device_keys(&user_id.clone(), device_id)? { + let metadata = db.users.get_device_metadata(user_id, device_id)?.ok_or( Error::BadRequest( ErrorKind::InvalidParam, "Tried to get keys for nonexistent device.", diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 4cec0af5..0a7f4bb5 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -1,5 +1,6 @@ use crate::{ - database::media::FileMeta, database::DatabaseGuard, utils, ConduitResult, Error, Ruma, + database::{media::FileMeta, DatabaseGuard}, + utils, ConduitResult, Error, Ruma, }; use ruma::api::client::{ error::ErrorKind, diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 01c19c21..10c052ea 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -56,7 +56,7 @@ pub async fn join_room_by_id_route( let mut servers = db .rooms - .invite_state(&sender_user, &body.room_id)? + .invite_state(sender_user, &body.room_id)? .unwrap_or_default() .iter() .filter_map(|event| { @@ -105,7 +105,7 @@ pub async fn join_room_by_id_or_alias_route( Ok(room_id) => { let mut servers = db .rooms - .invite_state(&sender_user, &room_id)? + .invite_state(sender_user, &room_id)? .unwrap_or_default() .iter() .filter_map(|event| { @@ -243,7 +243,7 @@ pub async fn kick_user_route( state_key: Some(body.user_id.to_string()), redacts: None, }, - &sender_user, + sender_user, &body.room_id, &db, &state_lock, @@ -319,7 +319,7 @@ pub async fn ban_user_route( state_key: Some(body.user_id.to_string()), redacts: None, }, - &sender_user, + sender_user, &body.room_id, &db, &state_lock, @@ -384,7 +384,7 @@ pub async fn unban_user_route( state_key: Some(body.user_id.to_string()), redacts: None, }, - &sender_user, + sender_user, &body.room_id, &db, &state_lock, @@ -416,7 +416,7 @@ pub async fn forget_room_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - db.rooms.forget(&body.room_id, &sender_user)?; + db.rooms.forget(&body.room_id, sender_user)?; db.flush()?; @@ -440,7 +440,7 @@ pub async fn joined_rooms_route( Ok(joined_rooms::Response { joined_rooms: db .rooms - .rooms_joined(&sender_user) + .rooms_joined(sender_user) .filter_map(|r| r.ok()) .collect(), } @@ -500,7 +500,7 @@ pub async fn joined_members_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if !db.rooms.is_joined(&sender_user, &body.room_id)? { + if !db.rooms.is_joined(sender_user, &body.room_id)? { return Err(Error::BadRequest( ErrorKind::Forbidden, "You aren't a member of the room.", @@ -545,7 +545,7 @@ async fn join_room_by_id_helper( let state_lock = mutex_state.lock().await; // Ask a remote server if we don't have this room - if !db.rooms.exists(&room_id)? && room_id.server_name() != db.globals.server_name() { + if !db.rooms.exists(room_id)? && room_id.server_name() != db.globals.server_name() { let mut make_join_response_and_server = Err(Error::BadServerResponse( "No server available to assist in joining.", )); @@ -606,11 +606,11 @@ async fn join_room_by_id_helper( "content".to_owned(), to_canonical_value(member::MemberEventContent { membership: member::MembershipState::Join, - displayname: db.users.displayname(&sender_user)?, - avatar_url: db.users.avatar_url(&sender_user)?, + displayname: db.users.displayname(sender_user)?, + avatar_url: db.users.avatar_url(sender_user)?, is_direct: None, third_party_invite: None, - blurhash: db.users.blurhash(&sender_user)?, + blurhash: db.users.blurhash(sender_user)?, reason: None, }) .expect("event is valid, we just created it"), @@ -658,7 +658,7 @@ async fn join_room_by_id_helper( ) .await?; - db.rooms.get_or_create_shortroomid(&room_id, &db.globals)?; + db.rooms.get_or_create_shortroomid(room_id, &db.globals)?; let pdu = PduEvent::from_id_val(&event_id, join_event.clone()) .map_err(|_| Error::BadServerResponse("Invalid join event PDU."))?; @@ -670,7 +670,7 @@ async fn join_room_by_id_helper( &send_join_response, &room_version, &pub_key_map, - &db, + db, ) .await?; @@ -678,7 +678,7 @@ async fn join_room_by_id_helper( .room_state .state .iter() - .map(|pdu| validate_and_add_event_id(pdu, &room_version, &pub_key_map, &db)) + .map(|pdu| validate_and_add_event_id(pdu, &room_version, &pub_key_map, db)) { let (event_id, value) = match result { Ok(t) => t, @@ -724,14 +724,14 @@ async fn join_room_by_id_helper( .into_iter() .map(|(k, id)| db.rooms.compress_state_event(k, &id, &db.globals)) .collect::>>()?, - &db, + db, )?; for result in send_join_response .room_state .auth_chain .iter() - .map(|pdu| validate_and_add_event_id(pdu, &room_version, &pub_key_map, &db)) + .map(|pdu| validate_and_add_event_id(pdu, &room_version, &pub_key_map, db)) { let (event_id, value) = match result { Ok(t) => t, @@ -754,15 +754,15 @@ async fn join_room_by_id_helper( // We set the room state after inserting the pdu, so that we never have a moment in time // where events in the current room state do not exist - db.rooms.set_room_state(&room_id, statehashid)?; + db.rooms.set_room_state(room_id, statehashid)?; } else { let event = member::MemberEventContent { membership: member::MembershipState::Join, - displayname: db.users.displayname(&sender_user)?, - avatar_url: db.users.avatar_url(&sender_user)?, + displayname: db.users.displayname(sender_user)?, + avatar_url: db.users.avatar_url(sender_user)?, is_direct: None, third_party_invite: None, - blurhash: db.users.blurhash(&sender_user)?, + blurhash: db.users.blurhash(sender_user)?, reason: None, }; @@ -774,9 +774,9 @@ async fn join_room_by_id_helper( state_key: Some(sender_user.to_string()), redacts: None, }, - &sender_user, - &room_id, - &db, + sender_user, + room_id, + db, &state_lock, )?; } @@ -800,7 +800,7 @@ fn validate_and_add_event_id( })?; let event_id = EventId::try_from(&*format!( "${}", - ruma::signatures::reference_hash(&value, &room_version) + ruma::signatures::reference_hash(&value, room_version) .expect("ruma can calculate reference hashes") )) .expect("ruma's reference hashes are valid event ids"); @@ -927,7 +927,7 @@ pub(crate) async fn invite_helper<'a>( let auth_events = db.rooms.get_auth_events( room_id, &kind, - &sender_user, + sender_user, Some(&state_key), &content, )?; @@ -1074,10 +1074,10 @@ pub(crate) async fn invite_helper<'a>( let pdu_id = server_server::handle_incoming_pdu( &origin, &event_id, - &room_id, + room_id, value, true, - &db, + db, &pub_key_map, ) .await @@ -1119,11 +1119,11 @@ pub(crate) async fn invite_helper<'a>( event_type: EventType::RoomMember, content: serde_json::to_value(member::MemberEventContent { membership: member::MembershipState::Invite, - displayname: db.users.displayname(&user_id)?, - avatar_url: db.users.avatar_url(&user_id)?, + displayname: db.users.displayname(user_id)?, + avatar_url: db.users.avatar_url(user_id)?, is_direct: Some(is_direct), third_party_invite: None, - blurhash: db.users.blurhash(&user_id)?, + blurhash: db.users.blurhash(user_id)?, reason: None, }) .expect("event is valid, we just created it"), @@ -1131,9 +1131,9 @@ pub(crate) async fn invite_helper<'a>( state_key: Some(user_id.to_string()), redacts: None, }, - &sender_user, + sender_user, room_id, - &db, + db, &state_lock, )?; diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 78008a59..93ead2c7 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -79,7 +79,7 @@ pub async fn send_message_event_route( state_key: None, redacts: None, }, - &sender_user, + sender_user, &body.room_id, &db, &state_lock, @@ -141,7 +141,7 @@ pub async fn get_message_events_route( get_message_events::Direction::Forward => { let events_after = db .rooms - .pdus_after(&sender_user, &body.room_id, from)? + .pdus_after(sender_user, &body.room_id, from)? .take(limit) .filter_map(|r| r.ok()) // Filter out buggy events .filter_map(|(pdu_id, pdu)| { @@ -171,7 +171,7 @@ pub async fn get_message_events_route( get_message_events::Direction::Backward => { let events_before = db .rooms - .pdus_until(&sender_user, &body.room_id, from)? + .pdus_until(sender_user, &body.room_id, from)? .take(limit) .filter_map(|r| r.ok()) // Filter out buggy events .filter_map(|(pdu_id, pdu)| { diff --git a/src/client_server/presence.rs b/src/client_server/presence.rs index 54eb210f..aaa78a92 100644 --- a/src/client_server/presence.rs +++ b/src/client_server/presence.rs @@ -19,17 +19,17 @@ pub async fn set_presence_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - for room_id in db.rooms.rooms_joined(&sender_user) { + for room_id in db.rooms.rooms_joined(sender_user) { let room_id = room_id?; db.rooms.edus.update_presence( - &sender_user, + sender_user, &room_id, ruma::events::presence::PresenceEvent { content: ruma::events::presence::PresenceEventContent { - avatar_url: db.users.avatar_url(&sender_user)?, + avatar_url: db.users.avatar_url(sender_user)?, currently_active: None, - displayname: db.users.displayname(&sender_user)?, + displayname: db.users.displayname(sender_user)?, last_active_ago: Some( utils::millis_since_unix_epoch() .try_into() @@ -76,7 +76,7 @@ pub async fn get_presence_route( if let Some(presence) = db .rooms .edus - .get_last_presence_event(&sender_user, &room_id)? + .get_last_presence_event(sender_user, &room_id)? { presence_event = Some(presence); break; diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index 5a8c7d2d..ab7fb024 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -34,12 +34,12 @@ pub async fn set_displayname_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); db.users - .set_displayname(&sender_user, body.displayname.clone())?; + .set_displayname(sender_user, body.displayname.clone())?; // Send a new membership event and presence update into all joined rooms let all_rooms_joined: Vec<_> = db .rooms - .rooms_joined(&sender_user) + .rooms_joined(sender_user) .filter_map(|r| r.ok()) .map(|room_id| { Ok::<_, Error>(( @@ -89,19 +89,19 @@ pub async fn set_displayname_route( ); let state_lock = mutex_state.lock().await; - let _ = - db.rooms - .build_and_append_pdu(pdu_builder, &sender_user, &room_id, &db, &state_lock); + let _ = db + .rooms + .build_and_append_pdu(pdu_builder, sender_user, &room_id, &db, &state_lock); // Presence update db.rooms.edus.update_presence( - &sender_user, + sender_user, &room_id, ruma::events::presence::PresenceEvent { content: ruma::events::presence::PresenceEventContent { - avatar_url: db.users.avatar_url(&sender_user)?, + avatar_url: db.users.avatar_url(sender_user)?, currently_active: None, - displayname: db.users.displayname(&sender_user)?, + displayname: db.users.displayname(sender_user)?, last_active_ago: Some( utils::millis_since_unix_epoch() .try_into() @@ -177,14 +177,14 @@ pub async fn set_avatar_url_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); db.users - .set_avatar_url(&sender_user, body.avatar_url.clone())?; + .set_avatar_url(sender_user, body.avatar_url.clone())?; - db.users.set_blurhash(&sender_user, body.blurhash.clone())?; + db.users.set_blurhash(sender_user, body.blurhash.clone())?; // Send a new membership event and presence update into all joined rooms let all_joined_rooms: Vec<_> = db .rooms - .rooms_joined(&sender_user) + .rooms_joined(sender_user) .filter_map(|r| r.ok()) .map(|room_id| { Ok::<_, Error>(( @@ -234,19 +234,19 @@ pub async fn set_avatar_url_route( ); let state_lock = mutex_state.lock().await; - let _ = - db.rooms - .build_and_append_pdu(pdu_builder, &sender_user, &room_id, &db, &state_lock); + let _ = db + .rooms + .build_and_append_pdu(pdu_builder, sender_user, &room_id, &db, &state_lock); // Presence update db.rooms.edus.update_presence( - &sender_user, + sender_user, &room_id, ruma::events::presence::PresenceEvent { content: ruma::events::presence::PresenceEventContent { - avatar_url: db.users.avatar_url(&sender_user)?, + avatar_url: db.users.avatar_url(sender_user)?, currently_active: None, - displayname: db.users.displayname(&sender_user)?, + displayname: db.users.displayname(sender_user)?, last_active_ago: Some( utils::millis_since_unix_epoch() .try_into() diff --git a/src/client_server/push.rs b/src/client_server/push.rs index 4e4611b5..98555d09 100644 --- a/src/client_server/push.rs +++ b/src/client_server/push.rs @@ -31,7 +31,7 @@ pub async fn get_pushrules_all_route( let event = db .account_data - .get::(None, &sender_user, EventType::PushRules)? + .get::(None, sender_user, EventType::PushRules)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "PushRules event not found.", @@ -59,7 +59,7 @@ pub async fn get_pushrule_route( let event = db .account_data - .get::(None, &sender_user, EventType::PushRules)? + .get::(None, sender_user, EventType::PushRules)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "PushRules event not found.", @@ -124,7 +124,7 @@ pub async fn set_pushrule_route( let mut event = db .account_data - .get::(None, &sender_user, EventType::PushRules)? + .get::(None, sender_user, EventType::PushRules)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "PushRules event not found.", @@ -193,13 +193,8 @@ pub async fn set_pushrule_route( _ => {} } - db.account_data.update( - None, - &sender_user, - EventType::PushRules, - &event, - &db.globals, - )?; + db.account_data + .update(None, sender_user, EventType::PushRules, &event, &db.globals)?; db.flush()?; @@ -229,7 +224,7 @@ pub async fn get_pushrule_actions_route( let mut event = db .account_data - .get::(None, &sender_user, EventType::PushRules)? + .get::(None, sender_user, EventType::PushRules)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "PushRules event not found.", @@ -291,7 +286,7 @@ pub async fn set_pushrule_actions_route( let mut event = db .account_data - .get::(None, &sender_user, EventType::PushRules)? + .get::(None, sender_user, EventType::PushRules)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "PushRules event not found.", @@ -332,13 +327,8 @@ pub async fn set_pushrule_actions_route( _ => {} }; - db.account_data.update( - None, - &sender_user, - EventType::PushRules, - &event, - &db.globals, - )?; + db.account_data + .update(None, sender_user, EventType::PushRules, &event, &db.globals)?; db.flush()?; @@ -368,7 +358,7 @@ pub async fn get_pushrule_enabled_route( let mut event = db .account_data - .get::(None, &sender_user, EventType::PushRules)? + .get::(None, sender_user, EventType::PushRules)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "PushRules event not found.", @@ -432,7 +422,7 @@ pub async fn set_pushrule_enabled_route( let mut event = db .account_data - .get::(None, &sender_user, EventType::PushRules)? + .get::(None, sender_user, EventType::PushRules)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "PushRules event not found.", @@ -478,13 +468,8 @@ pub async fn set_pushrule_enabled_route( _ => {} } - db.account_data.update( - None, - &sender_user, - EventType::PushRules, - &event, - &db.globals, - )?; + db.account_data + .update(None, sender_user, EventType::PushRules, &event, &db.globals)?; db.flush()?; @@ -514,7 +499,7 @@ pub async fn delete_pushrule_route( let mut event = db .account_data - .get::(None, &sender_user, EventType::PushRules)? + .get::(None, sender_user, EventType::PushRules)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "PushRules event not found.", @@ -550,13 +535,8 @@ pub async fn delete_pushrule_route( _ => {} } - db.account_data.update( - None, - &sender_user, - EventType::PushRules, - &event, - &db.globals, - )?; + db.account_data + .update(None, sender_user, EventType::PushRules, &event, &db.globals)?; db.flush()?; diff --git a/src/client_server/read_marker.rs b/src/client_server/read_marker.rs index 10298b9e..60aa4cef 100644 --- a/src/client_server/read_marker.rs +++ b/src/client_server/read_marker.rs @@ -37,7 +37,7 @@ pub async fn set_read_marker_route( }; db.account_data.update( Some(&body.room_id), - &sender_user, + sender_user, EventType::FullyRead, &fully_read_event, &db.globals, @@ -46,7 +46,7 @@ pub async fn set_read_marker_route( if let Some(event) = &body.read_receipt { db.rooms.edus.private_read_set( &body.room_id, - &sender_user, + sender_user, db.rooms.get_pdu_count(event)?.ok_or(Error::BadRequest( ErrorKind::InvalidParam, "Event does not exist.", @@ -54,7 +54,7 @@ pub async fn set_read_marker_route( &db.globals, )?; db.rooms - .reset_notification_counts(&sender_user, &body.room_id)?; + .reset_notification_counts(sender_user, &body.room_id)?; let mut user_receipts = BTreeMap::new(); user_receipts.insert( @@ -71,7 +71,7 @@ pub async fn set_read_marker_route( receipt_content.insert(event.to_owned(), receipts); db.rooms.edus.readreceipt_update( - &sender_user, + sender_user, &body.room_id, AnyEphemeralRoomEvent::Receipt(ruma::events::receipt::ReceiptEvent { content: ruma::events::receipt::ReceiptEventContent(receipt_content), @@ -102,7 +102,7 @@ pub async fn create_receipt_route( db.rooms.edus.private_read_set( &body.room_id, - &sender_user, + sender_user, db.rooms .get_pdu_count(&body.event_id)? .ok_or(Error::BadRequest( @@ -112,7 +112,7 @@ pub async fn create_receipt_route( &db.globals, )?; db.rooms - .reset_notification_counts(&sender_user, &body.room_id)?; + .reset_notification_counts(sender_user, &body.room_id)?; let mut user_receipts = BTreeMap::new(); user_receipts.insert( @@ -128,7 +128,7 @@ pub async fn create_receipt_route( receipt_content.insert(body.event_id.to_owned(), receipts); db.rooms.edus.readreceipt_update( - &sender_user, + sender_user, &body.room_id, AnyEphemeralRoomEvent::Receipt(ruma::events::receipt::ReceiptEvent { content: ruma::events::receipt::ReceiptEventContent(receipt_content), diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index 6d3e33ca..4b5219b2 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -46,7 +46,7 @@ pub async fn redact_event_route( state_key: None, redacts: Some(body.event_id.clone()), }, - &sender_user, + sender_user, &body.room_id, &db, &state_lock, diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 4ae8a3f8..5a026997 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -106,7 +106,7 @@ pub async fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, - &sender_user, + sender_user, &room_id, &db, &state_lock, @@ -118,11 +118,11 @@ pub async fn create_room_route( event_type: EventType::RoomMember, content: serde_json::to_value(member::MemberEventContent { membership: member::MembershipState::Join, - displayname: db.users.displayname(&sender_user)?, - avatar_url: db.users.avatar_url(&sender_user)?, + displayname: db.users.displayname(sender_user)?, + avatar_url: db.users.avatar_url(sender_user)?, is_direct: Some(body.is_direct), third_party_invite: None, - blurhash: db.users.blurhash(&sender_user)?, + blurhash: db.users.blurhash(sender_user)?, reason: None, }) .expect("event is valid, we just created it"), @@ -130,7 +130,7 @@ pub async fn create_room_route( state_key: Some(sender_user.to_string()), redacts: None, }, - &sender_user, + sender_user, &room_id, &db, &state_lock, @@ -185,7 +185,7 @@ pub async fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, - &sender_user, + sender_user, &room_id, &db, &state_lock, @@ -207,7 +207,7 @@ pub async fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, - &sender_user, + sender_user, &room_id, &db, &state_lock, @@ -235,7 +235,7 @@ pub async fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, - &sender_user, + sender_user, &room_id, &db, &state_lock, @@ -253,7 +253,7 @@ pub async fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, - &sender_user, + sender_user, &room_id, &db, &state_lock, @@ -279,7 +279,7 @@ pub async fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, - &sender_user, + sender_user, &room_id, &db, &state_lock, @@ -298,7 +298,7 @@ pub async fn create_room_route( } db.rooms - .build_and_append_pdu(pdu_builder, &sender_user, &room_id, &db, &state_lock)?; + .build_and_append_pdu(pdu_builder, sender_user, &room_id, &db, &state_lock)?; } // 7. Events implied by name and topic @@ -312,7 +312,7 @@ pub async fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, - &sender_user, + sender_user, &room_id, &db, &state_lock, @@ -331,7 +331,7 @@ pub async fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, - &sender_user, + sender_user, &room_id, &db, &state_lock, @@ -551,11 +551,11 @@ pub async fn upgrade_room_route( event_type: EventType::RoomMember, content: serde_json::to_value(member::MemberEventContent { membership: member::MembershipState::Join, - displayname: db.users.displayname(&sender_user)?, - avatar_url: db.users.avatar_url(&sender_user)?, + displayname: db.users.displayname(sender_user)?, + avatar_url: db.users.avatar_url(sender_user)?, is_direct: None, third_party_invite: None, - blurhash: db.users.blurhash(&sender_user)?, + blurhash: db.users.blurhash(sender_user)?, reason: None, }) .expect("event is valid, we just created it"), diff --git a/src/client_server/search.rs b/src/client_server/search.rs index cbd4ed79..9ff1a1bd 100644 --- a/src/client_server/search.rs +++ b/src/client_server/search.rs @@ -27,7 +27,7 @@ pub async fn search_events_route( let room_ids = filter.rooms.clone().unwrap_or_else(|| { db.rooms - .rooms_joined(&sender_user) + .rooms_joined(sender_user) .filter_map(|r| r.ok()) .collect() }); @@ -88,7 +88,7 @@ pub async fn search_events_route( rank: None, result: db .rooms - .get_pdu_from_id(&result)? + .get_pdu_from_id(result)? .map(|pdu| pdu.to_room_event()), }) }) diff --git a/src/client_server/session.rs b/src/client_server/session.rs index 94726272..b42689d2 100644 --- a/src/client_server/session.rs +++ b/src/client_server/session.rs @@ -100,8 +100,8 @@ pub async fn login_route( login::IncomingLoginInfo::Token { token } => { if let Some(jwt_decoding_key) = db.globals.jwt_decoding_key() { let token = jsonwebtoken::decode::( - &token, - &jwt_decoding_key, + token, + jwt_decoding_key, &jsonwebtoken::Validation::default(), ) .map_err(|_| Error::BadRequest(ErrorKind::InvalidUsername, "Token is invalid."))?; @@ -179,7 +179,7 @@ pub async fn logout_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); - db.users.remove_device(&sender_user, sender_device)?; + db.users.remove_device(sender_user, sender_device)?; db.flush()?; @@ -209,7 +209,7 @@ pub async fn logout_all_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); for device_id in db.users.all_device_ids(sender_user).flatten() { - db.users.remove_device(&sender_user, &device_id)?; + db.users.remove_device(sender_user, &device_id)?; } db.flush()?; diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 35553535..24cc2a18 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -308,9 +308,9 @@ async fn send_state_event_for_key_helper( state_key: Some(state_key), redacts: None, }, - &sender_user, - &room_id, - &db, + sender_user, + room_id, + db, &state_lock, )?; diff --git a/src/client_server/to_device.rs b/src/client_server/to_device.rs index bf2caefb..177b1234 100644 --- a/src/client_server/to_device.rs +++ b/src/client_server/to_device.rs @@ -68,8 +68,8 @@ pub async fn send_event_to_device_route( match target_device_id_maybe { DeviceIdOrAllDevices::DeviceId(target_device_id) => db.users.add_to_device_event( sender_user, - &target_user_id, - &target_device_id, + target_user_id, + target_device_id, &body.event_type, event.deserialize_as().map_err(|_| { Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid") @@ -78,10 +78,10 @@ pub async fn send_event_to_device_route( )?, DeviceIdOrAllDevices::AllDevices => { - for target_device_id in db.users.all_device_ids(&target_user_id) { + for target_device_id in db.users.all_device_ids(target_user_id) { db.users.add_to_device_event( sender_user, - &target_user_id, + target_user_id, &target_device_id?, &body.event_type, event.deserialize_as().map_err(|_| { diff --git a/src/client_server/typing.rs b/src/client_server/typing.rs index 4cf4bb11..15e74b35 100644 --- a/src/client_server/typing.rs +++ b/src/client_server/typing.rs @@ -21,7 +21,7 @@ pub fn create_typing_event_route( if let Typing::Yes(duration) = body.state { db.rooms.edus.typing_add( - &sender_user, + sender_user, &body.room_id, duration.as_millis() as u64 + utils::millis_since_unix_epoch(), &db.globals, @@ -29,7 +29,7 @@ pub fn create_typing_event_route( } else { db.rooms .edus - .typing_remove(&sender_user, &body.room_id, &db.globals)?; + .typing_remove(sender_user, &body.room_id, &db.globals)?; } Ok(create_typing_event::Response {}.into()) diff --git a/src/database.rs b/src/database.rs index dcba2abf..110d4d0c 100644 --- a/src/database.rs +++ b/src/database.rs @@ -196,14 +196,14 @@ impl Database { /// Load an existing database or create a new one. pub async fn load_or_create(config: &Config) -> Result>> { - Self::check_sled_or_sqlite_db(&config)?; + Self::check_sled_or_sqlite_db(config)?; if !Path::new(&config.database_path).exists() { std::fs::create_dir_all(&config.database_path) .map_err(|_| Error::BadConfig("Database folder doesn't exists and couldn't be created (e.g. due to missing permissions). Please create the database folder yourself."))?; } - let builder = Engine::open(&config)?; + let builder = Engine::open(config)?; if config.max_request_size < 1024 { eprintln!("ERROR: Max request size is less than 1KB. Please increase it."); @@ -618,7 +618,7 @@ impl Database { let short_room_id = db .rooms .roomid_shortroomid - .get(&room_id) + .get(room_id) .unwrap() .expect("shortroomid should exist"); @@ -641,7 +641,7 @@ impl Database { let short_room_id = db .rooms .roomid_shortroomid - .get(&room_id) + .get(room_id) .unwrap() .expect("shortroomid should exist"); @@ -677,7 +677,7 @@ impl Database { let short_room_id = db .rooms .roomid_shortroomid - .get(&room_id) + .get(room_id) .unwrap() .expect("shortroomid should exist"); let mut new_key = short_room_id; @@ -757,7 +757,7 @@ impl Database { #[cfg(feature = "sqlite")] { - Self::start_wal_clean_task(Arc::clone(&db), &config).await; + Self::start_wal_clean_task(Arc::clone(&db), config).await; } Ok(db) @@ -964,7 +964,7 @@ impl<'r> FromRequest<'r> for DatabaseGuard { async fn from_request(req: &'r Request<'_>) -> rocket::request::Outcome { let db = try_outcome!(req.guard::<&State>>>().await); - Ok(DatabaseGuard(Arc::clone(&db).read_owned().await)).or_forward(()) + Ok(DatabaseGuard(Arc::clone(db).read_owned().await)).or_forward(()) } } diff --git a/src/database/abstraction/sqlite.rs b/src/database/abstraction/sqlite.rs index 06e371e0..d1b7b5d2 100644 --- a/src/database/abstraction/sqlite.rs +++ b/src/database/abstraction/sqlite.rs @@ -192,7 +192,7 @@ impl SqliteTable { impl Tree for SqliteTable { #[tracing::instrument(skip(self, key))] fn get(&self, key: &[u8]) -> Result>> { - self.get_with_guard(&self.engine.read_lock(), key) + self.get_with_guard(self.engine.read_lock(), key) } #[tracing::instrument(skip(self, key, value))] @@ -275,7 +275,7 @@ impl Tree for SqliteTable { fn iter<'a>(&'a self) -> Box + 'a> { let guard = self.engine.read_lock_iterator(); - self.iter_with_guard(&guard) + self.iter_with_guard(guard) } #[tracing::instrument(skip(self, from, backwards))] diff --git a/src/database/account_data.rs b/src/database/account_data.rs index 1a8ad76c..456283bd 100644 --- a/src/database/account_data.rs +++ b/src/database/account_data.rs @@ -32,13 +32,13 @@ impl AccountData { .as_bytes() .to_vec(); prefix.push(0xff); - prefix.extend_from_slice(&user_id.as_bytes()); + prefix.extend_from_slice(user_id.as_bytes()); prefix.push(0xff); let mut roomuserdataid = prefix.clone(); roomuserdataid.extend_from_slice(&globals.next_count()?.to_be_bytes()); roomuserdataid.push(0xff); - roomuserdataid.extend_from_slice(&event_type.as_bytes()); + roomuserdataid.extend_from_slice(event_type.as_bytes()); let mut key = prefix; key.extend_from_slice(event_type.as_bytes()); @@ -83,7 +83,7 @@ impl AccountData { .as_bytes() .to_vec(); key.push(0xff); - key.extend_from_slice(&user_id.as_bytes()); + key.extend_from_slice(user_id.as_bytes()); key.push(0xff); key.extend_from_slice(kind.as_ref().as_bytes()); @@ -118,7 +118,7 @@ impl AccountData { .as_bytes() .to_vec(); prefix.push(0xff); - prefix.extend_from_slice(&user_id.as_bytes()); + prefix.extend_from_slice(user_id.as_bytes()); prefix.push(0xff); // Skip the data that's exactly at since, because we sent that last time diff --git a/src/database/globals.rs b/src/database/globals.rs index 048b9b89..2f1b45ad 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -113,7 +113,7 @@ impl Globals { .map(|key| (version, key)) }) .and_then(|(version, key)| { - ruma::signatures::Ed25519KeyPair::from_der(&key, version) + ruma::signatures::Ed25519KeyPair::from_der(key, version) .map_err(|_| Error::bad_database("Private or public keys are invalid.")) }); diff --git a/src/database/key_backups.rs b/src/database/key_backups.rs index 3315be3f..27d8030d 100644 --- a/src/database/key_backups.rs +++ b/src/database/key_backups.rs @@ -27,7 +27,7 @@ impl KeyBackups { let mut key = user_id.as_bytes().to_vec(); key.push(0xff); - key.extend_from_slice(&version.as_bytes()); + key.extend_from_slice(version.as_bytes()); self.backupid_algorithm.insert( &key, @@ -41,7 +41,7 @@ impl KeyBackups { pub fn delete_backup(&self, user_id: &UserId, version: &str) -> Result<()> { let mut key = user_id.as_bytes().to_vec(); key.push(0xff); - key.extend_from_slice(&version.as_bytes()); + key.extend_from_slice(version.as_bytes()); self.backupid_algorithm.remove(&key)?; self.backupid_etag.remove(&key)?; @@ -64,7 +64,7 @@ impl KeyBackups { ) -> Result { let mut key = user_id.as_bytes().to_vec(); key.push(0xff); - key.extend_from_slice(&version.as_bytes()); + key.extend_from_slice(version.as_bytes()); if self.backupid_algorithm.get(&key)?.is_none() { return Err(Error::BadRequest( @@ -75,7 +75,7 @@ impl KeyBackups { self.backupid_algorithm.insert( &key, - &serde_json::to_string(backup_metadata) + serde_json::to_string(backup_metadata) .expect("BackupAlgorithm::to_string always works") .as_bytes(), )?; @@ -192,7 +192,7 @@ impl KeyBackups { pub fn get_etag(&self, user_id: &UserId, version: &str) -> Result { let mut key = user_id.as_bytes().to_vec(); key.push(0xff); - key.extend_from_slice(&version.as_bytes()); + key.extend_from_slice(version.as_bytes()); Ok(utils::u64_from_bytes( &self @@ -223,7 +223,7 @@ impl KeyBackups { let mut parts = key.rsplit(|&b| b == 0xff); let session_id = - utils::string_from_bytes(&parts.next().ok_or_else(|| { + utils::string_from_bytes(parts.next().ok_or_else(|| { Error::bad_database("backupkeyid_backup key is invalid.") })?) .map_err(|_| { @@ -231,7 +231,7 @@ impl KeyBackups { })?; let room_id = RoomId::try_from( - utils::string_from_bytes(&parts.next().ok_or_else(|| { + utils::string_from_bytes(parts.next().ok_or_else(|| { Error::bad_database("backupkeyid_backup key is invalid.") })?) .map_err(|_| Error::bad_database("backupkeyid_backup room_id is invalid."))?, @@ -280,7 +280,7 @@ impl KeyBackups { let mut parts = key.rsplit(|&b| b == 0xff); let session_id = - utils::string_from_bytes(&parts.next().ok_or_else(|| { + utils::string_from_bytes(parts.next().ok_or_else(|| { Error::bad_database("backupkeyid_backup key is invalid.") })?) .map_err(|_| { @@ -325,7 +325,7 @@ impl KeyBackups { pub fn delete_all_keys(&self, user_id: &UserId, version: &str) -> Result<()> { let mut key = user_id.as_bytes().to_vec(); key.push(0xff); - key.extend_from_slice(&version.as_bytes()); + key.extend_from_slice(version.as_bytes()); key.push(0xff); for (outdated_key, _) in self.backupkeyid_backup.scan_prefix(key) { @@ -343,9 +343,9 @@ impl KeyBackups { ) -> Result<()> { let mut key = user_id.as_bytes().to_vec(); key.push(0xff); - key.extend_from_slice(&version.as_bytes()); + key.extend_from_slice(version.as_bytes()); key.push(0xff); - key.extend_from_slice(&room_id.as_bytes()); + key.extend_from_slice(room_id.as_bytes()); key.push(0xff); for (outdated_key, _) in self.backupkeyid_backup.scan_prefix(key) { @@ -364,11 +364,11 @@ impl KeyBackups { ) -> Result<()> { let mut key = user_id.as_bytes().to_vec(); key.push(0xff); - key.extend_from_slice(&version.as_bytes()); + key.extend_from_slice(version.as_bytes()); key.push(0xff); - key.extend_from_slice(&room_id.as_bytes()); + key.extend_from_slice(room_id.as_bytes()); key.push(0xff); - key.extend_from_slice(&session_id.as_bytes()); + key.extend_from_slice(session_id.as_bytes()); for (outdated_key, _) in self.backupkeyid_backup.scan_prefix(key) { self.backupkeyid_backup.remove(&outdated_key)?; diff --git a/src/database/media.rs b/src/database/media.rs index a9bb42b0..46630131 100644 --- a/src/database/media.rs +++ b/src/database/media.rs @@ -4,7 +4,10 @@ use image::{imageops::FilterType, GenericImageView}; use super::abstraction::Tree; use crate::{utils, Error, Result}; use std::{mem, sync::Arc}; -use tokio::{fs::File, io::AsyncReadExt, io::AsyncWriteExt}; +use tokio::{ + fs::File, + io::{AsyncReadExt, AsyncWriteExt}, +}; pub struct FileMeta { pub content_disposition: Option, diff --git a/src/database/pusher.rs b/src/database/pusher.rs index da4a6e75..b19f3392 100644 --- a/src/database/pusher.rs +++ b/src/database/pusher.rs @@ -236,7 +236,7 @@ pub fn get_actions<'a>( member_count: 10_u32.into(), // TODO: get member count efficiently user_display_name: db .users - .displayname(&user)? + .displayname(user)? .unwrap_or_else(|| user.localpart().to_owned()), users_power_levels: power_levels.users.clone(), default_power_level: power_levels.users_default, @@ -302,7 +302,7 @@ async fn send_notice( if event_id_only { send_request( &db.globals, - &url, + url, send_event_notification::v1::Request::new(notifi), ) .await?; @@ -332,7 +332,7 @@ async fn send_notice( send_request( &db.globals, - &url, + url, send_event_notification::v1::Request::new(notifi), ) .await?; diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 51023ba9..b272a5ce 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -252,7 +252,7 @@ impl Rooms { return Ok(HashMap::new()); }; - let auth_events = state_res::auth_types_for_event(kind, sender, state_key, &content); + let auth_events = state_res::auth_types_for_event(kind, sender, state_key, content); let mut sauthevents = auth_events .into_iter() @@ -339,7 +339,7 @@ impl Rooms { new_state_ids_compressed: HashSet, db: &Database, ) -> Result<()> { - let previous_shortstatehash = self.current_shortstatehash(&room_id)?; + let previous_shortstatehash = self.current_shortstatehash(room_id)?; let state_hash = self.calculate_hash( &new_state_ids_compressed @@ -424,7 +424,7 @@ impl Rooms { } } - self.update_joined_count(room_id, &db)?; + self.update_joined_count(room_id, db)?; self.roomid_shortstatehash .insert(room_id.as_bytes(), &new_shortstatehash.to_be_bytes())?; @@ -704,7 +704,7 @@ impl Rooms { event_id: &EventId, globals: &super::globals::Globals, ) -> Result { - if let Some(short) = self.eventidshort_cache.lock().unwrap().get_mut(&event_id) { + if let Some(short) = self.eventidshort_cache.lock().unwrap().get_mut(event_id) { return Ok(*short); } @@ -732,7 +732,7 @@ impl Rooms { #[tracing::instrument(skip(self))] pub fn get_shortroomid(&self, room_id: &RoomId) -> Result> { self.roomid_shortroomid - .get(&room_id.as_bytes())? + .get(room_id.as_bytes())? .map(|bytes| { utils::u64_from_bytes(&bytes) .map_err(|_| Error::bad_database("Invalid shortroomid in db.")) @@ -757,7 +757,7 @@ impl Rooms { let mut statekey = event_type.as_ref().as_bytes().to_vec(); statekey.push(0xff); - statekey.extend_from_slice(&state_key.as_bytes()); + statekey.extend_from_slice(state_key.as_bytes()); let short = self .statekey_shortstatekey @@ -784,13 +784,13 @@ impl Rooms { room_id: &RoomId, globals: &super::globals::Globals, ) -> Result { - Ok(match self.roomid_shortroomid.get(&room_id.as_bytes())? { + Ok(match self.roomid_shortroomid.get(room_id.as_bytes())? { Some(short) => utils::u64_from_bytes(&short) .map_err(|_| Error::bad_database("Invalid shortroomid in db."))?, None => { let short = globals.next_count()?; self.roomid_shortroomid - .insert(&room_id.as_bytes(), &short.to_be_bytes())?; + .insert(room_id.as_bytes(), &short.to_be_bytes())?; short } }) @@ -814,7 +814,7 @@ impl Rooms { let mut statekey = event_type.as_ref().as_bytes().to_vec(); statekey.push(0xff); - statekey.extend_from_slice(&state_key.as_bytes()); + statekey.extend_from_slice(state_key.as_bytes()); let short = match self.statekey_shortstatekey.get(&statekey)? { Some(shortstatekey) => utils::u64_from_bytes(&shortstatekey) @@ -891,12 +891,12 @@ impl Rooms { .ok_or_else(|| Error::bad_database("Invalid statekey in shortstatekey_statekey."))?; let event_type = - EventType::try_from(utils::string_from_bytes(&eventtype_bytes).map_err(|_| { + EventType::try_from(utils::string_from_bytes(eventtype_bytes).map_err(|_| { Error::bad_database("Event type in shortstatekey_statekey is invalid unicode.") })?) .map_err(|_| Error::bad_database("Event type in shortstatekey_statekey is invalid."))?; - let state_key = utils::string_from_bytes(&statekey_bytes).map_err(|_| { + let state_key = utils::string_from_bytes(statekey_bytes).map_err(|_| { Error::bad_database("Statekey in shortstatekey_statekey is invalid unicode.") })?; @@ -956,10 +956,8 @@ impl Rooms { /// Returns the `count` of this pdu's id. #[tracing::instrument(skip(self))] pub fn pdu_count(&self, pdu_id: &[u8]) -> Result { - Ok( - utils::u64_from_bytes(&pdu_id[pdu_id.len() - size_of::()..]) - .map_err(|_| Error::bad_database("PDU has invalid count bytes."))?, - ) + utils::u64_from_bytes(&pdu_id[pdu_id.len() - size_of::()..]) + .map_err(|_| Error::bad_database("PDU has invalid count bytes.")) } /// Returns the `count` of this pdu's id. @@ -1076,7 +1074,7 @@ impl Rooms { /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. #[tracing::instrument(skip(self))] pub fn get_pdu(&self, event_id: &EventId) -> Result>> { - if let Some(p) = self.pdu_cache.lock().unwrap().get_mut(&event_id) { + if let Some(p) = self.pdu_cache.lock().unwrap().get_mut(event_id) { return Ok(Some(Arc::clone(p))); } @@ -1138,9 +1136,9 @@ impl Rooms { /// Removes a pdu and creates a new one with the same id. #[tracing::instrument(skip(self))] fn replace_pdu(&self, pdu_id: &[u8], pdu: &PduEvent) -> Result<()> { - if self.pduid_pdu.get(&pdu_id)?.is_some() { + if self.pduid_pdu.get(pdu_id)?.is_some() { self.pduid_pdu.insert( - &pdu_id, + pdu_id, &serde_json::to_vec(pdu).expect("PduEvent::to_vec always works"), )?; Ok(()) @@ -1225,20 +1223,20 @@ impl Rooms { #[tracing::instrument(skip(self, pdu))] pub fn add_pdu_outlier(&self, event_id: &EventId, pdu: &CanonicalJsonObject) -> Result<()> { self.eventid_outlierpdu.insert( - &event_id.as_bytes(), + event_id.as_bytes(), &serde_json::to_vec(&pdu).expect("CanonicalJsonObject is valid"), ) } #[tracing::instrument(skip(self))] pub fn mark_event_soft_failed(&self, event_id: &EventId) -> Result<()> { - self.softfailedeventids.insert(&event_id.as_bytes(), &[]) + self.softfailedeventids.insert(event_id.as_bytes(), &[]) } #[tracing::instrument(skip(self))] pub fn is_event_soft_failed(&self, event_id: &EventId) -> Result { self.softfailedeventids - .get(&event_id.as_bytes()) + .get(event_id.as_bytes()) .map(|o| o.is_some()) } @@ -1268,7 +1266,7 @@ impl Rooms { { if let Some(shortstatehash) = self.pdu_shortstatehash(&pdu.event_id).unwrap() { if let Some(prev_state) = self - .state_get(shortstatehash, &pdu.kind, &state_key) + .state_get(shortstatehash, &pdu.kind, state_key) .unwrap() { unsigned.insert( @@ -1350,15 +1348,15 @@ impl Rooms { let rules_for_user = db .account_data - .get::(None, &user, EventType::PushRules)? + .get::(None, user, EventType::PushRules)? .map(|ev| ev.content.global) - .unwrap_or_else(|| push::Ruleset::server_default(&user)); + .unwrap_or_else(|| push::Ruleset::server_default(user)); let mut highlight = false; let mut notify = false; for action in pusher::get_actions( - &user, + user, &rules_for_user, &power_levels, &sync_pdu, @@ -1388,7 +1386,7 @@ impl Rooms { highlights.push(userroom_id); } - for senderkey in db.pusher.get_pusher_senderkeys(&user) { + for senderkey in db.pusher.get_pusher_senderkeys(user) { db.sending.send_push_pdu(&*pdu_id, senderkey)?; } } @@ -1401,7 +1399,7 @@ impl Rooms { match pdu.kind { EventType::RoomRedaction => { if let Some(redact_id) = &pdu.redacts { - self.redact_pdu(&redact_id, &pdu)?; + self.redact_pdu(redact_id, pdu)?; } } EventType::RoomMember => { @@ -1741,9 +1739,9 @@ impl Rooms { state_ids_compressed: HashSet, globals: &super::globals::Globals, ) -> Result<()> { - let shorteventid = self.get_or_create_shorteventid(&event_id, globals)?; + let shorteventid = self.get_or_create_shorteventid(event_id, globals)?; - let previous_shortstatehash = self.current_shortstatehash(&room_id)?; + let previous_shortstatehash = self.current_shortstatehash(room_id)?; let state_hash = self.calculate_hash( &state_ids_compressed @@ -1815,7 +1813,7 @@ impl Rooms { .map_or_else(|| Ok(Vec::new()), |p| self.load_shortstatehash_info(p))?; let shortstatekey = - self.get_or_create_shortstatekey(&new_pdu.kind, &state_key, globals)?; + self.get_or_create_shortstatekey(&new_pdu.kind, state_key, globals)?; let new = self.compress_state_event(shortstatekey, &new_pdu.event_id, globals)?; @@ -1840,7 +1838,7 @@ impl Rooms { let mut statediffremoved = HashSet::new(); if let Some(replaces) = replaces { - statediffremoved.insert(replaces.clone()); + statediffremoved.insert(*replaces); } self.save_state_from_diff( @@ -1953,12 +1951,12 @@ impl Rooms { } = pdu_builder; let prev_events = self - .get_pdu_leaves(&room_id)? + .get_pdu_leaves(room_id)? .into_iter() .take(20) .collect::>(); - let create_event = self.room_state_get(&room_id, &EventType::RoomCreate, "")?; + let create_event = self.room_state_get(room_id, &EventType::RoomCreate, "")?; let create_event_content = create_event .as_ref() @@ -1988,13 +1986,8 @@ impl Rooms { }); let room_version = RoomVersion::new(&room_version_id).expect("room version is supported"); - let auth_events = self.get_auth_events( - &room_id, - &event_type, - &sender, - state_key.as_deref(), - &content, - )?; + let auth_events = + self.get_auth_events(room_id, &event_type, sender, state_key.as_deref(), &content)?; // Our depth is the maximum depth of prev_events + 1 let depth = prev_events @@ -2006,7 +1999,7 @@ impl Rooms { let mut unsigned = unsigned.unwrap_or_default(); if let Some(state_key) = &state_key { - if let Some(prev_pdu) = self.room_state_get(&room_id, &event_type, &state_key)? { + if let Some(prev_pdu) = self.room_state_get(room_id, &event_type, state_key)? { unsigned.insert("prev_content".to_owned(), prev_pdu.content.clone()); unsigned.insert( "prev_sender".to_owned(), @@ -2109,7 +2102,7 @@ impl Rooms { // We set the room state after inserting the pdu, so that we never have a moment in time // where events in the current room state do not exist - self.set_room_state(&room_id, statehashid)?; + self.set_room_state(room_id, statehashid)?; for server in self .room_servers(room_id) @@ -2154,10 +2147,10 @@ impl Rooms { && pdu .state_key .as_ref() - .map_or(false, |state_key| users.is_match(&state_key)) + .map_or(false, |state_key| users.is_match(state_key)) }; let matching_aliases = |aliases: &Regex| { - self.room_aliases(&room_id) + self.room_aliases(room_id) .filter_map(|r| r.ok()) .any(|room_alias| aliases.is_match(room_alias.as_str())) }; @@ -2300,7 +2293,7 @@ impl Rooms { let mut pdu = self .get_pdu_from_id(&pdu_id)? .ok_or_else(|| Error::bad_database("PDU ID points to invalid PDU."))?; - pdu.redact(&reason)?; + pdu.redact(reason)?; self.replace_pdu(&pdu_id, &pdu)?; Ok(()) } else { @@ -2348,13 +2341,13 @@ impl Rooms { match &membership { member::MembershipState::Join => { // Check if the user never joined this room - if !self.once_joined(&user_id, &room_id)? { + if !self.once_joined(user_id, room_id)? { // Add the user ID to the join list then self.roomuseroncejoinedids.insert(&userroom_id, &[])?; // Check if the room has a predecessor if let Some(predecessor) = self - .room_state_get(&room_id, &EventType::RoomCreate, "")? + .room_state_get(room_id, &EventType::RoomCreate, "")? .and_then(|create| { serde_json::from_value::< Raw, @@ -2455,12 +2448,12 @@ impl Rooms { let is_ignored = db .account_data .get::( - None, // Ignored users are in global account data - &user_id, // Receiver + None, // Ignored users are in global account data + user_id, // Receiver EventType::IgnoredUserList, )? .map_or(false, |ignored| { - ignored.content.ignored_users.contains(&sender) + ignored.content.ignored_users.contains(sender) }); if is_ignored { @@ -2522,7 +2515,7 @@ impl Rooms { let mut joined_servers = HashSet::new(); let mut real_users = HashSet::new(); - for joined in self.room_members(&room_id).filter_map(|r| r.ok()) { + for joined in self.room_members(room_id).filter_map(|r| r.ok()) { joined_servers.insert(joined.server_name().to_owned()); if joined.server_name() == db.globals.server_name() && !db.users.is_deactivated(&joined).unwrap_or(true) @@ -2532,7 +2525,7 @@ impl Rooms { joinedcount += 1; } - for invited in self.room_members_invited(&room_id).filter_map(|r| r.ok()) { + for invited in self.room_members_invited(room_id).filter_map(|r| r.ok()) { joined_servers.insert(invited.server_name().to_owned()); invitedcount += 1; } @@ -2601,7 +2594,7 @@ impl Rooms { if let Some(users) = maybe { Ok(users) } else { - self.update_joined_count(room_id, &db)?; + self.update_joined_count(room_id, db)?; Ok(Arc::clone( self.our_real_users_cache .read() @@ -2650,7 +2643,7 @@ impl Rooms { let in_room = bridge_user_id .map_or(false, |id| self.is_joined(&id, room_id).unwrap_or(false)) - || self.room_members(&room_id).any(|userid| { + || self.room_members(room_id).any(|userid| { userid.map_or(false, |userid| { users.iter().any(|r| r.is_match(userid.as_str())) }) @@ -2890,21 +2883,21 @@ impl Rooms { if let Some(room_id) = room_id { // New alias self.alias_roomid - .insert(&alias.alias().as_bytes(), room_id.as_bytes())?; + .insert(alias.alias().as_bytes(), room_id.as_bytes())?; let mut aliasid = room_id.as_bytes().to_vec(); aliasid.push(0xff); aliasid.extend_from_slice(&globals.next_count()?.to_be_bytes()); self.aliasid_alias.insert(&aliasid, &*alias.as_bytes())?; } else { // room_id=None means remove alias - if let Some(room_id) = self.alias_roomid.get(&alias.alias().as_bytes())? { + if let Some(room_id) = self.alias_roomid.get(alias.alias().as_bytes())? { let mut prefix = room_id.to_vec(); prefix.push(0xff); for (key, _) in self.aliasid_alias.scan_prefix(prefix) { self.aliasid_alias.remove(&key)?; } - self.alias_roomid.remove(&alias.alias().as_bytes())?; + self.alias_roomid.remove(alias.alias().as_bytes())?; } else { return Err(Error::BadRequest( ErrorKind::NotFound, @@ -3077,7 +3070,7 @@ impl Rooms { self.roomserverids.scan_prefix(prefix).map(|(key, _)| { Box::::try_from( utils::string_from_bytes( - &key.rsplit(|&b| b == 0xff) + key.rsplit(|&b| b == 0xff) .next() .expect("rsplit always returns an element"), ) @@ -3110,7 +3103,7 @@ impl Rooms { self.serverroomids.scan_prefix(prefix).map(|(key, _)| { RoomId::try_from( utils::string_from_bytes( - &key.rsplit(|&b| b == 0xff) + key.rsplit(|&b| b == 0xff) .next() .expect("rsplit always returns an element"), ) @@ -3132,7 +3125,7 @@ impl Rooms { self.roomuserid_joined.scan_prefix(prefix).map(|(key, _)| { UserId::try_from( utils::string_from_bytes( - &key.rsplit(|&b| b == 0xff) + key.rsplit(|&b| b == 0xff) .next() .expect("rsplit always returns an element"), ) @@ -3146,26 +3139,24 @@ impl Rooms { #[tracing::instrument(skip(self))] pub fn room_joined_count(&self, room_id: &RoomId) -> Result> { - Ok(self - .roomid_joinedcount + self.roomid_joinedcount .get(room_id.as_bytes())? .map(|b| { utils::u64_from_bytes(&b) .map_err(|_| Error::bad_database("Invalid joinedcount in db.")) }) - .transpose()?) + .transpose() } #[tracing::instrument(skip(self))] pub fn room_invited_count(&self, room_id: &RoomId) -> Result> { - Ok(self - .roomid_invitedcount + self.roomid_invitedcount .get(room_id.as_bytes())? .map(|b| { utils::u64_from_bytes(&b) .map_err(|_| Error::bad_database("Invalid joinedcount in db.")) }) - .transpose()?) + .transpose() } /// Returns an iterator over all User IDs who ever joined a room. @@ -3182,7 +3173,7 @@ impl Rooms { .map(|(key, _)| { UserId::try_from( utils::string_from_bytes( - &key.rsplit(|&b| b == 0xff) + key.rsplit(|&b| b == 0xff) .next() .expect("rsplit always returns an element"), ) @@ -3208,7 +3199,7 @@ impl Rooms { .map(|(key, _)| { UserId::try_from( utils::string_from_bytes( - &key.rsplit(|&b| b == 0xff) + key.rsplit(|&b| b == 0xff) .next() .expect("rsplit always returns an element"), ) @@ -3261,7 +3252,7 @@ impl Rooms { .map(|(key, _)| { RoomId::try_from( utils::string_from_bytes( - &key.rsplit(|&b| b == 0xff) + key.rsplit(|&b| b == 0xff) .next() .expect("rsplit always returns an element"), ) @@ -3287,7 +3278,7 @@ impl Rooms { .map(|(key, state)| { let room_id = RoomId::try_from( utils::string_from_bytes( - &key.rsplit(|&b| b == 0xff) + key.rsplit(|&b| b == 0xff) .next() .expect("rsplit always returns an element"), ) @@ -3312,7 +3303,7 @@ impl Rooms { ) -> Result>>> { let mut key = user_id.as_bytes().to_vec(); key.push(0xff); - key.extend_from_slice(&room_id.as_bytes()); + key.extend_from_slice(room_id.as_bytes()); self.userroomid_invitestate .get(&key)? @@ -3333,7 +3324,7 @@ impl Rooms { ) -> Result>>> { let mut key = user_id.as_bytes().to_vec(); key.push(0xff); - key.extend_from_slice(&room_id.as_bytes()); + key.extend_from_slice(room_id.as_bytes()); self.userroomid_leftstate .get(&key)? @@ -3360,7 +3351,7 @@ impl Rooms { .map(|(key, state)| { let room_id = RoomId::try_from( utils::string_from_bytes( - &key.rsplit(|&b| b == 0xff) + key.rsplit(|&b| b == 0xff) .next() .expect("rsplit always returns an element"), ) diff --git a/src/database/rooms/edus.rs b/src/database/rooms/edus.rs index 14146fb9..e0639ffd 100644 --- a/src/database/rooms/edus.rs +++ b/src/database/rooms/edus.rs @@ -60,7 +60,7 @@ impl RoomEdus { let mut room_latest_id = prefix; room_latest_id.extend_from_slice(&globals.next_count()?.to_be_bytes()); room_latest_id.push(0xff); - room_latest_id.extend_from_slice(&user_id.as_bytes()); + room_latest_id.extend_from_slice(user_id.as_bytes()); self.readreceiptid_readreceipt.insert( &room_latest_id, @@ -126,7 +126,7 @@ impl RoomEdus { ) -> Result<()> { let mut key = room_id.as_bytes().to_vec(); key.push(0xff); - key.extend_from_slice(&user_id.as_bytes()); + key.extend_from_slice(user_id.as_bytes()); self.roomuserid_privateread .insert(&key, &count.to_be_bytes())?; @@ -142,7 +142,7 @@ impl RoomEdus { pub fn private_read_get(&self, room_id: &RoomId, user_id: &UserId) -> Result> { let mut key = room_id.as_bytes().to_vec(); key.push(0xff); - key.extend_from_slice(&user_id.as_bytes()); + key.extend_from_slice(user_id.as_bytes()); self.roomuserid_privateread .get(&key)? @@ -157,7 +157,7 @@ impl RoomEdus { pub fn last_privateread_update(&self, user_id: &UserId, room_id: &RoomId) -> Result { let mut key = room_id.as_bytes().to_vec(); key.push(0xff); - key.extend_from_slice(&user_id.as_bytes()); + key.extend_from_slice(user_id.as_bytes()); Ok(self .roomuserid_lastprivatereadupdate @@ -193,7 +193,7 @@ impl RoomEdus { .insert(&room_typing_id, &*user_id.as_bytes())?; self.roomid_lasttypingupdate - .insert(&room_id.as_bytes(), &count)?; + .insert(room_id.as_bytes(), &count)?; Ok(()) } @@ -224,7 +224,7 @@ impl RoomEdus { if found_outdated { self.roomid_lasttypingupdate - .insert(&room_id.as_bytes(), &globals.next_count()?.to_be_bytes())?; + .insert(room_id.as_bytes(), &globals.next_count()?.to_be_bytes())?; } Ok(()) @@ -268,7 +268,7 @@ impl RoomEdus { if found_outdated { self.roomid_lasttypingupdate - .insert(&room_id.as_bytes(), &globals.next_count()?.to_be_bytes())?; + .insert(room_id.as_bytes(), &globals.next_count()?.to_be_bytes())?; } Ok(()) @@ -285,7 +285,7 @@ impl RoomEdus { Ok(self .roomid_lasttypingupdate - .get(&room_id.as_bytes())? + .get(room_id.as_bytes())? .map_or(Ok::<_, Error>(None), |bytes| { Ok(Some(utils::u64_from_bytes(&bytes).map_err(|_| { Error::bad_database("Count in roomid_lastroomactiveupdate is invalid.") @@ -342,7 +342,7 @@ impl RoomEdus { presence_id.push(0xff); presence_id.extend_from_slice(&count); presence_id.push(0xff); - presence_id.extend_from_slice(&presence.sender.as_bytes()); + presence_id.extend_from_slice(presence.sender.as_bytes()); self.presenceid_presence.insert( &presence_id, @@ -361,7 +361,7 @@ impl RoomEdus { #[tracing::instrument(skip(self))] pub fn ping_presence(&self, user_id: &UserId) -> Result<()> { self.userid_lastpresenceupdate.insert( - &user_id.as_bytes(), + user_id.as_bytes(), &utils::millis_since_unix_epoch().to_be_bytes(), )?; @@ -371,7 +371,7 @@ impl RoomEdus { /// Returns the timestamp of the last presence update of this user in millis since the unix epoch. pub fn last_presence_update(&self, user_id: &UserId) -> Result> { self.userid_lastpresenceupdate - .get(&user_id.as_bytes())? + .get(user_id.as_bytes())? .map(|bytes| { utils::u64_from_bytes(&bytes).map_err(|_| { Error::bad_database("Invalid timestamp in userid_lastpresenceupdate.") @@ -394,7 +394,7 @@ impl RoomEdus { presence_id.push(0xff); presence_id.extend_from_slice(&last_update.to_be_bytes()); presence_id.push(0xff); - presence_id.extend_from_slice(&user_id.as_bytes()); + presence_id.extend_from_slice(user_id.as_bytes()); self.presenceid_presence .get(&presence_id)? @@ -480,7 +480,7 @@ impl RoomEdus { } self.userid_lastpresenceupdate.insert( - &user_id.as_bytes(), + user_id.as_bytes(), &utils::millis_since_unix_epoch().to_be_bytes(), )?; } diff --git a/src/database/sending.rs b/src/database/sending.rs index 1050c077..c14f5819 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -58,9 +58,9 @@ impl OutgoingKind { } OutgoingKind::Push(user, pushkey) => { let mut p = b"$".to_vec(); - p.extend_from_slice(&user); + p.extend_from_slice(user); p.push(0xff); - p.extend_from_slice(&pushkey); + p.extend_from_slice(pushkey); p } OutgoingKind::Normal(server) => { @@ -179,8 +179,8 @@ impl Sending { // Insert pdus we found for (e, key) in &new_events { let value = if let SendingEventType::Edu(value) = &e.1 { &**value } else { &[] }; - guard.sending.servercurrentevent_data.insert(&key, value).unwrap(); - guard.sending.servernameevent_data.remove(&key).unwrap(); + guard.sending.servercurrentevent_data.insert(key, value).unwrap(); + guard.sending.servernameevent_data.remove(key).unwrap(); } drop(guard); @@ -345,7 +345,7 @@ impl Sending { } let event = - serde_json::from_str::(&read_receipt.json().get()) + serde_json::from_str::(read_receipt.json().get()) .map_err(|_| Error::bad_database("Invalid edu event in read_receipts."))?; let federation_event = match event { AnySyncEphemeralRoomEvent::Receipt(r) => { @@ -486,7 +486,7 @@ impl Sending { match event { SendingEventType::Pdu(pdu_id) => { pdu_jsons.push(db.rooms - .get_pdu_from_id(&pdu_id) + .get_pdu_from_id(pdu_id) .map_err(|e| (kind.clone(), e))? .ok_or_else(|| { ( @@ -543,7 +543,7 @@ impl Sending { SendingEventType::Pdu(pdu_id) => { pdus.push( db.rooms - .get_pdu_from_id(&pdu_id) + .get_pdu_from_id(pdu_id) .map_err(|e| (kind.clone(), e))? .ok_or_else(|| { ( @@ -636,7 +636,7 @@ impl Sending { // TODO: check room version and remove event_id if needed let raw = PduEvent::convert_to_outgoing_federation_event( db.rooms - .get_pdu_json_from_id(&pdu_id) + .get_pdu_json_from_id(pdu_id) .map_err(|e| (OutgoingKind::Normal(server.clone()), e))? .ok_or_else(|| { ( @@ -711,7 +711,7 @@ impl Sending { let event = parts .next() .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?; - let server = utils::string_from_bytes(&server).map_err(|_| { + let server = utils::string_from_bytes(server).map_err(|_| { Error::bad_database("Invalid server bytes in server_currenttransaction") })?; @@ -750,7 +750,7 @@ impl Sending { let event = parts .next() .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?; - let server = utils::string_from_bytes(&server).map_err(|_| { + let server = utils::string_from_bytes(server).map_err(|_| { Error::bad_database("Invalid server bytes in server_currenttransaction") })?; diff --git a/src/database/uiaa.rs b/src/database/uiaa.rs index 8a3fe4f0..60b9bd31 100644 --- a/src/database/uiaa.rs +++ b/src/database/uiaa.rs @@ -54,7 +54,7 @@ impl Uiaa { ) -> Result<(bool, UiaaInfo)> { let mut uiaainfo = auth .session() - .map(|session| self.get_uiaa_session(&user_id, &device_id, session)) + .map(|session| self.get_uiaa_session(user_id, device_id, session)) .unwrap_or_else(|| Ok(uiaainfo.clone()))?; if uiaainfo.session.is_none() { diff --git a/src/database/users.rs b/src/database/users.rs index 88d66bea..63ed0710 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -81,13 +81,13 @@ impl Users { })?; Ok(Some(( - UserId::try_from(utils::string_from_bytes(&user_bytes).map_err(|_| { + UserId::try_from(utils::string_from_bytes(user_bytes).map_err(|_| { Error::bad_database("User ID in token_userdeviceid is invalid unicode.") })?) .map_err(|_| { Error::bad_database("User ID in token_userdeviceid is invalid.") })?, - utils::string_from_bytes(&device_bytes).map_err(|_| { + utils::string_from_bytes(device_bytes).map_err(|_| { Error::bad_database("Device ID in token_userdeviceid is invalid.") })?, ))) @@ -121,7 +121,7 @@ impl Users { #[tracing::instrument(skip(self, user_id, password))] pub fn set_password(&self, user_id: &UserId, password: Option<&str>) -> Result<()> { if let Some(password) = password { - if let Ok(hash) = utils::calculate_hash(&password) { + if let Ok(hash) = utils::calculate_hash(password) { self.userid_password .insert(user_id.as_bytes(), hash.as_bytes())?; Ok(()) @@ -245,7 +245,7 @@ impl Users { .expect("Device::to_string never fails."), )?; - self.set_token(user_id, &device_id, token)?; + self.set_token(user_id, device_id, token)?; Ok(()) } @@ -294,7 +294,7 @@ impl Users { .scan_prefix(prefix) .map(|(bytes, _)| { Ok(utils::string_from_bytes( - &bytes + bytes .rsplit(|&b| b == 0xff) .next() .ok_or_else(|| Error::bad_database("UserDevice ID in db is invalid."))?, @@ -357,7 +357,7 @@ impl Users { // TODO: Use DeviceKeyId::to_string when it's available (and update everything, // because there are no wrapping quotation marks anymore) key.extend_from_slice( - &serde_json::to_string(one_time_key_key) + serde_json::to_string(one_time_key_key) .expect("DeviceKeyId::to_string always works") .as_bytes(), ); @@ -368,7 +368,7 @@ impl Users { )?; self.userid_lastonetimekeyupdate - .insert(&user_id.as_bytes(), &globals.next_count()?.to_be_bytes())?; + .insert(user_id.as_bytes(), &globals.next_count()?.to_be_bytes())?; Ok(()) } @@ -376,7 +376,7 @@ impl Users { #[tracing::instrument(skip(self, user_id))] pub fn last_one_time_keys_update(&self, user_id: &UserId) -> Result { self.userid_lastonetimekeyupdate - .get(&user_id.as_bytes())? + .get(user_id.as_bytes())? .map(|bytes| { utils::u64_from_bytes(&bytes).map_err(|_| { Error::bad_database("Count in roomid_lastroomactiveupdate is invalid.") @@ -402,7 +402,7 @@ impl Users { prefix.push(b':'); self.userid_lastonetimekeyupdate - .insert(&user_id.as_bytes(), &globals.next_count()?.to_be_bytes())?; + .insert(user_id.as_bytes(), &globals.next_count()?.to_be_bytes())?; self.onetimekeyid_onetimekeys .scan_prefix(prefix) @@ -680,7 +680,7 @@ impl Users { globals: &super::globals::Globals, ) -> Result<()> { let count = globals.next_count()?.to_be_bytes(); - for room_id in rooms.rooms_joined(&user_id).filter_map(|r| r.ok()) { + for room_id in rooms.rooms_joined(user_id).filter_map(|r| r.ok()) { // Don't send key updates to unencrypted rooms if rooms .room_state_get(&room_id, &EventType::RoomEncryption, "")? @@ -961,7 +961,7 @@ impl Users { pub fn deactivate_account(&self, user_id: &UserId) -> Result<()> { // Remove all associated devices for device_id in self.all_device_ids(user_id) { - self.remove_device(&user_id, &device_id?)?; + self.remove_device(user_id, &device_id?)?; } // Set the password to "" to indicate a deactivated account. Hashes will never result in an diff --git a/src/ruma_wrapper.rs b/src/ruma_wrapper.rs index fa283790..4629de92 100644 --- a/src/ruma_wrapper.rs +++ b/src/ruma_wrapper.rs @@ -66,7 +66,7 @@ where let limit = db.globals.max_request_size(); let mut handle = data.open(ByteUnit::Byte(limit.into())); let mut body = Vec::new(); - if let Err(_) = handle.read_to_end(&mut body).await { + if handle.read_to_end(&mut body).await.is_err() { // Client disconnected // Missing Token return Failure((Status::new(582), ())); @@ -123,7 +123,7 @@ where match metadata.authentication { AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => { if let Some(token) = token { - match db.users.find_from_token(&token).unwrap() { + match db.users.find_from_token(token).unwrap() { // Unknown Token None => return Failure((Status::new(581), ())), Some((user_id, device_id)) => ( diff --git a/src/server_server.rs b/src/server_server.rs index 89b9c5cf..1d9ba61d 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -150,7 +150,7 @@ where } else { write_destination_to_cache = true; - let result = find_actual_destination(globals, &destination).await; + let result = find_actual_destination(globals, destination).await; (result.0, result.1.into_uri_string()) }; @@ -359,7 +359,7 @@ async fn find_actual_destination( let (host, port) = destination_str.split_at(pos); FedDest::Named(host.to_string(), port.to_string()) } else { - match request_well_known(globals, &destination.as_str()).await { + match request_well_known(globals, destination.as_str()).await { // 3: A .well-known file is available Some(delegated_hostname) => { hostname = add_port_to_hostname(&delegated_hostname).into_uri_string(); @@ -806,7 +806,7 @@ pub async fn send_transaction_message_route( .event_ids .iter() .filter_map(|id| { - db.rooms.get_pdu_count(&id).ok().flatten().map(|r| (id, r)) + db.rooms.get_pdu_count(id).ok().flatten().map(|r| (id, r)) }) .max_by_key(|(_, count)| *count) { @@ -875,8 +875,8 @@ pub async fn send_transaction_message_route( DeviceIdOrAllDevices::DeviceId(target_device_id) => { db.users.add_to_device_event( &sender, - &target_user_id, - &target_device_id, + target_user_id, + target_device_id, &ev_type.to_string(), event.deserialize_as().map_err(|_| { Error::BadRequest( @@ -889,10 +889,10 @@ pub async fn send_transaction_message_route( } DeviceIdOrAllDevices::AllDevices => { - for target_device_id in db.users.all_device_ids(&target_user_id) { + for target_device_id in db.users.all_device_ids(target_user_id) { db.users.add_to_device_event( &sender, - &target_user_id, + target_user_id, &target_device_id?, &ev_type.to_string(), event.deserialize_as().map_err(|_| { @@ -959,7 +959,7 @@ pub(crate) async fn handle_incoming_pdu<'a>( db: &'a Database, pub_key_map: &'a RwLock>>, ) -> StdResult>, String> { - match db.rooms.exists(&room_id) { + match db.rooms.exists(room_id) { Ok(true) => {} _ => { return Err("Room is unknown to this server.".to_string()); @@ -967,19 +967,19 @@ pub(crate) async fn handle_incoming_pdu<'a>( } // 1. Skip the PDU if we already have it as a timeline event - if let Ok(Some(pdu_id)) = db.rooms.get_pdu_id(&event_id) { + if let Ok(Some(pdu_id)) = db.rooms.get_pdu_id(event_id) { return Ok(Some(pdu_id.to_vec())); } let create_event = db .rooms - .room_state_get(&room_id, &EventType::RoomCreate, "") + .room_state_get(room_id, &EventType::RoomCreate, "") .map_err(|_| "Failed to ask database for event.".to_owned())? .ok_or_else(|| "Failed to find create event in db.".to_owned())?; let first_pdu_in_room = db .rooms - .first_pdu_in_room(&room_id) + .first_pdu_in_room(room_id) .map_err(|_| "Error loading first room event.".to_owned())? .expect("Room exists"); @@ -1021,7 +1021,7 @@ pub(crate) async fn handle_incoming_pdu<'a>( origin, &[prev_event_id.clone()], &create_event, - &room_id, + room_id, pub_key_map, ) .await @@ -1049,12 +1049,12 @@ pub(crate) async fn handle_incoming_pdu<'a>( (*prev_event_id).clone(), pdu.prev_events.iter().cloned().collect(), ); - eventid_info.insert(prev_event_id.clone(), (pdu, json)); } else { // Time based check failed graph.insert((*prev_event_id).clone(), HashSet::new()); - eventid_info.insert(prev_event_id.clone(), (pdu, json)); } + + eventid_info.insert(prev_event_id.clone(), (pdu, json)); } else { // Get json failed graph.insert((*prev_event_id).clone(), HashSet::new()); @@ -1146,7 +1146,7 @@ fn handle_outlier_pdu<'a>( // We go through all the signatures we see on the value and fetch the corresponding signing // keys - fetch_required_signing_keys(&value, &pub_key_map, db) + fetch_required_signing_keys(&value, pub_key_map, db) .await .map_err(|e| e.to_string())?; @@ -1210,8 +1210,8 @@ fn handle_outlier_pdu<'a>( .cloned() .map(Arc::new) .collect::>(), - &create_event, - &room_id, + create_event, + room_id, pub_key_map, ) .await; @@ -1256,7 +1256,7 @@ fn handle_outlier_pdu<'a>( if auth_events .get(&(EventType::RoomCreate, "".to_owned())) .map(|a| a.as_ref()) - != Some(&create_event) + != Some(create_event) { return Err("Incoming event refers to wrong create event.".to_owned()); } @@ -1273,8 +1273,6 @@ fn handle_outlier_pdu<'a>( None }; - let incoming_pdu = Arc::new(incoming_pdu.clone()); - if !state_res::event_auth::auth_check( &room_version, &incoming_pdu, @@ -1295,7 +1293,7 @@ fn handle_outlier_pdu<'a>( .map_err(|_| "Failed to add pdu as outlier.".to_owned())?; debug!("Added pdu as outlier."); - Ok((incoming_pdu, val)) + Ok((Arc::new(incoming_pdu), val)) }) } @@ -1427,7 +1425,7 @@ async fn upgrade_outlier_to_timeline_pdu( } auth_chain_sets.push( - get_auth_chain(&room_id, starting_events, db) + get_auth_chain(room_id, starting_events, db) .map_err(|_| "Failed to load auth chain.".to_owned())? .map(|event_id| (*event_id).clone()) .collect(), @@ -1478,7 +1476,7 @@ async fn upgrade_outlier_to_timeline_pdu( &db.globals, origin, get_room_state_ids::v1::Request { - room_id: &room_id, + room_id, event_id: &incoming_pdu.event_id, }, ) @@ -1487,15 +1485,15 @@ async fn upgrade_outlier_to_timeline_pdu( Ok(res) => { warn!("Fetching state events at event."); let state_vec = fetch_and_handle_outliers( - &db, + db, origin, &res.pdu_ids .iter() .cloned() .map(Arc::new) .collect::>(), - &create_event, - &room_id, + create_event, + room_id, pub_key_map, ) .await; @@ -1568,11 +1566,11 @@ async fn upgrade_outlier_to_timeline_pdu( None::, // TODO: third party invite |k, s| { db.rooms - .get_shortstatekey(&k, &s) + .get_shortstatekey(k, s) .ok() .flatten() .and_then(|shortstatekey| state_at_incoming_event.get(&shortstatekey)) - .and_then(|event_id| db.rooms.get_pdu(&event_id).ok().flatten()) + .and_then(|event_id| db.rooms.get_pdu(event_id).ok().flatten()) }, ) .map_err(|_e| "Auth check failed.".to_owned())?; @@ -1598,7 +1596,7 @@ async fn upgrade_outlier_to_timeline_pdu( // applied. We start with the previous extremities (aka leaves) let mut extremities = db .rooms - .get_pdu_leaves(&room_id) + .get_pdu_leaves(room_id) .map_err(|_| "Failed to load room leaves".to_owned())?; // Remove any forward extremities that are referenced by this incoming event's prev_events @@ -1609,11 +1607,11 @@ async fn upgrade_outlier_to_timeline_pdu( } // Only keep those extremities were not referenced yet - extremities.retain(|id| !matches!(db.rooms.is_event_referenced(&room_id, id), Ok(true))); + extremities.retain(|id| !matches!(db.rooms.is_event_referenced(room_id, id), Ok(true))); let current_sstatehash = db .rooms - .current_shortstatehash(&room_id) + .current_shortstatehash(room_id) .map_err(|_| "Failed to load current state hash.".to_owned())? .expect("every room has state"); @@ -1625,7 +1623,7 @@ async fn upgrade_outlier_to_timeline_pdu( let auth_events = db .rooms .get_auth_events( - &room_id, + room_id, &incoming_pdu.kind, &incoming_pdu.sender, incoming_pdu.state_key.as_deref(), @@ -1637,7 +1635,7 @@ async fn upgrade_outlier_to_timeline_pdu( .iter() .map(|(shortstatekey, id)| { db.rooms - .compress_state_event(*shortstatekey, &id, &db.globals) + .compress_state_event(*shortstatekey, id, &db.globals) .map_err(|_| "Failed to compress_state_event".to_owned()) }) .collect::>()?; @@ -1656,7 +1654,7 @@ async fn upgrade_outlier_to_timeline_pdu( if soft_fail { append_incoming_pdu( - &db, + db, &incoming_pdu, val, extremities, @@ -1680,7 +1678,7 @@ async fn upgrade_outlier_to_timeline_pdu( for id in dbg!(&extremities) { match db .rooms - .get_pdu(&id) + .get_pdu(id) .map_err(|_| "Failed to ask db for pdu.".to_owned())? { Some(leaf_pdu) => { @@ -1757,7 +1755,7 @@ async fn upgrade_outlier_to_timeline_pdu( .iter() .map(|(k, id)| { db.rooms - .compress_state_event(*k, &id, &db.globals) + .compress_state_event(*k, id, &db.globals) .map_err(|_| "Failed to compress_state_event.".to_owned()) }) .collect::>()? @@ -1769,7 +1767,7 @@ async fn upgrade_outlier_to_timeline_pdu( for state in &fork_states { auth_chain_sets.push( get_auth_chain( - &room_id, + room_id, state.iter().map(|(_, id)| id.clone()).collect(), db, ) @@ -1828,7 +1826,7 @@ async fn upgrade_outlier_to_timeline_pdu( // Set the new room state to the resolved state if update_state { db.rooms - .force_state(&room_id, new_room_state, &db) + .force_state(room_id, new_room_state, db) .map_err(|_| "Failed to set new room state.".to_owned())?; } debug!("Updated resolved state"); @@ -1841,7 +1839,7 @@ async fn upgrade_outlier_to_timeline_pdu( // represent the state for this event. let pdu_id = append_incoming_pdu( - &db, + db, &incoming_pdu, val, extremities, @@ -1886,7 +1884,7 @@ pub(crate) fn fetch_and_handle_outliers<'a>( let mut pdus = vec![]; for id in events { - if let Some((time, tries)) = db.globals.bad_event_ratelimiter.read().unwrap().get(&id) { + if let Some((time, tries)) = db.globals.bad_event_ratelimiter.read().unwrap().get(id) { // Exponential backoff let mut min_elapsed_duration = Duration::from_secs(5 * 60) * (*tries) * (*tries); if min_elapsed_duration > Duration::from_secs(60 * 60 * 24) { @@ -1902,7 +1900,7 @@ pub(crate) fn fetch_and_handle_outliers<'a>( // a. Look in the main timeline (pduid_pdu tree) // b. Look at outlier pdu tree // (get_pdu_json checks both) - let local_pdu = db.rooms.get_pdu(&id); + let local_pdu = db.rooms.get_pdu(id); let pdu = match local_pdu { Ok(Some(pdu)) => { trace!("Found {} in db", id); @@ -1916,7 +1914,7 @@ pub(crate) fn fetch_and_handle_outliers<'a>( .send_federation_request( &db.globals, origin, - get_event::v1::Request { event_id: &id }, + get_event::v1::Request { event_id: id }, ) .await { @@ -1940,8 +1938,8 @@ pub(crate) fn fetch_and_handle_outliers<'a>( match handle_outlier_pdu( origin, create_event, - &id, - &room_id, + id, + room_id, value.clone(), db, pub_key_map, @@ -2089,7 +2087,7 @@ pub(crate) async fn fetch_signing_keys( .sending .send_federation_request( &db.globals, - &server, + server, get_remote_server_keys::v2::Request::new( origin, MilliSecondsSinceUnixEpoch::from_system_time( @@ -2168,7 +2166,7 @@ fn append_incoming_pdu( pdu, pdu_json, &new_room_leaves.into_iter().collect::>(), - &db, + db, )?; for appservice in db.appservice.all()? { @@ -2206,7 +2204,7 @@ fn append_incoming_pdu( && pdu .state_key .as_ref() - .map_or(false, |state_key| users.is_match(&state_key)) + .map_or(false, |state_key| users.is_match(state_key)) }; let matching_aliases = |aliases: &Regex| { db.rooms @@ -2273,7 +2271,7 @@ pub(crate) fn get_auth_chain<'a>( chunk_cache.extend(cached.iter().cloned()); } else { misses2 += 1; - let auth_chain = Arc::new(get_auth_chain_inner(&room_id, &event_id, db)?); + let auth_chain = Arc::new(get_auth_chain_inner(room_id, &event_id, db)?); db.rooms .cache_auth_chain(vec![sevent_id], Arc::clone(&auth_chain))?; println!( @@ -2821,7 +2819,7 @@ async fn create_join_event( // We need to return the state prior to joining, let's keep a reference to that here let shortstatehash = db .rooms - .current_shortstatehash(&room_id)? + .current_shortstatehash(room_id)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "Pdu state not found.", @@ -2831,7 +2829,7 @@ async fn create_join_event( // let mut auth_cache = EventMap::new(); // We do not add the event_id field to the pdu here because of signature and hashes checks - let (event_id, value) = match crate::pdu::gen_event_id_canonical_json(&pdu) { + let (event_id, value) = match crate::pdu::gen_event_id_canonical_json(pdu) { Ok(t) => t, Err(_) => { // Event could not be converted to canonical json @@ -2860,7 +2858,7 @@ async fn create_join_event( .or_default(), ); let mutex_lock = mutex.lock().await; - let pdu_id = handle_incoming_pdu(&origin, &event_id, &room_id, value, true, &db, &pub_key_map) + let pdu_id = handle_incoming_pdu(&origin, &event_id, room_id, value, true, db, &pub_key_map) .await .map_err(|e| { warn!("Error while handling incoming send join PDU: {}", e); @@ -2877,14 +2875,14 @@ async fn create_join_event( let state_ids = db.rooms.state_full_ids(shortstatehash)?; let auth_chain_ids = get_auth_chain( - &room_id, + room_id, state_ids.iter().map(|(_, id)| id.clone()).collect(), - &db, + db, )?; for server in db .rooms - .room_servers(&room_id) + .room_servers(room_id) .filter_map(|r| r.ok()) .filter(|server| &**server != db.globals.server_name()) { @@ -2900,7 +2898,7 @@ async fn create_join_event( .collect(), state: state_ids .iter() - .filter_map(|(_, id)| db.rooms.get_pdu_json(&id).ok().flatten()) + .filter_map(|(_, id)| db.rooms.get_pdu_json(id).ok().flatten()) .map(PduEvent::convert_to_outgoing_federation_event) .collect(), }) @@ -3296,7 +3294,7 @@ fn get_server_keys_from_cache( let event_id = EventId::try_from(&*format!( "${}", - ruma::signatures::reference_hash(&value, &room_version) + ruma::signatures::reference_hash(&value, room_version) .expect("ruma can calculate reference hashes") )) .expect("ruma's reference hashes are valid event ids"); @@ -3388,10 +3386,10 @@ pub(crate) async fn fetch_join_signing_keys( // Try to fetch keys, failure is okay // Servers we couldn't find in the cache will be added to `servers` for pdu in &event.room_state.state { - let _ = get_server_keys_from_cache(pdu, &mut servers, &room_version, &mut pkm, &db); + let _ = get_server_keys_from_cache(pdu, &mut servers, room_version, &mut pkm, db); } for pdu in &event.room_state.auth_chain { - let _ = get_server_keys_from_cache(pdu, &mut servers, &room_version, &mut pkm, &db); + let _ = get_server_keys_from_cache(pdu, &mut servers, room_version, &mut pkm, db); } drop(pkm); From cbee7fe111cac04eff44776a66181bc32a747aef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 13 Sep 2021 23:19:00 +0200 Subject: [PATCH 012/201] improvement: deduplicate watchers --- src/database/abstraction/sqlite.rs | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/database/abstraction/sqlite.rs b/src/database/abstraction/sqlite.rs index 06e371e0..d5924b1c 100644 --- a/src/database/abstraction/sqlite.rs +++ b/src/database/abstraction/sqlite.rs @@ -4,14 +4,14 @@ use parking_lot::{Mutex, MutexGuard, RwLock}; use rusqlite::{Connection, DatabaseName::Main, OptionalExtension}; use std::{ cell::RefCell, - collections::HashMap, + collections::{hash_map, HashMap}, future::Future, path::{Path, PathBuf}, pin::Pin, sync::Arc, }; use thread_local::ThreadLocal; -use tokio::sync::oneshot::Sender; +use tokio::sync::watch; use tracing::debug; thread_local! { @@ -126,7 +126,7 @@ impl DatabaseEngine for Engine { pub struct SqliteTable { engine: Arc, name: String, - watchers: RwLock, Vec>>>, + watchers: RwLock, (watch::Sender<()>, watch::Receiver<()>)>>, } type TupleOfBytes = (Vec, Vec); @@ -215,10 +215,8 @@ impl Tree for SqliteTable { if !triggered.is_empty() { let mut watchers = self.watchers.write(); for prefix in triggered { - if let Some(txs) = watchers.remove(prefix) { - for tx in txs { - let _ = tx.send(()); - } + if let Some(tx) = watchers.remove(prefix) { + let _ = tx.0.send(()); } } }; @@ -367,17 +365,18 @@ impl Tree for SqliteTable { #[tracing::instrument(skip(self, prefix))] fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin + Send + 'a>> { - let (tx, rx) = tokio::sync::oneshot::channel(); - - self.watchers - .write() - .entry(prefix.to_vec()) - .or_default() - .push(tx); + let mut rx = match self.watchers.write().entry(prefix.to_vec()) { + hash_map::Entry::Occupied(o) => o.get().1.clone(), + hash_map::Entry::Vacant(v) => { + let (tx, rx) = tokio::sync::watch::channel(()); + v.insert((tx, rx.clone())); + rx + } + }; Box::pin(async move { // Tx is never destroyed - rx.await.unwrap(); + rx.changed().await.unwrap(); }) } From 422bd09e32f9f10f7705bfa1afeb291c97d579e7 Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Tue, 14 Sep 2021 09:44:15 +0000 Subject: [PATCH 013/201] Remove the "register an account with element" test Broke due to a timeout and Timo does not like broken tests. Less testing means less failing tests. Also, hopefully sytest is less broken now. --- .gitlab-ci.yml | 24 ----- .../test-element-web-registration.js | 101 ------------------ 2 files changed, 125 deletions(-) delete mode 100644 tests/client-element-web/test-element-web-registration.js diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 018e5a1e..dfe7198a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -340,30 +340,6 @@ test:sytest: junit: "$CI_PROJECT_DIR/sytest.xml" -test:register:element-web-stable: - stage: "test" - needs: - - "build:debug:cargo:x86_64-unknown-linux-gnu" - image: "buildkite/puppeteer:latest" - tags: [ "docker" ] - interruptible: true - script: - - "CONDUIT_CONFIG=tests/test-config.toml ./conduit-debug-x86_64-unknown-linux-gnu > conduit.log &" - - "cd tests/client-element-web/" - - "npm install puppeteer" - - "node test-element-web-registration.js \"https://app.element.io/\" \"http://localhost:6167\"" - - "killall --regexp \"conduit\"" - - "cd ../.." - - "cat conduit.log" - artifacts: - paths: - - "tests/client-element-web/*.png" - - "*.log" - expire_in: 1 week - when: always - retry: 1 - - # --------------------------------------------------------------------- # # Store binaries as package so they have download urls # # --------------------------------------------------------------------- # diff --git a/tests/client-element-web/test-element-web-registration.js b/tests/client-element-web/test-element-web-registration.js deleted file mode 100644 index 8f2e7f02..00000000 --- a/tests/client-element-web/test-element-web-registration.js +++ /dev/null @@ -1,101 +0,0 @@ -const puppeteer = require('puppeteer'); - -run().then(() => console.log('Done')).catch(error => { - console.error("Registration test failed."); - console.error("There might be a screenshot of the failure in the artifacts.\n"); - console.error(error); - process.exit(111); -}); - -async function run() { - - const elementUrl = process.argv[process.argv.length - 2]; - console.debug("Testing registration with ElementWeb hosted at "+ elementUrl); - - const homeserverUrl = process.argv[process.argv.length - 1]; - console.debug("Homeserver url: "+ homeserverUrl); - - const username = "testuser" + String(Math.floor(Math.random() * 100000)); - const password = "testpassword" + String(Math.floor(Math.random() * 100000)); - console.debug("Testuser for this run:\n User: " + username + "\n Password: " + password); - - const browser = await puppeteer.launch({ - headless: true, args: [ - "--no-sandbox" - ] - }); - - const page = await browser.newPage(); - await page.goto(elementUrl); - - await page.screenshot({ path: '01-element-web-opened.png' }); - - console.debug("Click [Create Account] button"); - await page.waitForSelector('a.mx_ButtonCreateAccount'); - await page.click('a.mx_ButtonCreateAccount'); - - await page.screenshot({ path: '02-clicked-create-account-button.png' }); - - // The webapp should have loaded right now, if anything takes more than 5 seconds, something probably broke - page.setDefaultTimeout(5000); - - console.debug("Click [Edit] to switch homeserver"); - await page.waitForSelector('div.mx_ServerPicker_change'); - await page.click('div.mx_ServerPicker_change'); - - await page.screenshot({ path: '03-clicked-edit-homeserver-button.png' }); - - console.debug("Type in local homeserver url"); - await page.waitForSelector('input#mx_homeserverInput'); - await page.click('input#mx_homeserverInput'); - await page.click('input#mx_homeserverInput'); - await page.keyboard.type(homeserverUrl); - - await page.screenshot({ path: '04-typed-in-homeserver.png' }); - - console.debug("[Continue] with changed homeserver"); - await page.waitForSelector("div.mx_ServerPickerDialog_continue"); - await page.click('div.mx_ServerPickerDialog_continue'); - - await page.screenshot({ path: '05-back-to-enter-user-credentials.png' }); - - console.debug("Type in username"); - await page.waitForSelector("input#mx_RegistrationForm_username"); - await page.click('input#mx_RegistrationForm_username'); - await page.keyboard.type(username); - - await page.screenshot({ path: '06-typed-in-username.png' }); - - console.debug("Type in password"); - await page.waitForSelector("input#mx_RegistrationForm_password"); - await page.click('input#mx_RegistrationForm_password'); - await page.keyboard.type(password); - - await page.screenshot({ path: '07-typed-in-password-once.png' }); - - console.debug("Type in password again"); - await page.waitForSelector("input#mx_RegistrationForm_passwordConfirm"); - await page.click('input#mx_RegistrationForm_passwordConfirm'); - await page.keyboard.type(password); - - await page.screenshot({ path: '08-typed-in-password-twice.png' }); - - console.debug("Click on [Register] to finish the account creation"); - await page.waitForSelector("input.mx_Login_submit"); - await page.click('input.mx_Login_submit'); - - await page.screenshot({ path: '09-clicked-on-register-button.png' }); - - // Waiting for the app to login can take some time, so be patient. - page.setDefaultTimeout(10000); - - console.debug("Wait for chat window to show up"); - await page.waitForSelector("div.mx_HomePage_default_buttons"); - console.debug("Apparently the registration worked."); - - await page.screenshot({ path: '10-logged-in-homescreen.png' }); - - - // Close the browser and exit the script - await browser.close(); -} \ No newline at end of file From ab472e9b7c58d27059d738e3d24d03d6dfc1cce6 Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Tue, 14 Sep 2021 12:25:35 +0200 Subject: [PATCH 014/201] fix(ci): Fix aarch64 build gcc-8-aarch64-linux-gnu is not available in debian 11 (which the rust image uses), so update to gcc-10 Signed-off-by: Jonas Zohren --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dfe7198a..19252720 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -75,12 +75,12 @@ build:release:cargo:aarch64-unknown-linux-gnu: extends: .build-cargo-shared-settings variables: TARGET: "aarch64-unknown-linux-gnu" - NEEDED_PACKAGES: "build-essential gcc-8-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross" + NEEDED_PACKAGES: "build-essential gcc-10-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross" CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc CC_aarch64_unknown_linux_gnu: aarch64-linux-gnu-gcc CXX_aarch64_unknown_linux_gnu: aarch64-linux-gnu-g++ - TARGET_CC: "/usr/bin/aarch64-linux-gnu-gcc-8" - TARGET_AR: "/usr/bin/aarch64-linux-gnu-gcc-ar-8" + TARGET_CC: "/usr/bin/aarch64-linux-gnu-gcc-10" + TARGET_AR: "/usr/bin/aarch64-linux-gnu-gcc-ar-10" artifacts: name: "conduit-aarch64-unknown-linux-gnu" paths: From 5c02dc783066e98b445206c18cebe86972620de2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 14 Sep 2021 14:23:43 +0200 Subject: [PATCH 015/201] improvement: batch inserts for inserting pdus --- src/client_server/membership.rs | 9 ++++----- src/database/rooms.rs | 9 ++++----- src/database/sending.rs | 26 ++++++++++++++++++-------- src/server_server.rs | 9 ++++----- 4 files changed, 30 insertions(+), 23 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 10c052ea..146af791 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -1092,14 +1092,13 @@ pub(crate) async fn invite_helper<'a>( "Could not accept incoming PDU as timeline event.", ))?; - for server in db + let servers = db .rooms .room_servers(room_id) .filter_map(|r| r.ok()) - .filter(|server| &**server != db.globals.server_name()) - { - db.sending.send_pdu(&server, &pdu_id)?; - } + .filter(|server| &**server != db.globals.server_name()); + + db.sending.send_pdu(servers, &pdu_id)?; return Ok(()); } diff --git a/src/database/rooms.rs b/src/database/rooms.rs index b272a5ce..ec03e3ac 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -2104,13 +2104,12 @@ impl Rooms { // where events in the current room state do not exist self.set_room_state(room_id, statehashid)?; - for server in self + let servers = self .room_servers(room_id) .filter_map(|r| r.ok()) - .filter(|server| &**server != db.globals.server_name()) - { - db.sending.send_pdu(&server, &pdu_id)?; - } + .filter(|server| &**server != db.globals.server_name()); + + db.sending.send_pdu(servers, &pdu_id)?; for appservice in db.appservice.all()? { if self.appservice_in_room(room_id, &appservice, db)? { diff --git a/src/database/sending.rs b/src/database/sending.rs index c14f5819..70ff1b6d 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -84,7 +84,7 @@ pub enum SendingEventType { pub struct Sending { /// The state for a given state hash. pub(super) servername_educount: Arc, // EduCount: Count of last EDU sync - pub(super) servernameevent_data: Arc, // ServernamEvent = (+ / $)SenderKey / ServerName / UserId + PduId / Id (for edus), Data = EDU content + pub(super) servernameevent_data: Arc, // ServernameEvent = (+ / $)SenderKey / ServerName / UserId + PduId / Id (for edus), Data = EDU content pub(super) servercurrentevent_data: Arc, // ServerCurrentEvents = (+ / $)ServerName / UserId + PduId / Id (for edus), Data = EDU content pub(super) maximum_requests: Arc, pub sender: mpsc::UnboundedSender<(Vec, Vec)>, @@ -423,13 +423,23 @@ impl Sending { Ok(()) } - #[tracing::instrument(skip(self, server, pdu_id))] - pub fn send_pdu(&self, server: &ServerName, pdu_id: &[u8]) -> Result<()> { - let mut key = server.as_bytes().to_vec(); - key.push(0xff); - key.extend_from_slice(pdu_id); - self.servernameevent_data.insert(&key, &[])?; - self.sender.unbounded_send((key, vec![])).unwrap(); + #[tracing::instrument(skip(self, servers, pdu_id))] + pub fn send_pdu>>( + &self, + servers: I, + pdu_id: &[u8], + ) -> Result<()> { + let mut batch = servers.map(|server| { + let mut key = server.as_bytes().to_vec(); + key.push(0xff); + key.extend_from_slice(pdu_id); + + self.sender.unbounded_send((key.clone(), vec![])).unwrap(); + + (key, Vec::new()) + }); + + self.servernameevent_data.insert_batch(&mut batch)?; Ok(()) } diff --git a/src/server_server.rs b/src/server_server.rs index 1d9ba61d..2b8b06c7 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -2880,14 +2880,13 @@ async fn create_join_event( db, )?; - for server in db + let servers = db .rooms .room_servers(room_id) .filter_map(|r| r.ok()) - .filter(|server| &**server != db.globals.server_name()) - { - db.sending.send_pdu(&server, &pdu_id)?; - } + .filter(|server| &**server != db.globals.server_name()); + + db.sending.send_pdu(servers, &pdu_id)?; db.flush()?; From e8d998cedfda8650c13410e165acc199ea40433e Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Tue, 14 Sep 2021 20:44:11 +0000 Subject: [PATCH 016/201] fix(ci): Convince kaniko that it is indeed running in a container by --force-ing it. --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 19252720..386986fd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -222,6 +222,7 @@ build:docker:next: - > /kaniko/executor $KANIKO_CACHE_ARGS + --force --context $CI_PROJECT_DIR --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ') --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml) From fa616342b610dfeea0f6e12dc633fe89258c9e1e Mon Sep 17 00:00:00 2001 From: Greg Sutcliffe Date: Mon, 13 Sep 2021 16:22:52 +0000 Subject: [PATCH 017/201] Add two flavours of change for the mautrix-signal patch --- APPSERVICES.md | 47 +++++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/APPSERVICES.md b/APPSERVICES.md index ba9ae89a..8e57bc2d 100644 --- a/APPSERVICES.md +++ b/APPSERVICES.md @@ -11,28 +11,35 @@ Here are some appservices we tested and that work with Conduit: - [mautrix-hangouts](https://github.com/mautrix/hangouts/) - [mautrix-telegram](https://github.com/mautrix/telegram/) - [mautrix-signal](https://github.com/mautrix/signal) - - There are a few things you need to do, in order for the bridge (at least up to version `0.2.0`) to work. Before following the bridge installation guide, you need to map apply a patch to bridges `portal.py`. Go to [portal.py](https://github.com/mautrix/signal/blob/master/mautrix_signal/portal.py) at [mautrix-signal](https://github.com/mautrix/signal) (don't forget to change to the correct commit/version of the file) and copy its content, create a `portal.py` on your host system and paste it in. Now you need to change two lines: - [Line 1020](https://github.com/mautrix/signal/blob/4ea831536f154aba6419d13292479eb383ea3308/mautrix_signal/portal.py#L1020) - ```diff - --- levels.users[self.main_intent.mxid] = 9001 if is_initial else 100 - +++ levels.users[self.main_intent.mxid] = 100 if is_initial else 100 - ``` +There are a few things you need to do, in order for the bridge (at least up to version `0.2.0`) to work. Before following the bridge installation guide, you need to map apply a patch to bridges `portal.py`. How you do this depends upon whether you are running the bridge in `Docker` or `virtualenv`. + - Find / create the changed file: + - For Docker: + - Go to [portal.py](https://github.com/mautrix/signal/blob/master/mautrix_signal/portal.py) at [mautrix-signal](https://github.com/mautrix/signal) (don't forget to change to the correct commit/version of the file) and copy its content, create a `portal.py` on your host system and paste it in + - For virtualenv + - Find `./lib/python3.7/site-packages/mautrix_signal/portal.py` (the exact version of Python may be different on your system). + - Once you have `portal.py` you now need to change two lines. Lines numbers given here are approximate, you may need to look nearby: + - [Edit Line 1020](https://github.com/mautrix/signal/blob/4ea831536f154aba6419d13292479eb383ea3308/mautrix_signal/portal.py#L1020) + ```diff + --- levels.users[self.main_intent.mxid] = 9001 if is_initial else 100 + +++ levels.users[self.main_intent.mxid] = 100 if is_initial else 100 + ``` + - Add a new line between [Lines 1041 and 1042](https://github.com/mautrix/signal/blob/4ea831536f154aba6419d13292479eb383ea3308/mautrix_signal/portal.py#L1041-L1042) - and add a new line between [Lines 1041 and 1042](https://github.com/mautrix/signal/blob/4ea831536f154aba6419d13292479eb383ea3308/mautrix_signal/portal.py#L1041-L1042) - - ```diff - "type": str(EventType.ROOM_POWER_LEVELS), - +++ "state_key": "", - "content": power_levels.serialize(), - ``` - - Now you just need to map the patched `portal.py` into the `mautrix-signal` container - ```yml - volumes: - - ./////portal.py:/usr/lib/python3.9/site-packages/mautrix_signal/portal.py - ``` - and then read below and start following the bridge [installation instructions](https://docs.mau.fi/bridges/index.html). + ```diff + "type": str(EventType.ROOM_POWER_LEVELS), + +++ "state_key": "", + "content": power_levels.serialize(), + ``` + - Deploy the change + - Docker: + - Now you just need to map the patched `portal.py` into the `mautrix-signal` container + ```yml + volumes: + - ./////portal.py:/usr/lib/python3.9/site-packages/mautrix_signal/portal.py + ``` + - For virtualenv, that's all you need to do - it uses the edited file directly + - Now continue with the bridge [installation instructions](https://docs.mau.fi/bridges/index.html) and the notes below. ## Set up the appservice From d38f9b5f0186b2e36d72296e6b842e1b36fbe75a Mon Sep 17 00:00:00 2001 From: Greg Sutcliffe Date: Wed, 15 Sep 2021 20:16:59 +0000 Subject: [PATCH 018/201] Move Generic instructions for APPSERVICES above notes for specific bridges and tidy up. --- APPSERVICES.md | 84 +++++++++++++++++++++++++++----------------------- 1 file changed, 46 insertions(+), 38 deletions(-) diff --git a/APPSERVICES.md b/APPSERVICES.md index 8e57bc2d..26c34cc4 100644 --- a/APPSERVICES.md +++ b/APPSERVICES.md @@ -4,44 +4,7 @@ If you run into any problems while setting up an Appservice, write an email to `timo@koesters.xyz`, ask us in `#conduit:matrix.org` or [open an issue on GitLab](https://gitlab.com/famedly/conduit/-/issues/new). -## Tested appservices - -Here are some appservices we tested and that work with Conduit: -- [matrix-appservice-discord](https://github.com/Half-Shot/matrix-appservice-discord) -- [mautrix-hangouts](https://github.com/mautrix/hangouts/) -- [mautrix-telegram](https://github.com/mautrix/telegram/) -- [mautrix-signal](https://github.com/mautrix/signal) - -There are a few things you need to do, in order for the bridge (at least up to version `0.2.0`) to work. Before following the bridge installation guide, you need to map apply a patch to bridges `portal.py`. How you do this depends upon whether you are running the bridge in `Docker` or `virtualenv`. - - Find / create the changed file: - - For Docker: - - Go to [portal.py](https://github.com/mautrix/signal/blob/master/mautrix_signal/portal.py) at [mautrix-signal](https://github.com/mautrix/signal) (don't forget to change to the correct commit/version of the file) and copy its content, create a `portal.py` on your host system and paste it in - - For virtualenv - - Find `./lib/python3.7/site-packages/mautrix_signal/portal.py` (the exact version of Python may be different on your system). - - Once you have `portal.py` you now need to change two lines. Lines numbers given here are approximate, you may need to look nearby: - - [Edit Line 1020](https://github.com/mautrix/signal/blob/4ea831536f154aba6419d13292479eb383ea3308/mautrix_signal/portal.py#L1020) - ```diff - --- levels.users[self.main_intent.mxid] = 9001 if is_initial else 100 - +++ levels.users[self.main_intent.mxid] = 100 if is_initial else 100 - ``` - - Add a new line between [Lines 1041 and 1042](https://github.com/mautrix/signal/blob/4ea831536f154aba6419d13292479eb383ea3308/mautrix_signal/portal.py#L1041-L1042) - - ```diff - "type": str(EventType.ROOM_POWER_LEVELS), - +++ "state_key": "", - "content": power_levels.serialize(), - ``` - - Deploy the change - - Docker: - - Now you just need to map the patched `portal.py` into the `mautrix-signal` container - ```yml - volumes: - - ./////portal.py:/usr/lib/python3.9/site-packages/mautrix_signal/portal.py - ``` - - For virtualenv, that's all you need to do - it uses the edited file directly - - Now continue with the bridge [installation instructions](https://docs.mau.fi/bridges/index.html) and the notes below. - -## Set up the appservice +## Set up the appservice - general instructions Follow whatever instructions are given by the appservice. This usually includes downloading, changing its config (setting domain, homeserver url, port etc.) @@ -76,3 +39,48 @@ Then you are done. Conduit will send messages to the appservices and the appservice can send requests to the homeserver. You don't need to restart Conduit, but if it doesn't work, restarting while the appservice is running could help. + +## Appservice-specific instructions + +### Tested appservices + +These appservices have been tested and work with Conduit without any extra steps: + +- [matrix-appservice-discord](https://github.com/Half-Shot/matrix-appservice-discord) +- [mautrix-hangouts](https://github.com/mautrix/hangouts/) +- [mautrix-telegram](https://github.com/mautrix/telegram/) + +### [mautrix-signal](https://github.com/mautrix/signal) + +There are a few things you need to do, in order for the Signal bridge (at least +up to version `0.2.0`) to work. How you do this depends on whether you use +Docker or `virtualenv` to run it. In either case you need to modify +[portal.py](https://github.com/mautrix/signal/blob/master/mautrix_signal/portal.py). +Do this **before** following the bridge installation guide. + +1. **Create a copy of `portal.py`**. Go to + [portal.py](https://github.com/mautrix/signal/blob/master/mautrix_signal/portal.py) +at [mautrix-signal](https://github.com/mautrix/signal) (make sure you change to +the correct commit/version of mautrix-signal you're using) and copy its +content. Create a new `portal.py` on your system and paste the content in. +2. **Patch the copy**. Exact line numbers may be slightly different, look nearby if they don't match: + - [Line 1020](https://github.com/mautrix/signal/blob/4ea831536f154aba6419d13292479eb383ea3308/mautrix_signal/portal.py#L1020) + ```diff + --- levels.users[self.main_intent.mxid] = 9001 if is_initial else 100 + +++ levels.users[self.main_intent.mxid] = 100 if is_initial else 100 + ``` + - [Between lines 1041 and 1042](https://github.com/mautrix/signal/blob/4ea831536f154aba6419d13292479eb383ea3308/mautrix_signal/portal.py#L1041-L1042) add a new line: + ```diff + "type": str(EventType.ROOM_POWER_LEVELS), + +++ "state_key": "", + "content": power_levels.serialize(), + ``` +3. **Deploy the patch**. This is different depending on how you have `mautrix-signal` deployed: + - [*If using virtualenv*] Copy your patched `portal.py` to `./lib/python3.7/site-packages/mautrix_signal/portal.py` (the exact version of Python may be different on your system). + - [*If using Docker*] Map the patched `portal.py` into the `mautrix-signal` container: + + ```yaml + volumes: + - ./your/path/on/host/portal.py:/usr/lib/python3.9/site-packages/mautrix_signal/portal.py + ``` +4. Now continue with the [bridge installation instructions ](https://docs.mau.fi/bridges/index.html) and the general bridge notes above. From 5b23d3d06e0d907301d857797b329c22ddc7dd14 Mon Sep 17 00:00:00 2001 From: Luc-pascal Ceccaldi Date: Thu, 23 Sep 2021 07:49:52 +0000 Subject: [PATCH 019/201] Change listen address when running inside a Container to prevent Bad Gateway error --- conduit-example.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/conduit-example.toml b/conduit-example.toml index 80082568..4275f528 100644 --- a/conduit-example.toml +++ b/conduit-example.toml @@ -40,6 +40,7 @@ trusted_servers = ["matrix.org"] #workers = 4 # default: cpu core count * 2 address = "127.0.0.1" # This makes sure Conduit can only be reached using the reverse proxy +#address = "0.0.0.0" # If Conduit is running in a container, make sure the reverse proxy (ie. Traefik) can reach it. proxy = "none" # more examples can be found at src/database/proxy.rs:6 From 6bc8fb2ae7a895628e52c88f2b347dd1389b8858 Mon Sep 17 00:00:00 2001 From: Nyaaori <+@nyaaori.cat> Date: Fri, 24 Sep 2021 07:16:34 +0000 Subject: [PATCH 020/201] Implement admin check and add config option for allowing room creation --- src/client_server/room.rs | 10 ++++++++++ src/database.rs | 2 ++ src/database/globals.rs | 4 ++++ src/database/users.rs | 19 ++++++++++++++++++- 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 5a026997..f6c3a50e 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -61,6 +61,16 @@ pub async fn create_room_route( ); let state_lock = mutex_state.lock().await; + if !db.globals.allow_room_creation() + && !body.from_appservice + && !db.users.is_admin(sender_user, &db.rooms, &db.globals)? + { + return Err(Error::BadRequest( + ErrorKind::Forbidden, + "Room creation has been disabled.", + )); + } + let alias: Option = body.room_alias_name .as_ref() diff --git a/src/database.rs b/src/database.rs index 110d4d0c..8a589299 100644 --- a/src/database.rs +++ b/src/database.rs @@ -61,6 +61,8 @@ pub struct Config { allow_encryption: bool, #[serde(default = "false_fn")] allow_federation: bool, + #[serde(default = "true_fn")] + allow_room_creation: bool, #[serde(default = "false_fn")] pub allow_jaeger: bool, #[serde(default = "false_fn")] diff --git a/src/database/globals.rs b/src/database/globals.rs index 2f1b45ad..c2ef1c36 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -211,6 +211,10 @@ impl Globals { self.config.allow_federation } + pub fn allow_room_creation(&self) -> bool { + self.config.allow_room_creation + } + pub fn trusted_servers(&self) -> &[Box] { &self.config.trusted_servers } diff --git a/src/database/users.rs b/src/database/users.rs index 63ed0710..ee064903 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -5,7 +5,8 @@ use ruma::{ events::{AnyToDeviceEvent, EventType}, identifiers::MxcUri, serde::Raw, - DeviceId, DeviceKeyAlgorithm, DeviceKeyId, MilliSecondsSinceUnixEpoch, UInt, UserId, + DeviceId, DeviceKeyAlgorithm, DeviceKeyId, MilliSecondsSinceUnixEpoch, RoomAliasId, UInt, + UserId, }; use std::{collections::BTreeMap, convert::TryFrom, mem, sync::Arc}; use tracing::warn; @@ -53,6 +54,22 @@ impl Users { .is_empty()) } + /// Check if a user is an admin + #[tracing::instrument(skip(self, user_id, rooms, globals))] + pub fn is_admin( + &self, + user_id: &UserId, + rooms: &super::rooms::Rooms, + globals: &super::globals::Globals, + ) -> Result { + let admin_room_alias_id = + RoomAliasId::try_from(format!("#admins:{}", globals.server_name())) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid alias."))?; + let admin_room_id = rooms.id_from_alias(&admin_room_alias_id)?.unwrap(); + + Ok(rooms.is_joined(user_id, &admin_room_id)?) + } + /// Create a new user account on this homeserver. #[tracing::instrument(skip(self, user_id, password))] pub fn create(&self, user_id: &UserId, password: Option<&str>) -> Result<()> { From 636db8cfaaedf60ff535acdfc545ffcfa26bb364 Mon Sep 17 00:00:00 2001 From: Nyaaori <+@nyaaori.cat> Date: Fri, 24 Sep 2021 22:44:26 +0000 Subject: [PATCH 021/201] Make allow_encryption work again, fixing #115 --- src/client_server/message.rs | 8 ++++++++ src/client_server/state.rs | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 93ead2c7..25964cc2 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -45,6 +45,14 @@ pub async fn send_message_event_route( ); let state_lock = mutex_state.lock().await; + // Forbid m.room.encrypted if encryption is disabled + if &body.event_type == "m.room.encrypted" && !db.globals.allow_encryption() { + return Err(Error::BadRequest( + ErrorKind::Forbidden, + "Encryption has been disabled", + )); + } + // Check if this is a new transaction id if let Some(response) = db.transaction_ids diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 24cc2a18..7618dcc4 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -73,6 +73,14 @@ pub async fn send_state_event_for_empty_key_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + // Forbid m.room.encryption if encryption is disabled + if &body.event_type == "m.room.encryption" && !db.globals.allow_encryption() { + return Err(Error::BadRequest( + ErrorKind::Forbidden, + "Encryption has been disabled", + )); + } + let event_id = send_state_event_for_key_helper( &db, sender_user, From 09895a20c8c3ffd5e4459db2fb5df0fc0dfc3338 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Wed, 13 Oct 2021 10:16:45 +0200 Subject: [PATCH 022/201] Upgrade Ruma MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Timo Kösters --- Cargo.lock | 50 +++--- Cargo.toml | 2 +- rust-toolchain | 2 +- src/client_server/account.rs | 81 +++++----- src/client_server/device.rs | 6 +- src/client_server/directory.rs | 274 ++++++++++++++++---------------- src/client_server/keys.rs | 4 +- src/client_server/membership.rs | 103 ++++++------ src/client_server/profile.rs | 20 +-- src/client_server/redact.rs | 5 +- src/client_server/room.rs | 125 +++++++-------- src/client_server/session.rs | 12 +- src/client_server/state.rs | 16 +- src/client_server/sync.rs | 34 ++-- src/database/admin.rs | 13 +- src/database/pusher.rs | 34 ++-- src/database/rooms.rs | 256 ++++++++++++++--------------- src/database/sending.rs | 12 +- src/database/uiaa.rs | 7 +- src/pdu.rs | 95 +++++------ src/server_server.rs | 100 ++++++------ 21 files changed, 628 insertions(+), 623 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 70d7f4b4..293bcff7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1968,7 +1968,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" dependencies = [ "assign", "js_int", @@ -1988,8 +1988,8 @@ dependencies = [ [[package]] name = "ruma-api" -version = "0.18.3" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +version = "0.18.5" +source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" dependencies = [ "bytes", "http", @@ -2004,8 +2004,8 @@ dependencies = [ [[package]] name = "ruma-api-macros" -version = "0.18.3" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +version = "0.18.5" +source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2016,7 +2016,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" dependencies = [ "ruma-api", "ruma-common", @@ -2029,8 +2029,8 @@ dependencies = [ [[package]] name = "ruma-client-api" -version = "0.12.2" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +version = "0.12.3" +source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" dependencies = [ "assign", "bytes", @@ -2050,7 +2050,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.6.0" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" dependencies = [ "indexmap", "js_int", @@ -2064,8 +2064,8 @@ dependencies = [ [[package]] name = "ruma-events" -version = "0.24.5" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +version = "0.24.6" +source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" dependencies = [ "indoc", "js_int", @@ -2080,8 +2080,8 @@ dependencies = [ [[package]] name = "ruma-events-macros" -version = "0.24.5" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +version = "0.24.6" +source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2092,7 +2092,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.3.1" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" dependencies = [ "js_int", "ruma-api", @@ -2107,9 +2107,10 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.20.0" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" dependencies = [ "paste", + "percent-encoding", "rand 0.8.4", "ruma-identifiers-macros", "ruma-identifiers-validation", @@ -2121,7 +2122,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.20.0" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" dependencies = [ "quote", "ruma-identifiers-validation", @@ -2131,12 +2132,15 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" +dependencies = [ + "thiserror", +] [[package]] name = "ruma-identity-service-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" dependencies = [ "js_int", "ruma-api", @@ -2149,7 +2153,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" dependencies = [ "js_int", "ruma-api", @@ -2164,7 +2168,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" dependencies = [ "bytes", "form_urlencoded", @@ -2178,7 +2182,7 @@ dependencies = [ [[package]] name = "ruma-serde-macros" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2189,7 +2193,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.9.0" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" dependencies = [ "base64 0.13.0", "ed25519-dalek", @@ -2206,7 +2210,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.4.1" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" dependencies = [ "itertools 0.10.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 593a1fd3..0f246733 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle request # Used for matrix spec type definitions and helpers #ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } -ruma = { git = "https://github.com/ruma/ruma", rev = "a6a1224652912a957b09f136ec5da2686be6e0e2", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +ruma = { git = "https://github.com/ruma/ruma", rev = "44cfd0adbc83303c19aef590ad0d71647e19f197", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { git = "https://github.com/timokoesters/ruma", rev = "50c1db7e0a3a21fc794b0cce3b64285a4c750c71", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } diff --git a/rust-toolchain b/rust-toolchain index d96ae405..74df8b16 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -1.52 +1.53 diff --git a/src/client_server/account.rs b/src/client_server/account.rs index fb338422..4b3ad0d4 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -15,19 +15,28 @@ use ruma::{ ThirdPartyIdRemovalStatus, }, contact::get_contacts, - uiaa::{AuthFlow, UiaaInfo}, + uiaa::{AuthFlow, AuthType, UiaaInfo}, }, }, events::{ room::{ - canonical_alias, guest_access, history_visibility, join_rules, member, message, name, - topic, + canonical_alias::RoomCanonicalAliasEventContent, + create::RoomCreateEventContent, + guest_access::{GuestAccess, RoomGuestAccessEventContent}, + history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, + join_rules::{JoinRule, RoomJoinRulesEventContent}, + member::{MembershipState, RoomMemberEventContent}, + message::RoomMessageEventContent, + name::RoomNameEventContent, + power_levels::RoomPowerLevelsEventContent, + topic::RoomTopicEventContent, }, EventType, }, identifiers::RoomName, push, RoomAliasId, RoomId, RoomVersionId, UserId, }; +use serde_json::value::to_raw_value; use tracing::info; use register::RegistrationKind; @@ -147,7 +156,7 @@ pub async fn register_route( // UIAA let mut uiaainfo = UiaaInfo { flows: vec![AuthFlow { - stages: vec!["m.login.dummy".to_owned()], + stages: vec![AuthType::Dummy], }], completed: Vec::new(), params: Default::default(), @@ -270,7 +279,7 @@ pub async fn register_route( ); let state_lock = mutex_state.lock().await; - let mut content = ruma::events::room::create::CreateEventContent::new(conduit_user.clone()); + let mut content = RoomCreateEventContent::new(conduit_user.clone()); content.federate = true; content.predecessor = None; content.room_version = RoomVersionId::Version6; @@ -279,7 +288,7 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomCreate, - content: serde_json::to_value(content).expect("event is valid, we just created it"), + content: to_raw_value(&content).expect("event is valid, we just created it"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -294,8 +303,8 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Join, + content: to_raw_value(&RoomMemberEventContent { + membership: MembershipState::Join, displayname: None, avatar_url: None, is_direct: None, @@ -322,12 +331,10 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomPowerLevels, - content: serde_json::to_value( - ruma::events::room::power_levels::PowerLevelsEventContent { - users, - ..Default::default() - }, - ) + content: to_raw_value(&RoomPowerLevelsEventContent { + users, + ..Default::default() + }) .expect("event is valid, we just created it"), unsigned: None, state_key: Some("".to_owned()), @@ -343,10 +350,8 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomJoinRules, - content: serde_json::to_value(join_rules::JoinRulesEventContent::new( - join_rules::JoinRule::Invite, - )) - .expect("event is valid, we just created it"), + content: to_raw_value(&RoomJoinRulesEventContent::new(JoinRule::Invite)) + .expect("event is valid, we just created it"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -361,11 +366,9 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomHistoryVisibility, - content: serde_json::to_value( - history_visibility::HistoryVisibilityEventContent::new( - history_visibility::HistoryVisibility::Shared, - ), - ) + content: to_raw_value(&RoomHistoryVisibilityEventContent::new( + HistoryVisibility::Shared, + )) .expect("event is valid, we just created it"), unsigned: None, state_key: Some("".to_owned()), @@ -381,10 +384,8 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomGuestAccess, - content: serde_json::to_value(guest_access::GuestAccessEventContent::new( - guest_access::GuestAccess::Forbidden, - )) - .expect("event is valid, we just created it"), + content: to_raw_value(&RoomGuestAccessEventContent::new(GuestAccess::Forbidden)) + .expect("event is valid, we just created it"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -402,7 +403,7 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomName, - content: serde_json::to_value(name::NameEventContent::new(Some(room_name))) + content: to_raw_value(&RoomNameEventContent::new(Some(room_name))) .expect("event is valid, we just created it"), unsigned: None, state_key: Some("".to_owned()), @@ -417,7 +418,7 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomTopic, - content: serde_json::to_value(topic::TopicEventContent { + content: to_raw_value(&RoomTopicEventContent { topic: format!("Manage {}", db.globals.server_name()), }) .expect("event is valid, we just created it"), @@ -439,7 +440,7 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomCanonicalAlias, - content: serde_json::to_value(canonical_alias::CanonicalAliasEventContent { + content: to_raw_value(&RoomCanonicalAliasEventContent { alias: Some(alias.clone()), alt_aliases: Vec::new(), }) @@ -460,8 +461,8 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Invite, + content: to_raw_value(&RoomMemberEventContent { + membership: MembershipState::Invite, displayname: None, avatar_url: None, is_direct: None, @@ -482,8 +483,8 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Join, + content: to_raw_value(&RoomMemberEventContent { + membership: MembershipState::Join, displayname: Some(displayname), avatar_url: None, is_direct: None, @@ -506,7 +507,7 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMessage, - content: serde_json::to_value(message::MessageEventContent::text_html( + content: to_raw_value(&RoomMessageEventContent::text_html( "## Thank you for trying out Conduit!\n\nConduit is currently in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.\n\nHelpful links:\n> Website: https://conduit.rs\n> Git and Documentation: https://gitlab.com/famedly/conduit\n> Report issues: https://gitlab.com/famedly/conduit/-/issues\n\nHere are some rooms you can join (by typing the command):\n\nConduit room (Ask questions and get notified on updates):\n`/join #conduit:fachschaften.org`\n\nConduit lounge (Off-topic, only Conduit users are allowed to join)\n`/join #conduit-lounge:conduit.rs`".to_owned(), "

Thank you for trying out Conduit!

\n

Conduit is currently in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.

\n

Helpful links:

\n
\n

Website: https://conduit.rs
Git and Documentation: https://gitlab.com/famedly/conduit
Report issues: https://gitlab.com/famedly/conduit/-/issues

\n
\n

Here are some rooms you can join (by typing the command):

\n

Conduit room (Ask questions and get notified on updates):
/join #conduit:fachschaften.org

\n

Conduit lounge (Off-topic, only Conduit users are allowed to join)
/join #conduit-lounge:conduit.rs

\n".to_owned(), )) @@ -562,7 +563,7 @@ pub async fn change_password_route( let mut uiaainfo = UiaaInfo { flows: vec![AuthFlow { - stages: vec!["m.login.password".to_owned()], + stages: vec![AuthType::Password], }], completed: Vec::new(), params: Default::default(), @@ -654,7 +655,7 @@ pub async fn deactivate_route( let mut uiaainfo = UiaaInfo { flows: vec![AuthFlow { - stages: vec!["m.login.password".to_owned()], + stages: vec![AuthType::Password], }], completed: Vec::new(), params: Default::default(), @@ -698,8 +699,8 @@ pub async fn deactivate_route( for room_id in all_rooms { let room_id = room_id?; - let event = member::MemberEventContent { - membership: member::MembershipState::Leave, + let event = RoomMemberEventContent { + membership: MembershipState::Leave, displayname: None, avatar_url: None, is_direct: None, @@ -721,7 +722,7 @@ pub async fn deactivate_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), + content: to_raw_value(&event).expect("event is valid, we just created it"), unsigned: None, state_key: Some(sender_user.to_string()), redacts: None, diff --git a/src/client_server/device.rs b/src/client_server/device.rs index 100b591f..b6fee377 100644 --- a/src/client_server/device.rs +++ b/src/client_server/device.rs @@ -3,7 +3,7 @@ use ruma::api::client::{ error::ErrorKind, r0::{ device::{self, delete_device, delete_devices, get_device, get_devices, update_device}, - uiaa::{AuthFlow, UiaaInfo}, + uiaa::{AuthFlow, AuthType, UiaaInfo}, }, }; @@ -109,7 +109,7 @@ pub async fn delete_device_route( // UIAA let mut uiaainfo = UiaaInfo { flows: vec![AuthFlow { - stages: vec!["m.login.password".to_owned()], + stages: vec![AuthType::Password], }], completed: Vec::new(), params: Default::default(), @@ -172,7 +172,7 @@ pub async fn delete_devices_route( // UIAA let mut uiaainfo = UiaaInfo { flows: vec![AuthFlow { - stages: vec!["m.login.password".to_owned()], + stages: vec![AuthType::Password], }], completed: Vec::new(), params: Default::default(), diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 0065e51a..835504cc 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -17,10 +17,16 @@ use ruma::{ }, directory::{Filter, IncomingFilter, IncomingRoomNetwork, PublicRoomsChunk, RoomNetwork}, events::{ - room::{avatar, canonical_alias, guest_access, history_visibility, name, topic}, + room::{ + avatar::RoomAvatarEventContent, + canonical_alias::RoomCanonicalAliasEventContent, + guest_access::{GuestAccess, RoomGuestAccessEventContent}, + history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, + name::RoomNameEventContent, + topic::RoomTopicEventContent, + }, EventType, }, - serde::Raw, ServerName, UInt, }; use tracing::{info, warn}; @@ -217,157 +223,143 @@ pub(crate) async fn get_public_rooms_filtered_helper( } } - let mut all_rooms = - db.rooms - .public_rooms() - .map(|room_id| { - let room_id = room_id?; + let mut all_rooms = db + .rooms + .public_rooms() + .map(|room_id| { + let room_id = room_id?; - let chunk = PublicRoomsChunk { - aliases: Vec::new(), - canonical_alias: db - .rooms - .room_state_get(&room_id, &EventType::RoomCanonicalAlias, "")? - .map_or(Ok::<_, Error>(None), |s| { - Ok(serde_json::from_value::< - Raw, - >(s.content.clone()) - .expect("from_value::> can never fail") - .deserialize() - .map_err(|_| { - Error::bad_database("Invalid canonical alias event in database.") - })? - .alias) - })?, - name: db - .rooms - .room_state_get(&room_id, &EventType::RoomName, "")? - .map_or(Ok::<_, Error>(None), |s| { - Ok(serde_json::from_value::>( - s.content.clone(), - ) - .expect("from_value::> can never fail") - .deserialize() - .map_err(|_| { - Error::bad_database("Invalid room name event in database.") - })? - .name) - })?, - num_joined_members: db - .rooms - .room_joined_count(&room_id)? - .unwrap_or_else(|| { - warn!("Room {} has no member count", room_id); - 0 - }) - .try_into() - .expect("user count should not be that big"), - topic: db - .rooms - .room_state_get(&room_id, &EventType::RoomTopic, "")? - .map_or(Ok::<_, Error>(None), |s| { - Ok(Some( - serde_json::from_value::>( - s.content.clone(), - ) - .expect("from_value::> can never fail") - .deserialize() + let chunk = PublicRoomsChunk { + aliases: Vec::new(), + canonical_alias: db + .rooms + .room_state_get(&room_id, &EventType::RoomCanonicalAlias, "")? + .map_or(Ok::<_, Error>(None), |s| { + Ok( + serde_json::from_str::(s.content.get()) + .map_err(|_| { + Error::bad_database( + "Invalid canonical alias event in database.", + ) + })? + .alias, + ) + })?, + name: db + .rooms + .room_state_get(&room_id, &EventType::RoomName, "")? + .map_or(Ok::<_, Error>(None), |s| { + Ok( + serde_json::from_str::(s.content.get()) + .map_err(|_| { + Error::bad_database("Invalid room name event in database.") + })? + .name, + ) + })?, + num_joined_members: db + .rooms + .room_joined_count(&room_id)? + .unwrap_or_else(|| { + warn!("Room {} has no member count", room_id); + 0 + }) + .try_into() + .expect("user count should not be that big"), + topic: db + .rooms + .room_state_get(&room_id, &EventType::RoomTopic, "")? + .map_or(Ok::<_, Error>(None), |s| { + Ok(Some( + serde_json::from_str::(s.content.get()) .map_err(|_| { Error::bad_database("Invalid room topic event in database.") })? .topic, - )) - })?, - world_readable: db - .rooms - .room_state_get(&room_id, &EventType::RoomHistoryVisibility, "")? - .map_or(Ok::<_, Error>(false), |s| { - Ok(serde_json::from_value::< - Raw, - >(s.content.clone()) - .expect("from_value::> can never fail") - .deserialize() - .map_err(|_| { - Error::bad_database( - "Invalid room history visibility event in database.", - ) - })? - .history_visibility - == history_visibility::HistoryVisibility::WorldReadable) - })?, - guest_can_join: db - .rooms - .room_state_get(&room_id, &EventType::RoomGuestAccess, "")? - .map_or(Ok::<_, Error>(false), |s| { - Ok( - serde_json::from_value::>( - s.content.clone(), - ) - .expect("from_value::> can never fail") - .deserialize() - .map_err(|_| { - Error::bad_database("Invalid room guest access event in database.") - })? - .guest_access - == guest_access::GuestAccess::CanJoin, + )) + })?, + world_readable: db + .rooms + .room_state_get(&room_id, &EventType::RoomHistoryVisibility, "")? + .map_or(Ok::<_, Error>(false), |s| { + Ok(serde_json::from_str::( + s.content.get(), ) - })?, - avatar_url: db - .rooms - .room_state_get(&room_id, &EventType::RoomAvatar, "")? - .map(|s| { - Ok::<_, Error>( - serde_json::from_value::>( - s.content.clone(), - ) - .expect("from_value::> can never fail") - .deserialize() + .map_err(|_| { + Error::bad_database( + "Invalid room history visibility event in database.", + ) + })? + .history_visibility + == HistoryVisibility::WorldReadable) + })?, + guest_can_join: db + .rooms + .room_state_get(&room_id, &EventType::RoomGuestAccess, "")? + .map_or(Ok::<_, Error>(false), |s| { + Ok( + serde_json::from_str::(s.content.get()) + .map_err(|_| { + Error::bad_database( + "Invalid room guest access event in database.", + ) + })? + .guest_access + == GuestAccess::CanJoin, + ) + })?, + avatar_url: db + .rooms + .room_state_get(&room_id, &EventType::RoomAvatar, "")? + .map(|s| { + Ok::<_, Error>( + serde_json::from_str::(s.content.get()) .map_err(|_| { Error::bad_database("Invalid room avatar event in database.") })? .url, - ) - }) - .transpose()? - // url is now an Option so we must flatten - .flatten(), - room_id, - }; - Ok(chunk) - }) - .filter_map(|r: Result<_>| r.ok()) // Filter out buggy rooms - .filter(|chunk| { - if let Some(query) = filter - .generic_search_term - .as_ref() - .map(|q| q.to_lowercase()) - { - if let Some(name) = &chunk.name { - if name.as_str().to_lowercase().contains(&query) { - return true; - } + ) + }) + .transpose()? + // url is now an Option so we must flatten + .flatten(), + room_id, + }; + Ok(chunk) + }) + .filter_map(|r: Result<_>| r.ok()) // Filter out buggy rooms + .filter(|chunk| { + if let Some(query) = filter + .generic_search_term + .as_ref() + .map(|q| q.to_lowercase()) + { + if let Some(name) = &chunk.name { + if name.as_str().to_lowercase().contains(&query) { + return true; } - - if let Some(topic) = &chunk.topic { - if topic.to_lowercase().contains(&query) { - return true; - } - } - - if let Some(canonical_alias) = &chunk.canonical_alias { - if canonical_alias.as_str().to_lowercase().contains(&query) { - return true; - } - } - - false - } else { - // No search term - true } - }) - // We need to collect all, so we can sort by member count - .collect::>(); + + if let Some(topic) = &chunk.topic { + if topic.to_lowercase().contains(&query) { + return true; + } + } + + if let Some(canonical_alias) = &chunk.canonical_alias { + if canonical_alias.as_str().to_lowercase().contains(&query) { + return true; + } + } + + false + } else { + // No search term + true + } + }) + // We need to collect all, so we can sort by member count + .collect::>(); all_rooms.sort_by(|l, r| r.num_joined_members.cmp(&l.num_joined_members)); diff --git a/src/client_server/keys.rs b/src/client_server/keys.rs index a74c4097..980acf03 100644 --- a/src/client_server/keys.rs +++ b/src/client_server/keys.rs @@ -10,7 +10,7 @@ use ruma::{ claim_keys, get_key_changes, get_keys, upload_keys, upload_signatures, upload_signing_keys, }, - uiaa::{AuthFlow, UiaaInfo}, + uiaa::{AuthFlow, AuthType, UiaaInfo}, }, }, federation, @@ -148,7 +148,7 @@ pub async fn upload_signing_keys_route( // UIAA let mut uiaainfo = UiaaInfo { flows: vec![AuthFlow { - stages: vec!["m.login.password".to_owned()], + stages: vec![AuthType::Password], }], completed: Vec::new(), params: Default::default(), diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 146af791..e37fe6c4 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -1,10 +1,9 @@ use crate::{ client_server, database::DatabaseGuard, - pdu::{PduBuilder, PduEvent}, + pdu::{EventHash, PduBuilder, PduEvent}, server_server, utils, ConduitResult, Database, Error, Result, Ruma, }; -use member::{MemberEventContent, MembershipState}; use ruma::{ api::{ client::{ @@ -18,14 +17,17 @@ use ruma::{ federation::{self, membership::create_invite}, }, events::{ - pdu::Pdu, - room::{create::CreateEventContent, member}, + room::{ + create::RoomCreateEventContent, + member::{MembershipState, RoomMemberEventContent}, + }, EventType, }, - serde::{to_canonical_value, CanonicalJsonObject, CanonicalJsonValue, Raw}, + serde::{to_canonical_value, CanonicalJsonObject, CanonicalJsonValue}, state_res::{self, RoomVersion}, uint, EventId, RoomId, RoomVersionId, ServerName, UserId, }; +use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use std::{ collections::{hash_map::Entry, BTreeMap, HashMap, HashSet}, convert::{TryFrom, TryInto}, @@ -204,7 +206,7 @@ pub async fn kick_user_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let mut event = serde_json::from_value::>( + let mut event: RoomMemberEventContent = serde_json::from_str( db.rooms .room_state_get( &body.room_id, @@ -216,13 +218,11 @@ pub async fn kick_user_route( "Cannot kick member that's not in the room.", ))? .content - .clone(), + .get(), ) - .expect("Raw::from_value always works") - .deserialize() .map_err(|_| Error::bad_database("Invalid member event in database."))?; - event.membership = ruma::events::room::member::MembershipState::Leave; + event.membership = MembershipState::Leave; // TODO: reason let mutex_state = Arc::clone( @@ -238,7 +238,7 @@ pub async fn kick_user_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), + content: to_raw_value(&event).expect("event is valid, we just created it"), unsigned: None, state_key: Some(body.user_id.to_string()), redacts: None, @@ -280,8 +280,8 @@ pub async fn ban_user_route( &body.user_id.to_string(), )? .map_or( - Ok::<_, Error>(member::MemberEventContent { - membership: member::MembershipState::Ban, + Ok::<_, Error>(RoomMemberEventContent { + membership: MembershipState::Ban, displayname: db.users.displayname(&body.user_id)?, avatar_url: db.users.avatar_url(&body.user_id)?, is_direct: None, @@ -290,13 +290,9 @@ pub async fn ban_user_route( reason: None, }), |event| { - let mut event = serde_json::from_value::>( - event.content.clone(), - ) - .expect("Raw::from_value always works") - .deserialize() - .map_err(|_| Error::bad_database("Invalid member event in database."))?; - event.membership = ruma::events::room::member::MembershipState::Ban; + let mut event = serde_json::from_str::(event.content.get()) + .map_err(|_| Error::bad_database("Invalid member event in database."))?; + event.membership = MembershipState::Ban; Ok(event) }, )?; @@ -314,7 +310,7 @@ pub async fn ban_user_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), + content: to_raw_value(&event).expect("event is valid, we just created it"), unsigned: None, state_key: Some(body.user_id.to_string()), redacts: None, @@ -346,7 +342,7 @@ pub async fn unban_user_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let mut event = serde_json::from_value::>( + let mut event = serde_json::from_str::( db.rooms .room_state_get( &body.room_id, @@ -358,13 +354,11 @@ pub async fn unban_user_route( "Cannot unban a user who is not banned.", ))? .content - .clone(), + .get(), ) - .expect("from_value::> can never fail") - .deserialize() .map_err(|_| Error::bad_database("Invalid member event in database."))?; - event.membership = ruma::events::room::member::MembershipState::Leave; + event.membership = MembershipState::Leave; let mutex_state = Arc::clone( db.globals @@ -379,7 +373,7 @@ pub async fn unban_user_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), + content: to_raw_value(&event).expect("event is valid, we just created it"), unsigned: None, state_key: Some(body.user_id.to_string()), redacts: None, @@ -584,10 +578,9 @@ async fn join_room_by_id_helper( }; let mut join_event_stub = - serde_json::from_str::(make_join_response.event.json().get()) - .map_err(|_| { - Error::BadServerResponse("Invalid make_join event json received from server.") - })?; + serde_json::from_str::(make_join_response.event.get()).map_err( + |_| Error::BadServerResponse("Invalid make_join event json received from server."), + )?; // TODO: Is origin needed? join_event_stub.insert( @@ -604,8 +597,8 @@ async fn join_room_by_id_helper( ); join_event_stub.insert( "content".to_owned(), - to_canonical_value(member::MemberEventContent { - membership: member::MembershipState::Join, + to_canonical_value(RoomMemberEventContent { + membership: MembershipState::Join, displayname: db.users.displayname(sender_user)?, avatar_url: db.users.avatar_url(sender_user)?, is_direct: None, @@ -653,7 +646,7 @@ async fn join_room_by_id_helper( federation::membership::create_join_event::v2::Request { room_id, event_id: &event_id, - pdu: PduEvent::convert_to_outgoing_federation_event(join_event.clone()), + pdu: &PduEvent::convert_to_outgoing_federation_event(join_event.clone()), }, ) .await?; @@ -756,8 +749,8 @@ async fn join_room_by_id_helper( // where events in the current room state do not exist db.rooms.set_room_state(room_id, statehashid)?; } else { - let event = member::MemberEventContent { - membership: member::MembershipState::Join, + let event = RoomMemberEventContent { + membership: MembershipState::Join, displayname: db.users.displayname(sender_user)?, avatar_url: db.users.avatar_url(sender_user)?, is_direct: None, @@ -769,7 +762,7 @@ async fn join_room_by_id_helper( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), + content: to_raw_value(&event).expect("event is valid, we just created it"), unsigned: None, state_key: Some(sender_user.to_string()), redacts: None, @@ -789,12 +782,12 @@ async fn join_room_by_id_helper( } fn validate_and_add_event_id( - pdu: &Raw, + pdu: &RawJsonValue, room_version: &RoomVersionId, pub_key_map: &RwLock>>, db: &Database, ) -> Result<(EventId, CanonicalJsonObject)> { - let mut value = serde_json::from_str::(pdu.json().get()).map_err(|e| { + let mut value = serde_json::from_str::(pdu.get()).map_err(|e| { error!("Invalid PDU in server response: {:?}: {:?}", pdu, e); Error::BadServerResponse("Invalid PDU in server response") })?; @@ -884,9 +877,7 @@ pub(crate) async fn invite_helper<'a>( let create_event_content = create_event .as_ref() .map(|create_event| { - serde_json::from_value::>(create_event.content.clone()) - .expect("Raw::from_value always works.") - .deserialize() + serde_json::from_str::(create_event.content.get()) .map_err(|e| { warn!("Invalid create event: {}", e); Error::bad_database("Invalid create event in db.") @@ -910,7 +901,7 @@ pub(crate) async fn invite_helper<'a>( let room_version = RoomVersion::new(&room_version_id).expect("room version is supported"); - let content = serde_json::to_value(MemberEventContent { + let content = to_raw_value(&RoomMemberEventContent { avatar_url: None, displayname: None, is_direct: Some(is_direct), @@ -946,7 +937,7 @@ pub(crate) async fn invite_helper<'a>( unsigned.insert("prev_content".to_owned(), prev_pdu.content.clone()); unsigned.insert( "prev_sender".to_owned(), - serde_json::to_value(&prev_pdu.sender).expect("UserId::to_value always works"), + serde_json::from_str(prev_pdu.sender.as_str()).expect("UserId is valid string"), ); } @@ -967,11 +958,15 @@ pub(crate) async fn invite_helper<'a>( .map(|(_, pdu)| pdu.event_id.clone()) .collect(), redacts: None, - unsigned, - hashes: ruma::events::pdu::EventHash { + unsigned: if unsigned.is_empty() { + None + } else { + Some(to_raw_value(&unsigned).expect("to_raw_value always works")) + }, + hashes: EventHash { sha256: "aaa".to_owned(), }, - signatures: BTreeMap::new(), + signatures: None, }; let auth_check = state_res::auth_check( @@ -1035,11 +1030,11 @@ pub(crate) async fn invite_helper<'a>( &db.globals, user_id.server_name(), create_invite::v2::Request { - room_id: room_id.clone(), - event_id: expected_event_id.clone(), - room_version: room_version_id, - event: PduEvent::convert_to_outgoing_federation_event(pdu_json.clone()), - invite_room_state, + room_id, + event_id: &expected_event_id, + room_version: &room_version_id, + event: &PduEvent::convert_to_outgoing_federation_event(pdu_json.clone()), + invite_room_state: &invite_room_state, }, ) .await?; @@ -1116,8 +1111,8 @@ pub(crate) async fn invite_helper<'a>( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Invite, + content: to_raw_value(&RoomMemberEventContent { + membership: MembershipState::Invite, displayname: db.users.displayname(user_id)?, avatar_url: db.users.avatar_url(user_id)?, is_direct: Some(is_direct), diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index ab7fb024..29b1ae87 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -9,9 +9,9 @@ use ruma::{ }, federation::{self, query::get_profile_information::v1::ProfileField}, }, - events::EventType, - serde::Raw, + events::{room::member::RoomMemberEventContent, EventType}, }; +use serde_json::value::to_raw_value; use std::{convert::TryInto, sync::Arc}; #[cfg(feature = "conduit_bin")] @@ -45,9 +45,9 @@ pub async fn set_displayname_route( Ok::<_, Error>(( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(ruma::events::room::member::MemberEventContent { + content: to_raw_value(&RoomMemberEventContent { displayname: body.displayname.clone(), - ..serde_json::from_value::>( + ..serde_json::from_str( db.rooms .room_state_get( &room_id, @@ -61,10 +61,8 @@ pub async fn set_displayname_route( ) })? .content - .clone(), + .get(), ) - .expect("from_value::> can never fail") - .deserialize() .map_err(|_| Error::bad_database("Database contains invalid PDU."))? }) .expect("event is valid, we just created it"), @@ -190,9 +188,9 @@ pub async fn set_avatar_url_route( Ok::<_, Error>(( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(ruma::events::room::member::MemberEventContent { + content: to_raw_value(&RoomMemberEventContent { avatar_url: body.avatar_url.clone(), - ..serde_json::from_value::>( + ..serde_json::from_str( db.rooms .room_state_get( &room_id, @@ -206,10 +204,8 @@ pub async fn set_avatar_url_route( ) })? .content - .clone(), + .get(), ) - .expect("from_value::> can never fail") - .deserialize() .map_err(|_| Error::bad_database("Database contains invalid PDU."))? }) .expect("event is valid, we just created it"), diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index 4b5219b2..7435c5c5 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -3,11 +3,12 @@ use std::sync::Arc; use crate::{database::DatabaseGuard, pdu::PduBuilder, ConduitResult, Ruma}; use ruma::{ api::client::r0::redact::redact_event, - events::{room::redaction, EventType}, + events::{room::redaction::RoomRedactionEventContent, EventType}, }; #[cfg(feature = "conduit_bin")] use rocket::put; +use serde_json::value::to_raw_value; /// # `PUT /_matrix/client/r0/rooms/{roomId}/redact/{eventId}/{txnId}` /// @@ -38,7 +39,7 @@ pub async fn redact_event_route( let event_id = db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomRedaction, - content: serde_json::to_value(redaction::RedactionEventContent { + content: to_raw_value(&RoomRedactionEventContent { reason: body.reason.clone(), }) .expect("event is valid, we just created it"), diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 5a026997..d1c79dff 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -8,12 +8,23 @@ use ruma::{ r0::room::{self, aliases, create_room, get_room_event, upgrade_room}, }, events::{ - room::{guest_access, history_visibility, join_rules, member, name, topic}, + room::{ + canonical_alias::RoomCanonicalAliasEventContent, + create::RoomCreateEventContent, + guest_access::{GuestAccess, RoomGuestAccessEventContent}, + history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, + join_rules::{JoinRule, RoomJoinRulesEventContent}, + member::{MembershipState, RoomMemberEventContent}, + name::RoomNameEventContent, + power_levels::RoomPowerLevelsEventContent, + tombstone::RoomTombstoneEventContent, + topic::RoomTopicEventContent, + }, EventType, }, - serde::Raw, RoomAliasId, RoomId, RoomVersionId, }; +use serde_json::value::to_raw_value; use std::{cmp::max, collections::BTreeMap, convert::TryFrom, sync::Arc}; use tracing::{info, warn}; @@ -80,7 +91,7 @@ pub async fn create_room_route( } })?; - let mut content = ruma::events::room::create::CreateEventContent::new(sender_user.clone()); + let mut content = RoomCreateEventContent::new(sender_user.clone()); content.federate = body.creation_content.federate; content.predecessor = body.creation_content.predecessor.clone(); content.room_version = match body.room_version.clone() { @@ -101,7 +112,7 @@ pub async fn create_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomCreate, - content: serde_json::to_value(content).expect("event is valid, we just created it"), + content: to_raw_value(&content).expect("event is valid, we just created it"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -116,8 +127,8 @@ pub async fn create_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Join, + content: to_raw_value(&RoomMemberEventContent { + membership: MembershipState::Join, displayname: db.users.displayname(sender_user)?, avatar_url: db.users.avatar_url(sender_user)?, is_direct: Some(body.is_direct), @@ -157,12 +168,11 @@ pub async fn create_room_route( } } - let mut power_levels_content = - serde_json::to_value(ruma::events::room::power_levels::PowerLevelsEventContent { - users, - ..Default::default() - }) - .expect("event is valid, we just created it"); + let mut power_levels_content = serde_json::to_value(RoomPowerLevelsEventContent { + users, + ..Default::default() + }) + .expect("event is valid, we just created it"); if let Some(power_level_content_override) = &body.power_level_content_override { let json = serde_json::from_str::>( @@ -180,7 +190,8 @@ pub async fn create_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomPowerLevels, - content: power_levels_content, + content: to_raw_value(&power_levels_content) + .expect("to_raw_value always works on serde_json::Value"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -196,12 +207,10 @@ pub async fn create_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomCanonicalAlias, - content: serde_json::to_value( - ruma::events::room::canonical_alias::CanonicalAliasEventContent { - alias: Some(room_alias_id.clone()), - alt_aliases: vec![], - }, - ) + content: to_raw_value(&RoomCanonicalAliasEventContent { + alias: Some(room_alias_id.clone()), + alt_aliases: vec![], + }) .expect("We checked that alias earlier, it must be fine"), unsigned: None, state_key: Some("".to_owned()), @@ -220,17 +229,12 @@ pub async fn create_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomJoinRules, - content: match preset { - create_room::RoomPreset::PublicChat => serde_json::to_value( - join_rules::JoinRulesEventContent::new(join_rules::JoinRule::Public), - ) - .expect("event is valid, we just created it"), + content: to_raw_value(&RoomJoinRulesEventContent::new(match preset { + create_room::RoomPreset::PublicChat => JoinRule::Public, // according to spec "invite" is the default - _ => serde_json::to_value(join_rules::JoinRulesEventContent::new( - join_rules::JoinRule::Invite, - )) - .expect("event is valid, we just created it"), - }, + _ => JoinRule::Invite, + })) + .expect("event is valid, we just created it"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -245,8 +249,8 @@ pub async fn create_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomHistoryVisibility, - content: serde_json::to_value(history_visibility::HistoryVisibilityEventContent::new( - history_visibility::HistoryVisibility::Shared, + content: to_raw_value(&RoomHistoryVisibilityEventContent::new( + HistoryVisibility::Shared, )) .expect("event is valid, we just created it"), unsigned: None, @@ -263,18 +267,11 @@ pub async fn create_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomGuestAccess, - content: match preset { - create_room::RoomPreset::PublicChat => { - serde_json::to_value(guest_access::GuestAccessEventContent::new( - guest_access::GuestAccess::Forbidden, - )) - .expect("event is valid, we just created it") - } - _ => serde_json::to_value(guest_access::GuestAccessEventContent::new( - guest_access::GuestAccess::CanJoin, - )) - .expect("event is valid, we just created it"), - }, + content: to_raw_value(&RoomGuestAccessEventContent::new(match preset { + create_room::RoomPreset::PublicChat => GuestAccess::Forbidden, + _ => GuestAccess::CanJoin, + })) + .expect("event is valid, we just created it"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -306,7 +303,7 @@ pub async fn create_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomName, - content: serde_json::to_value(name::NameEventContent::new(Some(name.clone()))) + content: to_raw_value(&RoomNameEventContent::new(Some(name.clone()))) .expect("event is valid, we just created it"), unsigned: None, state_key: Some("".to_owned()), @@ -323,7 +320,7 @@ pub async fn create_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomTopic, - content: serde_json::to_value(topic::TopicEventContent { + content: to_raw_value(&RoomTopicEventContent { topic: topic.clone(), }) .expect("event is valid, we just created it"), @@ -477,7 +474,7 @@ pub async fn upgrade_room_route( let tombstone_event_id = db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomTombstone, - content: serde_json::to_value(ruma::events::room::tombstone::TombstoneEventContent { + content: to_raw_value(&RoomTombstoneEventContent { body: "This room has been replaced".to_string(), replacement_room: replacement_room.clone(), }) @@ -505,15 +502,13 @@ pub async fn upgrade_room_route( let state_lock = mutex_state.lock().await; // Get the old room federations status - let federate = serde_json::from_value::>( + let federate = serde_json::from_str::( db.rooms .room_state_get(&body.room_id, &EventType::RoomCreate, "")? .ok_or_else(|| Error::bad_database("Found room without m.room.create event."))? .content - .clone(), + .get(), ) - .expect("Raw::from_value always works") - .deserialize() .map_err(|_| Error::bad_database("Invalid room event in database."))? .federate; @@ -524,8 +519,7 @@ pub async fn upgrade_room_route( )); // Send a m.room.create event containing a predecessor field and the applicable room_version - let mut create_event_content = - ruma::events::room::create::CreateEventContent::new(sender_user.clone()); + let mut create_event_content = RoomCreateEventContent::new(sender_user.clone()); create_event_content.federate = federate; create_event_content.room_version = body.new_version.clone(); create_event_content.predecessor = predecessor; @@ -533,7 +527,7 @@ pub async fn upgrade_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomCreate, - content: serde_json::to_value(create_event_content) + content: to_raw_value(&create_event_content) .expect("event is valid, we just created it"), unsigned: None, state_key: Some("".to_owned()), @@ -549,8 +543,8 @@ pub async fn upgrade_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Join, + content: to_raw_value(&RoomMemberEventContent { + membership: MembershipState::Join, displayname: db.users.displayname(sender_user)?, avatar_url: db.users.avatar_url(sender_user)?, is_direct: None, @@ -611,17 +605,14 @@ pub async fn upgrade_room_route( } // Get the old room power levels - let mut power_levels_event_content = - serde_json::from_value::>( - db.rooms - .room_state_get(&body.room_id, &EventType::RoomPowerLevels, "")? - .ok_or_else(|| Error::bad_database("Found room without m.room.create event."))? - .content - .clone(), - ) - .expect("database contains invalid PDU") - .deserialize() - .map_err(|_| Error::bad_database("Invalid room event in database."))?; + let mut power_levels_event_content = serde_json::from_str::( + db.rooms + .room_state_get(&body.room_id, &EventType::RoomPowerLevels, "")? + .ok_or_else(|| Error::bad_database("Found room without m.room.create event."))? + .content + .get(), + ) + .map_err(|_| Error::bad_database("Invalid room event in database."))?; // Setting events_default and invite to the greater of 50 and users_default + 1 let new_level = max( @@ -635,7 +626,7 @@ pub async fn upgrade_room_route( let _ = db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomPowerLevels, - content: serde_json::to_value(power_levels_event_content) + content: to_raw_value(&power_levels_event_content) .expect("event is valid, we just created it"), unsigned: None, state_key: Some("".to_owned()), diff --git a/src/client_server/session.rs b/src/client_server/session.rs index b42689d2..61e5519a 100644 --- a/src/client_server/session.rs +++ b/src/client_server/session.rs @@ -60,10 +60,10 @@ pub async fn login_route( // Validate login method // TODO: Other login methods let user_id = match &body.login_info { - login::IncomingLoginInfo::Password { + login::IncomingLoginInfo::Password(login::IncomingPassword { identifier, password, - } => { + }) => { let username = if let IncomingUserIdentifier::MatrixId(matrix_id) = identifier { matrix_id } else { @@ -97,7 +97,7 @@ pub async fn login_route( user_id } - login::IncomingLoginInfo::Token { token } => { + login::IncomingLoginInfo::Token(login::IncomingToken { token }) => { if let Some(jwt_decoding_key) = db.globals.jwt_decoding_key() { let token = jsonwebtoken::decode::( token, @@ -116,6 +116,12 @@ pub async fn login_route( )); } } + _ => { + return Err(Error::BadRequest( + ErrorKind::Unknown, + "Unsupported login type.", + )); + } }; // Generate new device id if the user didn't specify one diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 24cc2a18..35157332 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -10,8 +10,8 @@ use ruma::{ }, events::{ room::{ - canonical_alias::CanonicalAliasEventContent, - history_visibility::{HistoryVisibility, HistoryVisibilityEventContent}, + canonical_alias::RoomCanonicalAliasEventContent, + history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, }, AnyStateEventContent, EventType, }, @@ -112,7 +112,7 @@ pub async fn get_state_events_route( db.rooms .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")? .map(|event| { - serde_json::from_value::(event.content.clone()) + serde_json::from_str::(event.content.get()) .map_err(|_| { Error::bad_database( "Invalid room history visibility event in database.", @@ -164,7 +164,7 @@ pub async fn get_state_events_for_key_route( db.rooms .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")? .map(|event| { - serde_json::from_value::(event.content.clone()) + serde_json::from_str::(event.content.get()) .map_err(|_| { Error::bad_database( "Invalid room history visibility event in database.", @@ -190,7 +190,7 @@ pub async fn get_state_events_for_key_route( ))?; Ok(get_state_events_for_key::Response { - content: serde_json::from_value(event.content.clone()) + content: serde_json::from_str(event.content.get()) .map_err(|_| Error::bad_database("Invalid event content in database"))?, } .into()) @@ -220,7 +220,7 @@ pub async fn get_state_events_for_empty_key_route( db.rooms .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")? .map(|event| { - serde_json::from_value::(event.content.clone()) + serde_json::from_str::(event.content.get()) .map_err(|_| { Error::bad_database( "Invalid room history visibility event in database.", @@ -246,7 +246,7 @@ pub async fn get_state_events_for_empty_key_route( ))?; Ok(get_state_events_for_key::Response { - content: serde_json::from_value(event.content.clone()) + content: serde_json::from_str(event.content.get()) .map_err(|_| Error::bad_database("Invalid event content in database"))?, } .into()) @@ -265,7 +265,7 @@ async fn send_state_event_for_key_helper( // TODO: Review this check, error if event is unparsable, use event type, allow alias if it // previously existed if let Ok(canonical_alias) = - serde_json::from_str::(json.json().get()) + serde_json::from_str::(json.json().get()) { let mut aliases = canonical_alias.alt_aliases.clone(); diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 2d5ad27d..5b0dbaf7 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -1,7 +1,10 @@ use crate::{database::DatabaseGuard, ConduitResult, Database, Error, Result, Ruma, RumaResponse}; use ruma::{ api::client::r0::{sync::sync_events, uiaa::UiaaResponse}, - events::{room::member::MembershipState, AnySyncEphemeralRoomEvent, EventType}, + events::{ + room::member::{MembershipState, RoomMemberEventContent}, + AnySyncEphemeralRoomEvent, EventType, + }, serde::Raw, DeviceId, RoomId, UserId, }; @@ -287,10 +290,11 @@ async fn sync_helper( .filter_map(|pdu| pdu.ok()) // Ignore all broken pdus .filter(|(_, pdu)| pdu.kind == EventType::RoomMember) .map(|(_, pdu)| { - let content = serde_json::from_value::< - ruma::events::room::member::MemberEventContent, - >(pdu.content.clone()) - .map_err(|_| Error::bad_database("Invalid member event in database."))?; + let content = + serde_json::from_str::(pdu.content.get()) + .map_err(|_| { + Error::bad_database("Invalid member event in database.") + })?; if let Some(state_key) = &pdu.state_key { let user_id = UserId::try_from(state_key.clone()).map_err(|_| { @@ -371,13 +375,9 @@ async fn sync_helper( sender_user.as_str(), )? .and_then(|pdu| { - serde_json::from_value::>( - pdu.content.clone(), - ) - .expect("Raw::from_value always works") - .deserialize() - .map_err(|_| Error::bad_database("Invalid PDU in database.")) - .ok() + serde_json::from_str::(pdu.content.get()) + .map_err(|_| Error::bad_database("Invalid PDU in database.")) + .ok() }); let joined_since_last_sync = since_sender_member @@ -432,11 +432,9 @@ async fn sync_helper( continue; } - let new_membership = serde_json::from_value::< - Raw, - >(state_event.content.clone()) - .expect("Raw::from_value always works") - .deserialize() + let new_membership = serde_json::from_str::( + state_event.content.get(), + ) .map_err(|_| Error::bad_database("Invalid PDU in database."))? .membership; @@ -739,7 +737,7 @@ async fn sync_helper( presence: sync_events::Presence { events: presence_updates .into_iter() - .map(|(_, v)| Raw::from(v)) + .map(|(_, v)| Raw::new(&v).expect("PresenceEvent always serializes successfully")) .collect(), }, account_data: sync_events::GlobalAccountData { diff --git a/src/database/admin.rs b/src/database/admin.rs index 424e6746..8d8559a5 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -6,16 +6,17 @@ use std::{ use crate::{pdu::PduBuilder, Database}; use rocket::futures::{channel::mpsc, stream::StreamExt}; use ruma::{ - events::{room::message, EventType}, + events::{room::message::RoomMessageEventContent, EventType}, UserId, }; +use serde_json::value::to_raw_value; use tokio::sync::{MutexGuard, RwLock, RwLockReadGuard}; use tracing::warn; pub enum AdminCommand { RegisterAppservice(serde_yaml::Value), ListAppservices, - SendMessage(message::MessageEventContent), + SendMessage(RoomMessageEventContent), } #[derive(Clone)] @@ -58,7 +59,7 @@ impl Admin { drop(guard); - let send_message = |message: message::MessageEventContent, + let send_message = |message: RoomMessageEventContent, guard: RwLockReadGuard<'_, Database>, mutex_lock: &MutexGuard<'_, ()>| { guard @@ -66,7 +67,7 @@ impl Admin { .build_and_append_pdu( PduBuilder { event_type: EventType::RoomMessage, - content: serde_json::to_value(message) + content: to_raw_value(&message) .expect("event is valid, we just created it"), unsigned: None, state_key: None, @@ -106,9 +107,9 @@ impl Admin { count, appservices.into_iter().filter_map(|r| r.ok()).collect::>().join(", ") ); - send_message(message::MessageEventContent::text_plain(output), guard, &state_lock); + send_message(RoomMessageEventContent::text_plain(output), guard, &state_lock); } else { - send_message(message::MessageEventContent::text_plain("Failed to get appservices."), guard, &state_lock); + send_message(RoomMessageEventContent::text_plain("Failed to get appservices."), guard, &state_lock); } } AdminCommand::SendMessage(message) => { diff --git a/src/database/pusher.rs b/src/database/pusher.rs index b19f3392..f53f137b 100644 --- a/src/database/pusher.rs +++ b/src/database/pusher.rs @@ -9,8 +9,10 @@ use ruma::{ }, IncomingResponse, OutgoingRequest, SendAccessToken, }, - events::{room::power_levels::PowerLevelsEventContent, AnySyncRoomEvent, EventType}, - identifiers::RoomName, + events::{ + room::{name::RoomNameEventContent, power_levels::RoomPowerLevelsEventContent}, + AnySyncRoomEvent, EventType, + }, push::{Action, PushConditionRoomCtx, PushFormat, Ruleset, Tweak}, serde::Raw, uint, RoomId, UInt, UserId, @@ -177,11 +179,11 @@ pub async fn send_push_notice( let mut notify = None; let mut tweaks = Vec::new(); - let power_levels: PowerLevelsEventContent = db + let power_levels: RoomPowerLevelsEventContent = db .rooms .room_state_get(&pdu.room_id, &EventType::RoomPowerLevels, "")? .map(|ev| { - serde_json::from_value(ev.content.clone()) + serde_json::from_str(ev.content.get()) .map_err(|_| Error::bad_database("invalid m.room.power_levels event")) }) .transpose()? @@ -226,7 +228,7 @@ pub async fn send_push_notice( pub fn get_actions<'a>( user: &UserId, ruleset: &'a Ruleset, - power_levels: &PowerLevelsEventContent, + power_levels: &RoomPowerLevelsEventContent, pdu: &Raw, room_id: &RoomId, db: &Database, @@ -318,16 +320,18 @@ async fn send_notice( let user_name = db.users.displayname(&event.sender)?; notifi.sender_display_name = user_name.as_deref(); - let room_name = db - .rooms - .room_state_get(&event.room_id, &EventType::RoomName, "")? - .map(|pdu| match pdu.content.get("name") { - Some(serde_json::Value::String(s)) => { - Some(Box::::try_from(&**s).expect("room name is valid")) - } - _ => None, - }) - .flatten(); + + let room_name = if let Some(room_name_pdu) = + db.rooms + .room_state_get(&event.room_id, &EventType::RoomName, "")? + { + serde_json::from_str::(room_name_pdu.content.get()) + .map_err(|_| Error::bad_database("Invalid room name event in database."))? + .name + } else { + None + }; + notifi.room_name = room_name.as_deref(); send_request( diff --git a/src/database/rooms.rs b/src/database/rooms.rs index ec03e3ac..3096150d 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -1,9 +1,11 @@ mod edus; pub use edus::RoomEdus; -use member::MembershipState; -use crate::{pdu::PduBuilder, server_server, utils, Database, Error, PduEvent, Result}; +use crate::{ + pdu::{EventHash, PduBuilder}, + server_server, utils, Database, Error, PduEvent, Result, +}; use lru_cache::LruCache; use regex::Regex; use ring::digest; @@ -13,16 +15,22 @@ use ruma::{ events::{ ignored_user_list, push_rules, room::{ - create::CreateEventContent, member, message, power_levels::PowerLevelsEventContent, + create::RoomCreateEventContent, + member::{MembershipState, RoomMemberEventContent}, + message::RoomMessageEventContent, + power_levels::RoomPowerLevelsEventContent, }, AnyStrippedStateEvent, AnySyncStateEvent, EventType, }, - push::{self, Action, Tweak}, + push::{Action, Ruleset, Tweak}, serde::{CanonicalJsonObject, CanonicalJsonValue, Raw}, state_res::{self, RoomVersion, StateMap}, uint, EventId, RoomAliasId, RoomId, RoomVersionId, ServerName, UserId, }; +use serde::Deserialize; +use serde_json::value::to_raw_value; use std::{ + borrow::Cow, collections::{BTreeMap, HashMap, HashSet}, convert::{TryFrom, TryInto}, mem::size_of, @@ -243,7 +251,7 @@ impl Rooms { kind: &EventType, sender: &UserId, state_key: Option<&str>, - content: &serde_json::Value, + content: &serde_json::value::RawValue, ) -> Result>> { let shortstatehash = if let Some(current_shortstatehash) = self.current_shortstatehash(room_id)? { @@ -252,7 +260,8 @@ impl Rooms { return Ok(HashMap::new()); }; - let auth_events = state_res::auth_types_for_event(kind, sender, state_key, content); + let auth_events = state_res::auth_types_for_event(kind, sender, state_key, content) + .expect("content is a valid JSON object"); let mut sauthevents = auth_events .into_iter() @@ -391,37 +400,43 @@ impl Rooms { .ok() .map(|(_, id)| id) }) { - if let Some(pdu) = self.get_pdu_json(&event_id)? { - if pdu.get("type").and_then(|val| val.as_str()) == Some("m.room.member") { - if let Ok(pdu) = serde_json::from_value::( - serde_json::to_value(&pdu).expect("CanonicalJsonObj is a valid JsonValue"), - ) { - if let Some(membership) = - pdu.content.get("membership").and_then(|membership| { - serde_json::from_value::( - membership.clone(), - ) - .ok() - }) - { - if let Some(state_key) = pdu - .state_key - .and_then(|state_key| UserId::try_from(state_key).ok()) - { - self.update_membership( - room_id, - &state_key, - membership, - &pdu.sender, - None, - db, - false, - )?; - } - } - } - } + let pdu = match self.get_pdu_json(&event_id)? { + Some(pdu) => pdu, + None => continue, + }; + + if pdu.get("type").and_then(|val| val.as_str()) != Some("m.room.member") { + continue; } + + let pdu = match serde_json::from_str::( + &serde_json::to_string(&pdu).expect("CanonicalJsonObj can be serialized to JSON"), + ) { + Ok(pdu) => pdu, + Err(_) => continue, + }; + + #[derive(Deserialize)] + struct ExtractMembership { + membership: MembershipState, + } + + let membership = match serde_json::from_str::(pdu.content.get()) { + Ok(e) => e.membership, + Err(_) => continue, + }; + + let state_key = match pdu.state_key { + Some(k) => k, + None => continue, + }; + + let user_id = match UserId::try_from(state_key) { + Ok(id) => id, + Err(_) => continue, + }; + + self.update_membership(room_id, &user_id, membership, &pdu.sender, None, db, false)?; } self.update_joined_count(room_id, db)?; @@ -1325,11 +1340,11 @@ impl Rooms { drop(insert_lock); // See if the event matches any known pushers - let power_levels: PowerLevelsEventContent = db + let power_levels: RoomPowerLevelsEventContent = db .rooms .room_state_get(&pdu.room_id, &EventType::RoomPowerLevels, "")? .map(|ev| { - serde_json::from_value(ev.content.clone()) + serde_json::from_str(ev.content.get()) .map_err(|_| Error::bad_database("invalid m.room.power_levels event")) }) .transpose()? @@ -1350,7 +1365,7 @@ impl Rooms { .account_data .get::(None, user, EventType::PushRules)? .map(|ev| ev.content.global) - .unwrap_or_else(|| push::Ruleset::server_default(user)); + .unwrap_or_else(|| Ruleset::server_default(user)); let mut highlight = false; let mut notify = false; @@ -1404,30 +1419,21 @@ impl Rooms { } EventType::RoomMember => { if let Some(state_key) = &pdu.state_key { + #[derive(Deserialize)] + struct ExtractMembership { + membership: MembershipState, + } + // if the state_key fails let target_user_id = UserId::try_from(state_key.clone()) .expect("This state_key was previously validated"); - let membership = serde_json::from_value::( - pdu.content - .get("membership") - .ok_or(Error::BadRequest( - ErrorKind::InvalidParam, - "Invalid member event content", - ))? - .clone(), - ) - .map_err(|_| { - Error::BadRequest( - ErrorKind::InvalidParam, - "Invalid membership state content.", - ) - })?; + let content = serde_json::from_str::(pdu.content.get()) + .map_err(|_| Error::bad_database("Invalid content in pdu."))?; - let invite_state = match membership { - member::MembershipState::Invite => { + let invite_state = match content.membership { + MembershipState::Invite => { let state = self.calculate_invite_state(pdu)?; - Some(state) } _ => None, @@ -1438,7 +1444,7 @@ impl Rooms { self.update_membership( &pdu.room_id, &target_user_id, - membership, + content.membership, &pdu.sender, invite_state, db, @@ -1447,7 +1453,16 @@ impl Rooms { } } EventType::RoomMessage => { - if let Some(body) = pdu.content.get("body").and_then(|b| b.as_str()) { + #[derive(Deserialize)] + struct ExtractBody<'a> { + #[serde(borrow)] + body: Option>, + } + + let content = serde_json::from_str::>(pdu.content.get()) + .map_err(|_| Error::bad_database("Invalid content in pdu."))?; + + if let Some(body) = content.body { let mut batch = body .split_terminator(|c: char| !c.is_alphanumeric()) .filter(|s| !s.is_empty()) @@ -1498,18 +1513,16 @@ impl Rooms { } Err(e) => { db.admin.send(AdminCommand::SendMessage( - message::MessageEventContent::text_plain( - format!( - "Could not parse appservice config: {}", - e - ), - ), + RoomMessageEventContent::text_plain(format!( + "Could not parse appservice config: {}", + e + )), )); } } } else { db.admin.send(AdminCommand::SendMessage( - message::MessageEventContent::text_plain( + RoomMessageEventContent::text_plain( "Expected code block in command body.", ), )); @@ -1542,12 +1555,10 @@ impl Rooms { .count(); let elapsed = start.elapsed(); db.admin.send(AdminCommand::SendMessage( - message::MessageEventContent::text_plain( - format!( - "Loaded auth chain with length {} in {:?}", - count, elapsed - ), - ), + RoomMessageEventContent::text_plain(format!( + "Loaded auth chain with length {} in {:?}", + count, elapsed + )), )); } } @@ -1580,14 +1591,17 @@ impl Rooms { ) { Ok(pdu) => { db.admin.send(AdminCommand::SendMessage( - message::MessageEventContent::text_plain( - format!("EventId: {:?}\n{:#?}", event_id, pdu), + RoomMessageEventContent::text_plain( + format!( + "EventId: {:?}\n{:#?}", + event_id, pdu + ), ), )); } Err(e) => { db.admin.send(AdminCommand::SendMessage( - message::MessageEventContent::text_plain( + RoomMessageEventContent::text_plain( format!("EventId: {:?}\nCould not parse event: {}", event_id, e), ), )); @@ -1596,18 +1610,16 @@ impl Rooms { } Err(e) => { db.admin.send(AdminCommand::SendMessage( - message::MessageEventContent::text_plain( - format!( - "Invalid json in command body: {}", - e - ), - ), + RoomMessageEventContent::text_plain(format!( + "Invalid json in command body: {}", + e + )), )); } } } else { db.admin.send(AdminCommand::SendMessage( - message::MessageEventContent::text_plain( + RoomMessageEventContent::text_plain( "Expected code block in command body.", ), )); @@ -1629,7 +1641,7 @@ impl Rooms { serde_json::to_string_pretty(&json) .expect("canonical json is valid json"); db.admin.send(AdminCommand::SendMessage( - message::MessageEventContent::text_html( + RoomMessageEventContent::text_html( format!("{}\n```json\n{}\n```", if outlier { "PDU is outlier" @@ -1643,7 +1655,7 @@ impl Rooms { } None => { db.admin.send(AdminCommand::SendMessage( - message::MessageEventContent::text_plain( + RoomMessageEventContent::text_plain( "PDU not found.", ), )); @@ -1651,14 +1663,14 @@ impl Rooms { } } else { db.admin.send(AdminCommand::SendMessage( - message::MessageEventContent::text_plain( + RoomMessageEventContent::text_plain( "Event ID could not be parsed.", ), )); } } else { db.admin.send(AdminCommand::SendMessage( - message::MessageEventContent::text_plain( + RoomMessageEventContent::text_plain( "Usage: get_pdu ", ), )); @@ -1666,7 +1678,7 @@ impl Rooms { } _ => { db.admin.send(AdminCommand::SendMessage( - message::MessageEventContent::text_plain(format!( + RoomMessageEventContent::text_plain(format!( "Unrecognized command: {}", command )), @@ -1958,16 +1970,13 @@ impl Rooms { let create_event = self.room_state_get(room_id, &EventType::RoomCreate, "")?; - let create_event_content = create_event + let create_event_content: Option = create_event .as_ref() .map(|create_event| { - serde_json::from_value::>(create_event.content.clone()) - .expect("Raw::from_value always works.") - .deserialize() - .map_err(|e| { - warn!("Invalid create event: {}", e); - Error::bad_database("Invalid create event in db.") - }) + serde_json::from_str(create_event.content.get()).map_err(|e| { + warn!("Invalid create event: {}", e); + Error::bad_database("Invalid create event in db.") + }) }) .transpose()?; @@ -2000,7 +2009,10 @@ impl Rooms { let mut unsigned = unsigned.unwrap_or_default(); if let Some(state_key) = &state_key { if let Some(prev_pdu) = self.room_state_get(room_id, &event_type, state_key)? { - unsigned.insert("prev_content".to_owned(), prev_pdu.content.clone()); + unsigned.insert( + "prev_content".to_owned(), + serde_json::from_str(prev_pdu.content.get()).expect("string is valid json"), + ); unsigned.insert( "prev_sender".to_owned(), serde_json::to_value(&prev_pdu.sender).expect("UserId::to_value always works"), @@ -2025,11 +2037,15 @@ impl Rooms { .map(|(_, pdu)| pdu.event_id.clone()) .collect(), redacts, - unsigned, - hashes: ruma::events::pdu::EventHash { + unsigned: if unsigned.is_empty() { + None + } else { + Some(to_raw_value(&unsigned).expect("to_raw_value always works")) + }, + hashes: EventHash { sha256: "aaa".to_owned(), }, - signatures: BTreeMap::new(), + signatures: None, }; let auth_check = state_res::auth_check( @@ -2205,7 +2221,7 @@ impl Rooms { let mut pdu = serde_json::from_slice::(&v) .map_err(|_| Error::bad_database("PDU in db is invalid."))?; if pdu.sender != user_id { - pdu.unsigned.remove("transaction_id"); + pdu.remove_transaction_id()?; } Ok((pdu_id, pdu)) })) @@ -2242,7 +2258,7 @@ impl Rooms { let mut pdu = serde_json::from_slice::(&v) .map_err(|_| Error::bad_database("PDU in db is invalid."))?; if pdu.sender != user_id { - pdu.unsigned.remove("transaction_id"); + pdu.remove_transaction_id()?; } Ok((pdu_id, pdu)) })) @@ -2279,7 +2295,7 @@ impl Rooms { let mut pdu = serde_json::from_slice::(&v) .map_err(|_| Error::bad_database("PDU in db is invalid."))?; if pdu.sender != user_id { - pdu.unsigned.remove("transaction_id"); + pdu.remove_transaction_id()?; } Ok((pdu_id, pdu)) })) @@ -2309,7 +2325,7 @@ impl Rooms { &self, room_id: &RoomId, user_id: &UserId, - membership: member::MembershipState, + membership: MembershipState, sender: &UserId, last_state: Option>>, db: &Database, @@ -2338,7 +2354,7 @@ impl Rooms { roomuser_id.extend_from_slice(user_id.as_bytes()); match &membership { - member::MembershipState::Join => { + MembershipState::Join => { // Check if the user never joined this room if !self.once_joined(user_id, room_id)? { // Add the user ID to the join list then @@ -2348,12 +2364,8 @@ impl Rooms { if let Some(predecessor) = self .room_state_get(room_id, &EventType::RoomCreate, "")? .and_then(|create| { - serde_json::from_value::< - Raw, - >(create.content.clone()) - .expect("Raw::from_value always works") - .deserialize() - .ok() + serde_json::from_str::(create.content.get()) + .ok() }) .and_then(|content| content.predecessor) { @@ -2442,7 +2454,7 @@ impl Rooms { self.userroomid_leftstate.remove(&userroom_id)?; self.roomuserid_leftcount.remove(&roomuser_id)?; } - member::MembershipState::Invite => { + MembershipState::Invite => { // We want to know if the sender is ignored by the receiver let is_ignored = db .account_data @@ -2475,7 +2487,7 @@ impl Rooms { self.userroomid_leftstate.remove(&userroom_id)?; self.roomuserid_leftcount.remove(&roomuser_id)?; } - member::MembershipState::Leave | member::MembershipState::Ban => { + MembershipState::Leave | MembershipState::Ban => { if update_joined_count && self .room_members(room_id) @@ -2700,26 +2712,23 @@ impl Rooms { ); let state_lock = mutex_state.lock().await; - let mut event = serde_json::from_value::>( + let mut event = serde_json::from_str::( self.room_state_get(room_id, &EventType::RoomMember, &user_id.to_string())? .ok_or(Error::BadRequest( ErrorKind::BadState, "Cannot leave a room you are not a member of.", ))? .content - .clone(), + .get(), ) - .expect("from_value::> can never fail") - .deserialize() .map_err(|_| Error::bad_database("Invalid member event in database."))?; - event.membership = member::MembershipState::Leave; + event.membership = MembershipState::Leave; self.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(event) - .expect("event is valid, we just created it"), + content: to_raw_value(&event).expect("event is valid, we just created it"), unsigned: None, state_key: Some(user_id.to_string()), redacts: None, @@ -2793,10 +2802,9 @@ impl Rooms { }; let mut leave_event_stub = - serde_json::from_str::(make_leave_response.event.json().get()) - .map_err(|_| { - Error::BadServerResponse("Invalid make_leave event json received from server.") - })?; + serde_json::from_str::(make_leave_response.event.get()).map_err( + |_| Error::BadServerResponse("Invalid make_leave event json received from server."), + )?; // TODO: Is origin needed? leave_event_stub.insert( @@ -2847,7 +2855,7 @@ impl Rooms { federation::membership::create_leave_event::v2::Request { room_id, event_id: &event_id, - pdu: PduEvent::convert_to_outgoing_federation_event(leave_event.clone()), + pdu: &PduEvent::convert_to_outgoing_federation_event(leave_event.clone()), }, ) .await?; diff --git a/src/database/sending.rs b/src/database/sending.rs index 70ff1b6d..c1abcde2 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -398,7 +398,7 @@ impl Sending { let edu = Edu::DeviceListUpdate(DeviceListUpdateContent { user_id, device_id: device_id!("dummy"), - device_display_name: "Dummy".to_owned(), + device_display_name: Some("Dummy".to_owned()), stream_id: uint!(1), prev_id: Vec::new(), deleted: None, @@ -573,8 +573,14 @@ impl Sending { for pdu in pdus { // Redacted events are not notification targets (we don't send push for them) - if pdu.unsigned.get("redacted_because").is_some() { - continue; + if let Some(unsigned) = &pdu.unsigned { + if let Ok(unsigned) = + serde_json::from_str::(unsigned.get()) + { + if unsigned.get("redacted_because").is_some() { + continue; + } + } } let userid = diff --git a/src/database/uiaa.rs b/src/database/uiaa.rs index 60b9bd31..46796462 100644 --- a/src/database/uiaa.rs +++ b/src/database/uiaa.rs @@ -5,7 +5,8 @@ use ruma::{ api::client::{ error::ErrorKind, r0::uiaa::{ - IncomingAuthData, IncomingPassword, IncomingUserIdentifier::MatrixId, UiaaInfo, + AuthType, IncomingAuthData, IncomingPassword, IncomingUserIdentifier::MatrixId, + UiaaInfo, }, }, signatures::CanonicalJsonValue, @@ -99,10 +100,10 @@ impl Uiaa { } // Password was correct! Let's add it to `completed` - uiaainfo.completed.push("m.login.password".to_owned()); + uiaainfo.completed.push(AuthType::Password); } IncomingAuthData::Dummy(_) => { - uiaainfo.completed.push("m.login.dummy".to_owned()); + uiaainfo.completed.push(AuthType::Dummy); } k => error!("type not supported: {:?}", k), } diff --git a/src/pdu.rs b/src/pdu.rs index 8623b1a0..b74d079c 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -1,19 +1,28 @@ use crate::Error; use ruma::{ events::{ - pdu::EventHash, room::member::MemberEventContent, AnyEphemeralRoomEvent, - AnyInitialStateEvent, AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent, - AnySyncStateEvent, EventType, StateEvent, + room::member::RoomMemberEventContent, AnyEphemeralRoomEvent, AnyInitialStateEvent, + AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, + EventType, StateEvent, }, serde::{CanonicalJsonObject, CanonicalJsonValue, Raw}, - state_res, EventId, MilliSecondsSinceUnixEpoch, RoomId, RoomVersionId, ServerName, - ServerSigningKeyId, UInt, UserId, + state_res, EventId, MilliSecondsSinceUnixEpoch, RoomId, RoomVersionId, UInt, UserId, }; use serde::{Deserialize, Serialize}; -use serde_json::json; +use serde_json::{ + json, + value::{to_raw_value, RawValue as RawJsonValue}, +}; use std::{cmp::Ordering, collections::BTreeMap, convert::TryFrom}; use tracing::warn; +/// Content hashes of a PDU. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct EventHash { + /// The SHA-256 hash. + pub sha256: String, +} + #[derive(Clone, Deserialize, Serialize, Debug)] pub struct PduEvent { pub event_id: EventId, @@ -22,7 +31,7 @@ pub struct PduEvent { pub origin_server_ts: UInt, #[serde(rename = "type")] pub kind: EventType, - pub content: serde_json::Value, + pub content: Box, #[serde(skip_serializing_if = "Option::is_none")] pub state_key: Option, pub prev_events: Vec, @@ -30,16 +39,17 @@ pub struct PduEvent { pub auth_events: Vec, #[serde(skip_serializing_if = "Option::is_none")] pub redacts: Option, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub unsigned: BTreeMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub unsigned: Option>, pub hashes: EventHash, - pub signatures: BTreeMap, BTreeMap>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub signatures: Option>, // BTreeMap, BTreeMap> } impl PduEvent { #[tracing::instrument(skip(self))] pub fn redact(&mut self, reason: &PduEvent) -> crate::Result<()> { - self.unsigned.clear(); + self.unsigned = None; let allowed: &[&str] = match self.kind { EventType::RoomMember => &["membership"], @@ -59,10 +69,9 @@ impl PduEvent { _ => &[], }; - let old_content = self - .content - .as_object_mut() - .ok_or_else(|| Error::bad_database("PDU in db has invalid content."))?; + let mut old_content = + serde_json::from_str::>(self.content.get()) + .map_err(|_| Error::bad_database("PDU in db has invalid content."))?; let mut new_content = serde_json::Map::new(); @@ -72,12 +81,23 @@ impl PduEvent { } } - self.unsigned.insert( - "redacted_because".to_owned(), - serde_json::to_value(reason).expect("to_value(PduEvent) always works"), - ); + self.unsigned = Some(to_raw_value(&json!({ + "redacted_because": serde_json::to_value(reason).expect("to_value(PduEvent) always works") + })).expect("to string always works")); - self.content = new_content.into(); + self.content = to_raw_value(&new_content).expect("to string always works"); + + Ok(()) + } + + pub fn remove_transaction_id(&mut self) -> crate::Result<()> { + if let Some(unsigned) = &self.unsigned { + let mut unsigned = + serde_json::from_str::>>(unsigned.get()) + .map_err(|_| Error::bad_database("Invalid unsigned in pdu event"))?; + unsigned.remove("transaction_id"); + self.unsigned = Some(to_raw_value(&unsigned).expect("unsigned is valid")); + } Ok(()) } @@ -192,7 +212,7 @@ impl PduEvent { } #[tracing::instrument(skip(self))] - pub fn to_member_event(&self) -> Raw> { + pub fn to_member_event(&self) -> Raw> { let json = json!({ "content": self.content, "type": self.kind, @@ -212,7 +232,7 @@ impl PduEvent { #[tracing::instrument] pub fn convert_to_outgoing_federation_event( mut pdu_json: CanonicalJsonObject, - ) -> Raw { + ) -> Box { if let Some(unsigned) = pdu_json .get_mut("unsigned") .and_then(|val| val.as_object_mut()) @@ -229,10 +249,7 @@ impl PduEvent { // ) // .expect("Raw::from_value always works") - serde_json::from_value::>( - serde_json::to_value(pdu_json).expect("CanonicalJson is valid serde_json::Value"), - ) - .expect("Raw::from_value always works") + to_raw_value(&pdu_json).expect("CanonicalJson is valid serde_json::Value") } pub fn from_id_val( @@ -265,7 +282,7 @@ impl state_res::Event for PduEvent { &self.kind } - fn content(&self) -> &serde_json::Value { + fn content(&self) -> &RawJsonValue { &self.content } @@ -281,10 +298,6 @@ impl state_res::Event for PduEvent { Box::new(self.prev_events.iter()) } - fn depth(&self) -> &UInt { - &self.depth - } - fn auth_events(&self) -> Box + '_> { Box::new(self.auth_events.iter()) } @@ -292,18 +305,6 @@ impl state_res::Event for PduEvent { fn redacts(&self) -> Option<&EventId> { self.redacts.as_ref() } - - fn hashes(&self) -> &EventHash { - &self.hashes - } - - fn signatures(&self) -> BTreeMap, BTreeMap> { - self.signatures.clone() - } - - fn unsigned(&self) -> &BTreeMap { - &self.unsigned - } } // These impl's allow us to dedup state snapshots when resolving state @@ -329,9 +330,9 @@ impl Ord for PduEvent { /// /// Returns a tuple of the new `EventId` and the PDU as a `BTreeMap`. pub(crate) fn gen_event_id_canonical_json( - pdu: &Raw, + pdu: &RawJsonValue, ) -> crate::Result<(EventId, CanonicalJsonObject)> { - let value = serde_json::from_str(pdu.json().get()).map_err(|e| { + let value = serde_json::from_str(pdu.get()).map_err(|e| { warn!("Error parsing incoming event {:?}: {:?}", pdu, e); Error::BadServerResponse("Invalid PDU in server response") })?; @@ -352,7 +353,7 @@ pub(crate) fn gen_event_id_canonical_json( pub struct PduBuilder { #[serde(rename = "type")] pub event_type: EventType, - pub content: serde_json::Value, + pub content: Box, pub unsigned: Option>, pub state_key: Option, pub redacts: Option, @@ -363,7 +364,7 @@ impl From for PduBuilder { fn from(event: AnyInitialStateEvent) -> Self { Self { event_type: EventType::from(event.event_type()), - content: serde_json::value::to_value(event.content()) + content: to_raw_value(&event.content()) .expect("AnyStateEventContent came from JSON and can thus turn back into JSON."), unsigned: None, state_key: Some(event.state_key().to_owned()), diff --git a/src/server_server.rs b/src/server_server.rs index 2b8b06c7..805ae3a3 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1,6 +1,7 @@ use crate::{ client_server::{self, claim_keys_helper, get_keys_helper}, database::{rooms::CompressedStateEvent, DatabaseGuard}, + pdu::EventHash, utils, ConduitResult, Database, Error, PduEvent, Result, Ruma, }; use get_profile_information::v1::ProfileField; @@ -39,22 +40,22 @@ use ruma::{ }, directory::{IncomingFilter, IncomingRoomNetwork}, events::{ - pdu::Pdu, receipt::{ReceiptEvent, ReceiptEventContent}, room::{ - create::CreateEventContent, - member::{MemberEventContent, MembershipState}, + create::RoomCreateEventContent, + member::{MembershipState, RoomMemberEventContent}, }, AnyEphemeralRoomEvent, EventType, }, + int, receipt::ReceiptType, - serde::Raw, signatures::{CanonicalJsonObject, CanonicalJsonValue}, state_res::{self, RoomVersion, StateMap}, to_device::DeviceIdOrAllDevices, uint, EventId, MilliSecondsSinceUnixEpoch, RoomId, RoomVersionId, ServerName, ServerSigningKeyId, }; +use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use std::{ collections::{btree_map, hash_map, BTreeMap, BTreeSet, HashMap, HashSet}, convert::{TryFrom, TryInto}, @@ -1071,7 +1072,7 @@ pub(crate) async fn handle_incoming_pdu<'a>( // and lexically by event_id. println!("{}", event_id); Ok(( - 0, + int!(0), MilliSecondsSinceUnixEpoch( eventid_info .get(event_id) @@ -1153,14 +1154,13 @@ fn handle_outlier_pdu<'a>( // 2. Check signatures, otherwise drop // 3. check content hash, redact if doesn't match - let create_event_content = - serde_json::from_value::>(create_event.content.clone()) - .expect("Raw::from_value always works.") - .deserialize() - .map_err(|e| { - warn!("Invalid create event: {}", e); - "Invalid create event in db.".to_owned() - })?; + let create_event_content = serde_json::from_str::( + create_event.content.get(), + ) + .map_err(|e| { + warn!("Invalid create event: {}", e); + "Invalid create event in db.".to_owned() + })?; let room_version_id = &create_event_content.room_version; let room_version = RoomVersion::new(room_version_id).expect("room version is supported"); @@ -1241,7 +1241,7 @@ fn handle_outlier_pdu<'a>( .expect("all auth events have state keys"), )) { hash_map::Entry::Vacant(v) => { - v.insert(auth_event.clone()); + v.insert(auth_event); } hash_map::Entry::Occupied(_) => { return Err( @@ -1276,7 +1276,7 @@ fn handle_outlier_pdu<'a>( if !state_res::event_auth::auth_check( &room_version, &incoming_pdu, - previous_create, + previous_create.as_ref(), None::, // TODO: third party invite |k, s| auth_events.get(&(k.clone(), s.to_owned())), ) @@ -1319,14 +1319,13 @@ async fn upgrade_outlier_to_timeline_pdu( return Err("Event has been soft failed".into()); } - let create_event_content = - serde_json::from_value::>(create_event.content.clone()) - .expect("Raw::from_value always works.") - .deserialize() - .map_err(|e| { - warn!("Invalid create event: {}", e); - "Invalid create event in db.".to_owned() - })?; + let create_event_content = serde_json::from_str::( + create_event.content.get(), + ) + .map_err(|e| { + warn!("Invalid create event: {}", e); + "Invalid create event in db.".to_owned() + })?; let room_version_id = &create_event_content.room_version; let room_version = RoomVersion::new(room_version_id).expect("room version is supported"); @@ -1562,7 +1561,7 @@ async fn upgrade_outlier_to_timeline_pdu( let check_result = state_res::event_auth::auth_check( &room_version, &incoming_pdu, - previous_create.as_deref(), + previous_create.as_ref(), None::, // TODO: third party invite |k, s| { db.rooms @@ -1646,7 +1645,7 @@ async fn upgrade_outlier_to_timeline_pdu( let soft_fail = !state_res::event_auth::auth_check( &room_version, &incoming_pdu, - previous_create.as_deref(), + previous_create.as_ref(), None::, |k, s| auth_events.get(&(k.clone(), s.to_owned())), ) @@ -2669,13 +2668,12 @@ pub fn create_join_event_template_route( let create_event_content = create_event .as_ref() .map(|create_event| { - serde_json::from_value::>(create_event.content.clone()) - .expect("Raw::from_value always works.") - .deserialize() - .map_err(|e| { + serde_json::from_str::(create_event.content.get()).map_err( + |e| { warn!("Invalid create event: {}", e); Error::bad_database("Invalid create event in db.") - }) + }, + ) }) .transpose()?; @@ -2702,7 +2700,7 @@ pub fn create_join_event_template_route( )); } - let content = serde_json::to_value(MemberEventContent { + let content = to_raw_value(&RoomMemberEventContent { avatar_url: None, blurhash: None, displayname: None, @@ -2738,7 +2736,7 @@ pub fn create_join_event_template_route( unsigned.insert("prev_content".to_owned(), prev_pdu.content.clone()); unsigned.insert( "prev_sender".to_owned(), - serde_json::to_value(&prev_pdu.sender).expect("UserId::to_value always works"), + serde_json::from_str(prev_pdu.sender.as_str()).expect("UserId is valid string"), ); } @@ -2759,17 +2757,21 @@ pub fn create_join_event_template_route( .map(|(_, pdu)| pdu.event_id.clone()) .collect(), redacts: None, - unsigned, - hashes: ruma::events::pdu::EventHash { + unsigned: if unsigned.is_empty() { + None + } else { + Some(to_raw_value(&unsigned).expect("to_raw_value always works")) + }, + hashes: EventHash { sha256: "aaa".to_owned(), }, - signatures: BTreeMap::new(), + signatures: None, }; let auth_check = state_res::auth_check( &room_version, &pdu, - create_prev_event.as_deref(), + create_prev_event, None::, // TODO: third_party_invite |k, s| auth_events.get(&(k.clone(), s.to_owned())), ) @@ -2799,10 +2801,7 @@ pub fn create_join_event_template_route( Ok(create_join_event_template::v1::Response { room_version: Some(room_version_id), - event: serde_json::from_value::>( - serde_json::to_value(pdu_json).expect("CanonicalJson is valid serde_json::Value"), - ) - .expect("Raw::from_value always works"), + event: to_raw_value(&pdu_json).expect("CanonicalJson can be serialized to JSON"), } .into()) } @@ -2810,7 +2809,7 @@ pub fn create_join_event_template_route( async fn create_join_event( db: &DatabaseGuard, room_id: &RoomId, - pdu: &Raw, + pdu: &RawJsonValue, ) -> Result { if !db.globals.allow_federation() { return Err(Error::bad_config("Federation is disabled.")); @@ -2947,7 +2946,7 @@ pub async fn create_join_event_v2_route( #[tracing::instrument(skip(db, body))] pub async fn create_invite_route( db: DatabaseGuard, - body: Ruma, + body: Ruma>, ) -> ConduitResult { if !db.globals.allow_federation() { return Err(Error::bad_config("Federation is disabled.")); @@ -3014,10 +3013,11 @@ pub async fn create_invite_route( let mut invite_state = body.invite_room_state.clone(); - let mut event = serde_json::from_str::>( - &body.event.json().to_string(), - ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid invite event bytes."))?; + let mut event = + serde_json::from_str::>(body.event.get()) + .map_err(|_| { + Error::BadRequest(ErrorKind::InvalidParam, "Invalid invite event bytes.") + })?; event.insert("event_id".to_owned(), "$dummy".into()); @@ -3280,13 +3280,13 @@ pub(crate) async fn fetch_required_signing_keys( // Gets a list of servers for which we don't have the signing key yet. We go over // the PDUs and either cache the key or add it to the list that needs to be retrieved. fn get_server_keys_from_cache( - pdu: &Raw, + pdu: &RawJsonValue, servers: &mut BTreeMap, BTreeMap>, room_version: &RoomVersionId, pub_key_map: &mut RwLockWriteGuard<'_, BTreeMap>>, db: &Database, ) -> Result<()> { - let value = serde_json::from_str::(pdu.json().get()).map_err(|e| { + let value = serde_json::from_str::(pdu.get()).map_err(|e| { error!("Invalid PDU in server response: {:?}: {:?}", pdu, e); Error::BadServerResponse("Invalid PDU in server response") })?; @@ -3385,10 +3385,10 @@ pub(crate) async fn fetch_join_signing_keys( // Try to fetch keys, failure is okay // Servers we couldn't find in the cache will be added to `servers` for pdu in &event.room_state.state { - let _ = get_server_keys_from_cache(pdu, &mut servers, room_version, &mut pkm, db); + let _ = get_server_keys_from_cache(&pdu, &mut servers, room_version, &mut pkm, db); } for pdu in &event.room_state.auth_chain { - let _ = get_server_keys_from_cache(pdu, &mut servers, room_version, &mut pkm, db); + let _ = get_server_keys_from_cache(&pdu, &mut servers, room_version, &mut pkm, db); } drop(pkm); From 1c4d9af586611781ad6d5337778dbe2bea695ee4 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Wed, 13 Oct 2021 10:24:39 +0200 Subject: [PATCH 023/201] Enable more lints and apply their suggestions --- src/client_server/room.rs | 2 +- src/database.rs | 4 ++-- src/database/abstraction.rs | 4 ++-- src/database/key_backups.rs | 2 +- src/database/proxy.rs | 2 +- src/database/rooms.rs | 12 +++++------ src/database/rooms/edus.rs | 2 +- src/lib.rs | 6 ++++++ src/main.rs | 7 ++++++- src/pdu.rs | 2 +- src/ruma_wrapper.rs | 2 +- src/server_server.rs | 40 +++++++++++++++++-------------------- src/utils.rs | 2 +- 13 files changed, 47 insertions(+), 40 deletions(-) diff --git a/src/client_server/room.rs b/src/client_server/room.rs index d1c79dff..eb68135f 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -475,7 +475,7 @@ pub async fn upgrade_room_route( PduBuilder { event_type: EventType::RoomTombstone, content: to_raw_value(&RoomTombstoneEventContent { - body: "This room has been replaced".to_string(), + body: "This room has been replaced".to_owned(), replacement_room: replacement_room.clone(), }) .expect("event is valid, we just created it"), diff --git a/src/database.rs b/src/database.rs index 110d4d0c..63c4ebc4 100644 --- a/src/database.rs +++ b/src/database.rs @@ -499,13 +499,13 @@ impl Database { if let Some(parent_stateinfo) = states_parents.last() { let statediffnew = current_state .difference(&parent_stateinfo.1) - .cloned() + .copied() .collect::>(); let statediffremoved = parent_stateinfo .1 .difference(¤t_state) - .cloned() + .copied() .collect::>(); (statediffnew, statediffremoved) diff --git a/src/database/abstraction.rs b/src/database/abstraction.rs index 5b941fbe..11bbc3b1 100644 --- a/src/database/abstraction.rs +++ b/src/database/abstraction.rs @@ -22,7 +22,7 @@ pub trait Tree: Send + Sync { fn get(&self, key: &[u8]) -> Result>>; fn insert(&self, key: &[u8], value: &[u8]) -> Result<()>; - fn insert_batch<'a>(&self, iter: &mut dyn Iterator, Vec)>) -> Result<()>; + fn insert_batch(&self, iter: &mut dyn Iterator, Vec)>) -> Result<()>; fn remove(&self, key: &[u8]) -> Result<()>; @@ -35,7 +35,7 @@ pub trait Tree: Send + Sync { ) -> Box, Vec)> + 'a>; fn increment(&self, key: &[u8]) -> Result>; - fn increment_batch<'a>(&self, iter: &mut dyn Iterator>) -> Result<()>; + fn increment_batch(&self, iter: &mut dyn Iterator>) -> Result<()>; fn scan_prefix<'a>( &'a self, diff --git a/src/database/key_backups.rs b/src/database/key_backups.rs index 27d8030d..a960c72f 100644 --- a/src/database/key_backups.rs +++ b/src/database/key_backups.rs @@ -81,7 +81,7 @@ impl KeyBackups { )?; self.backupid_etag .insert(&key, &globals.next_count()?.to_be_bytes())?; - Ok(version.to_string()) + Ok(version.to_owned()) } pub fn get_latest_backup_version(&self, user_id: &UserId) -> Result> { diff --git a/src/database/proxy.rs b/src/database/proxy.rs index 78e9d2bf..33f7f3d9 100644 --- a/src/database/proxy.rs +++ b/src/database/proxy.rs @@ -136,7 +136,7 @@ impl std::str::FromStr for WildCardedDomain { }) } } -impl<'de> serde::de::Deserialize<'de> for WildCardedDomain { +impl<'de> Deserialize<'de> for WildCardedDomain { fn deserialize(deserializer: D) -> std::result::Result where D: serde::de::Deserializer<'de>, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 3096150d..1912e0c8 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -50,7 +50,7 @@ pub type StateHashId = Vec; pub type CompressedStateEvent = [u8; 2 * size_of::()]; pub struct Rooms { - pub edus: edus::RoomEdus, + pub edus: RoomEdus, pub(super) pduid_pdu: Arc, // PduId = ShortRoomId + Count pub(super) eventid_pduid: Arc, pub(super) roomid_pduleaves: Arc, @@ -371,13 +371,13 @@ impl Rooms { { let statediffnew = new_state_ids_compressed .difference(&parent_stateinfo.1) - .cloned() + .copied() .collect::>(); let statediffremoved = parent_stateinfo .1 .difference(&new_state_ids_compressed) - .cloned() + .copied() .collect::>(); (statediffnew, statediffremoved) @@ -498,7 +498,7 @@ impl Rooms { if parent != 0_u64 { let mut response = self.load_shortstatehash_info(parent)?; let mut state = response.last().unwrap().1.clone(); - state.extend(added.iter().cloned()); + state.extend(added.iter().copied()); for r in &removed { state.remove(r); } @@ -1773,13 +1773,13 @@ impl Rooms { if let Some(parent_stateinfo) = states_parents.last() { let statediffnew = state_ids_compressed .difference(&parent_stateinfo.1) - .cloned() + .copied() .collect::>(); let statediffremoved = parent_stateinfo .1 .difference(&state_ids_compressed) - .cloned() + .copied() .collect::>(); (statediffnew, statediffremoved) diff --git a/src/database/rooms/edus.rs b/src/database/rooms/edus.rs index e0639ffd..26f22bf4 100644 --- a/src/database/rooms/edus.rs +++ b/src/database/rooms/edus.rs @@ -331,7 +331,7 @@ impl RoomEdus { &self, user_id: &UserId, room_id: &RoomId, - presence: ruma::events::presence::PresenceEvent, + presence: PresenceEvent, globals: &super::super::globals::Globals, ) -> Result<()> { // TODO: Remove old entry? Or maybe just wipe completely from time to time? diff --git a/src/lib.rs b/src/lib.rs index fbffb7e3..82b8f340 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,9 @@ +#![warn( + rust_2018_idioms, + unused_qualifications, + clippy::cloned_instead_of_copied, + clippy::str_to_string +)] #![allow(clippy::suspicious_else_formatting)] #![deny(clippy::dbg_macro)] diff --git a/src/main.rs b/src/main.rs index 06409eeb..84dfb1fc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,9 @@ -#![warn(rust_2018_idioms)] +#![warn( + rust_2018_idioms, + unused_qualifications, + clippy::cloned_instead_of_copied, + clippy::str_to_string +)] #![allow(clippy::suspicious_else_formatting)] #![deny(clippy::dbg_macro)] diff --git a/src/pdu.rs b/src/pdu.rs index b74d079c..0a765e1f 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -257,7 +257,7 @@ impl PduEvent { mut json: CanonicalJsonObject, ) -> Result { json.insert( - "event_id".to_string(), + "event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned()), ); diff --git a/src/ruma_wrapper.rs b/src/ruma_wrapper.rs index 4629de92..03c115cd 100644 --- a/src/ruma_wrapper.rs +++ b/src/ruma_wrapper.rs @@ -344,7 +344,7 @@ impl Deref for Ruma { } /// This struct converts ruma responses into rocket http responses. -pub type ConduitResult = std::result::Result, Error>; +pub type ConduitResult = Result, Error>; pub fn response(response: RumaResponse) -> response::Result<'static> { let http_response = response diff --git a/src/server_server.rs b/src/server_server.rs index 805ae3a3..e9a94856 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -336,7 +336,7 @@ fn add_port_to_hostname(destination_str: &str) -> FedDest { None => (destination_str, ":8448"), Some(pos) => destination_str.split_at(pos), }; - FedDest::Named(host.to_string(), port.to_string()) + FedDest::Named(host.to_owned(), port.to_owned()) } /// Returns: actual_destination, host header @@ -358,7 +358,7 @@ async fn find_actual_destination( if let Some(pos) = destination_str.find(':') { // 2: Hostname with included port let (host, port) = destination_str.split_at(pos); - FedDest::Named(host.to_string(), port.to_string()) + FedDest::Named(host.to_owned(), port.to_owned()) } else { match request_well_known(globals, destination.as_str()).await { // 3: A .well-known file is available @@ -370,7 +370,7 @@ async fn find_actual_destination( if let Some(pos) = delegated_hostname.find(':') { // 3.2: Hostname with port in .well-known file let (host, port) = delegated_hostname.split_at(pos); - FedDest::Named(host.to_string(), port.to_string()) + FedDest::Named(host.to_owned(), port.to_owned()) } else { // Delegated hostname has no port in this branch if let Some(hostname_override) = @@ -454,12 +454,12 @@ async fn find_actual_destination( let hostname = if let Ok(addr) = hostname.parse::() { FedDest::Literal(addr) } else if let Ok(addr) = hostname.parse::() { - FedDest::Named(addr.to_string(), ":8448".to_string()) + FedDest::Named(addr.to_string(), ":8448".to_owned()) } else if let Some(pos) = hostname.find(':') { let (host, port) = hostname.split_at(pos); - FedDest::Named(host.to_string(), port.to_string()) + FedDest::Named(host.to_owned(), port.to_owned()) } else { - FedDest::Named(hostname, ":8448".to_string()) + FedDest::Named(hostname, ":8448".to_owned()) }; (actual_destination, hostname) } @@ -476,11 +476,7 @@ async fn query_srv_record( .map(|srv| { srv.iter().next().map(|result| { FedDest::Named( - result - .target() - .to_string() - .trim_end_matches('.') - .to_string(), + result.target().to_string().trim_end_matches('.').to_owned(), format!(":{}", result.port()), ) }) @@ -745,7 +741,7 @@ pub async fn send_transaction_message_route( Some(id) => id, None => { // Event is invalid - resolved_map.insert(event_id, Err("Event needs a valid RoomId.".to_string())); + resolved_map.insert(event_id, Err("Event needs a valid RoomId.".to_owned())); continue; } }; @@ -963,7 +959,7 @@ pub(crate) async fn handle_incoming_pdu<'a>( match db.rooms.exists(room_id) { Ok(true) => {} _ => { - return Err("Room is unknown to this server.".to_string()); + return Err("Room is unknown to this server.".to_owned()); } } @@ -1173,14 +1169,14 @@ fn handle_outlier_pdu<'a>( Err(e) => { // Drop warn!("Dropping bad event {}: {}", event_id, e); - return Err("Signature verification failed".to_string()); + return Err("Signature verification failed".to_owned()); } Ok(ruma::signatures::Verified::Signatures) => { // Redact warn!("Calculated hash does not match: {}", event_id); match ruma::signatures::redact(&value, room_version_id) { Ok(obj) => obj, - Err(_) => return Err("Redaction failed".to_string()), + Err(_) => return Err("Redaction failed".to_owned()), } } Ok(ruma::signatures::Verified::All) => value, @@ -1195,7 +1191,7 @@ fn handle_outlier_pdu<'a>( let incoming_pdu = serde_json::from_value::( serde_json::to_value(&val).expect("CanonicalJsonObj is a valid JsonValue"), ) - .map_err(|_| "Event is not a valid PDU.".to_string())?; + .map_err(|_| "Event is not a valid PDU.".to_owned())?; // 4. fetch any missing auth events doing all checks listed here starting at 1. These are not timeline events // 5. Reject "due to auth events" if can't get all the auth events or some of the auth events are also rejected "due to auth events" @@ -1280,9 +1276,9 @@ fn handle_outlier_pdu<'a>( None::, // TODO: third party invite |k, s| auth_events.get(&(k.clone(), s.to_owned())), ) - .map_err(|_e| "Auth check failed".to_string())? + .map_err(|_e| "Auth check failed".to_owned())? { - return Err("Event has failed auth check with auth events.".to_string()); + return Err("Event has failed auth check with auth events.".to_owned()); } debug!("Validation successful."); @@ -2256,7 +2252,7 @@ pub(crate) fn get_auth_chain<'a>( .collect::>(); if let Some(cached) = db.rooms.get_auth_chain_from_cache(&chunk_key)? { hits += 1; - full_auth_chain.extend(cached.iter().cloned()); + full_auth_chain.extend(cached.iter().copied()); continue; } misses += 1; @@ -2267,7 +2263,7 @@ pub(crate) fn get_auth_chain<'a>( for (sevent_id, event_id) in chunk { if let Some(cached) = db.rooms.get_auth_chain_from_cache(&[sevent_id])? { hits2 += 1; - chunk_cache.extend(cached.iter().cloned()); + chunk_cache.extend(cached.iter().copied()); } else { misses2 += 1; let auth_chain = Arc::new(get_auth_chain_inner(room_id, &event_id, db)?); @@ -3385,10 +3381,10 @@ pub(crate) async fn fetch_join_signing_keys( // Try to fetch keys, failure is okay // Servers we couldn't find in the cache will be added to `servers` for pdu in &event.room_state.state { - let _ = get_server_keys_from_cache(&pdu, &mut servers, room_version, &mut pkm, db); + let _ = get_server_keys_from_cache(pdu, &mut servers, room_version, &mut pkm, db); } for pdu in &event.room_state.auth_chain { - let _ = get_server_keys_from_cache(&pdu, &mut servers, room_version, &mut pkm, db); + let _ = get_server_keys_from_cache(pdu, &mut servers, room_version, &mut pkm, db); } drop(pkm); diff --git a/src/utils.rs b/src/utils.rs index d21395e1..26d71a8c 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -123,7 +123,7 @@ pub fn deserialize_from_str< E: std::fmt::Display, >( deserializer: D, -) -> std::result::Result { +) -> Result { struct Visitor, E>(std::marker::PhantomData); impl<'de, T: FromStr, Err: std::fmt::Display> serde::de::Visitor<'de> for Visitor From f2ef5677e0f016399dce4da66d45137e2e592e8c Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Wed, 13 Oct 2021 11:51:30 +0200 Subject: [PATCH 024/201] Reduce turbofish usage Should make the code a little bit easier to read. --- src/client_server/config.rs | 12 +-- src/client_server/context.rs | 16 ++-- src/client_server/device.rs | 4 +- src/client_server/directory.rs | 104 +++++++++----------- src/client_server/keys.rs | 4 +- src/client_server/membership.rs | 63 ++++++------ src/client_server/message.rs | 21 ++-- src/client_server/push.rs | 34 +++---- src/client_server/room.rs | 13 ++- src/client_server/search.rs | 6 +- src/client_server/state.rs | 12 +-- src/client_server/sync.rs | 37 ++++--- src/client_server/tag.rs | 23 +++-- src/database.rs | 4 +- src/database/globals.rs | 7 +- src/database/key_backups.rs | 11 ++- src/database/rooms.rs | 165 ++++++++++++++------------------ src/database/rooms/edus.rs | 22 +++-- src/database/sending.rs | 14 +-- src/database/uiaa.rs | 22 ++--- src/database/users.rs | 9 +- src/pdu.rs | 8 +- src/server_server.rs | 107 +++++++++------------ 23 files changed, 331 insertions(+), 387 deletions(-) diff --git a/src/client_server/config.rs b/src/client_server/config.rs index bd897bab..0c668ff1 100644 --- a/src/client_server/config.rs +++ b/src/client_server/config.rs @@ -30,7 +30,7 @@ pub async fn set_global_account_data_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let data = serde_json::from_str::(body.data.get()) + let data: serde_json::Value = serde_json::from_str(body.data.get()) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Data is invalid."))?; let event_type = body.event_type.to_string(); @@ -68,7 +68,7 @@ pub async fn set_room_account_data_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let data = serde_json::from_str::(body.data.get()) + let data: serde_json::Value = serde_json::from_str(body.data.get()) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Data is invalid."))?; let event_type = body.event_type.to_string(); @@ -103,9 +103,9 @@ pub async fn get_global_account_data_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let event = db + let event: Box = db .account_data - .get::>(None, sender_user, body.event_type.clone().into())? + .get(None, sender_user, body.event_type.clone().into())? .ok_or(Error::BadRequest(ErrorKind::NotFound, "Data not found."))?; let account_data = serde_json::from_str::(event.get()) @@ -132,9 +132,9 @@ pub async fn get_room_account_data_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let event = db + let event: Box = db .account_data - .get::>( + .get( Some(&body.room_id), sender_user, body.event_type.clone().into(), diff --git a/src/client_server/context.rs b/src/client_server/context.rs index b2346f5e..97fc4fd8 100644 --- a/src/client_server/context.rs +++ b/src/client_server/context.rs @@ -48,7 +48,7 @@ pub async fn get_context_route( ))? .to_room_event(); - let events_before = db + let events_before: Vec<_> = db .rooms .pdus_until(sender_user, &body.room_id, base_token)? .take( @@ -58,19 +58,19 @@ pub async fn get_context_route( / 2, ) .filter_map(|r| r.ok()) // Remove buggy events - .collect::>(); + .collect(); let start_token = events_before .last() .and_then(|(pdu_id, _)| db.rooms.pdu_count(pdu_id).ok()) .map(|count| count.to_string()); - let events_before = events_before + let events_before: Vec<_> = events_before .into_iter() .map(|(_, pdu)| pdu.to_room_event()) - .collect::>(); + .collect(); - let events_after = db + let events_after: Vec<_> = db .rooms .pdus_after(sender_user, &body.room_id, base_token)? .take( @@ -80,17 +80,17 @@ pub async fn get_context_route( / 2, ) .filter_map(|r| r.ok()) // Remove buggy events - .collect::>(); + .collect(); let end_token = events_after .last() .and_then(|(pdu_id, _)| db.rooms.pdu_count(pdu_id).ok()) .map(|count| count.to_string()); - let events_after = events_after + let events_after: Vec<_> = events_after .into_iter() .map(|(_, pdu)| pdu.to_room_event()) - .collect::>(); + .collect(); let mut resp = get_context::Response::new(); resp.start = start_token; diff --git a/src/client_server/device.rs b/src/client_server/device.rs index b6fee377..03a3004b 100644 --- a/src/client_server/device.rs +++ b/src/client_server/device.rs @@ -25,11 +25,11 @@ pub async fn get_devices_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let devices = db + let devices: Vec = db .users .all_devices_metadata(sender_user) .filter_map(|r| r.ok()) // Filter out buggy devices - .collect::>(); + .collect(); Ok(get_devices::Response { devices }.into()) } diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 835504cc..490f7524 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -223,7 +223,7 @@ pub(crate) async fn get_public_rooms_filtered_helper( } } - let mut all_rooms = db + let mut all_rooms: Vec<_> = db .rooms .public_rooms() .map(|room_id| { @@ -234,28 +234,22 @@ pub(crate) async fn get_public_rooms_filtered_helper( canonical_alias: db .rooms .room_state_get(&room_id, &EventType::RoomCanonicalAlias, "")? - .map_or(Ok::<_, Error>(None), |s| { - Ok( - serde_json::from_str::(s.content.get()) - .map_err(|_| { - Error::bad_database( - "Invalid canonical alias event in database.", - ) - })? - .alias, - ) + .map_or(Ok(None), |s| { + serde_json::from_str(s.content.get()) + .map(|c: RoomCanonicalAliasEventContent| c.alias) + .map_err(|_| { + Error::bad_database("Invalid canonical alias event in database.") + }) })?, name: db .rooms .room_state_get(&room_id, &EventType::RoomName, "")? - .map_or(Ok::<_, Error>(None), |s| { - Ok( - serde_json::from_str::(s.content.get()) - .map_err(|_| { - Error::bad_database("Invalid room name event in database.") - })? - .name, - ) + .map_or(Ok(None), |s| { + serde_json::from_str(s.content.get()) + .map(|c: RoomNameEventContent| c.name) + .map_err(|_| { + Error::bad_database("Invalid room name event in database.") + }) })?, num_joined_members: db .rooms @@ -269,56 +263,48 @@ pub(crate) async fn get_public_rooms_filtered_helper( topic: db .rooms .room_state_get(&room_id, &EventType::RoomTopic, "")? - .map_or(Ok::<_, Error>(None), |s| { - Ok(Some( - serde_json::from_str::(s.content.get()) - .map_err(|_| { - Error::bad_database("Invalid room topic event in database.") - })? - .topic, - )) + .map_or(Ok(None), |s| { + serde_json::from_str(s.content.get()) + .map(|c: RoomTopicEventContent| Some(c.topic)) + .map_err(|_| { + Error::bad_database("Invalid room topic event in database.") + }) })?, world_readable: db .rooms .room_state_get(&room_id, &EventType::RoomHistoryVisibility, "")? - .map_or(Ok::<_, Error>(false), |s| { - Ok(serde_json::from_str::( - s.content.get(), - ) - .map_err(|_| { - Error::bad_database( - "Invalid room history visibility event in database.", - ) - })? - .history_visibility - == HistoryVisibility::WorldReadable) + .map_or(Ok(false), |s| { + serde_json::from_str(s.content.get()) + .map(|c: RoomHistoryVisibilityEventContent| { + c.history_visibility == HistoryVisibility::WorldReadable + }) + .map_err(|_| { + Error::bad_database( + "Invalid room history visibility event in database.", + ) + }) })?, guest_can_join: db .rooms .room_state_get(&room_id, &EventType::RoomGuestAccess, "")? - .map_or(Ok::<_, Error>(false), |s| { - Ok( - serde_json::from_str::(s.content.get()) - .map_err(|_| { - Error::bad_database( - "Invalid room guest access event in database.", - ) - })? - .guest_access - == GuestAccess::CanJoin, - ) + .map_or(Ok(false), |s| { + serde_json::from_str(s.content.get()) + .map(|c: RoomGuestAccessEventContent| { + c.guest_access == GuestAccess::CanJoin + }) + .map_err(|_| { + Error::bad_database("Invalid room guest access event in database.") + }) })?, avatar_url: db .rooms .room_state_get(&room_id, &EventType::RoomAvatar, "")? .map(|s| { - Ok::<_, Error>( - serde_json::from_str::(s.content.get()) - .map_err(|_| { - Error::bad_database("Invalid room avatar event in database.") - })? - .url, - ) + serde_json::from_str(s.content.get()) + .map(|c: RoomAvatarEventContent| c.url) + .map_err(|_| { + Error::bad_database("Invalid room avatar event in database.") + }) }) .transpose()? // url is now an Option so we must flatten @@ -359,17 +345,17 @@ pub(crate) async fn get_public_rooms_filtered_helper( } }) // We need to collect all, so we can sort by member count - .collect::>(); + .collect(); all_rooms.sort_by(|l, r| r.num_joined_members.cmp(&l.num_joined_members)); let total_room_count_estimate = (all_rooms.len() as u32).into(); - let chunk = all_rooms + let chunk: Vec<_> = all_rooms .into_iter() .skip(num_since as usize) .take(limit as usize) - .collect::>(); + .collect(); let prev_batch = if num_since == 0 { None diff --git a/src/client_server/keys.rs b/src/client_server/keys.rs index 980acf03..a44f5e9c 100644 --- a/src/client_server/keys.rs +++ b/src/client_server/keys.rs @@ -395,7 +395,7 @@ pub(crate) async fn get_keys_helper bool>( let mut failures = BTreeMap::new(); - let mut futures = get_over_federation + let mut futures: FuturesUnordered<_> = get_over_federation .into_iter() .map(|(server, vec)| async move { let mut device_keys_input_fed = BTreeMap::new(); @@ -415,7 +415,7 @@ pub(crate) async fn get_keys_helper bool>( .await, ) }) - .collect::>(); + .collect(); while let Some((server, response)) = futures.next().await { match response { diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index e37fe6c4..732f6162 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -56,19 +56,17 @@ pub async fn join_room_by_id_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let mut servers = db + let mut servers: HashSet<_> = db .rooms .invite_state(sender_user, &body.room_id)? .unwrap_or_default() .iter() - .filter_map(|event| { - serde_json::from_str::(&event.json().to_string()).ok() - }) - .filter_map(|event| event.get("sender").cloned()) + .filter_map(|event| serde_json::from_str(event.json().get()).ok()) + .filter_map(|event: serde_json::Value| event.get("sender").cloned()) .filter_map(|sender| sender.as_str().map(|s| s.to_owned())) .filter_map(|sender| UserId::try_from(sender).ok()) .map(|user| user.server_name().to_owned()) - .collect::>(); + .collect(); servers.insert(body.room_id.server_name().to_owned()); @@ -105,19 +103,17 @@ pub async fn join_room_by_id_or_alias_route( let (servers, room_id) = match RoomId::try_from(body.room_id_or_alias.clone()) { Ok(room_id) => { - let mut servers = db + let mut servers: HashSet<_> = db .rooms .invite_state(sender_user, &room_id)? .unwrap_or_default() .iter() - .filter_map(|event| { - serde_json::from_str::(&event.json().to_string()).ok() - }) - .filter_map(|event| event.get("sender").cloned()) + .filter_map(|event| serde_json::from_str(event.json().get()).ok()) + .filter_map(|event: serde_json::Value| event.get("sender").cloned()) .filter_map(|sender| sender.as_str().map(|s| s.to_owned())) .filter_map(|sender| UserId::try_from(sender).ok()) .map(|user| user.server_name().to_owned()) - .collect::>(); + .collect(); servers.insert(room_id.server_name().to_owned()); (servers, room_id) @@ -280,7 +276,7 @@ pub async fn ban_user_route( &body.user_id.to_string(), )? .map_or( - Ok::<_, Error>(RoomMemberEventContent { + Ok(RoomMemberEventContent { membership: MembershipState::Ban, displayname: db.users.displayname(&body.user_id)?, avatar_url: db.users.avatar_url(&body.user_id)?, @@ -290,10 +286,12 @@ pub async fn ban_user_route( reason: None, }), |event| { - let mut event = serde_json::from_str::(event.content.get()) - .map_err(|_| Error::bad_database("Invalid member event in database."))?; - event.membership = MembershipState::Ban; - Ok(event) + serde_json::from_str(event.content.get()) + .map(|event: RoomMemberEventContent| RoomMemberEventContent { + membership: MembershipState::Ban, + ..event + }) + .map_err(|_| Error::bad_database("Invalid member event in database.")) }, )?; @@ -342,7 +340,7 @@ pub async fn unban_user_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let mut event = serde_json::from_str::( + let mut event: RoomMemberEventContent = serde_json::from_str( db.rooms .room_state_get( &body.room_id, @@ -577,10 +575,10 @@ async fn join_room_by_id_helper( _ => return Err(Error::BadServerResponse("Room version is not supported")), }; - let mut join_event_stub = - serde_json::from_str::(make_join_response.event.get()).map_err( - |_| Error::BadServerResponse("Invalid make_join event json received from server."), - )?; + let mut join_event_stub: CanonicalJsonObject = + serde_json::from_str(make_join_response.event.get()).map_err(|_| { + Error::BadServerResponse("Invalid make_join event json received from server.") + })?; // TODO: Is origin needed? join_event_stub.insert( @@ -716,7 +714,7 @@ async fn join_room_by_id_helper( state .into_iter() .map(|(k, id)| db.rooms.compress_state_event(k, &id, &db.globals)) - .collect::>>()?, + .collect::>()?, db, )?; @@ -787,7 +785,7 @@ fn validate_and_add_event_id( pub_key_map: &RwLock>>, db: &Database, ) -> Result<(EventId, CanonicalJsonObject)> { - let mut value = serde_json::from_str::(pdu.get()).map_err(|e| { + let mut value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| { error!("Invalid PDU in server response: {:?}: {:?}", pdu, e); Error::BadServerResponse("Invalid PDU in server response") })?; @@ -863,25 +861,24 @@ pub(crate) async fn invite_helper<'a>( ); let state_lock = mutex_state.lock().await; - let prev_events = db + let prev_events: Vec<_> = db .rooms .get_pdu_leaves(room_id)? .into_iter() .take(20) - .collect::>(); + .collect(); let create_event = db .rooms .room_state_get(room_id, &EventType::RoomCreate, "")?; - let create_event_content = create_event + let create_event_content: Option = create_event .as_ref() .map(|create_event| { - serde_json::from_str::(create_event.content.get()) - .map_err(|e| { - warn!("Invalid create event: {}", e); - Error::bad_database("Invalid create event in db.") - }) + serde_json::from_str(create_event.content.get()).map_err(|e| { + warn!("Invalid create event: {}", e); + Error::bad_database("Invalid create event in db.") + }) }) .transpose()?; @@ -1057,7 +1054,7 @@ pub(crate) async fn invite_helper<'a>( warn!("Server {} changed invite event, that's not allowed in the spec: ours: {:?}, theirs: {:?}", user_id.server_name(), pdu_json, value); } - let origin = serde_json::from_value::>( + let origin: Box = serde_json::from_value( serde_json::to_value(value.get("origin").ok_or(Error::BadRequest( ErrorKind::InvalidParam, "Event needs an origin field.", diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 93ead2c7..d778d6f1 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -132,14 +132,11 @@ pub async fn get_message_events_route( let to = body.to.as_ref().map(|t| t.parse()); // Use limit or else 10 - let limit = body - .limit - .try_into() - .map_or(Ok::<_, Error>(10_usize), |l: u32| Ok(l as usize))?; + let limit = body.limit.try_into().map_or(10_usize, |l: u32| l as usize); match body.dir { get_message_events::Direction::Forward => { - let events_after = db + let events_after: Vec<_> = db .rooms .pdus_after(sender_user, &body.room_id, from)? .take(limit) @@ -151,14 +148,14 @@ pub async fn get_message_events_route( .ok() }) .take_while(|&(k, _)| Some(Ok(k)) != to) // Stop at `to` - .collect::>(); + .collect(); let end_token = events_after.last().map(|(count, _)| count.to_string()); - let events_after = events_after + let events_after: Vec<_> = events_after .into_iter() .map(|(_, pdu)| pdu.to_room_event()) - .collect::>(); + .collect(); let mut resp = get_message_events::Response::new(); resp.start = Some(body.from.to_owned()); @@ -169,7 +166,7 @@ pub async fn get_message_events_route( Ok(resp.into()) } get_message_events::Direction::Backward => { - let events_before = db + let events_before: Vec<_> = db .rooms .pdus_until(sender_user, &body.room_id, from)? .take(limit) @@ -181,14 +178,14 @@ pub async fn get_message_events_route( .ok() }) .take_while(|&(k, _)| Some(Ok(k)) != to) // Stop at `to` - .collect::>(); + .collect(); let start_token = events_before.last().map(|(count, _)| count.to_string()); - let events_before = events_before + let events_before: Vec<_> = events_before .into_iter() .map(|(_, pdu)| pdu.to_room_event()) - .collect::>(); + .collect(); let mut resp = get_message_events::Response::new(); resp.start = Some(body.from.to_owned()); diff --git a/src/client_server/push.rs b/src/client_server/push.rs index 98555d09..64f27f1c 100644 --- a/src/client_server/push.rs +++ b/src/client_server/push.rs @@ -8,7 +8,7 @@ use ruma::{ set_pushrule_enabled, RuleKind, }, }, - events::{push_rules, EventType}, + events::{push_rules::PushRulesEvent, EventType}, push::{ConditionalPushRuleInit, PatternedPushRuleInit, SimplePushRuleInit}, }; @@ -29,9 +29,9 @@ pub async fn get_pushrules_all_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let event = db + let event: PushRulesEvent = db .account_data - .get::(None, sender_user, EventType::PushRules)? + .get(None, sender_user, EventType::PushRules)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "PushRules event not found.", @@ -57,9 +57,9 @@ pub async fn get_pushrule_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let event = db + let event: PushRulesEvent = db .account_data - .get::(None, sender_user, EventType::PushRules)? + .get(None, sender_user, EventType::PushRules)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "PushRules event not found.", @@ -122,9 +122,9 @@ pub async fn set_pushrule_route( )); } - let mut event = db + let mut event: PushRulesEvent = db .account_data - .get::(None, sender_user, EventType::PushRules)? + .get(None, sender_user, EventType::PushRules)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "PushRules event not found.", @@ -222,9 +222,9 @@ pub async fn get_pushrule_actions_route( )); } - let mut event = db + let mut event: PushRulesEvent = db .account_data - .get::(None, sender_user, EventType::PushRules)? + .get(None, sender_user, EventType::PushRules)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "PushRules event not found.", @@ -284,9 +284,9 @@ pub async fn set_pushrule_actions_route( )); } - let mut event = db + let mut event: PushRulesEvent = db .account_data - .get::(None, sender_user, EventType::PushRules)? + .get(None, sender_user, EventType::PushRules)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "PushRules event not found.", @@ -356,9 +356,9 @@ pub async fn get_pushrule_enabled_route( )); } - let mut event = db + let mut event: PushRulesEvent = db .account_data - .get::(None, sender_user, EventType::PushRules)? + .get(None, sender_user, EventType::PushRules)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "PushRules event not found.", @@ -420,9 +420,9 @@ pub async fn set_pushrule_enabled_route( )); } - let mut event = db + let mut event: PushRulesEvent = db .account_data - .get::(None, sender_user, EventType::PushRules)? + .get(None, sender_user, EventType::PushRules)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "PushRules event not found.", @@ -497,9 +497,9 @@ pub async fn delete_pushrule_route( )); } - let mut event = db + let mut event: PushRulesEvent = db .account_data - .get::(None, sender_user, EventType::PushRules)? + .get(None, sender_user, EventType::PushRules)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "PushRules event not found.", diff --git a/src/client_server/room.rs b/src/client_server/room.rs index eb68135f..47ffb0dc 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -22,6 +22,7 @@ use ruma::{ }, EventType, }, + serde::JsonObject, RoomAliasId, RoomId, RoomVersionId, }; use serde_json::value::to_raw_value; @@ -175,12 +176,10 @@ pub async fn create_room_route( .expect("event is valid, we just created it"); if let Some(power_level_content_override) = &body.power_level_content_override { - let json = serde_json::from_str::>( - power_level_content_override.json().get(), - ) - .map_err(|_| { - Error::BadRequest(ErrorKind::BadJson, "Invalid power_level_content_override.") - })?; + let json: JsonObject = serde_json::from_str(power_level_content_override.json().get()) + .map_err(|_| { + Error::BadRequest(ErrorKind::BadJson, "Invalid power_level_content_override.") + })?; for (key, value) in json { power_levels_content[key] = value; @@ -605,7 +604,7 @@ pub async fn upgrade_room_route( } // Get the old room power levels - let mut power_levels_event_content = serde_json::from_str::( + let mut power_levels_event_content: RoomPowerLevelsEventContent = serde_json::from_str( db.rooms .room_state_get(&body.room_id, &EventType::RoomPowerLevels, "")? .ok_or_else(|| Error::bad_database("Found room without m.room.create event."))? diff --git a/src/client_server/search.rs b/src/client_server/search.rs index 9ff1a1bd..59c9480a 100644 --- a/src/client_server/search.rs +++ b/src/client_server/search.rs @@ -74,7 +74,7 @@ pub async fn search_events_route( } } - let results = results + let results: Vec<_> = results .iter() .map(|result| { Ok::<_, Error>(SearchResult { @@ -95,7 +95,7 @@ pub async fn search_events_route( .filter_map(|r| r.ok()) .skip(skip) .take(limit) - .collect::>(); + .collect(); let next_batch = if results.len() < limit as usize { None @@ -114,7 +114,7 @@ pub async fn search_events_route( .search_term .split_terminator(|c: char| !c.is_alphanumeric()) .map(str::to_lowercase) - .collect::>(), + .collect(), }, }) .into()) diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 35157332..8581591a 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -112,13 +112,13 @@ pub async fn get_state_events_route( db.rooms .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")? .map(|event| { - serde_json::from_str::(event.content.get()) + serde_json::from_str(event.content.get()) + .map(|e: RoomHistoryVisibilityEventContent| e.history_visibility) .map_err(|_| { Error::bad_database( "Invalid room history visibility event in database.", ) }) - .map(|e| e.history_visibility) }), Some(Ok(HistoryVisibility::WorldReadable)) ) @@ -164,13 +164,13 @@ pub async fn get_state_events_for_key_route( db.rooms .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")? .map(|event| { - serde_json::from_str::(event.content.get()) + serde_json::from_str(event.content.get()) + .map(|e: RoomHistoryVisibilityEventContent| e.history_visibility) .map_err(|_| { Error::bad_database( "Invalid room history visibility event in database.", ) }) - .map(|e| e.history_visibility) }), Some(Ok(HistoryVisibility::WorldReadable)) ) @@ -220,13 +220,13 @@ pub async fn get_state_events_for_empty_key_route( db.rooms .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")? .map(|event| { - serde_json::from_str::(event.content.get()) + serde_json::from_str(event.content.get()) + .map(|e: RoomHistoryVisibilityEventContent| e.history_visibility) .map_err(|_| { Error::bad_database( "Invalid room history visibility event in database.", ) }) - .map(|e| e.history_visibility) }), Some(Ok(HistoryVisibility::WorldReadable)) ) diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 5b0dbaf7..284aeb06 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -244,13 +244,13 @@ async fn sync_helper( }); // Take the last 10 events for the timeline - let timeline_pdus = non_timeline_pdus + let timeline_pdus: Vec<_> = non_timeline_pdus .by_ref() .take(10) .collect::>() .into_iter() .rev() - .collect::>(); + .collect(); let send_notification_counts = !timeline_pdus.is_empty() || db @@ -290,11 +290,10 @@ async fn sync_helper( .filter_map(|pdu| pdu.ok()) // Ignore all broken pdus .filter(|(_, pdu)| pdu.kind == EventType::RoomMember) .map(|(_, pdu)| { - let content = - serde_json::from_str::(pdu.content.get()) - .map_err(|_| { - Error::bad_database("Invalid member event in database.") - })?; + let content: RoomMemberEventContent = + serde_json::from_str(pdu.content.get()).map_err(|_| { + Error::bad_database("Invalid member event in database.") + })?; if let Some(state_key) = &pdu.state_key { let user_id = UserId::try_from(state_key.clone()).map_err(|_| { @@ -347,11 +346,11 @@ async fn sync_helper( let (joined_member_count, invited_member_count, heroes) = calculate_counts()?; let current_state_ids = db.rooms.state_full_ids(current_shortstatehash)?; - let state_events = current_state_ids + let state_events: Vec<_> = current_state_ids .iter() .map(|(_, id)| db.rooms.get_pdu(id)) .filter_map(|r| r.ok().flatten()) - .collect::>(); + .collect(); ( heroes, @@ -367,7 +366,7 @@ async fn sync_helper( // Incremental /sync let since_shortstatehash = since_shortstatehash.unwrap(); - let since_sender_member = db + let since_sender_member: Option = db .rooms .state_get( since_shortstatehash, @@ -375,7 +374,7 @@ async fn sync_helper( sender_user.as_str(), )? .and_then(|pdu| { - serde_json::from_str::(pdu.content.get()) + serde_json::from_str(pdu.content.get()) .map_err(|_| Error::bad_database("Invalid PDU in database.")) .ok() }); @@ -523,18 +522,18 @@ async fn sync_helper( Ok(Some(db.rooms.pdu_count(pdu_id)?.to_string())) })?; - let room_events = timeline_pdus + let room_events: Vec<_> = timeline_pdus .iter() .map(|(_, pdu)| pdu.to_sync_room_event()) - .collect::>(); + .collect(); - let mut edus = db + let mut edus: Vec<_> = db .rooms .edus .readreceipts_since(&room_id, since) .filter_map(|r| r.ok()) // Filter out buggy events .map(|(_, _, v)| v) - .collect::>(); + .collect(); if db.rooms.edus.last_typing_update(&room_id, &db.globals)? > since { edus.push( @@ -563,7 +562,7 @@ async fn sync_helper( .map_err(|_| Error::bad_database("Invalid account event in database.")) .ok() }) - .collect::>(), + .collect(), }, summary: sync_events::RoomSummary { heroes, @@ -628,7 +627,7 @@ async fn sync_helper( } let mut left_rooms = BTreeMap::new(); - let all_left_rooms = db.rooms.rooms_left(&sender_user).collect::>(); + let all_left_rooms: Vec<_> = db.rooms.rooms_left(&sender_user).collect(); for result in all_left_rooms { let (room_id, left_state_events) = result?; @@ -668,7 +667,7 @@ async fn sync_helper( } let mut invited_rooms = BTreeMap::new(); - let all_invited_rooms = db.rooms.rooms_invited(&sender_user).collect::>(); + let all_invited_rooms: Vec<_> = db.rooms.rooms_invited(&sender_user).collect(); for result in all_invited_rooms { let (room_id, invite_state_events) = result?; @@ -750,7 +749,7 @@ async fn sync_helper( .map_err(|_| Error::bad_database("Invalid account event in database.")) .ok() }) - .collect::>(), + .collect(), }, device_lists: sync_events::DeviceLists { changed: device_list_updates.into_iter().collect(), diff --git a/src/client_server/tag.rs b/src/client_server/tag.rs index 1eb508c7..42bad4cf 100644 --- a/src/client_server/tag.rs +++ b/src/client_server/tag.rs @@ -1,7 +1,10 @@ use crate::{database::DatabaseGuard, ConduitResult, Ruma}; use ruma::{ api::client::r0::tag::{create_tag, delete_tag, get_tags}, - events::EventType, + events::{ + tag::{TagEvent, TagEventContent}, + EventType, + }, }; use std::collections::BTreeMap; @@ -26,9 +29,9 @@ pub async fn update_tag_route( let mut tags_event = db .account_data - .get::(Some(&body.room_id), sender_user, EventType::Tag)? - .unwrap_or_else(|| ruma::events::tag::TagEvent { - content: ruma::events::tag::TagEventContent { + .get(Some(&body.room_id), sender_user, EventType::Tag)? + .unwrap_or_else(|| TagEvent { + content: TagEventContent { tags: BTreeMap::new(), }, }); @@ -68,9 +71,9 @@ pub async fn delete_tag_route( let mut tags_event = db .account_data - .get::(Some(&body.room_id), sender_user, EventType::Tag)? - .unwrap_or_else(|| ruma::events::tag::TagEvent { - content: ruma::events::tag::TagEventContent { + .get(Some(&body.room_id), sender_user, EventType::Tag)? + .unwrap_or_else(|| TagEvent { + content: TagEventContent { tags: BTreeMap::new(), }, }); @@ -108,9 +111,9 @@ pub async fn get_tags_route( Ok(get_tags::Response { tags: db .account_data - .get::(Some(&body.room_id), sender_user, EventType::Tag)? - .unwrap_or_else(|| ruma::events::tag::TagEvent { - content: ruma::events::tag::TagEventContent { + .get(Some(&body.room_id), sender_user, EventType::Tag)? + .unwrap_or_else(|| TagEvent { + content: TagEventContent { tags: BTreeMap::new(), }, }) diff --git a/src/database.rs b/src/database.rs index 63c4ebc4..87190aa1 100644 --- a/src/database.rs +++ b/src/database.rs @@ -699,7 +699,7 @@ impl Database { println!("Deleting starts"); - let batch2 = db + let batch2: Vec<_> = db .rooms .tokenids .iter() @@ -711,7 +711,7 @@ impl Database { None } }) - .collect::>(); + .collect(); for key in batch2 { println!("del"); diff --git a/src/database/globals.rs b/src/database/globals.rs index 2f1b45ad..46eab636 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -57,8 +57,7 @@ pub struct RotationHandler(broadcast::Sender<()>, broadcast::Receiver<()>); impl RotationHandler { pub fn new() -> Self { - let (s, r) = broadcast::channel::<()>(1); - + let (s, r) = broadcast::channel(1); Self(s, r) } @@ -274,8 +273,8 @@ impl Globals { let signingkeys = self .server_signingkeys .get(origin.as_bytes())? - .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) - .map(|keys| { + .and_then(|bytes| serde_json::from_slice(&bytes).ok()) + .map(|keys: ServerSigningKeys| { let mut tree = keys.verify_keys; tree.extend( keys.old_verify_keys diff --git a/src/database/key_backups.rs b/src/database/key_backups.rs index a960c72f..98ea0111 100644 --- a/src/database/key_backups.rs +++ b/src/database/key_backups.rs @@ -94,15 +94,15 @@ impl KeyBackups { .iter_from(&last_possible_key, true) .take_while(move |(k, _)| k.starts_with(&prefix)) .next() - .map_or(Ok(None), |(key, _)| { + .map(|(key, _)| { utils::string_from_bytes( key.rsplit(|&b| b == 0xff) .next() .expect("rsplit always returns an element"), ) .map_err(|_| Error::bad_database("backupid_algorithm key is invalid.")) - .map(Some) }) + .transpose() } pub fn get_latest_backup(&self, user_id: &UserId) -> Result> { @@ -115,7 +115,7 @@ impl KeyBackups { .iter_from(&last_possible_key, true) .take_while(move |(k, _)| k.starts_with(&prefix)) .next() - .map_or(Ok(None), |(key, value)| { + .map(|(key, value)| { let version = utils::string_from_bytes( key.rsplit(|&b| b == 0xff) .next() @@ -123,13 +123,14 @@ impl KeyBackups { ) .map_err(|_| Error::bad_database("backupid_algorithm key is invalid."))?; - Ok(Some(( + Ok(( version, serde_json::from_slice(&value).map_err(|_| { Error::bad_database("Algorithm in backupid_algorithm is invalid.") })?, - ))) + )) }) + .transpose() } pub fn get_backup(&self, user_id: &UserId, version: &str) -> Result> { diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 1912e0c8..c5b795bd 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -13,13 +13,16 @@ use rocket::http::RawStr; use ruma::{ api::{client::error::ErrorKind, federation}, events::{ - ignored_user_list, push_rules, + direct::DirectEvent, + ignored_user_list::IgnoredUserListEvent, + push_rules::PushRulesEvent, room::{ create::RoomCreateEventContent, member::{MembershipState, RoomMemberEventContent}, message::RoomMessageEventContent, power_levels::RoomPowerLevelsEventContent, }, + tag::TagEvent, AnyStrippedStateEvent, AnySyncStateEvent, EventType, }, push::{Action, Ruleset, Tweak}, @@ -218,16 +221,16 @@ impl Rooms { self.eventid_shorteventid .get(event_id.as_bytes())? .map_or(Ok(None), |shorteventid| { - self.shorteventid_shortstatehash.get(&shorteventid)?.map_or( - Ok::<_, Error>(None), - |bytes| { - Ok(Some(utils::u64_from_bytes(&bytes).map_err(|_| { + self.shorteventid_shortstatehash + .get(&shorteventid)? + .map(|bytes| { + utils::u64_from_bytes(&bytes).map_err(|_| { Error::bad_database( "Invalid shortstatehash bytes in shorteventid_shortstatehash", ) - })?)) - }, - ) + }) + }) + .transpose() }) } @@ -369,16 +372,16 @@ impl Rooms { let (statediffnew, statediffremoved) = if let Some(parent_stateinfo) = states_parents.last() { - let statediffnew = new_state_ids_compressed + let statediffnew: HashSet<_> = new_state_ids_compressed .difference(&parent_stateinfo.1) .copied() - .collect::>(); + .collect(); - let statediffremoved = parent_stateinfo + let statediffremoved: HashSet<_> = parent_stateinfo .1 .difference(&new_state_ids_compressed) .copied() - .collect::>(); + .collect(); (statediffnew, statediffremoved) } else { @@ -409,7 +412,7 @@ impl Rooms { continue; } - let pdu = match serde_json::from_str::( + let pdu: PduEvent = match serde_json::from_str( &serde_json::to_string(&pdu).expect("CanonicalJsonObj can be serialized to JSON"), ) { Ok(pdu) => pdu, @@ -980,7 +983,8 @@ impl Rooms { pub fn get_pdu_count(&self, event_id: &EventId) -> Result> { self.eventid_pduid .get(event_id.as_bytes())? - .map_or(Ok(None), |pdu_id| self.pdu_count(&pdu_id).map(Some)) + .map(|pdu_id| self.pdu_count(&pdu_id)) + .transpose() } #[tracing::instrument(skip(self))] @@ -1008,7 +1012,7 @@ impl Rooms { pub fn get_pdu_json(&self, event_id: &EventId) -> Result> { self.eventid_pduid .get(event_id.as_bytes())? - .map_or_else::, _, _>( + .map_or_else( || self.eventid_outlierpdu.get(event_id.as_bytes()), |pduid| { Ok(Some(self.pduid_pdu.get(&pduid)?.ok_or_else(|| { @@ -1041,14 +1045,12 @@ impl Rooms { ) -> Result> { self.eventid_pduid .get(event_id.as_bytes())? - .map_or_else::, _, _>( - || Ok(None), - |pduid| { - Ok(Some(self.pduid_pdu.get(&pduid)?.ok_or_else(|| { - Error::bad_database("Invalid pduid in eventid_pduid.") - })?)) - }, - )? + .map(|pduid| { + self.pduid_pdu + .get(&pduid)? + .ok_or_else(|| Error::bad_database("Invalid pduid in eventid_pduid.")) + }) + .transpose()? .map(|pdu| { serde_json::from_slice(&pdu).map_err(|_| Error::bad_database("Invalid PDU in db.")) }) @@ -1058,9 +1060,7 @@ impl Rooms { /// Returns the pdu's id. #[tracing::instrument(skip(self))] pub fn get_pdu_id(&self, event_id: &EventId) -> Result>> { - self.eventid_pduid - .get(event_id.as_bytes())? - .map_or(Ok(None), |pdu_id| Ok(Some(pdu_id))) + self.eventid_pduid.get(event_id.as_bytes()) } /// Returns the pdu. @@ -1070,14 +1070,12 @@ impl Rooms { pub fn get_non_outlier_pdu(&self, event_id: &EventId) -> Result> { self.eventid_pduid .get(event_id.as_bytes())? - .map_or_else::, _, _>( - || Ok(None), - |pduid| { - Ok(Some(self.pduid_pdu.get(&pduid)?.ok_or_else(|| { - Error::bad_database("Invalid pduid in eventid_pduid.") - })?)) - }, - )? + .map(|pduid| { + self.pduid_pdu + .get(&pduid)? + .ok_or_else(|| Error::bad_database("Invalid pduid in eventid_pduid.")) + }) + .transpose()? .map(|pdu| { serde_json::from_slice(&pdu).map_err(|_| Error::bad_database("Invalid PDU in db.")) }) @@ -1096,11 +1094,8 @@ impl Rooms { if let Some(pdu) = self .eventid_pduid .get(event_id.as_bytes())? - .map_or_else::, _, _>( - || { - let r = self.eventid_outlierpdu.get(event_id.as_bytes()); - r - }, + .map_or_else( + || self.eventid_outlierpdu.get(event_id.as_bytes()), |pduid| { Ok(Some(self.pduid_pdu.get(&pduid)?.ok_or_else(|| { Error::bad_database("Invalid pduid in eventid_pduid.") @@ -1363,8 +1358,8 @@ impl Rooms { let rules_for_user = db .account_data - .get::(None, user, EventType::PushRules)? - .map(|ev| ev.content.global) + .get(None, user, EventType::PushRules)? + .map(|ev: PushRulesEvent| ev.content.global) .unwrap_or_else(|| Ruleset::server_default(user)); let mut highlight = false; @@ -1490,11 +1485,11 @@ impl Rooms { { let mut lines = body.lines(); let command_line = lines.next().expect("each string has at least one line"); - let body = lines.collect::>(); + let body: Vec<_> = lines.collect(); let mut parts = command_line.split_whitespace().skip(1); if let Some(command) = parts.next() { - let args = parts.collect::>(); + let args: Vec<_> = parts.collect(); match command { "register_appservice" => { @@ -1771,16 +1766,16 @@ impl Rooms { let (statediffnew, statediffremoved) = if let Some(parent_stateinfo) = states_parents.last() { - let statediffnew = state_ids_compressed + let statediffnew: HashSet<_> = state_ids_compressed .difference(&parent_stateinfo.1) .copied() - .collect::>(); + .collect(); - let statediffremoved = parent_stateinfo + let statediffremoved: HashSet<_> = parent_stateinfo .1 .difference(&state_ids_compressed) .copied() - .collect::>(); + .collect(); (statediffnew, statediffremoved) } else { @@ -2363,19 +2358,16 @@ impl Rooms { // Check if the room has a predecessor if let Some(predecessor) = self .room_state_get(room_id, &EventType::RoomCreate, "")? - .and_then(|create| { - serde_json::from_str::(create.content.get()) - .ok() - }) - .and_then(|content| content.predecessor) + .and_then(|create| serde_json::from_str(create.content.get()).ok()) + .and_then(|content: RoomCreateEventContent| content.predecessor) { // Copy user settings from predecessor to the current room: // - Push rules // // TODO: finish this once push rules are implemented. // - // let mut push_rules_event_content = account_data - // .get::( + // let mut push_rules_event_content: PushRulesEvent = account_data + // .get( // None, // user_id, // EventType::PushRules, @@ -2395,13 +2387,11 @@ impl Rooms { // .ok(); // Copy old tags to new room - if let Some(tag_event) = - db.account_data.get::( - Some(&predecessor.room_id), - user_id, - EventType::Tag, - )? - { + if let Some(tag_event) = db.account_data.get::( + Some(&predecessor.room_id), + user_id, + EventType::Tag, + )? { db.account_data .update( Some(room_id), @@ -2415,11 +2405,8 @@ impl Rooms { // Copy direct chat flag if let Some(mut direct_event) = - db.account_data.get::( - None, - user_id, - EventType::Direct, - )? + db.account_data + .get::(None, user_id, EventType::Direct)? { let mut room_ids_updated = false; @@ -2458,7 +2445,7 @@ impl Rooms { // We want to know if the sender is ignored by the receiver let is_ignored = db .account_data - .get::( + .get::( None, // Ignored users are in global account data user_id, // Receiver EventType::IgnoredUserList, @@ -2712,7 +2699,7 @@ impl Rooms { ); let state_lock = mutex_state.lock().await; - let mut event = serde_json::from_str::( + let mut event: RoomMemberEventContent = serde_json::from_str( self.room_state_get(room_id, &EventType::RoomMember, &user_id.to_string())? .ok_or(Error::BadRequest( ErrorKind::BadState, @@ -2762,16 +2749,14 @@ impl Rooms { "User is not invited.", ))?; - let servers = invite_state + let servers: HashSet<_> = invite_state .iter() - .filter_map(|event| { - serde_json::from_str::(&event.json().to_string()).ok() - }) - .filter_map(|event| event.get("sender").cloned()) + .filter_map(|event| serde_json::from_str(event.json().get()).ok()) + .filter_map(|event: serde_json::Value| event.get("sender").cloned()) .filter_map(|sender| sender.as_str().map(|s| s.to_owned())) .filter_map(|sender| UserId::try_from(sender).ok()) .map(|user| user.server_name().to_owned()) - .collect::>(); + .collect(); for remote_server in servers { let make_leave_response = db @@ -2920,14 +2905,13 @@ impl Rooms { pub fn id_from_alias(&self, alias: &RoomAliasId) -> Result> { self.alias_roomid .get(alias.alias().as_bytes())? - .map_or(Ok(None), |bytes| { - Ok(Some( - RoomId::try_from(utils::string_from_bytes(&bytes).map_err(|_| { - Error::bad_database("Room ID in alias_roomid is invalid unicode.") - })?) - .map_err(|_| Error::bad_database("Room ID in alias_roomid is invalid."))?, - )) + .map(|bytes| { + RoomId::try_from(utils::string_from_bytes(&bytes).map_err(|_| { + Error::bad_database("Room ID in alias_roomid is invalid unicode.") + })?) + .map_err(|_| Error::bad_database("Room ID in alias_roomid is invalid.")) }) + .transpose() } #[tracing::instrument(skip(self))] @@ -2987,11 +2971,11 @@ impl Rooms { .to_vec(); let prefix_clone = prefix.clone(); - let words = search_string + let words: Vec<_> = search_string .split_terminator(|c: char| !c.is_alphanumeric()) .filter(|s| !s.is_empty()) .map(str::to_lowercase) - .collect::>(); + .collect(); let iterators = words.clone().into_iter().map(move |word| { let mut prefix2 = prefix.clone(); @@ -3004,12 +2988,7 @@ impl Rooms { self.tokenids .iter_from(&last_possible_id, true) // Newest pdus first .take_while(move |(k, _)| k.starts_with(&prefix2)) - .map(|(key, _)| { - let pdu_id = key[key.len() - size_of::()..].to_vec(); - - Ok::<_, Error>(pdu_id) - }) - .filter_map(|r| r.ok()) + .map(|(key, _)| key[key.len() - size_of::()..].to_vec()) }); Ok(( @@ -3241,11 +3220,11 @@ impl Rooms { self.roomuserid_leftcount .get(&key)? - .map_or(Ok(None), |bytes| { - Ok(Some(utils::u64_from_bytes(&bytes).map_err(|_| { - Error::bad_database("Invalid leftcount in db.") - })?)) + .map(|bytes| { + utils::u64_from_bytes(&bytes) + .map_err(|_| Error::bad_database("Invalid leftcount in db.")) }) + .transpose() } /// Returns an iterator over all rooms this user joined. diff --git a/src/database/rooms/edus.rs b/src/database/rooms/edus.rs index 26f22bf4..9a27e437 100644 --- a/src/database/rooms/edus.rs +++ b/src/database/rooms/edus.rs @@ -162,11 +162,12 @@ impl RoomEdus { Ok(self .roomuserid_lastprivatereadupdate .get(&key)? - .map_or(Ok::<_, Error>(None), |bytes| { - Ok(Some(utils::u64_from_bytes(&bytes).map_err(|_| { + .map(|bytes| { + utils::u64_from_bytes(&bytes).map_err(|_| { Error::bad_database("Count in roomuserid_lastprivatereadupdate is invalid.") - })?)) - })? + }) + }) + .transpose()? .unwrap_or(0)) } @@ -286,11 +287,12 @@ impl RoomEdus { Ok(self .roomid_lasttypingupdate .get(room_id.as_bytes())? - .map_or(Ok::<_, Error>(None), |bytes| { - Ok(Some(utils::u64_from_bytes(&bytes).map_err(|_| { + .map(|bytes| { + utils::u64_from_bytes(&bytes).map_err(|_| { Error::bad_database("Count in roomid_lastroomactiveupdate is invalid.") - })?)) - })? + }) + }) + .transpose()? .unwrap_or(0)) } @@ -399,7 +401,7 @@ impl RoomEdus { self.presenceid_presence .get(&presence_id)? .map(|value| { - let mut presence = serde_json::from_slice::(&value) + let mut presence: PresenceEvent = serde_json::from_slice(&value) .map_err(|_| Error::bad_database("Invalid presence event in db."))?; let current_timestamp: UInt = utils::millis_since_unix_epoch() .try_into() @@ -521,7 +523,7 @@ impl RoomEdus { ) .map_err(|_| Error::bad_database("Invalid UserId in presenceid_presence."))?; - let mut presence = serde_json::from_slice::(&value) + let mut presence: PresenceEvent = serde_json::from_slice(&value) .map_err(|_| Error::bad_database("Invalid presence event in db."))?; let current_timestamp: UInt = utils::millis_since_unix_epoch() diff --git a/src/database/sending.rs b/src/database/sending.rs index c1abcde2..b4acce1d 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -27,7 +27,7 @@ use ruma::{ OutgoingRequest, }, device_id, - events::{push_rules, AnySyncEphemeralRoomEvent, EventType}, + events::{push_rules::PushRulesEvent, AnySyncEphemeralRoomEvent, EventType}, push, receipt::ReceiptType, uint, MilliSecondsSinceUnixEpoch, ServerName, UInt, UserId, @@ -165,13 +165,13 @@ impl Sending { } // Find events that have been added since starting the last request - let new_events = guard.sending.servernameevent_data + let new_events: Vec<_> = guard.sending.servernameevent_data .scan_prefix(prefix.clone()) .filter_map(|(k, v)| { Self::parse_servercurrentevent(&k, v).ok().map(|ev| (ev, k)) }) .take(30) - .collect::>(); + .collect::<>(); // TODO: find edus @@ -344,8 +344,8 @@ impl Sending { continue; } - let event = - serde_json::from_str::(read_receipt.json().get()) + let event: AnySyncEphemeralRoomEvent = + serde_json::from_str(read_receipt.json().get()) .map_err(|_| Error::bad_database("Invalid edu event in read_receipts."))?; let federation_event = match event { AnySyncEphemeralRoomEvent::Receipt(r) => { @@ -612,9 +612,9 @@ impl Sending { let rules_for_user = db .account_data - .get::(None, &userid, EventType::PushRules) + .get(None, &userid, EventType::PushRules) .unwrap_or_default() - .map(|ev| ev.content.global) + .map(|ev: PushRulesEvent| ev.content.global) .unwrap_or_else(|| push::Ruleset::server_default(&userid)); let unread: UInt = db diff --git a/src/database/uiaa.rs b/src/database/uiaa.rs index 46796462..1c0fb566 100644 --- a/src/database/uiaa.rs +++ b/src/database/uiaa.rs @@ -175,16 +175,14 @@ impl Uiaa { self.userdevicesessionid_uiaarequest .get(&userdevicesessionid)? - .map_or(Ok(None), |bytes| { - Ok::<_, Error>(Some( - serde_json::from_str::( - &utils::string_from_bytes(&bytes).map_err(|_| { - Error::bad_database("Invalid uiaa request bytes in db.") - })?, - ) - .map_err(|_| Error::bad_database("Invalid uiaa request in db."))?, - )) + .map(|bytes| { + serde_json::from_str::( + &utils::string_from_bytes(&bytes) + .map_err(|_| Error::bad_database("Invalid uiaa request bytes in db."))?, + ) + .map_err(|_| Error::bad_database("Invalid uiaa request in db.")) }) + .transpose() } fn update_uiaa_session( @@ -225,7 +223,7 @@ impl Uiaa { userdevicesessionid.push(0xff); userdevicesessionid.extend_from_slice(session.as_bytes()); - let uiaainfo = serde_json::from_slice::( + serde_json::from_slice( &self .userdevicesessionid_uiaainfo .get(&userdevicesessionid)? @@ -234,8 +232,6 @@ impl Uiaa { "UIAA session does not exist.", ))?, ) - .map_err(|_| Error::bad_database("UiaaInfo in userdeviceid_uiaainfo is invalid."))?; - - Ok(uiaainfo) + .map_err(|_| Error::bad_database("UiaaInfo in userdeviceid_uiaainfo is invalid.")) } } diff --git a/src/database/users.rs b/src/database/users.rs index 63ed0710..37a5dd32 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -603,10 +603,11 @@ impl Users { key.push(0xff); key.extend_from_slice(key_id.as_bytes()); - let mut cross_signing_key = - serde_json::from_slice::(&self.keyid_key.get(&key)?.ok_or( - Error::BadRequest(ErrorKind::InvalidParam, "Tried to sign nonexistent key."), - )?) + let mut cross_signing_key: serde_json::Value = + serde_json::from_slice(&self.keyid_key.get(&key)?.ok_or(Error::BadRequest( + ErrorKind::InvalidParam, + "Tried to sign nonexistent key.", + ))?) .map_err(|_| Error::bad_database("key in keyid_key is invalid."))?; let signatures = cross_signing_key diff --git a/src/pdu.rs b/src/pdu.rs index 0a765e1f..0f99f43b 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -69,8 +69,8 @@ impl PduEvent { _ => &[], }; - let mut old_content = - serde_json::from_str::>(self.content.get()) + let mut old_content: BTreeMap = + serde_json::from_str(self.content.get()) .map_err(|_| Error::bad_database("PDU in db has invalid content."))?; let mut new_content = serde_json::Map::new(); @@ -92,8 +92,8 @@ impl PduEvent { pub fn remove_transaction_id(&mut self) -> crate::Result<()> { if let Some(unsigned) = &self.unsigned { - let mut unsigned = - serde_json::from_str::>>(unsigned.get()) + let mut unsigned: BTreeMap> = + serde_json::from_str(unsigned.get()) .map_err(|_| Error::bad_database("Invalid unsigned in pdu event"))?; unsigned.remove("transaction_id"); self.unsigned = Some(to_raw_value(&unsigned).expect("unsigned is valid")); diff --git a/src/server_server.rs b/src/server_server.rs index e9a94856..cb00baaf 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -49,6 +49,7 @@ use ruma::{ }, int, receipt::ReceiptType, + serde::JsonObject, signatures::{CanonicalJsonObject, CanonicalJsonValue}, state_res::{self, RoomVersion, StateMap}, to_device::DeviceIdOrAllDevices, @@ -1003,12 +1004,12 @@ pub(crate) async fn handle_incoming_pdu<'a>( // 9. Fetch any missing prev events doing all checks listed here starting at 1. These are timeline events let mut graph = HashMap::new(); let mut eventid_info = HashMap::new(); - let mut todo_outlier_stack = incoming_pdu + let mut todo_outlier_stack: Vec<_> = incoming_pdu .prev_events .iter() .cloned() .map(Arc::new) - .collect::>(); + .collect(); let mut amount = 0; @@ -1150,13 +1151,11 @@ fn handle_outlier_pdu<'a>( // 2. Check signatures, otherwise drop // 3. check content hash, redact if doesn't match - let create_event_content = serde_json::from_str::( - create_event.content.get(), - ) - .map_err(|e| { - warn!("Invalid create event: {}", e); - "Invalid create event in db.".to_owned() - })?; + let create_event_content: RoomCreateEventContent = + serde_json::from_str(create_event.content.get()).map_err(|e| { + warn!("Invalid create event: {}", e); + "Invalid create event in db.".to_owned() + })?; let room_version_id = &create_event_content.room_version; let room_version = RoomVersion::new(room_version_id).expect("room version is supported"); @@ -1315,13 +1314,11 @@ async fn upgrade_outlier_to_timeline_pdu( return Err("Event has been soft failed".into()); } - let create_event_content = serde_json::from_str::( - create_event.content.get(), - ) - .map_err(|e| { - warn!("Invalid create event: {}", e); - "Invalid create event in db.".to_owned() - })?; + let create_event_content: RoomCreateEventContent = + serde_json::from_str(create_event.content.get()).map_err(|e| { + warn!("Invalid create event: {}", e); + "Invalid create event in db.".to_owned() + })?; let room_version_id = &create_event_content.room_version; let room_version = RoomVersion::new(room_version_id).expect("room version is supported"); @@ -1633,7 +1630,7 @@ async fn upgrade_outlier_to_timeline_pdu( .compress_state_event(*shortstatekey, id, &db.globals) .map_err(|_| "Failed to compress_state_event".to_owned()) }) - .collect::>()?; + .collect::>()?; // 13. Check if the event passes auth based on the "current state" of the room, if not "soft fail" it debug!("starting soft fail auth check"); @@ -1753,7 +1750,7 @@ async fn upgrade_outlier_to_timeline_pdu( .compress_state_event(*k, id, &db.globals) .map_err(|_| "Failed to compress_state_event.".to_owned()) }) - .collect::>()? + .collect::>()? } else { // We do need to force an update to this room's state update_state = true; @@ -1772,7 +1769,7 @@ async fn upgrade_outlier_to_timeline_pdu( ); } - let fork_states = &fork_states + let fork_states: Vec<_> = fork_states .into_iter() .map(|map| { map.into_iter() @@ -1783,12 +1780,12 @@ async fn upgrade_outlier_to_timeline_pdu( }) .collect::>>() }) - .collect::>>() + .collect::>() .map_err(|_| "Failed to get_statekey_from_short.".to_owned())?; let state = match state_res::resolve( room_version_id, - fork_states, + &fork_states, auth_chain_sets, |id| { let res = db.rooms.get_pdu(id); @@ -1815,7 +1812,7 @@ async fn upgrade_outlier_to_timeline_pdu( .compress_state_event(shortstatekey, &event_id, &db.globals) .map_err(|_| "Failed to compress state event".to_owned()) }) - .collect::>()? + .collect::>()? }; // Set the new room state to the resolved state @@ -2035,12 +2032,12 @@ pub(crate) async fn fetch_signing_keys( trace!("Loading signing keys for {}", origin); - let mut result = db + let mut result: BTreeMap<_, _> = db .globals .signing_keys_for(origin)? .into_iter() .map(|(k, v)| (k.to_string(), v.key)) - .collect::>(); + .collect(); if contains_all_ids(&result) { return Ok(result); @@ -2245,11 +2242,7 @@ pub(crate) fn get_auth_chain<'a>( continue; } - let chunk_key = chunk - .iter() - .map(|(short, _)| short) - .copied() - .collect::>(); + let chunk_key: Vec = chunk.iter().map(|(short, _)| short).copied().collect(); if let Some(cached) = db.rooms.get_auth_chain_from_cache(&chunk_key)? { hits += 1; full_auth_chain.extend(cached.iter().copied()); @@ -2564,9 +2557,9 @@ pub fn get_room_state_route( Ok(get_room_state::v1::Response { auth_chain: auth_chain_ids .map(|id| { - Ok::<_, Error>(PduEvent::convert_to_outgoing_federation_event( - db.rooms.get_pdu_json(&id)?.unwrap(), - )) + db.rooms.get_pdu_json(&id).map(|maybe_json| { + PduEvent::convert_to_outgoing_federation_event(maybe_json.unwrap()) + }) }) .filter_map(|r| r.ok()) .collect(), @@ -2650,26 +2643,24 @@ pub fn create_join_event_template_route( )); } - let prev_events = db + let prev_events: Vec<_> = db .rooms .get_pdu_leaves(&body.room_id)? .into_iter() .take(20) - .collect::>(); + .collect(); let create_event = db .rooms .room_state_get(&body.room_id, &EventType::RoomCreate, "")?; - let create_event_content = create_event + let create_event_content: Option = create_event .as_ref() .map(|create_event| { - serde_json::from_str::(create_event.content.get()).map_err( - |e| { - warn!("Invalid create event: {}", e); - Error::bad_database("Invalid create event in db.") - }, - ) + serde_json::from_str(create_event.content.get()).map_err(|e| { + warn!("Invalid create event: {}", e); + Error::bad_database("Invalid create event in db.") + }) }) .transpose()?; @@ -2835,7 +2826,7 @@ async fn create_join_event( } }; - let origin = serde_json::from_value::>( + let origin: Box = serde_json::from_value( serde_json::to_value(value.get("origin").ok_or(Error::BadRequest( ErrorKind::InvalidParam, "Event needs an origin field.", @@ -3009,15 +3000,12 @@ pub async fn create_invite_route( let mut invite_state = body.invite_room_state.clone(); - let mut event = - serde_json::from_str::>(body.event.get()) - .map_err(|_| { - Error::BadRequest(ErrorKind::InvalidParam, "Invalid invite event bytes.") - })?; + let mut event: JsonObject = serde_json::from_str(body.event.get()) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid invite event bytes."))?; event.insert("event_id".to_owned(), "$dummy".into()); - let pdu = serde_json::from_value::(event.into()).map_err(|e| { + let pdu: PduEvent = serde_json::from_value(event.into()).map_err(|e| { warn!("Invalid invite event: {}", e); Error::BadRequest(ErrorKind::InvalidParam, "Invalid invite event.") })?; @@ -3282,7 +3270,7 @@ fn get_server_keys_from_cache( pub_key_map: &mut RwLockWriteGuard<'_, BTreeMap>>, db: &Database, ) -> Result<()> { - let value = serde_json::from_str::(pdu.get()).map_err(|e| { + let value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| { error!("Invalid PDU in server response: {:?}: {:?}", pdu, e); Error::BadServerResponse("Invalid PDU in server response") })?; @@ -3343,19 +3331,16 @@ fn get_server_keys_from_cache( trace!("Loading signing keys for {}", origin); - let result = db + let result: BTreeMap<_, _> = db .globals .signing_keys_for(origin)? .into_iter() .map(|(k, v)| (k.to_string(), v.key)) - .collect::>(); + .collect(); if !contains_all_ids(&result) { trace!("Signing key not loaded for {}", origin); - servers.insert( - origin.clone(), - BTreeMap::::new(), - ); + servers.insert(origin.clone(), BTreeMap::new()); } pub_key_map.insert(origin.to_string(), result); @@ -3370,8 +3355,8 @@ pub(crate) async fn fetch_join_signing_keys( pub_key_map: &RwLock>>, db: &Database, ) -> Result<()> { - let mut servers = - BTreeMap::, BTreeMap>::new(); + let mut servers: BTreeMap, BTreeMap> = + BTreeMap::new(); { let mut pkm = pub_key_map @@ -3436,7 +3421,7 @@ pub(crate) async fn fetch_join_signing_keys( } } - let mut futures = servers + let mut futures: FuturesUnordered<_> = servers .into_iter() .map(|(server, _)| async move { ( @@ -3450,16 +3435,16 @@ pub(crate) async fn fetch_join_signing_keys( server, ) }) - .collect::>(); + .collect(); while let Some(result) = futures.next().await { if let (Ok(get_keys_response), origin) = result { - let result = db + let result: BTreeMap<_, _> = db .globals .add_signing_key(&origin, get_keys_response.server_key.clone())? .into_iter() .map(|(k, v)| (k.to_string(), v.key)) - .collect::>(); + .collect(); pub_key_map .write() From 91afa1e0e025695b677eec0ddade248c184efe8e Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Wed, 13 Oct 2021 11:56:46 +0200 Subject: [PATCH 025/201] Make Result alias usable with any error type --- src/client_server/sync.rs | 4 ++-- src/database/proxy.rs | 4 ++-- src/database/sending.rs | 2 +- src/error.rs | 2 +- src/server_server.rs | 16 +++++++--------- 5 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 284aeb06..e4c12c40 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -60,7 +60,7 @@ use rocket::{get, tokio}; pub async fn sync_events_route( db: DatabaseGuard, body: Ruma>, -) -> std::result::Result, RumaResponse> { +) -> Result, RumaResponse> { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); @@ -182,7 +182,7 @@ async fn sync_helper( full_state: bool, timeout: Option, // bool = caching allowed -) -> std::result::Result<(sync_events::Response, bool), Error> { +) -> Result<(sync_events::Response, bool), Error> { // TODO: match body.set_presence { db.rooms.edus.ping_presence(&sender_user)?; diff --git a/src/database/proxy.rs b/src/database/proxy.rs index 33f7f3d9..fb0387c9 100644 --- a/src/database/proxy.rs +++ b/src/database/proxy.rs @@ -125,7 +125,7 @@ impl WildCardedDomain { } impl std::str::FromStr for WildCardedDomain { type Err = std::convert::Infallible; - fn from_str(s: &str) -> std::result::Result { + fn from_str(s: &str) -> Result { // maybe do some domain validation? Ok(if s.starts_with("*.") { WildCardedDomain::WildCarded(s[1..].to_owned()) @@ -137,7 +137,7 @@ impl std::str::FromStr for WildCardedDomain { } } impl<'de> Deserialize<'de> for WildCardedDomain { - fn deserialize(deserializer: D) -> std::result::Result + fn deserialize(deserializer: D) -> Result where D: serde::de::Deserializer<'de>, { diff --git a/src/database/sending.rs b/src/database/sending.rs index b4acce1d..bf0cc2c1 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -485,7 +485,7 @@ impl Sending { kind: OutgoingKind, events: Vec, db: Arc>, - ) -> std::result::Result { + ) -> Result { let db = db.read().await; match &kind { diff --git a/src/error.rs b/src/error.rs index 1ecef3aa..7faddc91 100644 --- a/src/error.rs +++ b/src/error.rs @@ -20,7 +20,7 @@ use { tracing::error, }; -pub type Result = std::result::Result; +pub type Result = std::result::Result; #[derive(Error, Debug)] pub enum Error { diff --git a/src/server_server.rs b/src/server_server.rs index cb00baaf..68e262b4 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -65,7 +65,6 @@ use std::{ mem, net::{IpAddr, SocketAddr}, pin::Pin, - result::Result as StdResult, sync::{Arc, RwLock, RwLockWriteGuard}, time::{Duration, Instant, SystemTime}, }; @@ -956,7 +955,7 @@ pub(crate) async fn handle_incoming_pdu<'a>( is_timeline_event: bool, db: &'a Database, pub_key_map: &'a RwLock>>, -) -> StdResult>, String> { +) -> Result>, String> { match db.rooms.exists(room_id) { Ok(true) => {} _ => { @@ -1137,8 +1136,7 @@ fn handle_outlier_pdu<'a>( value: BTreeMap, db: &'a Database, pub_key_map: &'a RwLock>>, -) -> AsyncRecursiveType<'a, StdResult<(Arc, BTreeMap), String>> -{ +) -> AsyncRecursiveType<'a, Result<(Arc, BTreeMap), String>> { Box::pin(async move { // TODO: For RoomVersion6 we must check that Raw<..> is canonical do we anywhere?: https://matrix.org/docs/spec/rooms/v6#canonical-json @@ -1301,7 +1299,7 @@ async fn upgrade_outlier_to_timeline_pdu( db: &Database, room_id: &RoomId, pub_key_map: &RwLock>>, -) -> StdResult>, String> { +) -> Result>, String> { if let Ok(Some(pduid)) = db.rooms.get_pdu_id(&incoming_pdu.event_id) { return Ok(Some(pduid)); } @@ -1448,7 +1446,7 @@ async fn upgrade_outlier_to_timeline_pdu( .map_err(|_| "Failed to get_or_create_shortstatekey".to_owned())?; Ok((shortstatekey, Arc::new(event_id))) }) - .collect::>()?, + .collect::>()?, ), Err(e) => { warn!("State resolution on prev events failed, either an event could not be found or deserialization: {}", e); @@ -1630,7 +1628,7 @@ async fn upgrade_outlier_to_timeline_pdu( .compress_state_event(*shortstatekey, id, &db.globals) .map_err(|_| "Failed to compress_state_event".to_owned()) }) - .collect::>()?; + .collect::>()?; // 13. Check if the event passes auth based on the "current state" of the room, if not "soft fail" it debug!("starting soft fail auth check"); @@ -1750,7 +1748,7 @@ async fn upgrade_outlier_to_timeline_pdu( .compress_state_event(*k, id, &db.globals) .map_err(|_| "Failed to compress_state_event.".to_owned()) }) - .collect::>()? + .collect::>()? } else { // We do need to force an update to this room's state update_state = true; @@ -1812,7 +1810,7 @@ async fn upgrade_outlier_to_timeline_pdu( .compress_state_event(shortstatekey, &event_id, &db.globals) .map_err(|_| "Failed to compress state event".to_owned()) }) - .collect::>()? + .collect::>()? }; // Set the new room state to the resolved state From 9082a531c99781fba5dd1abadd4dfc4ada518bbd Mon Sep 17 00:00:00 2001 From: Nyaaori <+@nyaaori.cat> Date: Fri, 24 Sep 2021 22:44:26 +0000 Subject: [PATCH 026/201] Make allow_encryption work again, fixing #115 --- src/client_server/message.rs | 8 ++++++++ src/client_server/state.rs | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 93ead2c7..25964cc2 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -45,6 +45,14 @@ pub async fn send_message_event_route( ); let state_lock = mutex_state.lock().await; + // Forbid m.room.encrypted if encryption is disabled + if &body.event_type == "m.room.encrypted" && !db.globals.allow_encryption() { + return Err(Error::BadRequest( + ErrorKind::Forbidden, + "Encryption has been disabled", + )); + } + // Check if this is a new transaction id if let Some(response) = db.transaction_ids diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 24cc2a18..7618dcc4 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -73,6 +73,14 @@ pub async fn send_state_event_for_empty_key_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + // Forbid m.room.encryption if encryption is disabled + if &body.event_type == "m.room.encryption" && !db.globals.allow_encryption() { + return Err(Error::BadRequest( + ErrorKind::Forbidden, + "Encryption has been disabled", + )); + } + let event_id = send_state_event_for_key_helper( &db, sender_user, From d996d1b0e65ccce8d1336ef2b382b52d0df73997 Mon Sep 17 00:00:00 2001 From: Nyaaori <+@nyaaori.cat> Date: Fri, 15 Oct 2021 12:38:20 +0000 Subject: [PATCH 027/201] Always send device_one_time_keys_count, fixing #178 --- src/client_server/sync.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 2d5ad27d..fe8aad15 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -758,13 +758,7 @@ async fn sync_helper( changed: device_list_updates.into_iter().collect(), left: device_list_left.into_iter().collect(), }, - device_one_time_keys_count: if db.users.last_one_time_keys_update(&sender_user)? > since - || since == 0 - { - db.users.count_one_time_keys(&sender_user, &sender_device)? - } else { - BTreeMap::new() - }, + device_one_time_keys_count: db.users.count_one_time_keys(&sender_user, &sender_device)?, to_device: sync_events::ToDevice { events: db .users From 484a044b504dc3458799ee2eca87cd034f0ef8d5 Mon Sep 17 00:00:00 2001 From: Nyaaori <+@nyaaori.cat> Date: Fri, 15 Oct 2021 23:17:08 +0000 Subject: [PATCH 028/201] Remove device_one_time_keys_count from is_empty() sync checks, fixing sync issue as reported by Nekron --- src/client_server/sync.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index d98d759f..65c07bc9 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -769,7 +769,6 @@ async fn sync_helper( && response.presence.is_empty() && response.account_data.is_empty() && response.device_lists.is_empty() - && response.device_one_time_keys_count.is_empty() && response.to_device.is_empty() { // Hang a few seconds so requests are not spammed From 55d78b1914f9800867d4490a53deae00c1f86e31 Mon Sep 17 00:00:00 2001 From: Nyaaori <+@nyaaori.cat> Date: Sat, 16 Oct 2021 00:45:51 +0000 Subject: [PATCH 029/201] Bump Ruma version to fix M_BAD_JSON on login --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 0f246733..dae68bf1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle request # Used for matrix spec type definitions and helpers #ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } -ruma = { git = "https://github.com/ruma/ruma", rev = "44cfd0adbc83303c19aef590ad0d71647e19f197", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +ruma = { git = "https://github.com/ruma/ruma", rev = "58cdcae1f9a8f4824bcbec1de1bb13e659c66804", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { git = "https://github.com/timokoesters/ruma", rev = "50c1db7e0a3a21fc794b0cce3b64285a4c750c71", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } From ccf501a420d12d79b803ccf7334d0db978e4724e Mon Sep 17 00:00:00 2001 From: Nyaaori <+@nyaaori.cat> Date: Mon, 18 Oct 2021 04:51:11 +0000 Subject: [PATCH 030/201] Initial implementation of /report, fixing #13 --- src/client_server/mod.rs | 2 + src/client_server/report.rs | 75 +++++++++++++++++++++++++++++++++++++ src/main.rs | 1 + 3 files changed, 78 insertions(+) create mode 100644 src/client_server/report.rs diff --git a/src/client_server/mod.rs b/src/client_server/mod.rs index e0c340f1..115ddaf6 100644 --- a/src/client_server/mod.rs +++ b/src/client_server/mod.rs @@ -16,6 +16,7 @@ mod profile; mod push; mod read_marker; mod redact; +mod report; mod room; mod search; mod session; @@ -47,6 +48,7 @@ pub use profile::*; pub use push::*; pub use read_marker::*; pub use redact::*; +pub use report::*; pub use room::*; pub use search::*; pub use session::*; diff --git a/src/client_server/report.rs b/src/client_server/report.rs new file mode 100644 index 00000000..e56cbc9f --- /dev/null +++ b/src/client_server/report.rs @@ -0,0 +1,75 @@ +use std::sync::Arc; + +use crate::{database::admin::AdminCommand, database::DatabaseGuard, ConduitResult, Error, Ruma}; +use ruma::{ + api::client::{error::ErrorKind, r0::room::report_content}, + events::room::message, + Int, +}; + +#[cfg(feature = "conduit_bin")] +use rocket::post; + +/// # `POST /_matrix/client/r0/rooms/{roomId}/report/{eventId}` +/// +/// Reports an inappropriate event to homeserver admins +/// +#[cfg_attr( + feature = "conduit_bin", + post("/_matrix/client/r0/rooms/<_>/report/<_>", data = "") +)] +#[tracing::instrument(skip(db, body))] +pub async fn report_event_route( + db: DatabaseGuard, + body: Ruma>, +) -> ConduitResult { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + let pdu = match db.rooms.get_pdu(&body.event_id) { + Ok(pdu) if !pdu.is_none() => pdu, + _ => { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Invalid Event ID", + )) + } + } + .unwrap(); + + if body.score >= Int::from(0) && body.score <= Int::from(-100) { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Invalid score, must be within 0 to -100", + )); + }; + + if body.reason.chars().count() > 160 { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Reason too long, should be 160 characters or fewer", + )); + }; + + let mutex_state = Arc::clone( + db.globals + .roomid_mutex_state + .write() + .unwrap() + .entry(body.room_id.clone()) + .or_default(), + ); + let state_lock = mutex_state.lock().await; + + db.admin.send(AdminCommand::SendMessage( + message::RoomMessageEventContent::text_plain(format!( + "Report received from: {}\r\n\r\nEvent ID: {}\r\nRoom ID: {}\r\nSent By: {}\r\n\r\nReport Score: {}\r\nReport Reason: {}", + sender_user, pdu.event_id, pdu.room_id, pdu.sender, body.score, body.reason, + )), + )); + + drop(state_lock); + + db.flush()?; + + Ok(report_content::Response {}.into()) +} diff --git a/src/main.rs b/src/main.rs index 84dfb1fc..56faa3e7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -101,6 +101,7 @@ fn setup_rocket(config: Figment, data: Arc>) -> rocket::Rocket< client_server::create_typing_event_route, client_server::create_room_route, client_server::redact_event_route, + client_server::report_event_route, client_server::create_alias_route, client_server::delete_alias_route, client_server::get_alias_route, From 1541b93f457de2d5fb8c37739d6791fa3f60312b Mon Sep 17 00:00:00 2001 From: Nyaaori <+@nyaaori.cat> Date: Mon, 18 Oct 2021 05:38:41 +0000 Subject: [PATCH 031/201] Make reports look nicer and reduce spam potential, increase max report length to 1000 characters --- src/client_server/report.rs | 39 ++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/src/client_server/report.rs b/src/client_server/report.rs index e56cbc9f..7f66fa13 100644 --- a/src/client_server/report.rs +++ b/src/client_server/report.rs @@ -8,7 +8,7 @@ use ruma::{ }; #[cfg(feature = "conduit_bin")] -use rocket::post; +use rocket::{http::RawStr, post}; /// # `POST /_matrix/client/r0/rooms/{roomId}/report/{eventId}` /// @@ -43,10 +43,10 @@ pub async fn report_event_route( )); }; - if body.reason.chars().count() > 160 { + if body.reason.chars().count() > 1000 { return Err(Error::BadRequest( ErrorKind::InvalidParam, - "Reason too long, should be 160 characters or fewer", + "Reason too long, should be 1000 characters or fewer", )); }; @@ -61,10 +61,35 @@ pub async fn report_event_route( let state_lock = mutex_state.lock().await; db.admin.send(AdminCommand::SendMessage( - message::RoomMessageEventContent::text_plain(format!( - "Report received from: {}\r\n\r\nEvent ID: {}\r\nRoom ID: {}\r\nSent By: {}\r\n\r\nReport Score: {}\r\nReport Reason: {}", - sender_user, pdu.event_id, pdu.room_id, pdu.sender, body.score, body.reason, - )), + message::RoomMessageEventContent::text_html( + format!( + concat!( + "Report received from: {}\r\n\r\n", + "Event ID: {}\r\n", + "Room ID: {}\r\n", + "Sent By: {}\r\n\r\n", + "Report Score: {}\r\n", + "Report Reason: {}" + ), + sender_user, pdu.event_id, pdu.room_id, pdu.sender, body.score, body.reason + ) + .to_owned(), + format!( + concat!( + "
Report received from: {}
", + "Event Info

Event ID: {}
Room ID: {}
Sent By: {}", + "

Report Info

Report Score: {}", + "
Report Reason: {}

" + ), + sender_user, + pdu.event_id, + pdu.room_id, + pdu.sender, + body.score, + RawStr::new(&body.reason).html_escape() + ) + .to_owned(), + ), )); drop(state_lock); From 50f931a2fda72d94a6190092dac18f2268c96af1 Mon Sep 17 00:00:00 2001 From: Nyaaori <+@nyaaori.cat> Date: Wed, 20 Oct 2021 11:12:06 +0000 Subject: [PATCH 032/201] Cleanup and fix validation in report.rs, lower max report length, better html --- src/client_server/report.rs | 53 +++++++++++++------------------------ 1 file changed, 18 insertions(+), 35 deletions(-) diff --git a/src/client_server/report.rs b/src/client_server/report.rs index 7f66fa13..3dcb4d1c 100644 --- a/src/client_server/report.rs +++ b/src/client_server/report.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use crate::{database::admin::AdminCommand, database::DatabaseGuard, ConduitResult, Error, Ruma}; use ruma::{ api::client::{error::ErrorKind, r0::room::report_content}, @@ -25,62 +23,49 @@ pub async fn report_event_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let pdu = match db.rooms.get_pdu(&body.event_id) { - Ok(pdu) if !pdu.is_none() => pdu, + let pdu = match db.rooms.get_pdu(&body.event_id)? { + Some(pdu) => pdu, _ => { return Err(Error::BadRequest( ErrorKind::InvalidParam, "Invalid Event ID", )) } - } - .unwrap(); + }; - if body.score >= Int::from(0) && body.score <= Int::from(-100) { + if body.score > Int::from(0) || body.score < Int::from(-100) { return Err(Error::BadRequest( ErrorKind::InvalidParam, "Invalid score, must be within 0 to -100", )); }; - if body.reason.chars().count() > 1000 { + if body.reason.chars().count() > 250 { return Err(Error::BadRequest( ErrorKind::InvalidParam, - "Reason too long, should be 1000 characters or fewer", + "Reason too long, should be 250 characters or fewer", )); }; - let mutex_state = Arc::clone( - db.globals - .roomid_mutex_state - .write() - .unwrap() - .entry(body.room_id.clone()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; - db.admin.send(AdminCommand::SendMessage( message::RoomMessageEventContent::text_html( format!( - concat!( - "Report received from: {}\r\n\r\n", - "Event ID: {}\r\n", - "Room ID: {}\r\n", - "Sent By: {}\r\n\r\n", - "Report Score: {}\r\n", - "Report Reason: {}" - ), + "Report received from: {}\n\n\ + Event ID: {}\n\ + Room ID: {}\n\ + Sent By: {}\n\n\ + Report Score: {}\n\ + Report Reason: {}", sender_user, pdu.event_id, pdu.room_id, pdu.sender, body.score, body.reason ) .to_owned(), format!( - concat!( - "
Report received from: {}
", - "Event Info

Event ID: {}
Room ID: {}
Sent By: {}", - "

Report Info

Report Score: {}", - "
Report Reason: {}

" - ), + "
Report received from: {0}\ +
  • Event Info
  • \ + Report Info
    • Report Score: {4}
    • Report Reason: {5}
    • \ +
", sender_user, pdu.event_id, pdu.room_id, @@ -92,8 +77,6 @@ pub async fn report_event_route( ), )); - drop(state_lock); - db.flush()?; Ok(report_content::Response {}.into()) From bbe16f84679061f1f4af5c1ab76f519279a234c0 Mon Sep 17 00:00:00 2001 From: Nyaaori <+@nyaaori.cat> Date: Sun, 24 Oct 2021 00:45:02 +0000 Subject: [PATCH 033/201] Update Ruma --- Cargo.toml | 2 +- src/client_server/room.rs | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index dae68bf1..13a7af44 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle request # Used for matrix spec type definitions and helpers #ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } -ruma = { git = "https://github.com/ruma/ruma", rev = "58cdcae1f9a8f4824bcbec1de1bb13e659c66804", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +ruma = { git = "https://github.com/ruma/ruma", rev = "e7f01ca55a1eff437bad754bf0554cc09f44ec2a", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { git = "https://github.com/timokoesters/ruma", rev = "50c1db7e0a3a21fc794b0cce3b64285a4c750c71", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 2d1fe237..ec09eec8 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -22,10 +22,10 @@ use ruma::{ }, EventType, }, - serde::JsonObject, + serde::{JsonObject}, RoomAliasId, RoomId, RoomVersionId, }; -use serde_json::value::to_raw_value; +use serde_json::{value::to_raw_value}; use std::{cmp::max, collections::BTreeMap, convert::TryFrom, sync::Arc}; use tracing::{info, warn}; @@ -102,9 +102,14 @@ pub async fn create_room_route( } })?; + let creation_content = match body.creation_content.clone() { + Some(content) => content.deserialize().expect("Invalid creation content"), + None => create_room::CreationContent::new(), + }; + let mut content = RoomCreateEventContent::new(sender_user.clone()); - content.federate = body.creation_content.federate; - content.predecessor = body.creation_content.predecessor.clone(); + content.federate = creation_content.federate; + content.predecessor = creation_content.predecessor.clone(); content.room_version = match body.room_version.clone() { Some(room_version) => { if room_version == RoomVersionId::Version5 || room_version == RoomVersionId::Version6 { From 8087a26a35fdcd495e28e8bff401fa3ba2afd9ef Mon Sep 17 00:00:00 2001 From: Nyaaori <+@nyaaori.cat> Date: Sun, 24 Oct 2021 20:26:51 +0000 Subject: [PATCH 034/201] Make createRoom follow spec for m.room.create, allowing creation of spaces --- src/client_server/room.rs | 65 +++++++++++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/src/client_server/room.rs b/src/client_server/room.rs index ec09eec8..5e59e81d 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -22,11 +22,16 @@ use ruma::{ }, EventType, }, - serde::{JsonObject}, + serde::{CanonicalJsonObject, JsonObject, Raw}, RoomAliasId, RoomId, RoomVersionId, }; -use serde_json::{value::to_raw_value}; -use std::{cmp::max, collections::BTreeMap, convert::TryFrom, sync::Arc}; +use serde_json::{json, value::to_raw_value}; +use std::{ + cmp::max, + collections::BTreeMap, + convert::{TryFrom, TryInto}, + sync::Arc, +}; use tracing::{info, warn}; #[cfg(feature = "conduit_bin")] @@ -102,15 +107,7 @@ pub async fn create_room_route( } })?; - let creation_content = match body.creation_content.clone() { - Some(content) => content.deserialize().expect("Invalid creation content"), - None => create_room::CreationContent::new(), - }; - - let mut content = RoomCreateEventContent::new(sender_user.clone()); - content.federate = creation_content.federate; - content.predecessor = creation_content.predecessor.clone(); - content.room_version = match body.room_version.clone() { + let room_version = match body.room_version.clone() { Some(room_version) => { if room_version == RoomVersionId::Version5 || room_version == RoomVersionId::Version6 { room_version @@ -124,6 +121,50 @@ pub async fn create_room_route( None => RoomVersionId::Version6, }; + let content = match &body.creation_content { + Some(content) => { + let mut content = content + .deserialize_as::() + .expect("Invalid creation content"); + content.insert( + "creator".into(), + json!(sender_user.clone()).try_into().unwrap(), + ); + content.insert( + "room_version".into(), + json!(room_version.as_str()).try_into().unwrap(), + ); + content + } + None => { + let mut content = Raw::::from_json( + to_raw_value(&RoomCreateEventContent::new(sender_user.clone())).unwrap(), + ) + .deserialize_as::() + .unwrap(); + content.insert( + "room_version".into(), + json!(room_version.as_str()).try_into().unwrap(), + ); + content + } + }; + + // Validate creation content + match Raw::::from_json( + to_raw_value(&content).expect("Invalid creation content"), + ) + .deserialize_as::() + { + Ok(_t) => {} + Err(_e) => { + return Err(Error::BadRequest( + ErrorKind::BadJson, + "Invalid creation content", + )) + } + }; + // 1. The room create event db.rooms.build_and_append_pdu( PduBuilder { From d5d25fb064449cb42a0243248e6fc2020bf77fe2 Mon Sep 17 00:00:00 2001 From: Nyaaori <+@nyaaori.cat> Date: Sun, 24 Oct 2021 22:13:08 +0000 Subject: [PATCH 035/201] Preserve all m.room.create entries when performing room upgrades --- src/client_server/room.rs | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 5e59e81d..0c62d2d6 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -478,7 +478,7 @@ pub async fn get_room_aliases_route( .into()) } -/// # `GET /_matrix/client/r0/rooms/{roomId}/upgrade` +/// # `POST /_matrix/client/r0/rooms/{roomId}/upgrade` /// /// Upgrades the room. /// @@ -556,16 +556,15 @@ pub async fn upgrade_room_route( ); let state_lock = mutex_state.lock().await; - // Get the old room federations status - let federate = serde_json::from_str::( + // Get the old room creation event + let mut create_event_content = serde_json::from_str::( db.rooms .room_state_get(&body.room_id, &EventType::RoomCreate, "")? .ok_or_else(|| Error::bad_database("Found room without m.room.create event."))? .content .get(), ) - .map_err(|_| Error::bad_database("Invalid room event in database."))? - .federate; + .map_err(|_| Error::bad_database("Invalid room event in database."))?; // Use the m.room.tombstone event as the predecessor let predecessor = Some(ruma::events::room::create::PreviousRoom::new( @@ -574,10 +573,30 @@ pub async fn upgrade_room_route( )); // Send a m.room.create event containing a predecessor field and the applicable room_version - let mut create_event_content = RoomCreateEventContent::new(sender_user.clone()); - create_event_content.federate = federate; - create_event_content.room_version = body.new_version.clone(); - create_event_content.predecessor = predecessor; + create_event_content.insert( + "creator".into(), + json!(sender_user.clone()).try_into().unwrap(), + ); + create_event_content.insert( + "room_version".into(), + json!(body.new_version.clone()).try_into().unwrap(), + ); + create_event_content.insert("predecessor".into(), json!(predecessor).try_into().unwrap()); + + // Validate creation event content + match Raw::::from_json( + to_raw_value(&create_event_content).expect("Error forming creation event"), + ) + .deserialize_as::() + { + Ok(_t) => {} + Err(_e) => { + return Err(Error::BadRequest( + ErrorKind::BadJson, + "Error forming creation event", + )) + } + }; db.rooms.build_and_append_pdu( PduBuilder { From 743bdbe96125881418feb8583edb75ca703da4fc Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 26 Oct 2021 13:30:02 +0000 Subject: [PATCH 036/201] Add 'Federation publicRoom Name/topic keys are correct' test to sytest whitelist --- tests/sytest/sytest-whitelist | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/sytest/sytest-whitelist b/tests/sytest/sytest-whitelist index eda851ad..5afc3fd9 100644 --- a/tests/sytest/sytest-whitelist +++ b/tests/sytest/sytest-whitelist @@ -510,3 +510,4 @@ remote user can join room with version 5 remote user can join room with version 6 setting 'm.room.name' respects room powerlevel setting 'm.room.power_levels' respects room powerlevel +Federation publicRoom Name/topic keys are correct From 86177faae7f812136d02d08fe2f6533eabe28642 Mon Sep 17 00:00:00 2001 From: Nyaaori <+@nyaaori.cat> Date: Sun, 7 Nov 2021 07:57:15 +0000 Subject: [PATCH 037/201] Fix join panic bug --- src/client_server/membership.rs | 2 +- src/server_server.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 732f6162..ec685ec9 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -934,7 +934,7 @@ pub(crate) async fn invite_helper<'a>( unsigned.insert("prev_content".to_owned(), prev_pdu.content.clone()); unsigned.insert( "prev_sender".to_owned(), - serde_json::from_str(prev_pdu.sender.as_str()).expect("UserId is valid string"), + to_raw_value(&prev_pdu.sender).expect("UserId is valid"), ); } diff --git a/src/server_server.rs b/src/server_server.rs index 68e262b4..482edf0f 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -2721,7 +2721,7 @@ pub fn create_join_event_template_route( unsigned.insert("prev_content".to_owned(), prev_pdu.content.clone()); unsigned.insert( "prev_sender".to_owned(), - serde_json::from_str(prev_pdu.sender.as_str()).expect("UserId is valid string"), + to_raw_value(&prev_pdu.sender).expect("UserId is valid"), ); } From c4bce1d0c7ee0ba9c88fdccb11ac79112c19075b Mon Sep 17 00:00:00 2001 From: Nyaaori <+@nyaaori.cat> Date: Tue, 9 Nov 2021 16:12:44 +0000 Subject: [PATCH 038/201] Cleanup room.rs; replace unwraps with map_err --- src/client_server/room.rs | 86 +++++++++++++++++++++++---------------- 1 file changed, 50 insertions(+), 36 deletions(-) diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 0c62d2d6..47c7ee6f 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -22,7 +22,7 @@ use ruma::{ }, EventType, }, - serde::{CanonicalJsonObject, JsonObject, Raw}, + serde::{CanonicalJsonObject, JsonObject}, RoomAliasId, RoomId, RoomVersionId, }; use serde_json::{json, value::to_raw_value}; @@ -128,42 +128,48 @@ pub async fn create_room_route( .expect("Invalid creation content"); content.insert( "creator".into(), - json!(sender_user.clone()).try_into().unwrap(), + json!(&sender_user).try_into().map_err(|_| { + Error::BadRequest(ErrorKind::BadJson, "Invalid creation content") + })?, ); content.insert( "room_version".into(), - json!(room_version.as_str()).try_into().unwrap(), + json!(room_version.as_str()).try_into().map_err(|_| { + Error::BadRequest(ErrorKind::BadJson, "Invalid creation content") + })?, ); content } None => { - let mut content = Raw::::from_json( - to_raw_value(&RoomCreateEventContent::new(sender_user.clone())).unwrap(), + let mut content = serde_json::from_str::( + to_raw_value(&RoomCreateEventContent::new(sender_user.clone())) + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid creation content"))? + .get(), ) - .deserialize_as::() .unwrap(); content.insert( "room_version".into(), - json!(room_version.as_str()).try_into().unwrap(), + json!(room_version.as_str()).try_into().map_err(|_| { + Error::BadRequest(ErrorKind::BadJson, "Invalid creation content") + })?, ); content } }; // Validate creation content - match Raw::::from_json( - to_raw_value(&content).expect("Invalid creation content"), - ) - .deserialize_as::() - { - Ok(_t) => {} - Err(_e) => { - return Err(Error::BadRequest( - ErrorKind::BadJson, - "Invalid creation content", - )) - } - }; + let de_result = serde_json::from_str::( + to_raw_value(&content) + .expect("Invalid creation content") + .get(), + ); + + if let Err(_) = de_result { + return Err(Error::BadRequest( + ErrorKind::BadJson, + "Invalid creation content", + )); + } // 1. The room create event db.rooms.build_and_append_pdu( @@ -575,28 +581,36 @@ pub async fn upgrade_room_route( // Send a m.room.create event containing a predecessor field and the applicable room_version create_event_content.insert( "creator".into(), - json!(sender_user.clone()).try_into().unwrap(), + json!(&sender_user) + .try_into() + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Error forming creation event"))?, ); create_event_content.insert( "room_version".into(), - json!(body.new_version.clone()).try_into().unwrap(), + json!(&body.new_version) + .try_into() + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Error forming creation event"))?, + ); + create_event_content.insert( + "predecessor".into(), + json!(predecessor) + .try_into() + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Error forming creation event"))?, ); - create_event_content.insert("predecessor".into(), json!(predecessor).try_into().unwrap()); // Validate creation event content - match Raw::::from_json( - to_raw_value(&create_event_content).expect("Error forming creation event"), - ) - .deserialize_as::() - { - Ok(_t) => {} - Err(_e) => { - return Err(Error::BadRequest( - ErrorKind::BadJson, - "Error forming creation event", - )) - } - }; + let de_result = serde_json::from_str::( + to_raw_value(&create_event_content) + .expect("Error forming creation event") + .get(), + ); + + if let Err(_) = de_result { + return Err(Error::BadRequest( + ErrorKind::BadJson, + "Error forming creation event", + )); + } db.rooms.build_and_append_pdu( PduBuilder { From 109892b4b754e1666d4f00d9aec6356b46093668 Mon Sep 17 00:00:00 2001 From: Moritz Bitsch Date: Fri, 1 Oct 2021 15:53:16 +0200 Subject: [PATCH 039/201] Implement turn server settings this fills out the infos in /_matrix/client/r0/voip/turnServer with values specified in the server config --- src/client_server/voip.rs | 14 +++++++------- src/database.rs | 12 ++++++++++++ src/database/globals.rs | 16 ++++++++++++++++ 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/client_server/voip.rs b/src/client_server/voip.rs index 2a7f28e1..83f39a48 100644 --- a/src/client_server/voip.rs +++ b/src/client_server/voip.rs @@ -1,4 +1,4 @@ -use crate::ConduitResult; +use crate::{database::DatabaseGuard, ConduitResult}; use ruma::api::client::r0::voip::get_turn_server_info; use std::time::Duration; @@ -9,13 +9,13 @@ use rocket::get; /// /// TODO: Returns information about the recommended turn server. #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/voip/turnServer"))] -#[tracing::instrument] -pub async fn turn_server_route() -> ConduitResult { +#[tracing::instrument(skip(db))] +pub async fn turn_server_route(db: DatabaseGuard) -> ConduitResult { Ok(get_turn_server_info::Response { - username: "".to_owned(), - password: "".to_owned(), - uris: Vec::new(), - ttl: Duration::from_secs(60 * 60 * 24), + username: db.globals.turn_username().clone(), + password: db.globals.turn_password().clone(), + uris: db.globals.turn_uris().to_vec(), + ttl: Duration::from_secs(db.globals.turn_ttl()), } .into()) } diff --git a/src/database.rs b/src/database.rs index 8cf4f640..85213c00 100644 --- a/src/database.rs +++ b/src/database.rs @@ -74,6 +74,14 @@ pub struct Config { trusted_servers: Vec>, #[serde(default = "default_log")] pub log: String, + #[serde(default)] + turn_username: String, + #[serde(default)] + turn_password: String, + #[serde(default = "Vec::new")] + turn_uris: Vec, + #[serde(default = "default_turn_ttl")] + turn_ttl: u64, #[serde(flatten)] catchall: BTreeMap, @@ -131,6 +139,10 @@ fn default_log() -> String { "info,state_res=warn,rocket=off,_=off,sled=off".to_owned() } +fn default_turn_ttl() -> u64 { + 60 * 60 * 24 +} + #[cfg(feature = "sled")] pub type Engine = abstraction::sled::Engine; diff --git a/src/database/globals.rs b/src/database/globals.rs index f1cbbd92..7338f1ed 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -226,6 +226,22 @@ impl Globals { self.jwt_decoding_key.as_ref() } + pub fn turn_password(&self) -> &String { + &self.config.turn_password + } + + pub fn turn_ttl(&self) -> u64 { + self.config.turn_ttl + } + + pub fn turn_uris(&self) -> &[String] { + &self.config.turn_uris + } + + pub fn turn_username(&self) -> &String { + &self.config.turn_username + } + /// TODO: the key valid until timestamp is only honored in room version > 4 /// Remove the outdated keys and insert the new ones. /// From 9fccbb014a3297961fd169ce12363564e56afbc3 Mon Sep 17 00:00:00 2001 From: Moritz Bitsch Date: Sat, 2 Oct 2021 00:37:39 +0200 Subject: [PATCH 040/201] Implement TURN server authentication with hmac This is a prefered method to allow limited access to the TURN server --- Cargo.lock | 35 +++++++++++++++++++++++++++ Cargo.toml | 3 +++ src/client_server/voip.rs | 51 +++++++++++++++++++++++++++++++++------ src/database.rs | 2 ++ src/database/globals.rs | 4 +++ 5 files changed, 88 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 293bcff7..68293896 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -245,6 +245,7 @@ dependencies = [ "crossbeam", "directories", "heed", + "hmac", "http", "image", "jsonwebtoken", @@ -266,6 +267,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "sha-1", "sled", "thiserror", "thread_local", @@ -428,6 +430,16 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "crypto-mac" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "curve25519-dalek" version = "3.2.0" @@ -897,6 +909,16 @@ dependencies = [ "libc", ] +[[package]] +name = "hmac" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" +dependencies = [ + "crypto-mac", + "digest", +] + [[package]] name = "hostname" version = "0.3.1" @@ -2422,6 +2444,19 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "sha-1" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6" +dependencies = [ + "block-buffer", + "cfg-if 1.0.0", + "cpufeatures", + "digest", + "opaque-debug", +] + [[package]] name = "sha1" version = "0.6.0" diff --git a/Cargo.toml b/Cargo.toml index 13a7af44..fc83d11b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,6 +79,9 @@ num_cpus = "1.13.0" threadpool = "1.8.1" heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c758867e05ad973ef800a6fe1d5d", optional = true } thread_local = "1.1.3" +# used for TURN server authentication +hmac = "0.11.0" +sha-1 = "0.9.8" [features] default = ["conduit_bin", "backend_sqlite"] diff --git a/src/client_server/voip.rs b/src/client_server/voip.rs index 83f39a48..9c3b20d4 100644 --- a/src/client_server/voip.rs +++ b/src/client_server/voip.rs @@ -1,6 +1,11 @@ -use crate::{database::DatabaseGuard, ConduitResult}; +use crate::{database::DatabaseGuard, ConduitResult, Ruma}; +use hmac::{Hmac, Mac, NewMac}; use ruma::api::client::r0::voip::get_turn_server_info; -use std::time::Duration; +use ruma::SecondsSinceUnixEpoch; +use sha1::Sha1; +use std::time::{Duration, SystemTime}; + +type HmacSha1 = Hmac; #[cfg(feature = "conduit_bin")] use rocket::get; @@ -8,12 +13,44 @@ use rocket::get; /// # `GET /_matrix/client/r0/voip/turnServer` /// /// TODO: Returns information about the recommended turn server. -#[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/voip/turnServer"))] -#[tracing::instrument(skip(db))] -pub async fn turn_server_route(db: DatabaseGuard) -> ConduitResult { +#[cfg_attr( + feature = "conduit_bin", + get("/_matrix/client/r0/voip/turnServer", data = "") +)] +#[tracing::instrument(skip(body, db))] +pub async fn turn_server_route( + body: Ruma, + db: DatabaseGuard, +) -> ConduitResult { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + let turn_secret = db.globals.turn_secret(); + + let (username, password) = if turn_secret != "" { + let expiry = SecondsSinceUnixEpoch::from_system_time( + SystemTime::now() + Duration::from_secs(db.globals.turn_ttl()), + ) + .expect("time is valid"); + + let username: String = format!("{}:{}", expiry.get(), sender_user); + + let mut mac = HmacSha1::new_from_slice(turn_secret.as_bytes()) + .expect("HMAC can take key of any size"); + mac.update(username.as_bytes()); + + let password: String = base64::encode_config(mac.finalize().into_bytes(), base64::STANDARD); + + (username, password) + } else { + ( + db.globals.turn_username().clone(), + db.globals.turn_password().clone(), + ) + }; + Ok(get_turn_server_info::Response { - username: db.globals.turn_username().clone(), - password: db.globals.turn_password().clone(), + username: username, + password: password, uris: db.globals.turn_uris().to_vec(), ttl: Duration::from_secs(db.globals.turn_ttl()), } diff --git a/src/database.rs b/src/database.rs index 85213c00..080e24b3 100644 --- a/src/database.rs +++ b/src/database.rs @@ -80,6 +80,8 @@ pub struct Config { turn_password: String, #[serde(default = "Vec::new")] turn_uris: Vec, + #[serde(default)] + turn_secret: String, #[serde(default = "default_turn_ttl")] turn_ttl: u64, diff --git a/src/database/globals.rs b/src/database/globals.rs index 7338f1ed..05ecb568 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -242,6 +242,10 @@ impl Globals { &self.config.turn_username } + pub fn turn_secret(&self) -> &String { + &self.config.turn_secret + } + /// TODO: the key valid until timestamp is only honored in room version > 4 /// Remove the outdated keys and insert the new ones. /// From 2fff720df38c83673269fa597361c5631e991c9a Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Sun, 21 Nov 2021 17:34:08 +0000 Subject: [PATCH 041/201] CI: New Multiarch builds and Docker images + cargo clippy/test output now integrated into GitLab --- .dockerignore | 2 + .gitlab-ci.yml | 376 +++++++++++------------- Cargo.lock | 36 +-- Cargo.toml | 3 +- DEPLOY.md | 52 ++-- Dockerfile | 137 ++++----- docker/README.md | 105 +++---- docker/ci-binaries-packaging.Dockerfile | 48 +-- docker/healthcheck.sh | 6 +- 9 files changed, 356 insertions(+), 409 deletions(-) diff --git a/.dockerignore b/.dockerignore index 80b30721..933b380f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,6 +14,8 @@ docker-compose* # Git folder .git .gitea +.gitlab +.github # Dot files .env diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 386986fd..6f2e0fe3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,6 @@ variables: FF_USE_FASTZIP: 1 CACHE_COMPRESSION_LEVEL: fastest - # --------------------------------------------------------------------- # # Cargo: Compiling for different architectures # # --------------------------------------------------------------------- # @@ -20,7 +19,7 @@ variables: rules: - if: '$CI_COMMIT_BRANCH == "master"' - if: '$CI_COMMIT_BRANCH == "next"' - - if: '$CI_COMMIT_TAG' + - if: "$CI_COMMIT_TAG" interruptible: true image: "rust:latest" tags: ["docker"] @@ -28,258 +27,209 @@ variables: paths: - cargohome - target/ - key: "build_cache-$TARGET-release" + key: "build_cache--$TARGET--$CI_COMMIT_BRANCH--release" variables: - CARGO_PROFILE_RELEASE_LTO=true - CARGO_PROFILE_RELEASE_CODEGEN_UNITS=1 + CARGO_PROFILE_RELEASE_LTO: "true" + CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1" before_script: - 'echo "Building for target $TARGET"' - 'mkdir -p cargohome && CARGOHOME="cargohome"' - - "cat /etc/*-release && rustc --version && cargo --version" # Print version info for debugging - - 'apt-get update -yqq' - - 'echo "Installing packages: $NEEDED_PACKAGES"' - - "apt-get install -yqq --no-install-recommends $NEEDED_PACKAGES" + - "rustc --version && cargo --version && rustup show" # Print version info for debugging - "rustup target add $TARGET" script: - time cargo build --target $TARGET --release - - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"' + - 'cp "target/$TARGET/release/conduit" "conduit-$TARGET"' artifacts: expire_in: never - -build:release:cargo:x86_64-unknown-linux-gnu: +build:release:cargo:x86_64-unknown-linux-musl-with-debug: extends: .build-cargo-shared-settings + image: messense/rust-musl-cross:x86_64-musl variables: - TARGET: "x86_64-unknown-linux-gnu" + CARGO_PROFILE_RELEASE_DEBUG: 2 # Enable debug info for flamegraph profiling + TARGET: "x86_64-unknown-linux-musl" + after_script: + - "mv ./conduit-x86_64-unknown-linux-musl ./conduit-x86_64-unknown-linux-musl-with-debug" artifacts: - name: "conduit-x86_64-unknown-linux-gnu" + name: "conduit-x86_64-unknown-linux-musl-with-debug" paths: - - "conduit-x86_64-unknown-linux-gnu" - expose_as: "Conduit for x86_64-unknown-linux-gnu" - -build:release:cargo:armv7-unknown-linux-gnueabihf: - extends: .build-cargo-shared-settings - variables: - TARGET: "armv7-unknown-linux-gnueabihf" - NEEDED_PACKAGES: "build-essential gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf libc6-dev-armhf-cross" - CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER: arm-linux-gnueabihf-gcc - CC_armv7_unknown_linux_gnueabihf: arm-linux-gnueabihf-gcc - CXX_armv7_unknown_linux_gnueabihf: arm-linux-gnueabihf-g++ - artifacts: - name: "conduit-armv7-unknown-linux-gnueabihf" - paths: - - "conduit-armv7-unknown-linux-gnueabihf" - expose_as: "Conduit for armv7-unknown-linux-gnueabihf" - -build:release:cargo:aarch64-unknown-linux-gnu: - extends: .build-cargo-shared-settings - variables: - TARGET: "aarch64-unknown-linux-gnu" - NEEDED_PACKAGES: "build-essential gcc-10-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross" - CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc - CC_aarch64_unknown_linux_gnu: aarch64-linux-gnu-gcc - CXX_aarch64_unknown_linux_gnu: aarch64-linux-gnu-g++ - TARGET_CC: "/usr/bin/aarch64-linux-gnu-gcc-10" - TARGET_AR: "/usr/bin/aarch64-linux-gnu-gcc-ar-10" - artifacts: - name: "conduit-aarch64-unknown-linux-gnu" - paths: - - "conduit-aarch64-unknown-linux-gnu" - expose_as: "Conduit for aarch64-unknown-linux-gnu" + - "conduit-x86_64-unknown-linux-musl-with-debug" + expose_as: "Conduit for x86_64-unknown-linux-musl-with-debug" build:release:cargo:x86_64-unknown-linux-musl: extends: .build-cargo-shared-settings - image: "rust:alpine" + image: messense/rust-musl-cross:x86_64-musl variables: TARGET: "x86_64-unknown-linux-musl" - before_script: - - 'echo "Building for target $TARGET"' - - 'mkdir -p cargohome && CARGOHOME="cargohome"' - - "cat /etc/*-release && rustc --version && cargo --version" # Print version info for debugging - - "rustup target add $TARGET" - - "apk add libc-dev" artifacts: name: "conduit-x86_64-unknown-linux-musl" paths: - "conduit-x86_64-unknown-linux-musl" expose_as: "Conduit for x86_64-unknown-linux-musl" +build:release:cargo:arm-unknown-linux-musleabihf: + extends: .build-cargo-shared-settings + image: messense/rust-musl-cross:arm-musleabihf + variables: + TARGET: "arm-unknown-linux-musleabihf" + artifacts: + name: "conduit-arm-unknown-linux-musleabihf" + paths: + - "conduit-arm-unknown-linux-musleabihf" + expose_as: "Conduit for arm-unknown-linux-musleabihf" +build:release:cargo:armv7-unknown-linux-musleabihf: + extends: .build-cargo-shared-settings + image: messense/rust-musl-cross:armv7-musleabihf + variables: + TARGET: "armv7-unknown-linux-musleabihf" + artifacts: + name: "conduit-armv7-unknown-linux-musleabihf" + paths: + - "conduit-armv7-unknown-linux-musleabihf" + expose_as: "Conduit for armv7-unknown-linux-musleabihf" + +build:release:cargo:aarch64-unknown-linux-musl: + extends: .build-cargo-shared-settings + image: messense/rust-musl-cross:aarch64-musl + variables: + TARGET: "aarch64-unknown-linux-musl" + artifacts: + name: "conduit-aarch64-unknown-linux-musl" + paths: + - "conduit-aarch64-unknown-linux-musl" + expose_as: "Conduit for aarch64-unknown-linux-musl" .cargo-debug-shared-settings: extends: ".build-cargo-shared-settings" rules: - - if: '$CI_COMMIT_BRANCH' - - if: '$CI_COMMIT_TAG' + - if: '$CI_COMMIT_BRANCH != "master"' cache: - key: "build_cache-$TARGET-debug" + key: "build_cache--$TARGET--$CI_COMMIT_BRANCH--debug" script: - "time cargo build --target $TARGET" - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"' artifacts: expire_in: 4 weeks -build:debug:cargo:x86_64-unknown-linux-gnu: - extends: ".cargo-debug-shared-settings" - variables: - TARGET: "x86_64-unknown-linux-gnu" - artifacts: - name: "conduit-debug-x86_64-unknown-linux-gnu" - paths: - - "conduit-debug-x86_64-unknown-linux-gnu" - expose_as: "Conduit DEBUG for x86_64-unknown-linux-gnu" - build:debug:cargo:x86_64-unknown-linux-musl: extends: ".cargo-debug-shared-settings" - image: "rust:alpine" + image: messense/rust-musl-cross:x86_64-musl variables: TARGET: "x86_64-unknown-linux-musl" - before_script: - - 'echo "Building for target $TARGET"' - - 'mkdir -p cargohome && CARGOHOME="cargohome"' - - "cat /etc/*-release && rustc --version && cargo --version" # Print version info for debugging - - "rustup target add $TARGET" - - "apk add libc-dev" artifacts: name: "conduit-debug-x86_64-unknown-linux-musl" paths: - "conduit-debug-x86_64-unknown-linux-musl" expose_as: "Conduit DEBUG for x86_64-unknown-linux-musl" - - -# --------------------------------------------------------------------- # -# Cargo: Compiling deb packages for different architectures # -# --------------------------------------------------------------------- # - - -.build-cargo-deb-shared-settings: - stage: "build" - needs: [ ] - rules: - - if: '$CI_COMMIT_BRANCH == "master"' - - if: '$CI_COMMIT_BRANCH == "next"' - - if: '$CI_COMMIT_TAG' - interruptible: true - image: "rust:latest" - tags: ["docker"] - cache: - paths: - - cargohome - - target/ - key: "build_cache-deb-$TARGET" - before_script: - - 'echo "Building debian package for target $TARGET"' - - 'mkdir -p cargohome && CARGOHOME="cargohome"' - - "cat /etc/*-release && rustc --version && cargo --version" # Print version info for debugging - - 'apt-get update -yqq' - - 'echo "Installing packages: $NEEDED_PACKAGES"' - - "apt-get install -yqq --no-install-recommends $NEEDED_PACKAGES" - - "rustup target add $TARGET" - - "cargo install cargo-deb" - script: - - time cargo deb --target $TARGET - - 'mv target/$TARGET/debian/*.deb "conduit-$TARGET.deb"' - -build:cargo-deb:x86_64-unknown-linux-gnu: - extends: .build-cargo-deb-shared-settings - variables: - TARGET: "x86_64-unknown-linux-gnu" - NEEDED_PACKAGES: "" - artifacts: - name: "conduit-x86_64-unknown-linux-gnu.deb" - paths: - - "conduit-x86_64-unknown-linux-gnu.deb" - expose_as: "Debian Package x86_64" - - # --------------------------------------------------------------------- # # Create and publish docker image # # --------------------------------------------------------------------- # -# Build a docker image by packaging up the x86_64-unknown-linux-musl binary into an alpine image .docker-shared-settings: stage: "build docker image" - needs: [] - interruptible: true - image: - name: "gcr.io/kaniko-project/executor:debug" - entrypoint: [""] + image: jdrouet/docker-with-buildx:stable tags: ["docker"] + services: + - docker:dind + needs: + - "build:release:cargo:x86_64-unknown-linux-musl" + - "build:release:cargo:arm-unknown-linux-musleabihf" + - "build:release:cargo:armv7-unknown-linux-musleabihf" + - "build:release:cargo:aarch64-unknown-linux-musl" variables: - # Configure Kaniko Caching: https://cloud.google.com/build/docs/kaniko-cache - KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=120h --cache-repo $CI_REGISTRY_IMAGE/kaniko-ci-cache" + DOCKER_HOST: tcp://docker:2375/ + DOCKER_TLS_CERTDIR: "" + DOCKER_DRIVER: overlay2 + PLATFORMS: "linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/amd64" + DOCKER_FILE: "docker/ci-binaries-packaging.Dockerfile" before_script: - - "mkdir -p /kaniko/.docker" - - 'echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"},\"$DOCKER_HUB\":{\"username\":\"$DOCKER_HUB_USER\",\"password\":\"$DOCKER_HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json' - + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + # Only log in to Dockerhub if the credentials are given: + - if [ -n "${DOCKER_HUB}" ]; then docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD" "$DOCKER_HUB"; fi + script: + # Prepare buildx to build multiarch stuff: + - docker context create 'ci-context' + - docker buildx create --name 'multiarch-builder' --use 'ci-context' + # Copy binaries to their docker arch path + - mkdir -p linux/ && mv ./conduit-x86_64-unknown-linux-musl linux/amd64 + - mkdir -p linux/arm/ && mv ./conduit-arm-unknown-linux-musleabihf linux/arm/v6 + - mkdir -p linux/arm/ && mv ./conduit-armv7-unknown-linux-musleabihf linux/arm/v7 + - mkdir -p linux/arm64/ && mv ./conduit-aarch64-unknown-linux-musl linux/arm64/v8 + # Actually create multiarch image: + - > + docker buildx build + --pull + --push + --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ') + --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml) + --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA" + --platform "$PLATFORMS" + --tag "$GL_IMAGE_TAG" + --tag "$GL_IMAGE_TAG-commit-$CI_COMMIT_SHORT_SHA" + --file "$DOCKER_FILE" . + # Only try to push to docker hub, if auth data for dockerhub exists: + - if [ -n "${DOCKER_HUB}" ]; then docker push "$DH_IMAGE_TAG"; fi + - if [ -n "${DOCKER_HUB}" ]; then docker push "$DH_IMAGE_TAG-commit-$CI_COMMIT_SHORT_SHA"; fi build:docker:next: extends: .docker-shared-settings - needs: - - "build:release:cargo:x86_64-unknown-linux-musl" - script: - - > - /kaniko/executor - $KANIKO_CACHE_ARGS - --force - --context $CI_PROJECT_DIR - --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml) - --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA" - --dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile" - --destination "$CI_REGISTRY_IMAGE/conduit:next" - --destination "$CI_REGISTRY_IMAGE/conduit:next-alpine" - --destination "$CI_REGISTRY_IMAGE/conduit:commit-$CI_COMMIT_SHORT_SHA" - --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next" - --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next-alpine" - --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:commit-$CI_COMMIT_SHORT_SHA" rules: - if: '$CI_COMMIT_BRANCH == "next"' - + variables: + GL_IMAGE_TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:next" + DH_IMAGE_TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next" build:docker:master: extends: .docker-shared-settings - needs: - - "build:release:cargo:x86_64-unknown-linux-musl" - script: - - > - /kaniko/executor - $KANIKO_CACHE_ARGS - --context $CI_PROJECT_DIR - --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml) - --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA" - --dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile" - --destination "$CI_REGISTRY_IMAGE/conduit:latest" - --destination "$CI_REGISTRY_IMAGE/conduit:latest-alpine" - --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest" - --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest-alpine" rules: - if: '$CI_COMMIT_BRANCH == "master"' + variables: + GL_IMAGE_TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:latest" + DH_IMAGE_TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest" - -build:docker:tags: - extends: .docker-shared-settings - needs: - - "build:release:cargo:x86_64-unknown-linux-musl" - script: - - > - /kaniko/executor - $KANIKO_CACHE_ARGS - --context $CI_PROJECT_DIR - --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml) - --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA" - --dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile" - --destination "$CI_REGISTRY_IMAGE/conduit:$CI_COMMIT_TAG" - --destination "$CI_REGISTRY_IMAGE/conduit:$CI_COMMIT_TAG-alpine" - --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:$CI_COMMIT_TAG" - --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:$CI_COMMIT_TAG-alpine" - rules: - - if: '$CI_COMMIT_TAG' - - +## Build a docker image by packaging up the x86_64-unknown-linux-musl binary into an alpine image +#.docker-shared-settings: +# stage: "build docker image" +# needs: [] +# interruptible: true +# image: +# name: "gcr.io/kaniko-project/executor:debug" +# entrypoint: [""] +# tags: ["docker"] +# variables: +# # Configure Kaniko Caching: https://cloud.google.com/build/docs/kaniko-cache +# KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=120h --cache-repo $CI_REGISTRY_IMAGE/kaniko-ci-cache" +# before_script: +# - "mkdir -p /kaniko/.docker" +# - 'echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"},\"$DOCKER_HUB\":{\"username\":\"$DOCKER_HUB_USER\",\"password\":\"$DOCKER_HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json' +# +# +#build:docker:next: +# extends: .docker-shared-settings +# needs: +# - "build:release:cargo:x86_64-unknown-linux-musl" +# script: +# - > +# /kaniko/executor +# $KANIKO_CACHE_ARGS +# --force +# --context $CI_PROJECT_DIR +# --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ') +# --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml) +# --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA" +# --dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile" +# --destination "$CI_REGISTRY_IMAGE/conduit:next" +# --destination "$CI_REGISTRY_IMAGE/conduit:next-alpine" +# --destination "$CI_REGISTRY_IMAGE/conduit:commit-$CI_COMMIT_SHORT_SHA" +# --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next" +# --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next-alpine" +# --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:commit-$CI_COMMIT_SHORT_SHA" +# rules: +# - if: '$CI_COMMIT_BRANCH == "next"' +# +# # --------------------------------------------------------------------- # # Run tests # @@ -287,9 +237,9 @@ build:docker:tags: test:cargo: stage: "test" - needs: [ ] + needs: [] image: "rust:latest" - tags: [ "docker" ] + tags: ["docker"] variables: CARGO_HOME: "cargohome" cache: @@ -301,13 +251,20 @@ test:cargo: before_script: - mkdir -p $CARGO_HOME && echo "using $CARGO_HOME to cache cargo deps" - apt-get update -yqq - - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config + - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config wget - rustup component add clippy rustfmt + - wget "https://faulty-storage.de/gitlab-report" + - chmod +x ./gitlab-report script: - - rustc --version && cargo --version # Print version info for debugging + - rustc --version && cargo --version # Print version info for debugging - cargo fmt --all -- --check - - cargo test --workspace --verbose --locked - - cargo clippy + - "cargo test --color always --workspace --verbose --locked --no-fail-fast -- -Z unstable-options --format json | ./gitlab-report -p test > $CI_PROJECT_DIR/report.xml" + - "cargo clippy --color always --verbose --message-format=json | ./gitlab-report -p clippy > $CI_PROJECT_DIR/gl-code-quality-report.json" + artifacts: + when: always + reports: + junit: report.xml + codequality: gl-code-quality-report.json test:sytest: stage: "test" @@ -316,8 +273,8 @@ test:sytest: - "build:debug:cargo:x86_64-unknown-linux-musl" image: name: "valkum/sytest-conduit:latest" - entrypoint: [ "" ] - tags: [ "docker" ] + entrypoint: [""] + tags: ["docker"] variables: PLUGINS: "https://github.com/valkum/sytest_conduit/archive/master.tar.gz" before_script: @@ -330,7 +287,7 @@ test:sytest: script: - "SYTEST_EXIT_CODE=0" - "/bootstrap.sh conduit || SYTEST_EXIT_CODE=1" - - "perl /sytest/tap-to-junit-xml.pl --puretap --input /logs/results.tap --output $CI_PROJECT_DIR/sytest.xml \"Sytest\" && cp /logs/results.tap $CI_PROJECT_DIR/results.tap" + - 'perl /sytest/tap-to-junit-xml.pl --puretap --input /logs/results.tap --output $CI_PROJECT_DIR/sytest.xml "Sytest" && cp /logs/results.tap $CI_PROJECT_DIR/results.tap' - "exit $SYTEST_EXIT_CODE" artifacts: when: always @@ -340,7 +297,6 @@ test:sytest: reports: junit: "$CI_PROJECT_DIR/sytest.xml" - # --------------------------------------------------------------------- # # Store binaries as package so they have download urls # # --------------------------------------------------------------------- # @@ -348,25 +304,31 @@ test:sytest: publish:package: stage: "upload artifacts" needs: - - "build:release:cargo:x86_64-unknown-linux-gnu" - - "build:release:cargo:armv7-unknown-linux-gnueabihf" - - "build:release:cargo:aarch64-unknown-linux-gnu" - "build:release:cargo:x86_64-unknown-linux-musl" - - "build:cargo-deb:x86_64-unknown-linux-gnu" + - "build:release:cargo:arm-unknown-linux-musleabihf" + - "build:release:cargo:armv7-unknown-linux-musleabihf" + - "build:release:cargo:aarch64-unknown-linux-musl" + # - "build:cargo-deb:x86_64-unknown-linux-gnu" rules: - if: '$CI_COMMIT_BRANCH == "master"' - if: '$CI_COMMIT_BRANCH == "next"' - - if: '$CI_COMMIT_TAG' + - if: "$CI_COMMIT_TAG" image: curlimages/curl:latest tags: ["docker"] variables: GIT_STRATEGY: "none" # Don't need a clean copy of the code, we just operate on artifacts script: - 'BASE_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/conduit-${CI_COMMIT_REF_SLUG}/build-${CI_PIPELINE_ID}"' - - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-gnu "${BASE_URL}/conduit-x86_64-unknown-linux-gnu"' - - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-armv7-unknown-linux-gnueabihf "${BASE_URL}/conduit-armv7-unknown-linux-gnueabihf"' - - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-aarch64-unknown-linux-gnu "${BASE_URL}/conduit-aarch64-unknown-linux-gnu"' - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-musl "${BASE_URL}/conduit-x86_64-unknown-linux-musl"' - - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-gnu.deb "${BASE_URL}/conduit-x86_64-unknown-linux-gnu.deb"' - + - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-arm-unknown-linux-musleabihf "${BASE_URL}/conduit-arm-unknown-linux-musleabihf"' + - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-armv7-unknown-linux-musleabihf "${BASE_URL}/conduit-armv7-unknown-linux-musleabihf"' + - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-aarch64-unknown-linux-musl "${BASE_URL}/conduit-aarch64-unknown-linux-musl"' +# Avoid duplicate pipelines +# See: https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines +workflow: + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: "$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS" + when: never + - if: "$CI_COMMIT_BRANCH" diff --git a/Cargo.lock b/Cargo.lock index 293bcff7..166d67fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1968,7 +1968,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" +source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" dependencies = [ "assign", "js_int", @@ -1989,7 +1989,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.18.5" -source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" +source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" dependencies = [ "bytes", "http", @@ -2005,7 +2005,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.18.5" -source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" +source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2016,7 +2016,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" +source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" dependencies = [ "ruma-api", "ruma-common", @@ -2030,7 +2030,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.12.3" -source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" +source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" dependencies = [ "assign", "bytes", @@ -2050,7 +2050,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.6.0" -source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" +source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" dependencies = [ "indexmap", "js_int", @@ -2065,7 +2065,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.24.6" -source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" +source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" dependencies = [ "indoc", "js_int", @@ -2081,7 +2081,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.24.6" -source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" +source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2092,7 +2092,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.3.1" -source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" +source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" dependencies = [ "js_int", "ruma-api", @@ -2107,7 +2107,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.20.0" -source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" +source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" dependencies = [ "paste", "percent-encoding", @@ -2122,7 +2122,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.20.0" -source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" +source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" dependencies = [ "quote", "ruma-identifiers-validation", @@ -2132,7 +2132,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" +source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" dependencies = [ "thiserror", ] @@ -2140,7 +2140,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" +source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" dependencies = [ "js_int", "ruma-api", @@ -2153,7 +2153,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" +source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" dependencies = [ "js_int", "ruma-api", @@ -2168,7 +2168,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" +source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" dependencies = [ "bytes", "form_urlencoded", @@ -2182,7 +2182,7 @@ dependencies = [ [[package]] name = "ruma-serde-macros" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" +source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2193,7 +2193,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.9.0" -source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" +source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" dependencies = [ "base64 0.13.0", "ed25519-dalek", @@ -2210,7 +2210,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.4.1" -source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" +source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" dependencies = [ "itertools 0.10.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 13a7af44..d0dd6413 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -120,13 +120,12 @@ maintainer-scripts = "debian/" systemd-units = { unit-name = "matrix-conduit" } [profile.dev] -lto = 'thin' +lto = 'off' incremental = true [profile.release] lto = 'thin' incremental = true - codegen-units=32 # If you want to make flamegraphs, enable debug info: # debug = true diff --git a/DEPLOY.md b/DEPLOY.md index 84dd2beb..6470c902 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -2,25 +2,30 @@ ## Getting help -If you run into any problems while setting up Conduit, write an email to `timo@koesters.xyz`, ask us in `#conduit:matrix.org` or [open an issue on GitLab](https://gitlab.com/famedly/conduit/-/issues/new). +If you run into any problems while setting up Conduit, write an email to `timo@koesters.xyz`, ask us +in `#conduit:matrix.org` or [open an issue on GitLab](https://gitlab.com/famedly/conduit/-/issues/new). ## Installing Conduit +Although you might be able to compile Conduit for Windows, we do recommend running it on a linux server. We therefore +only offer Linux binaries. + You may simply download the binary that fits your machine. Run `uname -m` to see what you need. Now copy the right url: -| CPU Architecture | GNU (Ubuntu, Debian, ArchLinux, ...) | MUSL (Alpine, ... ) | -| -------------------- | ------------------------------------- | ----------------------- | -| x84_64 / amd64 | [Download][x84_64-gnu] | [Download][x84_64-musl] | -| armv7 (Raspberry Pi) | [Download][armv7-gnu] | - | -| armv8 / aarch64 | [Download][armv8-gnu] | - | - -[x84_64-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-gnu?job=build:release:cargo:x86_64-unknown-linux-gnu +| CPU Architecture | Download link | +| ------------------------------------------- | ----------------------- | +| x84_64 / amd64 (Most servers and computers) | [Download][x84_64-musl] | +| armv6 | [Download][armv6-musl] | +| armv7 (e.g. Raspberry Pi by default) | [Download][armv7-musl] | +| armv8 / aarch64 | [Download][armv8-musl] | [x84_64-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-musl?job=build:release:cargo:x86_64-unknown-linux-musl -[armv7-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-armv7-unknown-linux-gnueabihf?job=build:release:cargo:armv7-unknown-linux-gnueabihf +[armv6-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-arm-unknown-linux-musleabihf?job=build:release:cargo:arm-unknown-linux-musleabihf -[armv8-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-aarch64-unknown-linux-gnu?job=build:release:cargo:aarch64-unknown-linux-gnu +[armv7-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-armv7-unknown-linux-musleabihf?job=build:release:cargo:armv7-unknown-linux-musleabihf + +[armv8-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-aarch64-unknown-linux-musl?job=build:release:cargo:aarch64-unknown-linux-musl ```bash $ sudo wget -O /usr/local/bin/matrix-conduit @@ -32,15 +37,15 @@ Alternatively, you may compile the binary yourself using ```bash $ cargo build --release ``` + Note that this currently requires Rust 1.50. If you want to cross compile Conduit to another architecture, read the [Cross-Compile Guide](CROSS_COMPILE.md). - ## Adding a Conduit user -While Conduit can run as any user it is usually better to use dedicated users for different services. -This also allows you to make sure that the file permissions are correctly set up. +While Conduit can run as any user it is usually better to use dedicated users for different services. This also allows +you to make sure that the file permissions are correctly set up. In Debian you can use this command to create a Conduit user: @@ -50,9 +55,8 @@ sudo adduser --system conduit --no-create-home ## Setting up a systemd service -Now we'll set up a systemd service for Conduit, so it's easy to start/stop -Conduit and set it to autostart when your server reboots. Simply paste the -default systemd service you can find below into +Now we'll set up a systemd service for Conduit, so it's easy to start/stop Conduit and set it to autostart when your +server reboots. Simply paste the default systemd service you can find below into `/etc/systemd/system/conduit.service`. ```systemd @@ -77,10 +81,10 @@ Finally, run $ sudo systemctl daemon-reload ``` - ## Creating the Conduit configuration file -Now we need to create the Conduit's config file in `/etc/matrix-conduit/conduit.toml`. Paste this in **and take a moment to read it. You need to change at least the server name.** +Now we need to create the Conduit's config file in `/etc/matrix-conduit/conduit.toml`. Paste this in **and take a moment +to read it. You need to change at least the server name.** ```toml [global] @@ -128,8 +132,8 @@ address = "127.0.0.1" # This makes sure Conduit can only be reached using the re ## Setting the correct file permissions -As we are using a Conduit specific user we need to allow it to read the config. -To do that you can run this command on Debian: +As we are using a Conduit specific user we need to allow it to read the config. To do that you can run this command on +Debian: ```bash sudo chown -R conduit:nogroup /etc/matrix-conduit @@ -142,7 +146,6 @@ sudo mkdir -p /var/lib/matrix-conduit/conduit_db sudo chown -R conduit:nogroup /var/lib/matrix-conduit/conduit_db ``` - ## Setting up the Reverse Proxy This depends on whether you use Apache, Nginx or another web server. @@ -171,11 +174,9 @@ ProxyPassReverse /_matrix/ http://127.0.0.1:6167/_matrix/ $ sudo systemctl reload apache2 ``` - ### Nginx -If you use Nginx and not Apache, add the following server section inside the -http section of `/etc/nginx/nginx.conf` +If you use Nginx and not Apache, add the following server section inside the http section of `/etc/nginx/nginx.conf` ```nginx server { @@ -198,13 +199,13 @@ server { include /etc/letsencrypt/options-ssl-nginx.conf; } ``` + **You need to make some edits again.** When you are done, run ```bash $ sudo systemctl reload nginx ``` - ## SSL Certificate The easiest way to get an SSL certificate, if you don't have one already, is to install `certbot` and run this: @@ -213,7 +214,6 @@ The easiest way to get an SSL certificate, if you don't have one already, is to $ sudo certbot -d your.server.name ``` - ## You're done! Now you can start Conduit with: diff --git a/Dockerfile b/Dockerfile index f4b176f5..d137353a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,75 +1,66 @@ -# Using multistage build: -# https://docs.docker.com/develop/develop-images/multistage-build/ -# https://whitfin.io/speeding-up-rust-docker-builds/ +# syntax=docker/dockerfile:1 +FROM docker.io/rust:1.53-alpine AS builder +WORKDIR /usr/src/conduit + +# Install required packages to build Conduit and it's dependencies +RUN apk add musl-dev + +# == Build dependencies without our own code separately for caching == +# +# Need a fake main.rs since Cargo refuses to build anything otherwise. +# +# See https://github.com/rust-lang/cargo/issues/2644 for a Cargo feature +# request that would allow just dependencies to be compiled, presumably +# regardless of whether source files are available. +RUN mkdir src && touch src/lib.rs && echo 'fn main() {}' > src/main.rs +COPY Cargo.toml Cargo.lock ./ +RUN cargo build --release && rm -r src + +# Copy over actual Conduit sources +COPY src src + +# main.rs and lib.rs need their timestamp updated for this to work correctly since +# otherwise the build with the fake main.rs from above is newer than the +# source files (COPY preserves timestamps). +# +# Builds conduit and places the binary at /usr/src/conduit/target/release/conduit +RUN touch src/main.rs && touch src/lib.rs && cargo build --release -########################## BUILD IMAGE ########################## -# Alpine build image to build Conduit's statically compiled binary -FROM alpine:3.14 as builder -# Install packages needed for building all crates -RUN apk add --no-cache \ - cargo \ - openssl-dev -# Specifies if the local project is build or if Conduit gets build -# from the official git repository. Defaults to the git repo. -ARG LOCAL=false -# Specifies which revision/commit is build. Defaults to HEAD -ARG GIT_REF=origin/master +# --------------------------------------------------------------------------------------------------------------- +# Stuff below this line actually ends up in the resulting docker image +# --------------------------------------------------------------------------------------------------------------- +FROM docker.io/alpine:3.14 AS runner -# Copy project files from current folder -COPY . . -# Build it from the copied local files or from the official git repository -RUN if [[ $LOCAL == "true" ]]; then \ - mv ./docker/healthcheck.sh . ; \ - echo "Building from local source..." ; \ - cargo install --path . ; \ - else \ - echo "Building revision '${GIT_REF}' from online source..." ; \ - cargo install --git "https://gitlab.com/famedly/conduit.git" --rev ${GIT_REF} ; \ - echo "Loadings healthcheck script from online source..." ; \ - wget "https://gitlab.com/famedly/conduit/-/raw/${GIT_REF#origin/}/docker/healthcheck.sh" ; \ - fi - -########################## RUNTIME IMAGE ########################## -# Create new stage with a minimal image for the actual -# runtime image/container -FROM alpine:3.14 - -ARG CREATED -ARG VERSION -ARG GIT_REF=origin/master - -ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" - -# Labels according to https://github.com/opencontainers/image-spec/blob/master/annotations.md -# including a custom label specifying the build command -LABEL org.opencontainers.image.created=${CREATED} \ - org.opencontainers.image.authors="Conduit Contributors" \ - org.opencontainers.image.title="Conduit" \ - org.opencontainers.image.version=${VERSION} \ - org.opencontainers.image.vendor="Conduit Contributors" \ - org.opencontainers.image.description="A Matrix homeserver written in Rust" \ - org.opencontainers.image.url="https://conduit.rs/" \ - org.opencontainers.image.revision=${GIT_REF} \ - org.opencontainers.image.source="https://gitlab.com/famedly/conduit.git" \ - org.opencontainers.image.licenses="Apache-2.0" \ - org.opencontainers.image.documentation="" \ - org.opencontainers.image.ref.name="" \ - org.label-schema.docker.build="docker build . -t matrixconduit/matrix-conduit:latest --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ') --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)" \ - maintainer="Weasy666" - -# Standard port on which Conduit launches. You still need to map the port when using the docker command or docker-compose. +# Standard port on which Conduit launches. +# You still need to map the port when using the docker command or docker-compose. EXPOSE 6167 -# Copy config files from context and the binary from -# the "builder" stage to the current stage into folder -# /srv/conduit and create data folder for database -RUN mkdir -p /srv/conduit/.local/share/conduit -COPY --from=builder /root/.cargo/bin/conduit /srv/conduit/ -COPY --from=builder ./healthcheck.sh /srv/conduit/ +# Note from @jfowl: I would like to remove this in the future and just have the Docker version be configured with envs. +ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" +# Conduit needs: +# ca-certificates: for https +# libgcc: Apparently this is needed, even if I (@jfowl) don't know exactly why. But whatever, it's not that big. +RUN apk add --no-cache \ + ca-certificates \ + curl \ + libgcc + + +# Created directory for the database and media files +RUN mkdir -p /srv/conduit/.local/share/conduit + +# Test if Conduit is still alive, uses the same endpoint as Element +COPY ./docker/healthcheck.sh /srv/conduit/ +HEALTHCHECK --start-period=5s --interval=5s CMD ./healthcheck.sh + +# Copy over the actual Conduit binary from the builder stage +COPY --from=builder /usr/src/conduit/target/release/conduit /srv/conduit/ + +# Improve security: Don't run stuff as root, that does not need to run as root: # Add www-data user and group with UID 82, as used by alpine # https://git.alpinelinux.org/aports/tree/main/nginx/nginx.pre-install RUN set -x ; \ @@ -79,19 +70,13 @@ RUN set -x ; \ # Change ownership of Conduit files to www-data user and group RUN chown -cR www-data:www-data /srv/conduit +RUN chmod +x /srv/conduit/healthcheck.sh -# Install packages needed to run Conduit -RUN apk add --no-cache \ - ca-certificates \ - curl \ - libgcc - -# Test if Conduit is still alive, uses the same endpoint as Element -HEALTHCHECK --start-period=5s --interval=60s CMD ./healthcheck.sh - -# Set user to www-data +# Change user to www-data USER www-data # Set container home directory WORKDIR /srv/conduit -# Run Conduit -ENTRYPOINT [ "/srv/conduit/conduit" ] + +# Run Conduit and print backtraces on panics +ENV RUST_BACKTRACE=1 +ENTRYPOINT [ "/srv/conduit/conduit" ] \ No newline at end of file diff --git a/docker/README.md b/docker/README.md index 0e834820..19d9dca6 100644 --- a/docker/README.md +++ b/docker/README.md @@ -2,53 +2,41 @@ > **Note:** To run and use Conduit you should probably use it with a Domain or Subdomain behind a reverse proxy (like Nginx, Traefik, Apache, ...) with a Lets Encrypt certificate. - ## Docker ### Build & Dockerfile The Dockerfile provided by Conduit has two stages, each of which creates an image. + 1. **Builder:** Builds the binary from local context or by cloning a git revision from the official repository. -2. **Runtime:** Copies the built binary from **Builder** and sets up the runtime environment, like creating a volume to persist the database and applying the correct permissions. - -The Dockerfile includes a few build arguments that should be supplied when building it. - -``` Dockerfile -ARG LOCAL=false -ARG CREATED -ARG VERSION -ARG GIT_REF=origin/master -``` - -- **CREATED:** Date and time as string (date-time as defined by RFC 3339). Will be used to create the Open Container Initiative compliant label `org.opencontainers.image.created`. Supply by it like this `$(date -u +'%Y-%m-%dT%H:%M:%SZ')` -- **VERSION:** The SemVer version of Conduit, which is in the image. Will be used to create the Open Container Initiative compliant label `org.opencontainers.image.version`. If you have a `Cargo.toml` in your build context, you can get it with `$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)` -- **LOCAL:** *(Optional)* A boolean value, specifies if the local build context should be used, or if the official repository will be cloned. If not supplied with the build command, it will default to `false`. -- **GIT_REF:** *(Optional)* A git ref, like `HEAD` or a commit ID. The supplied ref will be used to create the Open Container Initiative compliant label `org.opencontainers.image.revision` and will be the ref that is cloned from the repository when not building from the local context. If not supplied with the build command, it will default to `origin/master`. +2. **Runner:** Copies the built binary from **Builder** and sets up the runtime environment, like creating a volume to persist the database and applying the correct permissions. To build the image you can use the following command -``` bash -docker build . -t matrixconduit/matrix-conduit:latest --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ') --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml) +```bash +docker build --tag matrixconduit/matrix-conduit:latest . ``` which also will tag the resulting image as `matrixconduit/matrix-conduit:latest`. -**Note:** it ommits the two optional `build-arg`s. - ### Run After building the image you can simply run it with -``` bash +```bash docker run -d -p 8448:6167 -v ~/conduit.toml:/srv/conduit/conduit.toml -v db:/srv/conduit/.local/share/conduit matrixconduit/matrix-conduit:latest ``` or you can skip the build step and pull the image from one of the following registries: -| Registry | Image | Size | -| --------------- | ------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- | -| Docker Hub | [matrixconduit/matrix-conduit:latest](https://hub.docker.com/r/matrixconduit/matrix-conduit) | ![Image Size](https://img.shields.io/docker/image-size/matrixconduit/matrix-conduit/latest) | -| GitLab Registry | [registry.gitlab.com/famedly/conduit/conduit:latest](https://gitlab.com/famedly/conduit/container_registry/2134341) | ![Image Size](https://img.shields.io/docker/image-size/matrixconduit/matrix-conduit/latest) | +| Registry | Image | Size | +| --------------- | --------------------------------------------------------------- | --------------------- | +| Docker Hub | [matrixconduit/matrix-conduit:latest][dh] | ![Image Size][shield] | +| GitLab Registry | [registry.gitlab.com/famedly/conduit/matrix-conduit:latest][gl] | ![Image Size][shield] | + +[dh]: https://hub.docker.com/r/matrixconduit/matrix-conduit +[gl]: https://gitlab.com/famedly/conduit/container_registry/ +[shield]: https://img.shields.io/docker/image-size/matrixconduit/matrix-conduit/latest The `-d` flag lets the container run in detached mode. You now need to supply a `conduit.toml` config file, an example can be found [here](../conduit-example.toml). You can pass in different env vars to change config values on the fly. You can even configure Conduit completely by using env vars, but for that you need @@ -56,29 +44,26 @@ to pass `-e CONDUIT_CONFIG=""` into your container. For an overview of possible If you just want to test Conduit for a short time, you can use the `--rm` flag, which will clean up everything related to your container after you stop it. - ## Docker-compose If the docker command is not for you or your setup, you can also use one of the provided `docker-compose` files. Depending on your proxy setup, use the [`docker-compose.traefik.yml`](docker-compose.traefik.yml) and [`docker-compose.override.traefik.yml`](docker-compose.override.traefik.yml) for Traefik (don't forget to remove `.traefik` from the filenames) or the normal [`docker-compose.yml`](../docker-compose.yml) for every other reverse proxy. Additional info about deploying Conduit can be found [here](../DEPLOY.md). - ### Build To build the Conduit image with docker-compose, you first need to open and modify the `docker-compose.yml` file. There you need to comment the `image:` option and uncomment the `build:` option. Then call docker-compose with: -``` bash -CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ') VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml) docker-compose up +```bash +docker-compose up ``` -This will also start the container right afterwards, so if want it to run in detached mode, you also should use the `-d` flag. For possible `build-args`, please take a look at the above `Build & Dockerfile` section. - +This will also start the container right afterwards, so if want it to run in detached mode, you also should use the `-d` flag. ### Run If you already have built the image or want to use one from the registries, you can just start the container and everything else in the compose file in detached mode with: -``` bash +```bash docker-compose up -d ``` @@ -101,32 +86,36 @@ So...step by step: 3. Create the `conduit.toml` config file, an example can be found [here](../conduit-example.toml), or set `CONDUIT_CONFIG=""` and configure Conduit per env vars. 4. Uncomment the `element-web` service if you want to host your own Element Web Client and create a `element_config.json`. 5. Create the files needed by the `well-known` service. - - `./nginx/matrix.conf` (relative to the compose file, you can change this, but then also need to change the volume mapping) - ```nginx - server { - server_name .; - listen 80 default_server; - location /.well-known/matrix/ { - root /var/www; - default_type application/json; - add_header Access-Control-Allow-Origin *; - } - } - ``` - - `./nginx/www/.well-known/matrix/client` (relative to the compose file, you can change this, but then also need to change the volume mapping) - ```json - { - "m.homeserver": { - "base_url": "https://." - } - } - ``` - - `./nginx/www/.well-known/matrix/server` (relative to the compose file, you can change this, but then also need to change the volume mapping) - ```json - { - "m.server": ".:443" - } - ``` + - `./nginx/matrix.conf` (relative to the compose file, you can change this, but then also need to change the volume mapping) + + ```nginx + server { + server_name .; + listen 80 default_server; + + location /.well-known/matrix/ { + root /var/www; + default_type application/json; + add_header Access-Control-Allow-Origin *; + } + } + ``` + + - `./nginx/www/.well-known/matrix/client` (relative to the compose file, you can change this, but then also need to change the volume mapping) + ```json + { + "m.homeserver": { + "base_url": "https://." + } + } + ``` + - `./nginx/www/.well-known/matrix/server` (relative to the compose file, you can change this, but then also need to change the volume mapping) + ```json + { + "m.server": ".:443" + } + ``` + 6. Run `docker-compose up -d` 7. Connect to your homeserver with your preferred client and create a user. You should do this immediatly after starting Conduit, because the first created user is the admin. diff --git a/docker/ci-binaries-packaging.Dockerfile b/docker/ci-binaries-packaging.Dockerfile index fb674396..b51df7c1 100644 --- a/docker/ci-binaries-packaging.Dockerfile +++ b/docker/ci-binaries-packaging.Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 # --------------------------------------------------------------------------------------------------------- # This Dockerfile is intended to be built as part of Conduit's CI pipeline. # It does not build Conduit in Docker, but just copies the matching build artifact from the build job. @@ -7,20 +8,26 @@ # Credit's for the original Dockerfile: Weasy666. # --------------------------------------------------------------------------------------------------------- -FROM alpine:3.14 +FROM docker.io/alpine:3.14 AS runner -# Install packages needed to run Conduit +# Standard port on which Conduit launches. +# You still need to map the port when using the docker command or docker-compose. +EXPOSE 6167 + +# Note from @jfowl: I would like to remove this in the future and just have the Docker version be configured with envs. +ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" + +# Conduit needs: +# ca-certificates: for https +# libgcc: Apparently this is needed, even if I (@jfowl) don't know exactly why. But whatever, it's not that big. RUN apk add --no-cache \ ca-certificates \ - curl \ libgcc + ARG CREATED ARG VERSION ARG GIT_REF - -ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" - # Labels according to https://github.com/opencontainers/image-spec/blob/master/annotations.md # including a custom label specifying the build command LABEL org.opencontainers.image.created=${CREATED} \ @@ -33,19 +40,24 @@ LABEL org.opencontainers.image.created=${CREATED} \ org.opencontainers.image.revision=${GIT_REF} \ org.opencontainers.image.source="https://gitlab.com/famedly/conduit.git" \ org.opencontainers.image.licenses="Apache-2.0" \ - org.opencontainers.image.documentation="" \ + org.opencontainers.image.documentation="https://gitlab.com/famedly/conduit" \ org.opencontainers.image.ref.name="" -# Standard port on which Conduit launches. You still need to map the port when using the docker command or docker-compose. -EXPOSE 6167 - -# create data folder for database +# Created directory for the database and media files RUN mkdir -p /srv/conduit/.local/share/conduit -# Copy the Conduit binary into the image at the latest possible moment to maximise caching: -COPY ./conduit-x86_64-unknown-linux-musl /srv/conduit/conduit +# Test if Conduit is still alive, uses the same endpoint as Element COPY ./docker/healthcheck.sh /srv/conduit/ +HEALTHCHECK --start-period=5s --interval=5s CMD ./healthcheck.sh + +# Depending on the target platform (e.g. "linux/arm/v7", "linux/arm64/v8", or "linux/amd64") +# copy the matching binary into this docker image +ARG TARGETPLATFORM +COPY ./$TARGETPLATFORM /srv/conduit/conduit + + +# Improve security: Don't run stuff as root, that does not need to run as root: # Add www-data user and group with UID 82, as used by alpine # https://git.alpinelinux.org/aports/tree/main/nginx/nginx.pre-install RUN set -x ; \ @@ -57,13 +69,11 @@ RUN set -x ; \ RUN chown -cR www-data:www-data /srv/conduit RUN chmod +x /srv/conduit/healthcheck.sh - -# Test if Conduit is still alive, uses the same endpoint as Element -HEALTHCHECK --start-period=5s --interval=60s CMD ./healthcheck.sh - -# Set user to www-data +# Change user to www-data USER www-data # Set container home directory WORKDIR /srv/conduit -# Run Conduit + +# Run Conduit and print backtraces on panics +ENV RUST_BACKTRACE=1 ENTRYPOINT [ "/srv/conduit/conduit" ] diff --git a/docker/healthcheck.sh b/docker/healthcheck.sh index 568838ec..7ca04602 100644 --- a/docker/healthcheck.sh +++ b/docker/healthcheck.sh @@ -7,7 +7,7 @@ fi # The actual health check. # We try to first get a response on HTTP and when that fails on HTTPS and when that fails, we exit with code 1. -# TODO: Change this to a single curl call. Do we have a config value that we can check for that? -curl --fail -s "http://localhost:${CONDUIT_PORT}/_matrix/client/versions" || \ - curl -k --fail -s "https://localhost:${CONDUIT_PORT}/_matrix/client/versions" || \ +# TODO: Change this to a single wget call. Do we have a config value that we can check for that? +wget --no-verbose --tries=1 --spider "http://localhost:${CONDUIT_PORT}/_matrix/client/versions" || \ + wget --no-verbose --tries=1 --spider "https://localhost:${CONDUIT_PORT}/_matrix/client/versions" || \ exit 1 From 9bfc7b34b6d72def7da19ccd1decbe1ac2c7e6db Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Thu, 25 Nov 2021 22:36:44 +0000 Subject: [PATCH 042/201] Fixes for !225 --- .gitlab-ci.yml | 87 +++++++++---------------- DEPLOY.md | 23 +++---- Dockerfile | 4 +- docker/ci-binaries-packaging.Dockerfile | 2 +- 4 files changed, 44 insertions(+), 72 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6f2e0fe3..a8d43842 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -142,8 +142,12 @@ build:debug:cargo:x86_64-unknown-linux-musl: DOCKER_HOST: tcp://docker:2375/ DOCKER_TLS_CERTDIR: "" DOCKER_DRIVER: overlay2 - PLATFORMS: "linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/amd64" + PLATFORMS: "linux/arm/v6,linux/arm/v7,linux/arm64,linux/amd64" DOCKER_FILE: "docker/ci-binaries-packaging.Dockerfile" + cache: + paths: + - docker_cache + key: "$CI_JOB_NAME" before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY # Only log in to Dockerhub if the credentials are given: @@ -156,80 +160,51 @@ build:debug:cargo:x86_64-unknown-linux-musl: - mkdir -p linux/ && mv ./conduit-x86_64-unknown-linux-musl linux/amd64 - mkdir -p linux/arm/ && mv ./conduit-arm-unknown-linux-musleabihf linux/arm/v6 - mkdir -p linux/arm/ && mv ./conduit-armv7-unknown-linux-musleabihf linux/arm/v7 - - mkdir -p linux/arm64/ && mv ./conduit-aarch64-unknown-linux-musl linux/arm64/v8 - # Actually create multiarch image: + - mv ./conduit-aarch64-unknown-linux-musl linux/arm64 + - 'export CREATED=$(date -u +''%Y-%m-%dT%H:%M:%SZ'') && echo "Docker image creation date: $CREATED"' + # Build and push image: - > docker buildx build --pull --push - --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ') + --cache-from=type=local,src=$CI_PROJECT_DIR/docker_cache + --cache-to=type=local,dest=$CI_PROJECT_DIR/docker_cache + --build-arg CREATED=$CREATED --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml) --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA" --platform "$PLATFORMS" - --tag "$GL_IMAGE_TAG" - --tag "$GL_IMAGE_TAG-commit-$CI_COMMIT_SHORT_SHA" + --tag "$TAG" + --tag "$TAG-alpine" + --tag "$TAG-commit-$CI_COMMIT_SHORT_SHA" --file "$DOCKER_FILE" . - # Only try to push to docker hub, if auth data for dockerhub exists: - - if [ -n "${DOCKER_HUB}" ]; then docker push "$DH_IMAGE_TAG"; fi - - if [ -n "${DOCKER_HUB}" ]; then docker push "$DH_IMAGE_TAG-commit-$CI_COMMIT_SHORT_SHA"; fi -build:docker:next: +docker:next:gitlab: extends: .docker-shared-settings rules: - if: '$CI_COMMIT_BRANCH == "next"' variables: - GL_IMAGE_TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:next" - DH_IMAGE_TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next" + TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:next" -build:docker:master: +docker:next:dockerhub: + extends: .docker-shared-settings + rules: + - if: '$CI_COMMIT_BRANCH == "next" && $DOCKER_HUB' + variables: + TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next" + +docker:master:gitlab: extends: .docker-shared-settings rules: - if: '$CI_COMMIT_BRANCH == "master"' variables: - GL_IMAGE_TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:latest" - DH_IMAGE_TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest" + TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:latest" -## Build a docker image by packaging up the x86_64-unknown-linux-musl binary into an alpine image -#.docker-shared-settings: -# stage: "build docker image" -# needs: [] -# interruptible: true -# image: -# name: "gcr.io/kaniko-project/executor:debug" -# entrypoint: [""] -# tags: ["docker"] -# variables: -# # Configure Kaniko Caching: https://cloud.google.com/build/docs/kaniko-cache -# KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=120h --cache-repo $CI_REGISTRY_IMAGE/kaniko-ci-cache" -# before_script: -# - "mkdir -p /kaniko/.docker" -# - 'echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"},\"$DOCKER_HUB\":{\"username\":\"$DOCKER_HUB_USER\",\"password\":\"$DOCKER_HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json' -# -# -#build:docker:next: -# extends: .docker-shared-settings -# needs: -# - "build:release:cargo:x86_64-unknown-linux-musl" -# script: -# - > -# /kaniko/executor -# $KANIKO_CACHE_ARGS -# --force -# --context $CI_PROJECT_DIR -# --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ') -# --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml) -# --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA" -# --dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile" -# --destination "$CI_REGISTRY_IMAGE/conduit:next" -# --destination "$CI_REGISTRY_IMAGE/conduit:next-alpine" -# --destination "$CI_REGISTRY_IMAGE/conduit:commit-$CI_COMMIT_SHORT_SHA" -# --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next" -# --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next-alpine" -# --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:commit-$CI_COMMIT_SHORT_SHA" -# rules: -# - if: '$CI_COMMIT_BRANCH == "next"' -# -# +docker:master:dockerhub: + extends: .docker-shared-settings + rules: + - if: '$CI_COMMIT_BRANCH == "master" && $DOCKER_HUB' + variables: + TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest" # --------------------------------------------------------------------- # # Run tests # diff --git a/DEPLOY.md b/DEPLOY.md index 6470c902..0058b93d 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -12,20 +12,17 @@ only offer Linux binaries. You may simply download the binary that fits your machine. Run `uname -m` to see what you need. Now copy the right url: -| CPU Architecture | Download link | -| ------------------------------------------- | ----------------------- | -| x84_64 / amd64 (Most servers and computers) | [Download][x84_64-musl] | -| armv6 | [Download][armv6-musl] | -| armv7 (e.g. Raspberry Pi by default) | [Download][armv7-musl] | -| armv8 / aarch64 | [Download][armv8-musl] | +| CPU Architecture | Download stable version | +| ------------------------------------------- | ------------------------------ | +| x84_64 / amd64 (Most servers and computers) | [Download][x84_64-musl-master] | +| armv6 | [Download][armv6-musl-master] | +| armv7 (e.g. Raspberry Pi by default) | [Download][armv7-musl-master] | +| armv8 / aarch64 | [Download][armv8-musl-master] | -[x84_64-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-musl?job=build:release:cargo:x86_64-unknown-linux-musl - -[armv6-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-arm-unknown-linux-musleabihf?job=build:release:cargo:arm-unknown-linux-musleabihf - -[armv7-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-armv7-unknown-linux-musleabihf?job=build:release:cargo:armv7-unknown-linux-musleabihf - -[armv8-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-aarch64-unknown-linux-musl?job=build:release:cargo:aarch64-unknown-linux-musl +[x84_64-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-musl?job=build:release:cargo:x86_64-unknown-linux-musl +[armv6-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-arm-unknown-linux-musleabihf?job=build:release:cargo:arm-unknown-linux-musleabihf +[armv7-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-armv7-unknown-linux-musleabihf?job=build:release:cargo:armv7-unknown-linux-musleabihf +[armv8-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-aarch64-unknown-linux-musl?job=build:release:cargo:aarch64-unknown-linux-musl ```bash $ sudo wget -O /usr/local/bin/matrix-conduit diff --git a/Dockerfile b/Dockerfile index d137353a..6a9ea732 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,11 +54,11 @@ RUN apk add --no-cache \ RUN mkdir -p /srv/conduit/.local/share/conduit # Test if Conduit is still alive, uses the same endpoint as Element -COPY ./docker/healthcheck.sh /srv/conduit/ +COPY ./docker/healthcheck.sh /srv/conduit/healthcheck.sh HEALTHCHECK --start-period=5s --interval=5s CMD ./healthcheck.sh # Copy over the actual Conduit binary from the builder stage -COPY --from=builder /usr/src/conduit/target/release/conduit /srv/conduit/ +COPY --from=builder /usr/src/conduit/target/release/conduit /srv/conduit/conduit # Improve security: Don't run stuff as root, that does not need to run as root: # Add www-data user and group with UID 82, as used by alpine diff --git a/docker/ci-binaries-packaging.Dockerfile b/docker/ci-binaries-packaging.Dockerfile index b51df7c1..4ab874dd 100644 --- a/docker/ci-binaries-packaging.Dockerfile +++ b/docker/ci-binaries-packaging.Dockerfile @@ -47,7 +47,7 @@ LABEL org.opencontainers.image.created=${CREATED} \ RUN mkdir -p /srv/conduit/.local/share/conduit # Test if Conduit is still alive, uses the same endpoint as Element -COPY ./docker/healthcheck.sh /srv/conduit/ +COPY ./docker/healthcheck.sh /srv/conduit/healthcheck.sh HEALTHCHECK --start-period=5s --interval=5s CMD ./healthcheck.sh From f91216dd3ce5f842c1c441d0bae5a852e689bccf Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Tue, 14 Dec 2021 11:16:02 +0100 Subject: [PATCH 043/201] CI: Optionally use sccache for compilation This moves compiler caching for incremental builds away from GitLab caching the whole target/ folder to caching each code unit in S3. This aleviates the need to zip and unzip and just caches on the fly. This feature is optional and gated behind the SCCACHE_BIN_URL env --- .gitlab-ci.yml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a8d43842..664b5ea3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,16 +26,19 @@ variables: cache: paths: - cargohome - - target/ - key: "build_cache--$TARGET--$CI_COMMIT_BRANCH--release" + key: "build_cache--$TARGET--$CI_COMMIT_BRANCH" variables: CARGO_PROFILE_RELEASE_LTO: "true" CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1" + CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow + CARGO_HOME: $CI_PROJECT_DIR/cargohome before_script: - 'echo "Building for target $TARGET"' - - 'mkdir -p cargohome && CARGOHOME="cargohome"' + - "mkdir -p $CARGO_HOME" - "rustc --version && cargo --version && rustup show" # Print version info for debugging - "rustup target add $TARGET" + # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: + - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi script: - time cargo build --target $TARGET --release - 'cp "target/$TARGET/release/conduit" "conduit-$TARGET"' @@ -216,20 +219,20 @@ test:cargo: image: "rust:latest" tags: ["docker"] variables: - CARGO_HOME: "cargohome" + CARGO_HOME: "$CI_PROJECT_DIR/cargohome" + CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow cache: paths: - - target - cargohome - key: test_cache + key: "test_cache--$CI_COMMIT_BRANCH" interruptible: true before_script: - - mkdir -p $CARGO_HOME && echo "using $CARGO_HOME to cache cargo deps" + - mkdir -p $CARGO_HOME - apt-get update -yqq - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config wget - rustup component add clippy rustfmt - - wget "https://faulty-storage.de/gitlab-report" - - chmod +x ./gitlab-report + # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: + - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi script: - rustc --version && cargo --version # Print version info for debugging - cargo fmt --all -- --check From adb518fa0df35ba85c2ff1c96a539dda085f8991 Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Tue, 14 Dec 2021 11:16:40 +0100 Subject: [PATCH 044/201] CI: Use curl instead of wget The rust docker image already comes with curl, no need to install wget. --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 664b5ea3..1dedd8ff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -229,8 +229,9 @@ test:cargo: before_script: - mkdir -p $CARGO_HOME - apt-get update -yqq - - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config wget + - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config - rustup component add clippy rustfmt + - curl "https://faulty-storage.de/gitlab-report" --output ./gitlab-report && chmod +x ./gitlab-report # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi script: From 339a26f56c84da242d753a1894589f5923b0fd7e Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Wed, 15 Dec 2021 10:14:20 +0000 Subject: [PATCH 045/201] Update docker images --- Dockerfile | 7 +++--- docker/ci-binaries-packaging.Dockerfile | 31 ++++++++++++------------- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6a9ea732..5812fdf9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,7 +32,7 @@ RUN touch src/main.rs && touch src/lib.rs && cargo build --release # --------------------------------------------------------------------------------------------------------------- # Stuff below this line actually ends up in the resulting docker image # --------------------------------------------------------------------------------------------------------------- -FROM docker.io/alpine:3.14 AS runner +FROM docker.io/alpine:3.15.0 AS runner # Standard port on which Conduit launches. # You still need to map the port when using the docker command or docker-compose. @@ -45,9 +45,8 @@ ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" # ca-certificates: for https # libgcc: Apparently this is needed, even if I (@jfowl) don't know exactly why. But whatever, it's not that big. RUN apk add --no-cache \ - ca-certificates \ - curl \ - libgcc + ca-certificates \ + libgcc # Created directory for the database and media files diff --git a/docker/ci-binaries-packaging.Dockerfile b/docker/ci-binaries-packaging.Dockerfile index 4ab874dd..f4603105 100644 --- a/docker/ci-binaries-packaging.Dockerfile +++ b/docker/ci-binaries-packaging.Dockerfile @@ -1,14 +1,13 @@ # syntax=docker/dockerfile:1 # --------------------------------------------------------------------------------------------------------- # This Dockerfile is intended to be built as part of Conduit's CI pipeline. -# It does not build Conduit in Docker, but just copies the matching build artifact from the build job. -# As a consequence, this is not a multiarch capable image. It always expects and packages a x86_64 binary. +# It does not build Conduit in Docker, but just copies the matching build artifact from the build jobs. # # It is mostly based on the normal Conduit Dockerfile, but adjusted in a few places to maximise caching. # Credit's for the original Dockerfile: Weasy666. # --------------------------------------------------------------------------------------------------------- -FROM docker.io/alpine:3.14 AS runner +FROM docker.io/alpine:3.15.0 AS runner # Standard port on which Conduit launches. # You still need to map the port when using the docker command or docker-compose. @@ -21,8 +20,8 @@ ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" # ca-certificates: for https # libgcc: Apparently this is needed, even if I (@jfowl) don't know exactly why. But whatever, it's not that big. RUN apk add --no-cache \ - ca-certificates \ - libgcc + ca-certificates \ + libgcc ARG CREATED @@ -31,17 +30,17 @@ ARG GIT_REF # Labels according to https://github.com/opencontainers/image-spec/blob/master/annotations.md # including a custom label specifying the build command LABEL org.opencontainers.image.created=${CREATED} \ - org.opencontainers.image.authors="Conduit Contributors" \ - org.opencontainers.image.title="Conduit" \ - org.opencontainers.image.version=${VERSION} \ - org.opencontainers.image.vendor="Conduit Contributors" \ - org.opencontainers.image.description="A Matrix homeserver written in Rust" \ - org.opencontainers.image.url="https://conduit.rs/" \ - org.opencontainers.image.revision=${GIT_REF} \ - org.opencontainers.image.source="https://gitlab.com/famedly/conduit.git" \ - org.opencontainers.image.licenses="Apache-2.0" \ - org.opencontainers.image.documentation="https://gitlab.com/famedly/conduit" \ - org.opencontainers.image.ref.name="" + org.opencontainers.image.authors="Conduit Contributors" \ + org.opencontainers.image.title="Conduit" \ + org.opencontainers.image.version=${VERSION} \ + org.opencontainers.image.vendor="Conduit Contributors" \ + org.opencontainers.image.description="A Matrix homeserver written in Rust" \ + org.opencontainers.image.url="https://conduit.rs/" \ + org.opencontainers.image.revision=${GIT_REF} \ + org.opencontainers.image.source="https://gitlab.com/famedly/conduit.git" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.documentation="https://gitlab.com/famedly/conduit" \ + org.opencontainers.image.ref.name="" # Created directory for the database and media files RUN mkdir -p /srv/conduit/.local/share/conduit From 1fc616320a2aa8ab02edbfca7620773f69abf797 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Fri, 26 Nov 2021 19:28:47 +0100 Subject: [PATCH 046/201] Use struct init shorthand --- src/client_server/voip.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client_server/voip.rs b/src/client_server/voip.rs index 9c3b20d4..c9a98d9f 100644 --- a/src/client_server/voip.rs +++ b/src/client_server/voip.rs @@ -49,8 +49,8 @@ pub async fn turn_server_route( }; Ok(get_turn_server_info::Response { - username: username, - password: password, + username, + password, uris: db.globals.turn_uris().to_vec(), ttl: Duration::from_secs(db.globals.turn_ttl()), } From 892a0525f20a0a815e7d12f45a7c5a623de7844d Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Fri, 26 Nov 2021 20:36:40 +0100 Subject: [PATCH 047/201] Upgrade Ruma --- Cargo.lock | 43 ++++------ Cargo.toml | 2 +- src/client_server/account.rs | 15 ++-- src/client_server/capabilities.rs | 6 +- src/client_server/directory.rs | 2 +- src/client_server/keys.rs | 22 ++--- src/client_server/membership.rs | 41 ++++----- src/client_server/message.rs | 2 +- src/client_server/report.rs | 6 +- src/client_server/room.rs | 26 +++--- src/client_server/state.rs | 4 +- src/client_server/sync.rs | 27 +++--- src/client_server/voip.rs | 2 +- src/database.rs | 11 +-- src/database/admin.rs | 17 ++-- src/database/globals.rs | 14 +-- src/database/key_backups.rs | 6 +- src/database/pusher.rs | 4 +- src/database/rooms.rs | 138 +++++++++++++++--------------- src/database/rooms/edus.rs | 21 +++-- src/database/sending.rs | 4 +- src/database/users.rs | 35 ++++---- src/pdu.rs | 28 +++--- src/ruma_wrapper.rs | 6 +- src/server_server.rs | 119 ++++++++++++-------------- 25 files changed, 297 insertions(+), 304 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9682f2fe..8b25b478 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1516,12 +1516,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "paste" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58" - [[package]] name = "pear" version = "0.2.3" @@ -1990,7 +1984,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" +source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" dependencies = [ "assign", "js_int", @@ -2011,7 +2005,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.18.5" -source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" +source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" dependencies = [ "bytes", "http", @@ -2027,7 +2021,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.18.5" -source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" +source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2038,7 +2032,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" +source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" dependencies = [ "ruma-api", "ruma-common", @@ -2052,7 +2046,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.12.3" -source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" +source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" dependencies = [ "assign", "bytes", @@ -2072,7 +2066,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.6.0" -source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" +source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" dependencies = [ "indexmap", "js_int", @@ -2087,7 +2081,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.24.6" -source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" +source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" dependencies = [ "indoc", "js_int", @@ -2103,7 +2097,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.24.6" -source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" +source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2114,7 +2108,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.3.1" -source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" +source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" dependencies = [ "js_int", "ruma-api", @@ -2129,9 +2123,8 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.20.0" -source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" +source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" dependencies = [ - "paste", "percent-encoding", "rand 0.8.4", "ruma-identifiers-macros", @@ -2144,7 +2137,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.20.0" -source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" +source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" dependencies = [ "quote", "ruma-identifiers-validation", @@ -2154,7 +2147,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" +source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" dependencies = [ "thiserror", ] @@ -2162,7 +2155,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" +source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" dependencies = [ "js_int", "ruma-api", @@ -2175,7 +2168,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" +source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" dependencies = [ "js_int", "ruma-api", @@ -2190,7 +2183,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" +source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" dependencies = [ "bytes", "form_urlencoded", @@ -2204,7 +2197,7 @@ dependencies = [ [[package]] name = "ruma-serde-macros" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" +source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2215,7 +2208,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.9.0" -source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" +source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" dependencies = [ "base64 0.13.0", "ed25519-dalek", @@ -2232,7 +2225,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.4.1" -source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a" +source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" dependencies = [ "itertools 0.10.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 91c7e259..b24afb5c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle request # Used for matrix spec type definitions and helpers #ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } -ruma = { git = "https://github.com/ruma/ruma", rev = "e7f01ca55a1eff437bad754bf0554cc09f44ec2a", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +ruma = { git = "https://github.com/ruma/ruma", rev = "bba7d624425da2c65a834bbd0e633b7577488cdf", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { git = "https://github.com/timokoesters/ruma", rev = "50c1db7e0a3a21fc794b0cce3b64285a4c750c71", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 4b3ad0d4..d7c2f63e 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -11,10 +11,9 @@ use ruma::{ error::ErrorKind, r0::{ account::{ - change_password, deactivate, get_username_availability, register, whoami, - ThirdPartyIdRemovalStatus, + change_password, deactivate, get_3pids, get_username_availability, register, + whoami, ThirdPartyIdRemovalStatus, }, - contact::get_contacts, uiaa::{AuthFlow, AuthType, UiaaInfo}, }, }, @@ -282,7 +281,7 @@ pub async fn register_route( let mut content = RoomCreateEventContent::new(conduit_user.clone()); content.federate = true; content.predecessor = None; - content.room_version = RoomVersionId::Version6; + content.room_version = RoomVersionId::V6; // 1. The room create event db.rooms.build_and_append_pdu( @@ -433,7 +432,7 @@ pub async fn register_route( )?; // Room alias - let alias: RoomAliasId = format!("#admins:{}", db.globals.server_name()) + let alias: Box = format!("#admins:{}", db.globals.server_name()) .try_into() .expect("#admins:server_name is a valid alias name"); @@ -757,9 +756,9 @@ pub async fn deactivate_route( get("/_matrix/client/r0/account/3pid", data = "") )] pub async fn third_party_route( - body: Ruma, -) -> ConduitResult { + body: Ruma, +) -> ConduitResult { let _sender_user = body.sender_user.as_ref().expect("user is authenticated"); - Ok(get_contacts::Response::new(Vec::new()).into()) + Ok(get_3pids::Response::new(Vec::new()).into()) } diff --git a/src/client_server/capabilities.rs b/src/client_server/capabilities.rs index f86b23b5..c69b7cb2 100644 --- a/src/client_server/capabilities.rs +++ b/src/client_server/capabilities.rs @@ -22,12 +22,12 @@ pub async fn get_capabilities_route( _body: Ruma, ) -> ConduitResult { let mut available = BTreeMap::new(); - available.insert(RoomVersionId::Version5, RoomVersionStability::Stable); - available.insert(RoomVersionId::Version6, RoomVersionStability::Stable); + available.insert(RoomVersionId::V5, RoomVersionStability::Stable); + available.insert(RoomVersionId::V6, RoomVersionStability::Stable); let mut capabilities = Capabilities::new(); capabilities.room_versions = RoomVersionsCapability { - default: RoomVersionId::Version6, + default: RoomVersionId::V6, available, }; diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 490f7524..5a1bc494 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -167,7 +167,7 @@ pub(crate) async fn get_public_rooms_filtered_helper( other_server, federation::directory::get_public_rooms_filtered::v1::Request { limit, - since: since.as_deref(), + since, filter: Filter { generic_search_term: filter.generic_search_term.as_deref(), }, diff --git a/src/client_server/keys.rs b/src/client_server/keys.rs index a44f5e9c..08ea6e76 100644 --- a/src/client_server/keys.rs +++ b/src/client_server/keys.rs @@ -316,7 +316,7 @@ pub async fn get_key_changes_route( pub(crate) async fn get_keys_helper bool>( sender_user: Option<&UserId>, - device_keys_input: &BTreeMap>>, + device_keys_input: &BTreeMap, Vec>>, allowed_signatures: F, db: &Database, ) -> Result { @@ -328,6 +328,8 @@ pub(crate) async fn get_keys_helper bool>( let mut get_over_federation = HashMap::new(); for (user_id, device_ids) in device_keys_input { + let user_id: &UserId = &**user_id; + if user_id.server_name() != db.globals.server_name() { get_over_federation .entry(user_id.server_name()) @@ -355,11 +357,11 @@ pub(crate) async fn get_keys_helper bool>( container.insert(device_id, keys); } } - device_keys.insert(user_id.clone(), container); + device_keys.insert(user_id.to_owned(), container); } else { for device_id in device_ids { let mut container = BTreeMap::new(); - if let Some(mut keys) = db.users.get_device_keys(&user_id.clone(), device_id)? { + if let Some(mut keys) = db.users.get_device_keys(user_id, device_id)? { let metadata = db.users.get_device_metadata(user_id, device_id)?.ok_or( Error::BadRequest( ErrorKind::InvalidParam, @@ -371,24 +373,24 @@ pub(crate) async fn get_keys_helper bool>( device_display_name: metadata.display_name, }; - container.insert(device_id.clone(), keys); + container.insert(device_id.to_owned(), keys); } - device_keys.insert(user_id.clone(), container); + device_keys.insert(user_id.to_owned(), container); } } if let Some(master_key) = db.users.get_master_key(user_id, &allowed_signatures)? { - master_keys.insert(user_id.clone(), master_key); + master_keys.insert(user_id.to_owned(), master_key); } if let Some(self_signing_key) = db .users .get_self_signing_key(user_id, &allowed_signatures)? { - self_signing_keys.insert(user_id.clone(), self_signing_key); + self_signing_keys.insert(user_id.to_owned(), self_signing_key); } if Some(user_id) == sender_user { if let Some(user_signing_key) = db.users.get_user_signing_key(user_id)? { - user_signing_keys.insert(user_id.clone(), user_signing_key); + user_signing_keys.insert(user_id.to_owned(), user_signing_key); } } } @@ -400,7 +402,7 @@ pub(crate) async fn get_keys_helper bool>( .map(|(server, vec)| async move { let mut device_keys_input_fed = BTreeMap::new(); for (user_id, keys) in vec { - device_keys_input_fed.insert(user_id.clone(), keys.clone()); + device_keys_input_fed.insert(user_id.to_owned(), keys.clone()); } ( server, @@ -440,7 +442,7 @@ pub(crate) async fn get_keys_helper bool>( } pub(crate) async fn claim_keys_helper( - one_time_keys_input: &BTreeMap, DeviceKeyAlgorithm>>, + one_time_keys_input: &BTreeMap, BTreeMap, DeviceKeyAlgorithm>>, db: &Database, ) -> Result { let mut one_time_keys = BTreeMap::new(); diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index ec685ec9..f65287da 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -64,7 +64,7 @@ pub async fn join_room_by_id_route( .filter_map(|event| serde_json::from_str(event.json().get()).ok()) .filter_map(|event: serde_json::Value| event.get("sender").cloned()) .filter_map(|sender| sender.as_str().map(|s| s.to_owned())) - .filter_map(|sender| UserId::try_from(sender).ok()) + .filter_map(|sender| Box::::try_from(sender).ok()) .map(|user| user.server_name().to_owned()) .collect(); @@ -72,7 +72,7 @@ pub async fn join_room_by_id_route( let ret = join_room_by_id_helper( &db, - body.sender_user.as_ref(), + body.sender_user.as_deref(), &body.room_id, &servers, body.third_party_signed.as_ref(), @@ -101,7 +101,7 @@ pub async fn join_room_by_id_or_alias_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let (servers, room_id) = match RoomId::try_from(body.room_id_or_alias.clone()) { + let (servers, room_id) = match Box::::try_from(body.room_id_or_alias.clone()) { Ok(room_id) => { let mut servers: HashSet<_> = db .rooms @@ -111,7 +111,7 @@ pub async fn join_room_by_id_or_alias_route( .filter_map(|event| serde_json::from_str(event.json().get()).ok()) .filter_map(|event: serde_json::Value| event.get("sender").cloned()) .filter_map(|sender| sender.as_str().map(|s| s.to_owned())) - .filter_map(|sender| UserId::try_from(sender).ok()) + .filter_map(|sender| Box::::try_from(sender).ok()) .map(|user| user.server_name().to_owned()) .collect(); @@ -127,7 +127,7 @@ pub async fn join_room_by_id_or_alias_route( let join_room_response = join_room_by_id_helper( &db, - body.sender_user.as_ref(), + body.sender_user.as_deref(), &room_id, &servers, body.third_party_signed.as_ref(), @@ -531,7 +531,7 @@ async fn join_room_by_id_helper( .roomid_mutex_state .write() .unwrap() - .entry(room_id.clone()) + .entry(room_id.to_owned()) .or_default(), ); let state_lock = mutex_state.lock().await; @@ -551,7 +551,7 @@ async fn join_room_by_id_helper( federation::membership::create_join_event_template::v1::Request { room_id, user_id: sender_user, - ver: &[RoomVersionId::Version5, RoomVersionId::Version6], + ver: &[RoomVersionId::V5, RoomVersionId::V6], }, ) .await; @@ -567,8 +567,7 @@ async fn join_room_by_id_helper( let room_version = match make_join_response.room_version { Some(room_version) - if room_version == RoomVersionId::Version5 - || room_version == RoomVersionId::Version6 => + if room_version == RoomVersionId::V5 || room_version == RoomVersionId::V6 => { room_version } @@ -620,7 +619,7 @@ async fn join_room_by_id_helper( .expect("event is valid, we just created it"); // Generate event id - let event_id = EventId::try_from(&*format!( + let event_id = Box::::try_from(&*format!( "${}", ruma::signatures::reference_hash(&join_event_stub, &room_version) .expect("ruma can calculate reference hashes") @@ -776,7 +775,7 @@ async fn join_room_by_id_helper( db.flush()?; - Ok(join_room_by_id::Response::new(room_id.clone()).into()) + Ok(join_room_by_id::Response::new(room_id.to_owned()).into()) } fn validate_and_add_event_id( @@ -784,12 +783,12 @@ fn validate_and_add_event_id( room_version: &RoomVersionId, pub_key_map: &RwLock>>, db: &Database, -) -> Result<(EventId, CanonicalJsonObject)> { +) -> Result<(Box, CanonicalJsonObject)> { let mut value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| { error!("Invalid PDU in server response: {:?}: {:?}", pdu, e); Error::BadServerResponse("Invalid PDU in server response") })?; - let event_id = EventId::try_from(&*format!( + let event_id = Box::::try_from(&*format!( "${}", ruma::signatures::reference_hash(&value, room_version) .expect("ruma can calculate reference hashes") @@ -856,7 +855,7 @@ pub(crate) async fn invite_helper<'a>( .roomid_mutex_state .write() .unwrap() - .entry(room_id.clone()) + .entry(room_id.to_owned()) .or_default(), ); let state_lock = mutex_state.lock().await; @@ -892,9 +891,7 @@ pub(crate) async fn invite_helper<'a>( // If there was no create event yet, assume we are creating a version 6 room right now let room_version_id = create_event_content - .map_or(RoomVersionId::Version6, |create_event| { - create_event.room_version - }); + .map_or(RoomVersionId::V6, |create_event| create_event.room_version); let room_version = RoomVersion::new(&room_version_id).expect("room version is supported"); @@ -939,9 +936,9 @@ pub(crate) async fn invite_helper<'a>( } let pdu = PduEvent { - event_id: ruma::event_id!("$thiswillbefilledinlater"), - room_id: room_id.clone(), - sender: sender_user.clone(), + event_id: ruma::event_id!("$thiswillbefilledinlater").to_owned(), + room_id: room_id.to_owned(), + sender: sender_user.to_owned(), origin_server_ts: utils::millis_since_unix_epoch() .try_into() .expect("time is valid"), @@ -1014,7 +1011,7 @@ pub(crate) async fn invite_helper<'a>( }; // Generate event id - let expected_event_id = EventId::try_from(&*format!( + let expected_event_id = Box::::try_from(&*format!( "${}", ruma::signatures::reference_hash(&pdu_json, &room_version_id) .expect("ruma can calculate reference hashes") @@ -1100,7 +1097,7 @@ pub(crate) async fn invite_helper<'a>( .roomid_mutex_state .write() .unwrap() - .entry(room_id.clone()) + .entry(room_id.to_owned()) .or_default(), ); let state_lock = mutex_state.lock().await; diff --git a/src/client_server/message.rs b/src/client_server/message.rs index abbbe8ea..0d006101 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -67,7 +67,7 @@ pub async fn send_message_event_route( )); } - let event_id = EventId::try_from( + let event_id = Box::::try_from( utils::string_from_bytes(&response) .map_err(|_| Error::bad_database("Invalid txnid bytes in database."))?, ) diff --git a/src/client_server/report.rs b/src/client_server/report.rs index 3dcb4d1c..2e6527d4 100644 --- a/src/client_server/report.rs +++ b/src/client_server/report.rs @@ -57,8 +57,7 @@ pub async fn report_event_route( Report Score: {}\n\ Report Reason: {}", sender_user, pdu.event_id, pdu.room_id, pdu.sender, body.score, body.reason - ) - .to_owned(), + ), format!( "
Report received from: {0}\
  • Event Info
    • Event ID: {1}\ @@ -72,8 +71,7 @@ pub async fn report_event_route( pdu.sender, body.score, RawStr::new(&body.reason).html_escape() - ) - .to_owned(), + ), ), )); diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 47c7ee6f..97b3f482 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -88,14 +88,17 @@ pub async fn create_room_route( )); } - let alias: Option = + let alias: Option> = body.room_alias_name .as_ref() .map_or(Ok(None), |localpart| { // TODO: Check for invalid characters and maximum length - let alias = - RoomAliasId::try_from(format!("#{}:{}", localpart, db.globals.server_name())) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid alias."))?; + let alias = Box::::try_from(format!( + "#{}:{}", + localpart, + db.globals.server_name(), + )) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid alias."))?; if db.rooms.id_from_alias(&alias)?.is_some() { Err(Error::BadRequest( @@ -109,7 +112,7 @@ pub async fn create_room_route( let room_version = match body.room_version.clone() { Some(room_version) => { - if room_version == RoomVersionId::Version5 || room_version == RoomVersionId::Version6 { + if room_version == RoomVersionId::V5 || room_version == RoomVersionId::V6 { room_version } else { return Err(Error::BadRequest( @@ -118,7 +121,7 @@ pub async fn create_room_route( )); } } - None => RoomVersionId::Version6, + None => RoomVersionId::V6, }; let content = match &body.creation_content { @@ -164,7 +167,7 @@ pub async fn create_room_route( .get(), ); - if let Err(_) = de_result { + if de_result.is_err() { return Err(Error::BadRequest( ErrorKind::BadJson, "Invalid creation content", @@ -269,7 +272,7 @@ pub async fn create_room_route( PduBuilder { event_type: EventType::RoomCanonicalAlias, content: to_raw_value(&RoomCanonicalAliasEventContent { - alias: Some(room_alias_id.clone()), + alias: Some(room_alias_id.to_owned()), alt_aliases: vec![], }) .expect("We checked that alias earlier, it must be fine"), @@ -505,10 +508,7 @@ pub async fn upgrade_room_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if !matches!( - body.new_version, - RoomVersionId::Version5 | RoomVersionId::Version6 - ) { + if !matches!(body.new_version, RoomVersionId::V5 | RoomVersionId::V6) { return Err(Error::BadRequest( ErrorKind::UnsupportedRoomVersion, "This server does not support that room version.", @@ -605,7 +605,7 @@ pub async fn upgrade_room_route( .get(), ); - if let Err(_) = de_result { + if de_result.is_err() { return Err(Error::BadRequest( ErrorKind::BadJson, "Error forming creation event", diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 307bccab..0ba20620 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -267,7 +267,7 @@ async fn send_state_event_for_key_helper( event_type: EventType, json: &Raw, state_key: String, -) -> Result { +) -> Result> { let sender_user = sender; // TODO: Review this check, error if event is unparsable, use event type, allow alias if it @@ -303,7 +303,7 @@ async fn send_state_event_for_key_helper( .roomid_mutex_state .write() .unwrap() - .entry(room_id.clone()) + .entry(room_id.to_owned()) .or_default(), ); let state_lock = mutex_state.lock().await; diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 65c07bc9..1060d917 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -54,15 +54,17 @@ use rocket::{get, tokio}; /// `since` will be cached #[cfg_attr( feature = "conduit_bin", - get("/_matrix/client/r0/sync", data = "") + get("/_matrix/client/r0/sync", data = "") )] -#[tracing::instrument(skip(db, body))] +#[tracing::instrument(skip(db, req))] pub async fn sync_events_route( db: DatabaseGuard, - body: Ruma>, + req: Ruma>, ) -> Result, RumaResponse> { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let sender_device = body.sender_device.as_ref().expect("user is authenticated"); + let body = req.body; + + let sender_user = req.sender_user.expect("user is authenticated"); + let sender_device = req.sender_device.expect("user is authenticated"); let arc_db = Arc::new(db); @@ -132,7 +134,7 @@ pub async fn sync_events_route( async fn sync_helper_wrapper( db: Arc, - sender_user: UserId, + sender_user: Box, sender_device: Box, since: Option, full_state: bool, @@ -176,7 +178,7 @@ async fn sync_helper_wrapper( async fn sync_helper( db: Arc, - sender_user: UserId, + sender_user: Box, sender_device: Box, since: Option, full_state: bool, @@ -296,9 +298,10 @@ async fn sync_helper( })?; if let Some(state_key) = &pdu.state_key { - let user_id = UserId::try_from(state_key.clone()).map_err(|_| { - Error::bad_database("Invalid UserId in member PDU.") - })?; + let user_id = + Box::::try_from(state_key.clone()).map_err(|_| { + Error::bad_database("Invalid UserId in member PDU.") + })?; // The membership was and still is invite or join if matches!( @@ -424,7 +427,7 @@ async fn sync_helper( } if let Some(state_key) = &state_event.state_key { - let user_id = UserId::try_from(state_key.clone()) + let user_id = Box::::try_from(state_key.clone()) .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; if user_id == sender_user { @@ -793,7 +796,7 @@ fn share_encrypted_room( ) -> Result { Ok(db .rooms - .get_shared_rooms(vec![sender_user.clone(), user_id.clone()])? + .get_shared_rooms(vec![sender_user.to_owned(), user_id.to_owned()])? .filter_map(|r| r.ok()) .filter(|room_id| room_id != ignore_room) .filter_map(|other_room_id| { diff --git a/src/client_server/voip.rs b/src/client_server/voip.rs index c9a98d9f..66a85f0f 100644 --- a/src/client_server/voip.rs +++ b/src/client_server/voip.rs @@ -26,7 +26,7 @@ pub async fn turn_server_route( let turn_secret = db.globals.turn_secret(); - let (username, password) = if turn_secret != "" { + let (username, password) = if !turn_secret.is_empty() { let expiry = SecondsSinceUnixEpoch::from_system_time( SystemTime::now() + Duration::from_secs(db.globals.turn_ttl()), ) diff --git a/src/database.rs b/src/database.rs index 080e24b3..056d49ad 100644 --- a/src/database.rs +++ b/src/database.rs @@ -477,7 +477,8 @@ impl Database { // Set room member count for (roomid, _) in db.rooms.roomid_shortstatehash.iter() { let room_id = - RoomId::try_from(utils::string_from_bytes(&roomid).unwrap()).unwrap(); + Box::::try_from(utils::string_from_bytes(&roomid).unwrap()) + .unwrap(); db.rooms.update_joined_count(&room_id, &db)?; } @@ -489,7 +490,7 @@ impl Database { if db.globals.database_version()? < 7 { // Upgrade state store - let mut last_roomstates: HashMap = HashMap::new(); + let mut last_roomstates: HashMap, u64> = HashMap::new(); let mut current_sstatehash: Option = None; let mut current_room = None; let mut current_state = HashSet::new(); @@ -570,7 +571,7 @@ impl Database { if let Some(current_sstatehash) = current_sstatehash { handle_state( current_sstatehash, - current_room.as_ref().unwrap(), + current_room.as_deref().unwrap(), current_state, &mut last_roomstates, )?; @@ -587,7 +588,7 @@ impl Database { .unwrap() .unwrap(); let event_id = - EventId::try_from(utils::string_from_bytes(&event_id).unwrap()) + Box::::try_from(utils::string_from_bytes(&event_id).unwrap()) .unwrap(); let pdu = db.rooms.get_pdu(&event_id).unwrap().unwrap(); @@ -604,7 +605,7 @@ impl Database { if let Some(current_sstatehash) = current_sstatehash { handle_state( current_sstatehash, - current_room.as_ref().unwrap(), + current_room.as_deref().unwrap(), current_state, &mut last_roomstates, )?; diff --git a/src/database/admin.rs b/src/database/admin.rs index 8d8559a5..07a487e2 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -1,13 +1,10 @@ -use std::{ - convert::{TryFrom, TryInto}, - sync::Arc, -}; +use std::{convert::TryFrom, sync::Arc}; use crate::{pdu::PduBuilder, Database}; use rocket::futures::{channel::mpsc, stream::StreamExt}; use ruma::{ events::{room::message::RoomMessageEventContent, EventType}, - UserId, + RoomAliasId, UserId, }; use serde_json::value::to_raw_value; use tokio::sync::{MutexGuard, RwLock, RwLockReadGuard}; @@ -37,15 +34,17 @@ impl Admin { let guard = db.read().await; let conduit_user = - UserId::try_from(format!("@conduit:{}", guard.globals.server_name())) + Box::::try_from(format!("@conduit:{}", guard.globals.server_name())) .expect("@conduit:server_name is valid"); let conduit_room = guard .rooms .id_from_alias( - &format!("#admins:{}", guard.globals.server_name()) - .try_into() - .expect("#admins:server_name is a valid room alias"), + &Box::::try_from(format!( + "#admins:{}", + guard.globals.server_name() + )) + .expect("#admins:server_name is a valid room alias"), ) .unwrap(); diff --git a/src/database/globals.rs b/src/database/globals.rs index 05ecb568..098d8197 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -40,13 +40,13 @@ pub struct Globals { dns_resolver: TokioAsyncResolver, jwt_decoding_key: Option>, pub(super) server_signingkeys: Arc, - pub bad_event_ratelimiter: Arc>>, + pub bad_event_ratelimiter: Arc, RateLimitState>>>, pub bad_signature_ratelimiter: Arc, RateLimitState>>>, pub servername_ratelimiter: Arc, Arc>>>, - pub sync_receivers: RwLock), SyncHandle>>, - pub roomid_mutex_insert: RwLock>>>, - pub roomid_mutex_state: RwLock>>>, - pub roomid_mutex_federation: RwLock>>>, // this lock will be held longer + pub sync_receivers: RwLock, Box), SyncHandle>>, + pub roomid_mutex_insert: RwLock, Arc>>>, + pub roomid_mutex_state: RwLock, Arc>>>, + pub roomid_mutex_federation: RwLock, Arc>>>, // this lock will be held longer pub rotate: RotationHandler, } @@ -254,7 +254,7 @@ impl Globals { &self, origin: &ServerName, new_keys: ServerSigningKeys, - ) -> Result> { + ) -> Result, VerifyKey>> { // Not atomic, but this is not critical let signingkeys = self.server_signingkeys.get(origin.as_bytes())?; @@ -293,7 +293,7 @@ impl Globals { pub fn signing_keys_for( &self, origin: &ServerName, - ) -> Result> { + ) -> Result, VerifyKey>> { let signingkeys = self .server_signingkeys .get(origin.as_bytes())? diff --git a/src/database/key_backups.rs b/src/database/key_backups.rs index 98ea0111..3010a37b 100644 --- a/src/database/key_backups.rs +++ b/src/database/key_backups.rs @@ -209,13 +209,13 @@ impl KeyBackups { &self, user_id: &UserId, version: &str, - ) -> Result> { + ) -> Result, RoomKeyBackup>> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xff); prefix.extend_from_slice(version.as_bytes()); prefix.push(0xff); - let mut rooms = BTreeMap::::new(); + let mut rooms = BTreeMap::, RoomKeyBackup>::new(); for result in self .backupkeyid_backup @@ -231,7 +231,7 @@ impl KeyBackups { Error::bad_database("backupkeyid_backup session_id is invalid.") })?; - let room_id = RoomId::try_from( + let room_id = Box::::try_from( utils::string_from_bytes(parts.next().ok_or_else(|| { Error::bad_database("backupkeyid_backup key is invalid.") })?) diff --git a/src/database/pusher.rs b/src/database/pusher.rs index f53f137b..97ca85d8 100644 --- a/src/database/pusher.rs +++ b/src/database/pusher.rs @@ -234,7 +234,7 @@ pub fn get_actions<'a>( db: &Database, ) -> Result<&'a [Action]> { let ctx = PushConditionRoomCtx { - room_id: room_id.clone(), + room_id: room_id.to_owned(), member_count: 10_u32.into(), // TODO: get member count efficiently user_display_name: db .users @@ -277,7 +277,7 @@ async fn send_notice( let mut data_minus_url = pusher.data.clone(); // The url must be stripped off according to spec data_minus_url.url = None; - device.data = Some(data_minus_url); + device.data = data_minus_url; // Tweaks are only added if the format is NOT event_id_only if !event_id_only { diff --git a/src/database/rooms.rs b/src/database/rooms.rs index c5b795bd..ebd0941b 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -107,14 +107,14 @@ pub struct Rooms { /// RoomId + EventId -> Parent PDU EventId. pub(super) referencedevents: Arc, - pub(super) pdu_cache: Mutex>>, + pub(super) pdu_cache: Mutex, Arc>>, pub(super) shorteventid_cache: Mutex>>, pub(super) auth_chain_cache: Mutex, Arc>>>, - pub(super) eventidshort_cache: Mutex>, + pub(super) eventidshort_cache: Mutex, u64>>, pub(super) statekeyshort_cache: Mutex>, pub(super) shortstatekey_cache: Mutex>, - pub(super) our_real_users_cache: RwLock>>>, - pub(super) appservice_in_room_cache: RwLock>>, + pub(super) our_real_users_cache: RwLock, Arc>>>>, + pub(super) appservice_in_room_cache: RwLock, HashMap>>, pub(super) stateinfo_cache: Mutex< LruCache< u64, @@ -434,7 +434,7 @@ impl Rooms { None => continue, }; - let user_id = match UserId::try_from(state_key) { + let user_id = match Box::::try_from(state_key) { Ok(id) => id, Err(_) => continue, }; @@ -742,7 +742,7 @@ impl Rooms { self.eventidshort_cache .lock() .unwrap() - .insert(event_id.clone(), short); + .insert(event_id.to_owned(), short); Ok(short) } @@ -871,8 +871,8 @@ impl Rooms { .get(&shorteventid.to_be_bytes())? .ok_or_else(|| Error::bad_database("Shorteventid does not exist"))?; - let event_id = Arc::new( - EventId::try_from(utils::string_from_bytes(&bytes).map_err(|_| { + let event_id = Arc::from( + Box::::try_from(utils::string_from_bytes(&bytes).map_err(|_| { Error::bad_database("EventID in shorteventid_eventid is invalid unicode.") })?) .map_err(|_| Error::bad_database("EventId in shorteventid_eventid is invalid."))?, @@ -1112,7 +1112,7 @@ impl Rooms { self.pdu_cache .lock() .unwrap() - .insert(event_id.clone(), Arc::clone(&pdu)); + .insert(event_id.to_owned(), Arc::clone(&pdu)); Ok(Some(pdu)) } else { Ok(None) @@ -1162,14 +1162,14 @@ impl Rooms { /// Returns the leaf pdus of a room. #[tracing::instrument(skip(self))] - pub fn get_pdu_leaves(&self, room_id: &RoomId) -> Result> { + pub fn get_pdu_leaves(&self, room_id: &RoomId) -> Result>> { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); self.roomid_pduleaves .scan_prefix(prefix) .map(|(_, bytes)| { - EventId::try_from(utils::string_from_bytes(&bytes).map_err(|_| { + Box::::try_from(utils::string_from_bytes(&bytes).map_err(|_| { Error::bad_database("EventID in roomid_pduleaves is invalid unicode.") })?) .map_err(|_| Error::bad_database("EventId in roomid_pduleaves is invalid.")) @@ -1178,7 +1178,7 @@ impl Rooms { } #[tracing::instrument(skip(self, room_id, event_ids))] - pub fn mark_as_referenced(&self, room_id: &RoomId, event_ids: &[EventId]) -> Result<()> { + pub fn mark_as_referenced(&self, room_id: &RoomId, event_ids: &[Box]) -> Result<()> { for prev in event_ids { let mut key = room_id.as_bytes().to_vec(); key.extend_from_slice(prev.as_bytes()); @@ -1193,7 +1193,7 @@ impl Rooms { /// The provided `event_ids` become the new leaves, this allows a room to have multiple /// `prev_events`. #[tracing::instrument(skip(self))] - pub fn replace_pdu_leaves(&self, room_id: &RoomId, event_ids: &[EventId]) -> Result<()> { + pub fn replace_pdu_leaves(&self, room_id: &RoomId, event_ids: &[Box]) -> Result<()> { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); @@ -1261,7 +1261,7 @@ impl Rooms { &self, pdu: &PduEvent, mut pdu_json: CanonicalJsonObject, - leaves: &[EventId], + leaves: &[Box], db: &Database, ) -> Result> { let shortroomid = self.get_shortroomid(&pdu.room_id)?.expect("room exists"); @@ -1420,7 +1420,7 @@ impl Rooms { } // if the state_key fails - let target_user_id = UserId::try_from(state_key.clone()) + let target_user_id = Box::::try_from(state_key.clone()) .expect("This state_key was previously validated"); let content = serde_json::from_str::(pdu.content.get()) @@ -1476,9 +1476,11 @@ impl Rooms { if body.starts_with(&format!("@conduit:{}: ", db.globals.server_name())) && self .id_from_alias( - &format!("#admins:{}", db.globals.server_name()) - .try_into() - .expect("#admins:server_name is a valid room alias"), + &Box::::try_from(format!( + "#admins:{}", + db.globals.server_name() + )) + .expect("#admins:server_name is a valid room alias"), )? .as_ref() == Some(&pdu.room_id) @@ -1528,7 +1530,7 @@ impl Rooms { } "get_auth_chain" => { if args.len() == 1 { - if let Ok(event_id) = EventId::try_from(args[0]) { + if let Ok(event_id) = Box::::try_from(args[0]) { if let Some(event) = db.rooms.get_pdu_json(&event_id)? { let room_id_str = event .get("room_id") @@ -1539,12 +1541,12 @@ impl Rooms { ) })?; - let room_id = RoomId::try_from(room_id_str) + let room_id = Box::::try_from(room_id_str) .map_err(|_| Error::bad_database("Invalid room id field in event in database"))?; let start = Instant::now(); let count = server_server::get_auth_chain( &room_id, - vec![Arc::new(event_id)], + vec![Arc::from(event_id)], db, )? .count(); @@ -1567,12 +1569,12 @@ impl Rooms { let string = body[1..body.len() - 1].join("\n"); match serde_json::from_str(&string) { Ok(value) => { - let event_id = EventId::try_from(&*format!( + let event_id = Box::::try_from(&*format!( "${}", // Anything higher than version3 behaves the same ruma::signatures::reference_hash( &value, - &RoomVersionId::Version6 + &RoomVersionId::V6 ) .expect("ruma can calculate reference hashes") )) @@ -1622,7 +1624,7 @@ impl Rooms { } "get_pdu" => { if args.len() == 1 { - if let Ok(event_id) = EventId::try_from(args[0]) { + if let Ok(event_id) = Box::::try_from(args[0]) { let mut outlier = false; let mut pdu_json = db.rooms.get_non_outlier_pdu_json(&event_id)?; @@ -1948,7 +1950,7 @@ impl Rooms { room_id: &RoomId, db: &Database, _mutex_lock: &MutexGuard<'_, ()>, // Take mutex guard to make sure users get the room mutex - ) -> Result { + ) -> Result> { let PduBuilder { event_type, content, @@ -1985,9 +1987,7 @@ impl Rooms { // If there was no create event yet, assume we are creating a version 6 room right now let room_version_id = create_event_content - .map_or(RoomVersionId::Version6, |create_event| { - create_event.room_version - }); + .map_or(RoomVersionId::V6, |create_event| create_event.room_version); let room_version = RoomVersion::new(&room_version_id).expect("room version is supported"); let auth_events = @@ -2016,9 +2016,9 @@ impl Rooms { } let mut pdu = PduEvent { - event_id: ruma::event_id!("$thiswillbefilledinlater"), - room_id: room_id.clone(), - sender: sender.clone(), + event_id: ruma::event_id!("$thiswillbefilledinlater").to_owned(), + room_id: room_id.to_owned(), + sender: sender.to_owned(), origin_server_ts: utils::millis_since_unix_epoch() .try_into() .expect("time is valid"), @@ -2083,7 +2083,7 @@ impl Rooms { .expect("event is valid, we just created it"); // Generate event id - pdu.event_id = EventId::try_from(&*format!( + pdu.event_id = Box::::try_from(&*format!( "${}", ruma::signatures::reference_hash(&pdu_json, &room_version_id) .expect("ruma can calculate reference hashes") @@ -2206,7 +2206,7 @@ impl Rooms { let mut first_pdu_id = prefix.clone(); first_pdu_id.extend_from_slice(&(since + 1).to_be_bytes()); - let user_id = user_id.clone(); + let user_id = user_id.to_owned(); Ok(self .pduid_pdu @@ -2243,7 +2243,7 @@ impl Rooms { let current: &[u8] = ¤t; - let user_id = user_id.clone(); + let user_id = user_id.to_owned(); Ok(self .pduid_pdu @@ -2280,7 +2280,7 @@ impl Rooms { let current: &[u8] = ¤t; - let user_id = user_id.clone(); + let user_id = user_id.to_owned(); Ok(self .pduid_pdu @@ -2412,7 +2412,7 @@ impl Rooms { for room_ids in direct_event.content.0.values_mut() { if room_ids.iter().any(|r| r == &predecessor.room_id) { - room_ids.push(room_id.clone()); + room_ids.push(room_id.to_owned()); room_ids_updated = true; } } @@ -2451,7 +2451,11 @@ impl Rooms { EventType::IgnoredUserList, )? .map_or(false, |ignored| { - ignored.content.ignored_users.contains(sender) + ignored + .content + .ignored_users + .iter() + .any(|user| user == sender) }); if is_ignored { @@ -2537,7 +2541,7 @@ impl Rooms { self.our_real_users_cache .write() .unwrap() - .insert(room_id.clone(), Arc::new(real_users)); + .insert(room_id.to_owned(), Arc::new(real_users)); for old_joined_server in self.room_servers(room_id).filter_map(|r| r.ok()) { if !joined_servers.remove(&old_joined_server) { @@ -2582,7 +2586,7 @@ impl Rooms { &self, room_id: &RoomId, db: &Database, - ) -> Result>> { + ) -> Result>>> { let maybe = self .our_real_users_cache .read() @@ -2650,7 +2654,7 @@ impl Rooms { self.appservice_in_room_cache .write() .unwrap() - .entry(room_id.clone()) + .entry(room_id.to_owned()) .or_default() .insert(appservice.0.clone(), in_room); @@ -2694,7 +2698,7 @@ impl Rooms { .roomid_mutex_state .write() .unwrap() - .entry(room_id.clone()) + .entry(room_id.to_owned()) .or_default(), ); let state_lock = mutex_state.lock().await; @@ -2754,7 +2758,7 @@ impl Rooms { .filter_map(|event| serde_json::from_str(event.json().get()).ok()) .filter_map(|event: serde_json::Value| event.get("sender").cloned()) .filter_map(|sender| sender.as_str().map(|s| s.to_owned())) - .filter_map(|sender| UserId::try_from(sender).ok()) + .filter_map(|sender| Box::::try_from(sender).ok()) .map(|user| user.server_name().to_owned()) .collect(); @@ -2778,9 +2782,7 @@ impl Rooms { let (make_leave_response, remote_server) = make_leave_response_and_server?; let room_version_id = match make_leave_response.room_version { - Some(version) - if version == RoomVersionId::Version5 || version == RoomVersionId::Version6 => - { + Some(version) if version == RoomVersionId::V5 || version == RoomVersionId::V6 => { version } _ => return Err(Error::BadServerResponse("Room version is not supported")), @@ -2817,7 +2819,7 @@ impl Rooms { .expect("event is valid, we just created it"); // Generate event id - let event_id = EventId::try_from(&*format!( + let event_id = Box::::try_from(&*format!( "${}", ruma::signatures::reference_hash(&leave_event_stub, &room_version_id) .expect("ruma can calculate reference hashes") @@ -2902,11 +2904,11 @@ impl Rooms { } #[tracing::instrument(skip(self))] - pub fn id_from_alias(&self, alias: &RoomAliasId) -> Result> { + pub fn id_from_alias(&self, alias: &RoomAliasId) -> Result>> { self.alias_roomid .get(alias.alias().as_bytes())? .map(|bytes| { - RoomId::try_from(utils::string_from_bytes(&bytes).map_err(|_| { + Box::::try_from(utils::string_from_bytes(&bytes).map_err(|_| { Error::bad_database("Room ID in alias_roomid is invalid unicode.") })?) .map_err(|_| Error::bad_database("Room ID in alias_roomid is invalid.")) @@ -2918,7 +2920,7 @@ impl Rooms { pub fn room_aliases<'a>( &'a self, room_id: &RoomId, - ) -> impl Iterator> + 'a { + ) -> impl Iterator>> + 'a { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); @@ -2947,9 +2949,9 @@ impl Rooms { } #[tracing::instrument(skip(self))] - pub fn public_rooms(&self) -> impl Iterator> + '_ { + pub fn public_rooms(&self) -> impl Iterator>> + '_ { self.publicroomids.iter().map(|(bytes, _)| { - RoomId::try_from( + Box::::try_from( utils::string_from_bytes(&bytes).map_err(|_| { Error::bad_database("Room ID in publicroomids is invalid unicode.") })?, @@ -3009,8 +3011,8 @@ impl Rooms { #[tracing::instrument(skip(self))] pub fn get_shared_rooms<'a>( &'a self, - users: Vec, - ) -> Result> + 'a> { + users: Vec>, + ) -> Result>> + 'a> { let iterators = users.into_iter().map(move |user_id| { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xff); @@ -3037,7 +3039,7 @@ impl Rooms { Ok(utils::common_elements(iterators, Ord::cmp) .expect("users is not empty") .map(|bytes| { - RoomId::try_from(utils::string_from_bytes(&*bytes).map_err(|_| { + Box::::try_from(utils::string_from_bytes(&*bytes).map_err(|_| { Error::bad_database("Invalid RoomId bytes in userroomid_joined") })?) .map_err(|_| Error::bad_database("Invalid RoomId in userroomid_joined.")) @@ -3082,12 +3084,12 @@ impl Rooms { pub fn server_rooms<'a>( &'a self, server: &ServerName, - ) -> impl Iterator> + 'a { + ) -> impl Iterator>> + 'a { let mut prefix = server.as_bytes().to_vec(); prefix.push(0xff); self.serverroomids.scan_prefix(prefix).map(|(key, _)| { - RoomId::try_from( + Box::::try_from( utils::string_from_bytes( key.rsplit(|&b| b == 0xff) .next() @@ -3104,12 +3106,12 @@ impl Rooms { pub fn room_members<'a>( &'a self, room_id: &RoomId, - ) -> impl Iterator> + 'a { + ) -> impl Iterator>> + 'a { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); self.roomuserid_joined.scan_prefix(prefix).map(|(key, _)| { - UserId::try_from( + Box::::try_from( utils::string_from_bytes( key.rsplit(|&b| b == 0xff) .next() @@ -3150,14 +3152,14 @@ impl Rooms { pub fn room_useroncejoined<'a>( &'a self, room_id: &RoomId, - ) -> impl Iterator> + 'a { + ) -> impl Iterator>> + 'a { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); self.roomuseroncejoinedids .scan_prefix(prefix) .map(|(key, _)| { - UserId::try_from( + Box::::try_from( utils::string_from_bytes( key.rsplit(|&b| b == 0xff) .next() @@ -3176,14 +3178,14 @@ impl Rooms { pub fn room_members_invited<'a>( &'a self, room_id: &RoomId, - ) -> impl Iterator> + 'a { + ) -> impl Iterator>> + 'a { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); self.roomuserid_invitecount .scan_prefix(prefix) .map(|(key, _)| { - UserId::try_from( + Box::::try_from( utils::string_from_bytes( key.rsplit(|&b| b == 0xff) .next() @@ -3232,11 +3234,11 @@ impl Rooms { pub fn rooms_joined<'a>( &'a self, user_id: &UserId, - ) -> impl Iterator> + 'a { + ) -> impl Iterator>> + 'a { self.userroomid_joined .scan_prefix(user_id.as_bytes().to_vec()) .map(|(key, _)| { - RoomId::try_from( + Box::::try_from( utils::string_from_bytes( key.rsplit(|&b| b == 0xff) .next() @@ -3255,14 +3257,14 @@ impl Rooms { pub fn rooms_invited<'a>( &'a self, user_id: &UserId, - ) -> impl Iterator>)>> + 'a { + ) -> impl Iterator, Vec>)>> + 'a { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xff); self.userroomid_invitestate .scan_prefix(prefix) .map(|(key, state)| { - let room_id = RoomId::try_from( + let room_id = Box::::try_from( utils::string_from_bytes( key.rsplit(|&b| b == 0xff) .next() @@ -3328,14 +3330,14 @@ impl Rooms { pub fn rooms_left<'a>( &'a self, user_id: &UserId, - ) -> impl Iterator>)>> + 'a { + ) -> impl Iterator, Vec>)>> + 'a { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xff); self.userroomid_leftstate .scan_prefix(prefix) .map(|(key, state)| { - let room_id = RoomId::try_from( + let room_id = Box::::try_from( utils::string_from_bytes( key.rsplit(|&b| b == 0xff) .next() diff --git a/src/database/rooms/edus.rs b/src/database/rooms/edus.rs index 9a27e437..365211b6 100644 --- a/src/database/rooms/edus.rs +++ b/src/database/rooms/edus.rs @@ -76,8 +76,13 @@ impl RoomEdus { &'a self, room_id: &RoomId, since: u64, - ) -> impl Iterator)>> + 'a - { + ) -> impl Iterator< + Item = Result<( + Box, + u64, + Raw, + )>, + > + 'a { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); let prefix2 = prefix.clone(); @@ -92,7 +97,7 @@ impl RoomEdus { let count = utils::u64_from_bytes(&k[prefix.len()..prefix.len() + mem::size_of::()]) .map_err(|_| Error::bad_database("Invalid readreceiptid count in db."))?; - let user_id = UserId::try_from( + let user_id = Box::::try_from( utils::string_from_bytes(&k[prefix.len() + mem::size_of::() + 1..]) .map_err(|_| { Error::bad_database("Invalid readreceiptid userid bytes in db.") @@ -309,7 +314,7 @@ impl RoomEdus { .typingid_userid .scan_prefix(prefix) .map(|(_, user_id)| { - UserId::try_from(utils::string_from_bytes(&user_id).map_err(|_| { + Box::::try_from(utils::string_from_bytes(&user_id).map_err(|_| { Error::bad_database("User ID in typingid_userid is invalid unicode.") })?) .map_err(|_| Error::bad_database("User ID in typingid_userid is invalid.")) @@ -449,7 +454,7 @@ impl RoomEdus { { // Send new presence events to set the user offline let count = globals.next_count()?.to_be_bytes(); - let user_id = utils::string_from_bytes(&user_id_bytes) + let user_id: Box<_> = utils::string_from_bytes(&user_id_bytes) .map_err(|_| { Error::bad_database("Invalid UserId bytes in userid_lastpresenceupdate.") })? @@ -475,7 +480,7 @@ impl RoomEdus { presence: PresenceState::Offline, status_msg: None, }, - sender: user_id.clone(), + sender: user_id.to_owned(), }) .expect("PresenceEvent can be serialized"), )?; @@ -498,7 +503,7 @@ impl RoomEdus { since: u64, _rooms: &super::Rooms, _globals: &super::super::globals::Globals, - ) -> Result> { + ) -> Result, PresenceEvent>> { //self.presence_maintain(rooms, globals)?; let mut prefix = room_id.as_bytes().to_vec(); @@ -513,7 +518,7 @@ impl RoomEdus { .iter_from(&*first_possible_edu, false) .take_while(|(key, _)| key.starts_with(&prefix)) { - let user_id = UserId::try_from( + let user_id = Box::::try_from( utils::string_from_bytes( key.rsplit(|&b| b == 0xff) .next() diff --git a/src/database/sending.rs b/src/database/sending.rs index bf0cc2c1..c27b5731 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -397,7 +397,7 @@ impl Sending { // Because synapse resyncs, we can just insert dummy data let edu = Edu::DeviceListUpdate(DeviceListUpdateContent { user_id, - device_id: device_id!("dummy"), + device_id: device_id!("dummy").to_owned(), device_display_name: Some("Dummy".to_owned()), stream_id: uint!(1), prev_id: Vec::new(), @@ -584,7 +584,7 @@ impl Sending { } let userid = - UserId::try_from(utils::string_from_bytes(user).map_err(|_| { + Box::::try_from(utils::string_from_bytes(user).map_err(|_| { ( kind.clone(), Error::bad_database("Invalid push user string in db."), diff --git a/src/database/users.rs b/src/database/users.rs index d0da0714..4a084722 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -63,11 +63,11 @@ impl Users { globals: &super::globals::Globals, ) -> Result { let admin_room_alias_id = - RoomAliasId::try_from(format!("#admins:{}", globals.server_name())) + Box::::try_from(format!("#admins:{}", globals.server_name())) .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid alias."))?; let admin_room_id = rooms.id_from_alias(&admin_room_alias_id)?.unwrap(); - Ok(rooms.is_joined(user_id, &admin_room_id)?) + rooms.is_joined(user_id, &admin_room_id) } /// Create a new user account on this homeserver. @@ -85,7 +85,7 @@ impl Users { /// Find out which user an access token belongs to. #[tracing::instrument(skip(self, token))] - pub fn find_from_token(&self, token: &str) -> Result> { + pub fn find_from_token(&self, token: &str) -> Result, String)>> { self.token_userdeviceid .get(token.as_bytes())? .map_or(Ok(None), |bytes| { @@ -98,9 +98,11 @@ impl Users { })?; Ok(Some(( - UserId::try_from(utils::string_from_bytes(user_bytes).map_err(|_| { - Error::bad_database("User ID in token_userdeviceid is invalid unicode.") - })?) + Box::::try_from(utils::string_from_bytes(user_bytes).map_err( + |_| { + Error::bad_database("User ID in token_userdeviceid is invalid unicode.") + }, + )?) .map_err(|_| { Error::bad_database("User ID in token_userdeviceid is invalid.") })?, @@ -113,9 +115,9 @@ impl Users { /// Returns an iterator over all users on this homeserver. #[tracing::instrument(skip(self))] - pub fn iter(&self) -> impl Iterator> + '_ { + pub fn iter(&self) -> impl Iterator>> + '_ { self.userid_password.iter().map(|(bytes, _)| { - UserId::try_from(utils::string_from_bytes(&bytes).map_err(|_| { + Box::::try_from(utils::string_from_bytes(&bytes).map_err(|_| { Error::bad_database("User ID in userid_password is invalid unicode.") })?) .map_err(|_| Error::bad_database("User ID in userid_password is invalid.")) @@ -181,20 +183,21 @@ impl Users { /// Get the avatar_url of a user. #[tracing::instrument(skip(self, user_id))] - pub fn avatar_url(&self, user_id: &UserId) -> Result> { + pub fn avatar_url(&self, user_id: &UserId) -> Result>> { self.userid_avatarurl .get(user_id.as_bytes())? .map(|bytes| { let s = utils::string_from_bytes(&bytes) .map_err(|_| Error::bad_database("Avatar URL in db is invalid."))?; - MxcUri::try_from(s).map_err(|_| Error::bad_database("Avatar URL in db is invalid.")) + Box::::try_from(s) + .map_err(|_| Error::bad_database("Avatar URL in db is invalid.")) }) .transpose() } /// Sets a new avatar_url or removes it if avatar_url is None. #[tracing::instrument(skip(self, user_id, avatar_url))] - pub fn set_avatar_url(&self, user_id: &UserId, avatar_url: Option) -> Result<()> { + pub fn set_avatar_url(&self, user_id: &UserId, avatar_url: Option>) -> Result<()> { if let Some(avatar_url) = avatar_url { self.userid_avatarurl .insert(user_id.as_bytes(), avatar_url.to_string().as_bytes())?; @@ -409,7 +412,7 @@ impl Users { device_id: &DeviceId, key_algorithm: &DeviceKeyAlgorithm, globals: &super::globals::Globals, - ) -> Result> { + ) -> Result, OneTimeKey)>> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xff); prefix.extend_from_slice(device_id.as_bytes()); @@ -459,7 +462,7 @@ impl Users { .scan_prefix(userdeviceid) .map(|(bytes, _)| { Ok::<_, Error>( - serde_json::from_slice::( + serde_json::from_slice::>( &*bytes.rsplit(|&b| b == 0xff).next().ok_or_else(|| { Error::bad_database("OneTimeKey ID in db is invalid.") })?, @@ -632,7 +635,7 @@ impl Users { .ok_or_else(|| Error::bad_database("key in keyid_key has no signatures field."))? .as_object_mut() .ok_or_else(|| Error::bad_database("key in keyid_key has invalid signatures field."))? - .entry(sender_id.clone()) + .entry(sender_id.to_owned()) .or_insert_with(|| serde_json::Map::new().into()); signatures @@ -657,7 +660,7 @@ impl Users { user_or_room_id: &str, from: u64, to: Option, - ) -> impl Iterator> + 'a { + ) -> impl Iterator>> + 'a { let mut prefix = user_or_room_id.as_bytes().to_vec(); prefix.push(0xff); @@ -683,7 +686,7 @@ impl Users { } }) .map(|(_, bytes)| { - UserId::try_from(utils::string_from_bytes(&bytes).map_err(|_| { + Box::::try_from(utils::string_from_bytes(&bytes).map_err(|_| { Error::bad_database("User ID in devicekeychangeid_userid is invalid unicode.") })?) .map_err(|_| Error::bad_database("User ID in devicekeychangeid_userid is invalid.")) diff --git a/src/pdu.rs b/src/pdu.rs index 0f99f43b..3c955976 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -13,7 +13,7 @@ use serde_json::{ json, value::{to_raw_value, RawValue as RawJsonValue}, }; -use std::{cmp::Ordering, collections::BTreeMap, convert::TryFrom}; +use std::{cmp::Ordering, collections::BTreeMap, convert::TryFrom, ops::Deref}; use tracing::warn; /// Content hashes of a PDU. @@ -25,20 +25,20 @@ pub struct EventHash { #[derive(Clone, Deserialize, Serialize, Debug)] pub struct PduEvent { - pub event_id: EventId, - pub room_id: RoomId, - pub sender: UserId, + pub event_id: Box, + pub room_id: Box, + pub sender: Box, pub origin_server_ts: UInt, #[serde(rename = "type")] pub kind: EventType, pub content: Box, #[serde(skip_serializing_if = "Option::is_none")] pub state_key: Option, - pub prev_events: Vec, + pub prev_events: Vec>, pub depth: UInt, - pub auth_events: Vec, + pub auth_events: Vec>, #[serde(skip_serializing_if = "Option::is_none")] - pub redacts: Option, + pub redacts: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub unsigned: Option>, pub hashes: EventHash, @@ -295,15 +295,15 @@ impl state_res::Event for PduEvent { } fn prev_events(&self) -> Box + '_> { - Box::new(self.prev_events.iter()) + Box::new(self.prev_events.iter().map(Deref::deref)) } fn auth_events(&self) -> Box + '_> { - Box::new(self.auth_events.iter()) + Box::new(self.auth_events.iter().map(Deref::deref)) } fn redacts(&self) -> Option<&EventId> { - self.redacts.as_ref() + self.redacts.as_deref() } } @@ -331,16 +331,16 @@ impl Ord for PduEvent { /// Returns a tuple of the new `EventId` and the PDU as a `BTreeMap`. pub(crate) fn gen_event_id_canonical_json( pdu: &RawJsonValue, -) -> crate::Result<(EventId, CanonicalJsonObject)> { +) -> crate::Result<(Box, CanonicalJsonObject)> { let value = serde_json::from_str(pdu.get()).map_err(|e| { warn!("Error parsing incoming event {:?}: {:?}", pdu, e); Error::BadServerResponse("Invalid PDU in server response") })?; - let event_id = EventId::try_from(&*format!( + let event_id = Box::::try_from(&*format!( "${}", // Anything higher than version3 behaves the same - ruma::signatures::reference_hash(&value, &RoomVersionId::Version6) + ruma::signatures::reference_hash(&value, &RoomVersionId::V6) .expect("ruma can calculate reference hashes") )) .expect("ruma's reference hashes are valid event ids"); @@ -356,7 +356,7 @@ pub struct PduBuilder { pub content: Box, pub unsigned: Option>, pub state_key: Option, - pub redacts: Option, + pub redacts: Option>, } /// Direct conversion prevents loss of the empty `state_key` that ruma requires. diff --git a/src/ruma_wrapper.rs b/src/ruma_wrapper.rs index 03c115cd..2cff2f5a 100644 --- a/src/ruma_wrapper.rs +++ b/src/ruma_wrapper.rs @@ -29,7 +29,7 @@ use { /// first. pub struct Ruma { pub body: T::Incoming, - pub sender_user: Option, + pub sender_user: Option>, pub sender_device: Option>, pub sender_servername: Option>, // This is None when body is not a valid string @@ -86,7 +86,7 @@ where registration .get("as_token") .and_then(|as_token| as_token.as_str()) - .map_or(false, |as_token| token.as_deref() == Some(as_token)) + .map_or(false, |as_token| token == Some(as_token)) }) { match metadata.authentication { AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => { @@ -103,7 +103,7 @@ where .unwrap() }, |string| { - UserId::try_from(string.expect("parsing to string always works")) + Box::::try_from(string.expect("parsing to string always works")) .unwrap() }, ); diff --git a/src/server_server.rs b/src/server_server.rs index 482edf0f..ec5bc345 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -552,7 +552,7 @@ pub fn get_server_keys_route(db: DatabaseGuard) -> Json { let mut verify_keys = BTreeMap::new(); verify_keys.insert( - ServerSigningKeyId::try_from( + Box::::try_from( format!("ed25519:{}", db.globals.keypair().version()).as_str(), ) .expect("found invalid server signing keys in DB"), @@ -736,7 +736,7 @@ pub async fn send_transaction_message_route( // 0. Check the server is in the room let room_id = match value .get("room_id") - .and_then(|id| RoomId::try_from(id.as_str()?).ok()) + .and_then(|id| Box::::try_from(id.as_str()?).ok()) { Some(id) => id, None => { @@ -1003,11 +1003,10 @@ pub(crate) async fn handle_incoming_pdu<'a>( // 9. Fetch any missing prev events doing all checks listed here starting at 1. These are timeline events let mut graph = HashMap::new(); let mut eventid_info = HashMap::new(); - let mut todo_outlier_stack: Vec<_> = incoming_pdu + let mut todo_outlier_stack: Vec> = incoming_pdu .prev_events .iter() - .cloned() - .map(Arc::new) + .map(|x| Arc::from(&**x)) .collect(); let mut amount = 0; @@ -1027,7 +1026,7 @@ pub(crate) async fn handle_incoming_pdu<'a>( if amount > 100 { // Max limit reached warn!("Max prev event limit reached!"); - graph.insert((*prev_event_id).clone(), HashSet::new()); + graph.insert((*prev_event_id).to_owned(), HashSet::new()); continue; } @@ -1038,27 +1037,27 @@ pub(crate) async fn handle_incoming_pdu<'a>( amount += 1; for prev_prev in &pdu.prev_events { if !graph.contains_key(prev_prev) { - todo_outlier_stack.push(dbg!(Arc::new(prev_prev.clone()))); + todo_outlier_stack.push(dbg!(Arc::from(&**prev_prev))); } } graph.insert( - (*prev_event_id).clone(), + (*prev_event_id).to_owned(), pdu.prev_events.iter().cloned().collect(), ); } else { // Time based check failed - graph.insert((*prev_event_id).clone(), HashSet::new()); + graph.insert((*prev_event_id).to_owned(), HashSet::new()); } eventid_info.insert(prev_event_id.clone(), (pdu, json)); } else { // Get json failed - graph.insert((*prev_event_id).clone(), HashSet::new()); + graph.insert((*prev_event_id).to_owned(), HashSet::new()); } } else { // Fetch and handle failed - graph.insert((*prev_event_id).clone(), HashSet::new()); + graph.insert((*prev_event_id).to_owned(), HashSet::new()); } } @@ -1074,7 +1073,6 @@ pub(crate) async fn handle_incoming_pdu<'a>( .get(event_id) .map_or_else(|| uint!(0), |info| info.0.origin_server_ts), ), - ruma::event_id!("$notimportant"), )) }) .map_err(|_| "Error sorting prev events".to_owned())?; @@ -1084,7 +1082,7 @@ pub(crate) async fn handle_incoming_pdu<'a>( if errors >= 5 { break; } - if let Some((pdu, json)) = eventid_info.remove(&prev_id) { + if let Some((pdu, json)) = eventid_info.remove(&*prev_id) { if pdu.origin_server_ts < first_pdu_in_room.origin_server_ts { continue; } @@ -1200,8 +1198,7 @@ fn handle_outlier_pdu<'a>( &incoming_pdu .auth_events .iter() - .cloned() - .map(Arc::new) + .map(|x| Arc::from(&**x)) .collect::>(), create_event, room_id, @@ -1331,7 +1328,7 @@ async fn upgrade_outlier_to_timeline_pdu( let mut state_at_incoming_event = None; if incoming_pdu.prev_events.len() == 1 { - let prev_event = &incoming_pdu.prev_events[0]; + let prev_event = &*incoming_pdu.prev_events[0]; let prev_event_sstatehash = db .rooms .pdu_shortstatehash(prev_event) @@ -1353,7 +1350,7 @@ async fn upgrade_outlier_to_timeline_pdu( .get_or_create_shortstatekey(&prev_pdu.kind, state_key, &db.globals) .map_err(|_| "Failed to create shortstatekey.".to_owned())?; - state.insert(shortstatekey, Arc::new(prev_event.clone())); + state.insert(shortstatekey, Arc::from(prev_event)); // Now it's the state after the pdu } @@ -1397,7 +1394,7 @@ async fn upgrade_outlier_to_timeline_pdu( .rooms .get_or_create_shortstatekey(&prev_event.kind, state_key, &db.globals) .map_err(|_| "Failed to create shortstatekey.".to_owned())?; - leaf_state.insert(shortstatekey, Arc::new(prev_event.event_id.clone())); + leaf_state.insert(shortstatekey, Arc::from(&*prev_event.event_id)); // Now it's the state after the pdu } @@ -1410,14 +1407,14 @@ async fn upgrade_outlier_to_timeline_pdu( .get_statekey_from_short(k) .map_err(|_| "Failed to get_statekey_from_short.".to_owned())?; - state.insert(k, (*id).clone()); + state.insert(k, (*id).to_owned()); starting_events.push(id); } auth_chain_sets.push( get_auth_chain(room_id, starting_events, db) .map_err(|_| "Failed to load auth chain.".to_owned())? - .map(|event_id| (*event_id).clone()) + .map(|event_id| (*event_id).to_owned()) .collect(), ); @@ -1444,7 +1441,7 @@ async fn upgrade_outlier_to_timeline_pdu( .rooms .get_or_create_shortstatekey(&event_type, &state_key, &db.globals) .map_err(|_| "Failed to get_or_create_shortstatekey".to_owned())?; - Ok((shortstatekey, Arc::new(event_id))) + Ok((shortstatekey, Arc::from(event_id))) }) .collect::>()?, ), @@ -1479,8 +1476,7 @@ async fn upgrade_outlier_to_timeline_pdu( origin, &res.pdu_ids .iter() - .cloned() - .map(Arc::new) + .map(|x| Arc::from(&**x)) .collect::>(), create_event, room_id, @@ -1488,7 +1484,7 @@ async fn upgrade_outlier_to_timeline_pdu( ) .await; - let mut state = BTreeMap::new(); + let mut state: BTreeMap<_, Arc> = BTreeMap::new(); for (pdu, _) in state_vec { let state_key = pdu .state_key @@ -1502,7 +1498,7 @@ async fn upgrade_outlier_to_timeline_pdu( match state.entry(shortstatekey) { btree_map::Entry::Vacant(v) => { - v.insert(Arc::new(pdu.event_id.clone())); + v.insert(Arc::from(&*pdu.event_id)); } btree_map::Entry::Occupied(_) => return Err( "State event's type and state_key combination exists multiple times." @@ -1577,7 +1573,7 @@ async fn upgrade_outlier_to_timeline_pdu( .roomid_mutex_state .write() .unwrap() - .entry(room_id.clone()) + .entry(room_id.to_owned()) .or_default(), ); let state_lock = mutex_state.lock().await; @@ -1715,7 +1711,7 @@ async fn upgrade_outlier_to_timeline_pdu( .rooms .get_or_create_shortstatekey(&leaf_pdu.kind, state_key, &db.globals) .map_err(|_| "Failed to create shortstatekey.".to_owned())?; - leaf_state.insert(shortstatekey, Arc::new(leaf_pdu.event_id.clone())); + leaf_state.insert(shortstatekey, Arc::from(&*leaf_pdu.event_id)); // Now it's the state after the pdu } @@ -1730,7 +1726,7 @@ async fn upgrade_outlier_to_timeline_pdu( .get_or_create_shortstatekey(&incoming_pdu.kind, state_key, &db.globals) .map_err(|_| "Failed to create shortstatekey.".to_owned())?; - state_after.insert(shortstatekey, Arc::new(incoming_pdu.event_id.clone())); + state_after.insert(shortstatekey, Arc::from(&*incoming_pdu.event_id)); } fork_states.push(state_after); @@ -1762,7 +1758,7 @@ async fn upgrade_outlier_to_timeline_pdu( db, ) .map_err(|_| "Failed to load auth chain.".to_owned())? - .map(|event_id| (*event_id).clone()) + .map(|event_id| (*event_id).to_owned()) .collect(), ); } @@ -1774,7 +1770,7 @@ async fn upgrade_outlier_to_timeline_pdu( .map(|(k, id)| { db.rooms .get_statekey_from_short(k) - .map(|k| (k, (*id).clone())) + .map(|k| (k, (*id).to_owned())) }) .collect::>>() }) @@ -1874,7 +1870,8 @@ pub(crate) fn fetch_and_handle_outliers<'a>( let mut pdus = vec![]; for id in events { - if let Some((time, tries)) = db.globals.bad_event_ratelimiter.read().unwrap().get(id) { + if let Some((time, tries)) = db.globals.bad_event_ratelimiter.read().unwrap().get(&**id) + { // Exponential backoff let mut min_elapsed_duration = Duration::from_secs(5 * 60) * (*tries) * (*tries); if min_elapsed_duration > Duration::from_secs(60 * 60 * 24) { @@ -1914,7 +1911,7 @@ pub(crate) fn fetch_and_handle_outliers<'a>( match crate::pdu::gen_event_id_canonical_json(&res.pdu) { Ok(t) => t, Err(_) => { - back_off((**id).clone()); + back_off((**id).to_owned()); continue; } }; @@ -1939,14 +1936,14 @@ pub(crate) fn fetch_and_handle_outliers<'a>( Ok((pdu, json)) => (pdu, Some(json)), Err(e) => { warn!("Authentication of event {} failed: {:?}", id, e); - back_off((**id).clone()); + back_off((**id).to_owned()); continue; } } } Err(_) => { warn!("Failed to fetch event: {}", id); - back_off((**id).clone()); + back_off((**id).to_owned()); continue; } } @@ -2128,7 +2125,7 @@ fn append_incoming_pdu( db: &Database, pdu: &PduEvent, pdu_json: CanonicalJsonObject, - new_room_leaves: HashSet, + new_room_leaves: HashSet>, state_ids_compressed: HashSet, soft_fail: bool, _mutex_lock: &MutexGuard<'_, ()>, // Take mutex guard to make sure users get the room mutex @@ -2298,13 +2295,13 @@ fn get_auth_chain_inner( event_id: &EventId, db: &Database, ) -> Result> { - let mut todo = vec![event_id.clone()]; + let mut todo = vec![event_id.to_owned()]; let mut found = HashSet::new(); while let Some(event_id) = todo.pop() { match db.rooms.get_pdu(&event_id) { Ok(Some(pdu)) => { - if &pdu.room_id != room_id { + if pdu.room_id != room_id { return Err(Error::BadRequest(ErrorKind::Forbidden, "Evil event in db")); } for auth_event in &pdu.auth_events { @@ -2314,7 +2311,7 @@ fn get_auth_chain_inner( if !found.contains(&sauthevent) { found.insert(sauthevent); - todo.push(auth_event.clone()); + todo.push(auth_event.to_owned()); } } } @@ -2363,7 +2360,7 @@ pub fn get_event_route( .and_then(|val| val.as_str()) .ok_or_else(|| Error::bad_database("Invalid event in database"))?; - let room_id = RoomId::try_from(room_id_str) + let room_id = Box::::try_from(room_id_str) .map_err(|_| Error::bad_database("Invalid room id field in event in database"))?; if !db.rooms.server_in_room(sender_servername, &room_id)? { @@ -2417,7 +2414,7 @@ pub fn get_missing_events_route( .and_then(|val| val.as_str()) .ok_or_else(|| Error::bad_database("Invalid event in database"))?; - let event_room_id = RoomId::try_from(room_id_str) + let event_room_id = Box::::try_from(room_id_str) .map_err(|_| Error::bad_database("Invalid room id field in event in database"))?; if event_room_id != body.room_id { @@ -2436,7 +2433,7 @@ pub fn get_missing_events_route( continue; } queued_events.extend_from_slice( - &serde_json::from_value::>( + &serde_json::from_value::>>( serde_json::to_value(pdu.get("prev_events").cloned().ok_or_else(|| { Error::bad_database("Event in db has no prev_events field.") })?) @@ -2485,14 +2482,14 @@ pub fn get_event_authorization_route( .and_then(|val| val.as_str()) .ok_or_else(|| Error::bad_database("Invalid event in database"))?; - let room_id = RoomId::try_from(room_id_str) + let room_id = Box::::try_from(room_id_str) .map_err(|_| Error::bad_database("Invalid room id field in event in database"))?; if !db.rooms.server_in_room(sender_servername, &room_id)? { return Err(Error::BadRequest(ErrorKind::NotFound, "Event not found.")); } - let auth_chain_ids = get_auth_chain(&room_id, vec![Arc::new(body.event_id.clone())], &db)?; + let auth_chain_ids = get_auth_chain(&room_id, vec![Arc::from(&*body.event_id)], &db)?; Ok(get_event_authorization::v1::Response { auth_chain: auth_chain_ids @@ -2550,7 +2547,7 @@ pub fn get_room_state_route( }) .collect(); - let auth_chain_ids = get_auth_chain(&body.room_id, vec![Arc::new(body.event_id.clone())], &db)?; + let auth_chain_ids = get_auth_chain(&body.room_id, vec![Arc::from(&*body.event_id)], &db)?; Ok(get_room_state::v1::Response { auth_chain: auth_chain_ids @@ -2606,13 +2603,13 @@ pub fn get_room_state_ids_route( .rooms .state_full_ids(shortstatehash)? .into_iter() - .map(|(_, id)| (*id).clone()) + .map(|(_, id)| (*id).to_owned()) .collect(); - let auth_chain_ids = get_auth_chain(&body.room_id, vec![Arc::new(body.event_id.clone())], &db)?; + let auth_chain_ids = get_auth_chain(&body.room_id, vec![Arc::from(&*body.event_id)], &db)?; Ok(get_room_state_ids::v1::Response { - auth_chain_ids: auth_chain_ids.map(|id| (*id).clone()).collect(), + auth_chain_ids: auth_chain_ids.map(|id| (*id).to_owned()).collect(), pdu_ids, } .into()) @@ -2671,9 +2668,8 @@ pub fn create_join_event_template_route( }; // If there was no create event yet, assume we are creating a version 6 room right now - let room_version_id = create_event_content.map_or(RoomVersionId::Version6, |create_event| { - create_event.room_version - }); + let room_version_id = + create_event_content.map_or(RoomVersionId::V6, |create_event| create_event.room_version); let room_version = RoomVersion::new(&room_version_id).expect("room version is supported"); if !body.ver.contains(&room_version_id) { @@ -2726,7 +2722,7 @@ pub fn create_join_event_template_route( } let pdu = PduEvent { - event_id: ruma::event_id!("$thiswillbefilledinlater"), + event_id: ruma::event_id!("$thiswillbefilledinlater").to_owned(), room_id: body.room_id.clone(), sender: body.user_id.clone(), origin_server_ts: utils::millis_since_unix_epoch() @@ -2838,7 +2834,7 @@ async fn create_join_event( .roomid_mutex_federation .write() .unwrap() - .entry(room_id.clone()) + .entry(room_id.to_owned()) .or_default(), ); let mutex_lock = mutex.lock().await; @@ -2937,8 +2933,7 @@ pub async fn create_invite_route( return Err(Error::bad_config("Federation is disabled.")); } - if body.room_version != RoomVersionId::Version5 && body.room_version != RoomVersionId::Version6 - { + if body.room_version != RoomVersionId::V5 && body.room_version != RoomVersionId::V6 { return Err(Error::BadRequest( ErrorKind::IncompatibleRoomVersion { room_version: body.room_version.clone(), @@ -2959,7 +2954,7 @@ pub async fn create_invite_route( .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Failed to sign event."))?; // Generate event id - let event_id = EventId::try_from(&*format!( + let event_id = Box::::try_from(&*format!( "${}", ruma::signatures::reference_hash(&signed_event, &body.room_version) .expect("ruma can calculate reference hashes") @@ -2972,7 +2967,7 @@ pub async fn create_invite_route( CanonicalJsonValue::String(event_id.into()), ); - let sender = serde_json::from_value( + let sender: Box<_> = serde_json::from_value( signed_event .get("sender") .ok_or(Error::BadRequest( @@ -2984,7 +2979,7 @@ pub async fn create_invite_route( ) .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "sender is not a user id."))?; - let invited_user = serde_json::from_value( + let invited_user: Box<_> = serde_json::from_value( signed_event .get("state_key") .ok_or(Error::BadRequest( @@ -3263,7 +3258,7 @@ pub(crate) async fn fetch_required_signing_keys( // the PDUs and either cache the key or add it to the list that needs to be retrieved. fn get_server_keys_from_cache( pdu: &RawJsonValue, - servers: &mut BTreeMap, BTreeMap>, + servers: &mut BTreeMap, BTreeMap, QueryCriteria>>, room_version: &RoomVersionId, pub_key_map: &mut RwLockWriteGuard<'_, BTreeMap>>, db: &Database, @@ -3273,7 +3268,7 @@ fn get_server_keys_from_cache( Error::BadServerResponse("Invalid PDU in server response") })?; - let event_id = EventId::try_from(&*format!( + let event_id = Box::::try_from(&*format!( "${}", ruma::signatures::reference_hash(&value, room_version) .expect("ruma can calculate reference hashes") @@ -3353,7 +3348,7 @@ pub(crate) async fn fetch_join_signing_keys( pub_key_map: &RwLock>>, db: &Database, ) -> Result<()> { - let mut servers: BTreeMap, BTreeMap> = + let mut servers: BTreeMap, BTreeMap, QueryCriteria>> = BTreeMap::new(); { @@ -3387,10 +3382,6 @@ pub(crate) async fn fetch_join_signing_keys( server, get_remote_server_keys_batch::v2::Request { server_keys: servers.clone(), - minimum_valid_until_ts: MilliSecondsSinceUnixEpoch::from_system_time( - SystemTime::now() + Duration::from_secs(60), - ) - .expect("time is valid"), }, ) .await From 41fef1da64ea792b9ae8827f04d72cf7bbc1c960 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Sat, 27 Nov 2021 00:30:00 +0100 Subject: [PATCH 048/201] Remove unnecessary .to_string() calls --- src/server_server.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/server_server.rs b/src/server_server.rs index ec5bc345..8a50d234 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -396,10 +396,7 @@ async fn find_actual_destination( } if let Some(port) = force_port { - FedDest::Named( - delegated_hostname, - format!(":{}", port.to_string()), - ) + FedDest::Named(delegated_hostname, format!(":{}", port)) } else { add_port_to_hostname(&delegated_hostname) } @@ -432,10 +429,7 @@ async fn find_actual_destination( } if let Some(port) = force_port { - FedDest::Named( - hostname.clone(), - format!(":{}", port.to_string()), - ) + FedDest::Named(hostname.clone(), format!(":{}", port)) } else { add_port_to_hostname(&hostname) } From bffddbd4879e950a52647126284acafde4c46df4 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Sat, 27 Nov 2021 00:30:28 +0100 Subject: [PATCH 049/201] Simplify identifier parsing code --- src/client_server/account.rs | 11 ++---- src/client_server/membership.rs | 39 +++++++++++--------- src/client_server/message.rs | 16 +++----- src/client_server/room.rs | 18 +++------ src/client_server/sync.rs | 11 +++--- src/database.rs | 15 +++----- src/database/admin.rs | 18 ++++----- src/database/key_backups.rs | 4 +- src/database/rooms.rs | 65 ++++++++++++++++----------------- src/database/rooms/edus.rs | 24 +++++------- src/database/sending.rs | 31 ++++++++-------- src/database/users.rs | 21 +++++------ src/pdu.rs | 7 ++-- src/ruma_wrapper.rs | 6 +-- src/server_server.rs | 40 ++++++++++---------- 15 files changed, 147 insertions(+), 179 deletions(-) diff --git a/src/client_server/account.rs b/src/client_server/account.rs index d7c2f63e..3149187f 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -1,8 +1,4 @@ -use std::{ - collections::BTreeMap, - convert::{TryFrom, TryInto}, - sync::Arc, -}; +use std::{collections::BTreeMap, convert::TryInto, sync::Arc}; use super::{DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH}; use crate::{database::DatabaseGuard, pdu::PduBuilder, utils, ConduitResult, Error, Ruma}; @@ -396,9 +392,8 @@ pub async fn register_route( )?; // 6. Events implied by name and topic - let room_name = - Box::::try_from(format!("{} Admin Room", db.globals.server_name())) - .expect("Room name is valid"); + let room_name = RoomName::parse(format!("{} Admin Room", db.globals.server_name())) + .expect("Room name is valid"); db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomName, diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index f65287da..6c7b7211 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -64,7 +64,7 @@ pub async fn join_room_by_id_route( .filter_map(|event| serde_json::from_str(event.json().get()).ok()) .filter_map(|event: serde_json::Value| event.get("sender").cloned()) .filter_map(|sender| sender.as_str().map(|s| s.to_owned())) - .filter_map(|sender| Box::::try_from(sender).ok()) + .filter_map(|sender| UserId::parse(sender).ok()) .map(|user| user.server_name().to_owned()) .collect(); @@ -92,16 +92,17 @@ pub async fn join_room_by_id_route( /// - If the server does not know about the room: asks other servers over federation #[cfg_attr( feature = "conduit_bin", - post("/_matrix/client/r0/join/<_>", data = "") + post("/_matrix/client/r0/join/<_>", data = "") )] -#[tracing::instrument(skip(db, body))] +#[tracing::instrument(skip(db, req))] pub async fn join_room_by_id_or_alias_route( db: DatabaseGuard, - body: Ruma>, + req: Ruma>, ) -> ConduitResult { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let body = req.body; + let sender_user = req.sender_user.as_ref().expect("user is authenticated"); - let (servers, room_id) = match Box::::try_from(body.room_id_or_alias.clone()) { + let (servers, room_id) = match Box::::try_from(body.room_id_or_alias) { Ok(room_id) => { let mut servers: HashSet<_> = db .rooms @@ -111,7 +112,7 @@ pub async fn join_room_by_id_or_alias_route( .filter_map(|event| serde_json::from_str(event.json().get()).ok()) .filter_map(|event: serde_json::Value| event.get("sender").cloned()) .filter_map(|sender| sender.as_str().map(|s| s.to_owned())) - .filter_map(|sender| Box::::try_from(sender).ok()) + .filter_map(|sender| UserId::parse(sender).ok()) .map(|user| user.server_name().to_owned()) .collect(); @@ -127,7 +128,7 @@ pub async fn join_room_by_id_or_alias_route( let join_room_response = join_room_by_id_helper( &db, - body.sender_user.as_deref(), + req.sender_user.as_deref(), &room_id, &servers, body.third_party_signed.as_ref(), @@ -619,12 +620,13 @@ async fn join_room_by_id_helper( .expect("event is valid, we just created it"); // Generate event id - let event_id = Box::::try_from(&*format!( + let event_id = format!( "${}", ruma::signatures::reference_hash(&join_event_stub, &room_version) .expect("ruma can calculate reference hashes") - )) - .expect("ruma's reference hashes are valid event ids"); + ); + let event_id = <&EventId>::try_from(event_id.as_str()) + .expect("ruma's reference hashes are valid event ids"); // Add event_id back join_event_stub.insert( @@ -642,7 +644,7 @@ async fn join_room_by_id_helper( remote_server, federation::membership::create_join_event::v2::Request { room_id, - event_id: &event_id, + event_id, pdu: &PduEvent::convert_to_outgoing_federation_event(join_event.clone()), }, ) @@ -650,7 +652,7 @@ async fn join_room_by_id_helper( db.rooms.get_or_create_shortroomid(room_id, &db.globals)?; - let pdu = PduEvent::from_id_val(&event_id, join_event.clone()) + let pdu = PduEvent::from_id_val(event_id, join_event.clone()) .map_err(|_| Error::BadServerResponse("Invalid join event PDU."))?; let mut state = HashMap::new(); @@ -788,7 +790,7 @@ fn validate_and_add_event_id( error!("Invalid PDU in server response: {:?}: {:?}", pdu, e); Error::BadServerResponse("Invalid PDU in server response") })?; - let event_id = Box::::try_from(&*format!( + let event_id = EventId::parse(format!( "${}", ruma::signatures::reference_hash(&value, room_version) .expect("ruma can calculate reference hashes") @@ -1011,12 +1013,13 @@ pub(crate) async fn invite_helper<'a>( }; // Generate event id - let expected_event_id = Box::::try_from(&*format!( + let expected_event_id = format!( "${}", ruma::signatures::reference_hash(&pdu_json, &room_version_id) .expect("ruma can calculate reference hashes") - )) - .expect("ruma's reference hashes are valid event ids"); + ); + let expected_event_id = <&EventId>::try_from(expected_event_id.as_str()) + .expect("ruma's reference hashes are valid event ids"); let response = db .sending @@ -1025,7 +1028,7 @@ pub(crate) async fn invite_helper<'a>( user_id.server_name(), create_invite::v2::Request { room_id, - event_id: &expected_event_id, + event_id: expected_event_id, room_version: &room_version_id, event: &PduEvent::convert_to_outgoing_federation_event(pdu_json.clone()), invite_room_state: &invite_room_state, diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 0d006101..e5219433 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -5,13 +5,8 @@ use ruma::{ r0::message::{get_message_events, send_message_event}, }, events::EventType, - EventId, -}; -use std::{ - collections::BTreeMap, - convert::{TryFrom, TryInto}, - sync::Arc, }; +use std::{collections::BTreeMap, convert::TryInto, sync::Arc}; #[cfg(feature = "conduit_bin")] use rocket::{get, put}; @@ -67,11 +62,10 @@ pub async fn send_message_event_route( )); } - let event_id = Box::::try_from( - utils::string_from_bytes(&response) - .map_err(|_| Error::bad_database("Invalid txnid bytes in database."))?, - ) - .map_err(|_| Error::bad_database("Invalid event id in txnid data."))?; + let event_id = utils::string_from_bytes(&response) + .map_err(|_| Error::bad_database("Invalid txnid bytes in database."))? + .try_into() + .map_err(|_| Error::bad_database("Invalid event id in txnid data."))?; return Ok(send_message_event::Response { event_id }.into()); } diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 97b3f482..83571f1d 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -26,12 +26,7 @@ use ruma::{ RoomAliasId, RoomId, RoomVersionId, }; use serde_json::{json, value::to_raw_value}; -use std::{ - cmp::max, - collections::BTreeMap, - convert::{TryFrom, TryInto}, - sync::Arc, -}; +use std::{cmp::max, collections::BTreeMap, convert::TryInto, sync::Arc}; use tracing::{info, warn}; #[cfg(feature = "conduit_bin")] @@ -93,12 +88,11 @@ pub async fn create_room_route( .as_ref() .map_or(Ok(None), |localpart| { // TODO: Check for invalid characters and maximum length - let alias = Box::::try_from(format!( - "#{}:{}", - localpart, - db.globals.server_name(), - )) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid alias."))?; + let alias = + RoomAliasId::parse(format!("#{}:{}", localpart, db.globals.server_name())) + .map_err(|_| { + Error::BadRequest(ErrorKind::InvalidParam, "Invalid alias.") + })?; if db.rooms.id_from_alias(&alias)?.is_some() { Err(Error::BadRequest( diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 1060d917..2e372f91 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -10,7 +10,7 @@ use ruma::{ }; use std::{ collections::{hash_map::Entry, BTreeMap, HashMap, HashSet}, - convert::{TryFrom, TryInto}, + convert::TryInto, sync::Arc, time::Duration, }; @@ -298,10 +298,9 @@ async fn sync_helper( })?; if let Some(state_key) = &pdu.state_key { - let user_id = - Box::::try_from(state_key.clone()).map_err(|_| { - Error::bad_database("Invalid UserId in member PDU.") - })?; + let user_id = UserId::parse(state_key.clone()).map_err(|_| { + Error::bad_database("Invalid UserId in member PDU.") + })?; // The membership was and still is invite or join if matches!( @@ -427,7 +426,7 @@ async fn sync_helper( } if let Some(state_key) = &state_event.state_key { - let user_id = Box::::try_from(state_key.clone()) + let user_id = UserId::parse(state_key.clone()) .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; if user_id == sender_user { diff --git a/src/database.rs b/src/database.rs index 056d49ad..84ca68dc 100644 --- a/src/database.rs +++ b/src/database.rs @@ -476,11 +476,9 @@ impl Database { if db.globals.database_version()? < 6 { // Set room member count for (roomid, _) in db.rooms.roomid_shortstatehash.iter() { - let room_id = - Box::::try_from(utils::string_from_bytes(&roomid).unwrap()) - .unwrap(); - - db.rooms.update_joined_count(&room_id, &db)?; + let string = utils::string_from_bytes(&roomid).unwrap(); + let room_id = <&RoomId>::try_from(string.as_str()).unwrap(); + db.rooms.update_joined_count(room_id, &db)?; } db.globals.bump_database_version(6)?; @@ -587,10 +585,9 @@ impl Database { .get(&seventid) .unwrap() .unwrap(); - let event_id = - Box::::try_from(utils::string_from_bytes(&event_id).unwrap()) - .unwrap(); - let pdu = db.rooms.get_pdu(&event_id).unwrap().unwrap(); + let string = utils::string_from_bytes(&event_id).unwrap(); + let event_id = <&EventId>::try_from(string.as_str()).unwrap(); + let pdu = db.rooms.get_pdu(event_id).unwrap().unwrap(); if Some(&pdu.room_id) != current_room.as_ref() { current_room = Some(pdu.room_id.clone()); diff --git a/src/database/admin.rs b/src/database/admin.rs index 07a487e2..1e5c47c9 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -1,10 +1,10 @@ -use std::{convert::TryFrom, sync::Arc}; +use std::{convert::TryInto, sync::Arc}; use crate::{pdu::PduBuilder, Database}; use rocket::futures::{channel::mpsc, stream::StreamExt}; use ruma::{ events::{room::message::RoomMessageEventContent, EventType}, - RoomAliasId, UserId, + UserId, }; use serde_json::value::to_raw_value; use tokio::sync::{MutexGuard, RwLock, RwLockReadGuard}; @@ -33,18 +33,16 @@ impl Admin { let guard = db.read().await; - let conduit_user = - Box::::try_from(format!("@conduit:{}", guard.globals.server_name())) - .expect("@conduit:server_name is valid"); + let conduit_user = UserId::parse(format!("@conduit:{}", guard.globals.server_name())) + .expect("@conduit:server_name is valid"); let conduit_room = guard .rooms .id_from_alias( - &Box::::try_from(format!( - "#admins:{}", - guard.globals.server_name() - )) - .expect("#admins:server_name is a valid room alias"), + format!("#admins:{}", guard.globals.server_name()) + .as_str() + .try_into() + .expect("#admins:server_name is a valid room alias"), ) .unwrap(); diff --git a/src/database/key_backups.rs b/src/database/key_backups.rs index 3010a37b..56963c08 100644 --- a/src/database/key_backups.rs +++ b/src/database/key_backups.rs @@ -6,7 +6,7 @@ use ruma::{ }, RoomId, UserId, }; -use std::{collections::BTreeMap, convert::TryFrom, sync::Arc}; +use std::{collections::BTreeMap, sync::Arc}; use super::abstraction::Tree; @@ -231,7 +231,7 @@ impl KeyBackups { Error::bad_database("backupkeyid_backup session_id is invalid.") })?; - let room_id = Box::::try_from( + let room_id = RoomId::parse( utils::string_from_bytes(parts.next().ok_or_else(|| { Error::bad_database("backupkeyid_backup key is invalid.") })?) diff --git a/src/database/rooms.rs b/src/database/rooms.rs index ebd0941b..f8d2cad8 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -434,7 +434,7 @@ impl Rooms { None => continue, }; - let user_id = match Box::::try_from(state_key) { + let user_id = match UserId::parse(state_key) { Ok(id) => id, Err(_) => continue, }; @@ -871,12 +871,10 @@ impl Rooms { .get(&shorteventid.to_be_bytes())? .ok_or_else(|| Error::bad_database("Shorteventid does not exist"))?; - let event_id = Arc::from( - Box::::try_from(utils::string_from_bytes(&bytes).map_err(|_| { - Error::bad_database("EventID in shorteventid_eventid is invalid unicode.") - })?) - .map_err(|_| Error::bad_database("EventId in shorteventid_eventid is invalid."))?, - ); + let event_id = EventId::parse_arc(utils::string_from_bytes(&bytes).map_err(|_| { + Error::bad_database("EventID in shorteventid_eventid is invalid unicode.") + })?) + .map_err(|_| Error::bad_database("EventId in shorteventid_eventid is invalid."))?; self.shorteventid_cache .lock() @@ -1169,7 +1167,7 @@ impl Rooms { self.roomid_pduleaves .scan_prefix(prefix) .map(|(_, bytes)| { - Box::::try_from(utils::string_from_bytes(&bytes).map_err(|_| { + EventId::parse(utils::string_from_bytes(&bytes).map_err(|_| { Error::bad_database("EventID in roomid_pduleaves is invalid unicode.") })?) .map_err(|_| Error::bad_database("EventId in roomid_pduleaves is invalid.")) @@ -1420,7 +1418,7 @@ impl Rooms { } // if the state_key fails - let target_user_id = Box::::try_from(state_key.clone()) + let target_user_id = UserId::parse(state_key.clone()) .expect("This state_key was previously validated"); let content = serde_json::from_str::(pdu.content.get()) @@ -1476,10 +1474,9 @@ impl Rooms { if body.starts_with(&format!("@conduit:{}: ", db.globals.server_name())) && self .id_from_alias( - &Box::::try_from(format!( - "#admins:{}", - db.globals.server_name() - )) + <&RoomAliasId>::try_from( + format!("#admins:{}", db.globals.server_name()).as_str(), + ) .expect("#admins:server_name is a valid room alias"), )? .as_ref() @@ -1530,7 +1527,7 @@ impl Rooms { } "get_auth_chain" => { if args.len() == 1 { - if let Ok(event_id) = Box::::try_from(args[0]) { + if let Ok(event_id) = EventId::parse_arc(args[0]) { if let Some(event) = db.rooms.get_pdu_json(&event_id)? { let room_id_str = event .get("room_id") @@ -1541,12 +1538,12 @@ impl Rooms { ) })?; - let room_id = Box::::try_from(room_id_str) - .map_err(|_| Error::bad_database("Invalid room id field in event in database"))?; + let room_id = <&RoomId>::try_from(room_id_str) + .map_err(|_| Error::bad_database("Invalid room id field in event in database"))?; let start = Instant::now(); let count = server_server::get_auth_chain( - &room_id, - vec![Arc::from(event_id)], + room_id, + vec![event_id], db, )? .count(); @@ -1569,7 +1566,7 @@ impl Rooms { let string = body[1..body.len() - 1].join("\n"); match serde_json::from_str(&string) { Ok(value) => { - let event_id = Box::::try_from(&*format!( + let event_id = EventId::parse(format!( "${}", // Anything higher than version3 behaves the same ruma::signatures::reference_hash( @@ -1624,7 +1621,7 @@ impl Rooms { } "get_pdu" => { if args.len() == 1 { - if let Ok(event_id) = Box::::try_from(args[0]) { + if let Ok(event_id) = EventId::parse(args[0]) { let mut outlier = false; let mut pdu_json = db.rooms.get_non_outlier_pdu_json(&event_id)?; @@ -2083,7 +2080,7 @@ impl Rooms { .expect("event is valid, we just created it"); // Generate event id - pdu.event_id = Box::::try_from(&*format!( + pdu.event_id = EventId::parse(format!( "${}", ruma::signatures::reference_hash(&pdu_json, &room_version_id) .expect("ruma can calculate reference hashes") @@ -2758,7 +2755,7 @@ impl Rooms { .filter_map(|event| serde_json::from_str(event.json().get()).ok()) .filter_map(|event: serde_json::Value| event.get("sender").cloned()) .filter_map(|sender| sender.as_str().map(|s| s.to_owned())) - .filter_map(|sender| Box::::try_from(sender).ok()) + .filter_map(|sender| UserId::parse(sender).ok()) .map(|user| user.server_name().to_owned()) .collect(); @@ -2819,7 +2816,7 @@ impl Rooms { .expect("event is valid, we just created it"); // Generate event id - let event_id = Box::::try_from(&*format!( + let event_id = EventId::parse(format!( "${}", ruma::signatures::reference_hash(&leave_event_stub, &room_version_id) .expect("ruma can calculate reference hashes") @@ -2908,7 +2905,7 @@ impl Rooms { self.alias_roomid .get(alias.alias().as_bytes())? .map(|bytes| { - Box::::try_from(utils::string_from_bytes(&bytes).map_err(|_| { + RoomId::parse(utils::string_from_bytes(&bytes).map_err(|_| { Error::bad_database("Room ID in alias_roomid is invalid unicode.") })?) .map_err(|_| Error::bad_database("Room ID in alias_roomid is invalid.")) @@ -2951,7 +2948,7 @@ impl Rooms { #[tracing::instrument(skip(self))] pub fn public_rooms(&self) -> impl Iterator>> + '_ { self.publicroomids.iter().map(|(bytes, _)| { - Box::::try_from( + RoomId::parse( utils::string_from_bytes(&bytes).map_err(|_| { Error::bad_database("Room ID in publicroomids is invalid unicode.") })?, @@ -3039,7 +3036,7 @@ impl Rooms { Ok(utils::common_elements(iterators, Ord::cmp) .expect("users is not empty") .map(|bytes| { - Box::::try_from(utils::string_from_bytes(&*bytes).map_err(|_| { + RoomId::parse(utils::string_from_bytes(&*bytes).map_err(|_| { Error::bad_database("Invalid RoomId bytes in userroomid_joined") })?) .map_err(|_| Error::bad_database("Invalid RoomId in userroomid_joined.")) @@ -3056,7 +3053,7 @@ impl Rooms { prefix.push(0xff); self.roomserverids.scan_prefix(prefix).map(|(key, _)| { - Box::::try_from( + ServerName::parse( utils::string_from_bytes( key.rsplit(|&b| b == 0xff) .next() @@ -3089,7 +3086,7 @@ impl Rooms { prefix.push(0xff); self.serverroomids.scan_prefix(prefix).map(|(key, _)| { - Box::::try_from( + RoomId::parse( utils::string_from_bytes( key.rsplit(|&b| b == 0xff) .next() @@ -3111,7 +3108,7 @@ impl Rooms { prefix.push(0xff); self.roomuserid_joined.scan_prefix(prefix).map(|(key, _)| { - Box::::try_from( + UserId::parse( utils::string_from_bytes( key.rsplit(|&b| b == 0xff) .next() @@ -3159,7 +3156,7 @@ impl Rooms { self.roomuseroncejoinedids .scan_prefix(prefix) .map(|(key, _)| { - Box::::try_from( + UserId::parse( utils::string_from_bytes( key.rsplit(|&b| b == 0xff) .next() @@ -3185,7 +3182,7 @@ impl Rooms { self.roomuserid_invitecount .scan_prefix(prefix) .map(|(key, _)| { - Box::::try_from( + UserId::parse( utils::string_from_bytes( key.rsplit(|&b| b == 0xff) .next() @@ -3238,7 +3235,7 @@ impl Rooms { self.userroomid_joined .scan_prefix(user_id.as_bytes().to_vec()) .map(|(key, _)| { - Box::::try_from( + RoomId::parse( utils::string_from_bytes( key.rsplit(|&b| b == 0xff) .next() @@ -3264,7 +3261,7 @@ impl Rooms { self.userroomid_invitestate .scan_prefix(prefix) .map(|(key, state)| { - let room_id = Box::::try_from( + let room_id = RoomId::parse( utils::string_from_bytes( key.rsplit(|&b| b == 0xff) .next() @@ -3337,7 +3334,7 @@ impl Rooms { self.userroomid_leftstate .scan_prefix(prefix) .map(|(key, state)| { - let room_id = Box::::try_from( + let room_id = RoomId::parse( utils::string_from_bytes( key.rsplit(|&b| b == 0xff) .next() diff --git a/src/database/rooms/edus.rs b/src/database/rooms/edus.rs index 365211b6..eb2d3427 100644 --- a/src/database/rooms/edus.rs +++ b/src/database/rooms/edus.rs @@ -11,7 +11,7 @@ use ruma::{ }; use std::{ collections::{HashMap, HashSet}, - convert::{TryFrom, TryInto}, + convert::TryInto, mem, sync::Arc, }; @@ -97,7 +97,7 @@ impl RoomEdus { let count = utils::u64_from_bytes(&k[prefix.len()..prefix.len() + mem::size_of::()]) .map_err(|_| Error::bad_database("Invalid readreceiptid count in db."))?; - let user_id = Box::::try_from( + let user_id = UserId::parse( utils::string_from_bytes(&k[prefix.len() + mem::size_of::() + 1..]) .map_err(|_| { Error::bad_database("Invalid readreceiptid userid bytes in db.") @@ -310,17 +310,13 @@ impl RoomEdus { let mut user_ids = HashSet::new(); - for user_id in self - .typingid_userid - .scan_prefix(prefix) - .map(|(_, user_id)| { - Box::::try_from(utils::string_from_bytes(&user_id).map_err(|_| { - Error::bad_database("User ID in typingid_userid is invalid unicode.") - })?) - .map_err(|_| Error::bad_database("User ID in typingid_userid is invalid.")) - }) - { - user_ids.insert(user_id?); + for (_, user_id) in self.typingid_userid.scan_prefix(prefix) { + let user_id = UserId::parse(utils::string_from_bytes(&user_id).map_err(|_| { + Error::bad_database("User ID in typingid_userid is invalid unicode.") + })?) + .map_err(|_| Error::bad_database("User ID in typingid_userid is invalid."))?; + + user_ids.insert(user_id); } Ok(SyncEphemeralRoomEvent { @@ -518,7 +514,7 @@ impl RoomEdus { .iter_from(&*first_possible_edu, false) .take_while(|(key, _)| key.starts_with(&prefix)) { - let user_id = Box::::try_from( + let user_id = UserId::parse( utils::string_from_bytes( key.rsplit(|&b| b == 0xff) .next() diff --git a/src/database/sending.rs b/src/database/sending.rs index c27b5731..1e180d43 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -1,6 +1,6 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, - convert::{TryFrom, TryInto}, + convert::TryInto, fmt::Debug, sync::Arc, time::{Duration, Instant}, @@ -583,19 +583,18 @@ impl Sending { } } - let userid = - Box::::try_from(utils::string_from_bytes(user).map_err(|_| { - ( - kind.clone(), - Error::bad_database("Invalid push user string in db."), - ) - })?) - .map_err(|_| { - ( - kind.clone(), - Error::bad_database("Invalid push user id in db."), - ) - })?; + let userid = UserId::parse(utils::string_from_bytes(user).map_err(|_| { + ( + kind.clone(), + Error::bad_database("Invalid push user string in db."), + ) + })?) + .map_err(|_| { + ( + kind.clone(), + Error::bad_database("Invalid push user id in db."), + ) + })?; let mut senderkey = user.clone(); senderkey.push(0xff); @@ -732,7 +731,7 @@ impl Sending { })?; ( - OutgoingKind::Appservice(Box::::try_from(server).map_err(|_| { + OutgoingKind::Appservice(ServerName::parse(server).map_err(|_| { Error::bad_database("Invalid server string in server_currenttransaction") })?), if value.is_empty() { @@ -771,7 +770,7 @@ impl Sending { })?; ( - OutgoingKind::Normal(Box::::try_from(server).map_err(|_| { + OutgoingKind::Normal(ServerName::parse(server).map_err(|_| { Error::bad_database("Invalid server string in server_currenttransaction") })?), if value.is_empty() { diff --git a/src/database/users.rs b/src/database/users.rs index 4a084722..d4bf4890 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -8,7 +8,7 @@ use ruma::{ DeviceId, DeviceKeyAlgorithm, DeviceKeyId, MilliSecondsSinceUnixEpoch, RoomAliasId, UInt, UserId, }; -use std::{collections::BTreeMap, convert::TryFrom, mem, sync::Arc}; +use std::{collections::BTreeMap, convert::TryInto, mem, sync::Arc}; use tracing::warn; use super::abstraction::Tree; @@ -62,9 +62,8 @@ impl Users { rooms: &super::rooms::Rooms, globals: &super::globals::Globals, ) -> Result { - let admin_room_alias_id = - Box::::try_from(format!("#admins:{}", globals.server_name())) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid alias."))?; + let admin_room_alias_id = RoomAliasId::parse(format!("#admins:{}", globals.server_name())) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid alias."))?; let admin_room_id = rooms.id_from_alias(&admin_room_alias_id)?.unwrap(); rooms.is_joined(user_id, &admin_room_id) @@ -98,11 +97,9 @@ impl Users { })?; Ok(Some(( - Box::::try_from(utils::string_from_bytes(user_bytes).map_err( - |_| { - Error::bad_database("User ID in token_userdeviceid is invalid unicode.") - }, - )?) + UserId::parse(utils::string_from_bytes(user_bytes).map_err(|_| { + Error::bad_database("User ID in token_userdeviceid is invalid unicode.") + })?) .map_err(|_| { Error::bad_database("User ID in token_userdeviceid is invalid.") })?, @@ -117,7 +114,7 @@ impl Users { #[tracing::instrument(skip(self))] pub fn iter(&self) -> impl Iterator>> + '_ { self.userid_password.iter().map(|(bytes, _)| { - Box::::try_from(utils::string_from_bytes(&bytes).map_err(|_| { + UserId::parse(utils::string_from_bytes(&bytes).map_err(|_| { Error::bad_database("User ID in userid_password is invalid unicode.") })?) .map_err(|_| Error::bad_database("User ID in userid_password is invalid.")) @@ -189,7 +186,7 @@ impl Users { .map(|bytes| { let s = utils::string_from_bytes(&bytes) .map_err(|_| Error::bad_database("Avatar URL in db is invalid."))?; - Box::::try_from(s) + s.try_into() .map_err(|_| Error::bad_database("Avatar URL in db is invalid.")) }) .transpose() @@ -686,7 +683,7 @@ impl Users { } }) .map(|(_, bytes)| { - Box::::try_from(utils::string_from_bytes(&bytes).map_err(|_| { + UserId::parse(utils::string_from_bytes(&bytes).map_err(|_| { Error::bad_database("User ID in devicekeychangeid_userid is invalid unicode.") })?) .map_err(|_| Error::bad_database("User ID in devicekeychangeid_userid is invalid.")) diff --git a/src/pdu.rs b/src/pdu.rs index 3c955976..c1f3d27d 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -13,7 +13,7 @@ use serde_json::{ json, value::{to_raw_value, RawValue as RawJsonValue}, }; -use std::{cmp::Ordering, collections::BTreeMap, convert::TryFrom, ops::Deref}; +use std::{cmp::Ordering, collections::BTreeMap, convert::TryInto, ops::Deref}; use tracing::warn; /// Content hashes of a PDU. @@ -337,12 +337,13 @@ pub(crate) fn gen_event_id_canonical_json( Error::BadServerResponse("Invalid PDU in server response") })?; - let event_id = Box::::try_from(&*format!( + let event_id = format!( "${}", // Anything higher than version3 behaves the same ruma::signatures::reference_hash(&value, &RoomVersionId::V6) .expect("ruma can calculate reference hashes") - )) + ) + .try_into() .expect("ruma's reference hashes are valid event ids"); Ok((event_id, value)) diff --git a/src/ruma_wrapper.rs b/src/ruma_wrapper.rs index 2cff2f5a..4b8d5dea 100644 --- a/src/ruma_wrapper.rs +++ b/src/ruma_wrapper.rs @@ -20,7 +20,6 @@ use { }, ruma::api::{AuthScheme, IncomingRequest}, std::collections::BTreeMap, - std::convert::TryFrom, std::io::Cursor, tracing::{debug, warn}, }; @@ -103,8 +102,7 @@ where .unwrap() }, |string| { - Box::::try_from(string.expect("parsing to string always works")) - .unwrap() + UserId::parse(string.expect("parsing to string always works")).unwrap() }, ); @@ -171,7 +169,7 @@ where } }; - let origin = match Box::::try_from(origin_str) { + let origin = match ServerName::parse(origin_str) { Ok(s) => s, _ => { warn!( diff --git a/src/server_server.rs b/src/server_server.rs index 8a50d234..b0e3f0f6 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -544,12 +544,11 @@ pub fn get_server_keys_route(db: DatabaseGuard) -> Json { return Json("Federation is disabled.".to_owned()); } - let mut verify_keys = BTreeMap::new(); + let mut verify_keys: BTreeMap, VerifyKey> = BTreeMap::new(); verify_keys.insert( - Box::::try_from( - format!("ed25519:{}", db.globals.keypair().version()).as_str(), - ) - .expect("found invalid server signing keys in DB"), + format!("ed25519:{}", db.globals.keypair().version()) + .try_into() + .expect("found invalid server signing keys in DB"), VerifyKey { key: base64::encode_config(db.globals.keypair().public_key(), base64::STANDARD_NO_PAD), }, @@ -730,7 +729,7 @@ pub async fn send_transaction_message_route( // 0. Check the server is in the room let room_id = match value .get("room_id") - .and_then(|id| Box::::try_from(id.as_str()?).ok()) + .and_then(|id| RoomId::parse(id.as_str()?).ok()) { Some(id) => id, None => { @@ -2354,10 +2353,10 @@ pub fn get_event_route( .and_then(|val| val.as_str()) .ok_or_else(|| Error::bad_database("Invalid event in database"))?; - let room_id = Box::::try_from(room_id_str) + let room_id = <&RoomId>::try_from(room_id_str) .map_err(|_| Error::bad_database("Invalid room id field in event in database"))?; - if !db.rooms.server_in_room(sender_servername, &room_id)? { + if !db.rooms.server_in_room(sender_servername, room_id)? { return Err(Error::BadRequest(ErrorKind::NotFound, "Event not found.")); } @@ -2408,7 +2407,7 @@ pub fn get_missing_events_route( .and_then(|val| val.as_str()) .ok_or_else(|| Error::bad_database("Invalid event in database"))?; - let event_room_id = Box::::try_from(room_id_str) + let event_room_id = <&RoomId>::try_from(room_id_str) .map_err(|_| Error::bad_database("Invalid room id field in event in database"))?; if event_room_id != body.room_id { @@ -2476,14 +2475,14 @@ pub fn get_event_authorization_route( .and_then(|val| val.as_str()) .ok_or_else(|| Error::bad_database("Invalid event in database"))?; - let room_id = Box::::try_from(room_id_str) + let room_id = <&RoomId>::try_from(room_id_str) .map_err(|_| Error::bad_database("Invalid room id field in event in database"))?; - if !db.rooms.server_in_room(sender_servername, &room_id)? { + if !db.rooms.server_in_room(sender_servername, room_id)? { return Err(Error::BadRequest(ErrorKind::NotFound, "Event not found.")); } - let auth_chain_ids = get_auth_chain(&room_id, vec![Arc::from(&*body.event_id)], &db)?; + let auth_chain_ids = get_auth_chain(room_id, vec![Arc::from(&*body.event_id)], &db)?; Ok(get_event_authorization::v1::Response { auth_chain: auth_chain_ids @@ -2948,7 +2947,7 @@ pub async fn create_invite_route( .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Failed to sign event."))?; // Generate event id - let event_id = Box::::try_from(&*format!( + let event_id = EventId::parse(format!( "${}", ruma::signatures::reference_hash(&signed_event, &body.room_version) .expect("ruma can calculate reference hashes") @@ -3224,7 +3223,7 @@ pub(crate) async fn fetch_required_signing_keys( let fetch_res = fetch_signing_keys( db, - &Box::::try_from(&**signature_server).map_err(|_| { + signature_server.as_str().try_into().map_err(|_| { Error::BadServerResponse("Invalid servername in signatures of server response pdu.") })?, signature_ids, @@ -3262,19 +3261,20 @@ fn get_server_keys_from_cache( Error::BadServerResponse("Invalid PDU in server response") })?; - let event_id = Box::::try_from(&*format!( + let event_id = format!( "${}", ruma::signatures::reference_hash(&value, room_version) .expect("ruma can calculate reference hashes") - )) - .expect("ruma's reference hashes are valid event ids"); + ); + let event_id = <&EventId>::try_from(event_id.as_str()) + .expect("ruma's reference hashes are valid event ids"); if let Some((time, tries)) = db .globals .bad_event_ratelimiter .read() .unwrap() - .get(&event_id) + .get(event_id) { // Exponential backoff let mut min_elapsed_duration = Duration::from_secs(30) * (*tries) * (*tries); @@ -3308,7 +3308,7 @@ fn get_server_keys_from_cache( let contains_all_ids = |keys: &BTreeMap| signature_ids.iter().all(|id| keys.contains_key(id)); - let origin = &Box::::try_from(&**signature_server).map_err(|_| { + let origin = <&ServerName>::try_from(signature_server.as_str()).map_err(|_| { Error::BadServerResponse("Invalid servername in signatures of server response pdu.") })?; @@ -3327,7 +3327,7 @@ fn get_server_keys_from_cache( if !contains_all_ids(&result) { trace!("Signing key not loaded for {}", origin); - servers.insert(origin.clone(), BTreeMap::new()); + servers.insert(origin.to_owned(), BTreeMap::new()); } pub_key_map.insert(origin.to_string(), result); From 58ea081762adb5f14ecaadc3e16f7b6dddcaed43 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Sat, 27 Nov 2021 16:04:19 +0100 Subject: [PATCH 050/201] Use int! macro instead of Int::from --- src/client_server/report.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/client_server/report.rs b/src/client_server/report.rs index 2e6527d4..ae069849 100644 --- a/src/client_server/report.rs +++ b/src/client_server/report.rs @@ -1,8 +1,11 @@ -use crate::{database::admin::AdminCommand, database::DatabaseGuard, ConduitResult, Error, Ruma}; +use crate::{ + database::{admin::AdminCommand, DatabaseGuard}, + ConduitResult, Error, Ruma, +}; use ruma::{ api::client::{error::ErrorKind, r0::room::report_content}, events::room::message, - Int, + int, }; #[cfg(feature = "conduit_bin")] @@ -33,7 +36,7 @@ pub async fn report_event_route( } }; - if body.score > Int::from(0) || body.score < Int::from(-100) { + if body.score > int!(0) || body.score < int!(-100) { return Err(Error::BadRequest( ErrorKind::InvalidParam, "Invalid score, must be within 0 to -100", From f71245504726a717c8a85ecd99f83a7395bd3c2b Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Sat, 27 Nov 2021 16:35:59 +0100 Subject: [PATCH 051/201] Reduce EventId copying --- src/client_server/membership.rs | 3 ++- src/database/rooms.rs | 14 ++++++++++---- src/server_server.rs | 22 ++++++++-------------- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 6c7b7211..e6c9d4b6 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -31,6 +31,7 @@ use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use std::{ collections::{hash_map::Entry, BTreeMap, HashMap, HashSet}, convert::{TryFrom, TryInto}, + iter, sync::{Arc, RwLock}, time::{Duration, Instant}, }; @@ -740,7 +741,7 @@ async fn join_room_by_id_helper( db.rooms.append_pdu( &pdu, utils::to_canonical_object(&pdu).expect("Pdu is valid canonical object"), - &[pdu.event_id.clone()], + iter::once(&*pdu.event_id), db, )?; diff --git a/src/database/rooms.rs b/src/database/rooms.rs index f8d2cad8..4c092bf7 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -36,6 +36,8 @@ use std::{ borrow::Cow, collections::{BTreeMap, HashMap, HashSet}, convert::{TryFrom, TryInto}, + fmt::Debug, + iter, mem::size_of, sync::{Arc, Mutex, RwLock}, time::Instant, @@ -1191,7 +1193,11 @@ impl Rooms { /// The provided `event_ids` become the new leaves, this allows a room to have multiple /// `prev_events`. #[tracing::instrument(skip(self))] - pub fn replace_pdu_leaves(&self, room_id: &RoomId, event_ids: &[Box]) -> Result<()> { + pub fn replace_pdu_leaves<'a>( + &self, + room_id: &RoomId, + event_ids: impl IntoIterator + Debug, + ) -> Result<()> { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); @@ -1255,11 +1261,11 @@ impl Rooms { /// /// Returns pdu id #[tracing::instrument(skip(self, pdu, pdu_json, leaves, db))] - pub fn append_pdu( + pub fn append_pdu<'a>( &self, pdu: &PduEvent, mut pdu_json: CanonicalJsonObject, - leaves: &[Box], + leaves: impl IntoIterator + Debug, db: &Database, ) -> Result> { let shortroomid = self.get_shortroomid(&pdu.room_id)?.expect("room exists"); @@ -2104,7 +2110,7 @@ impl Rooms { pdu_json, // Since this PDU references all pdu_leaves we can update the leaves // of the room - &[pdu.event_id.clone()], + iter::once(&*pdu.event_id), db, )?; diff --git a/src/server_server.rs b/src/server_server.rs index b0e3f0f6..ca6bb3fd 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -64,6 +64,7 @@ use std::{ future::Future, mem, net::{IpAddr, SocketAddr}, + ops::Deref, pin::Pin, sync::{Arc, RwLock, RwLockWriteGuard}, time::{Duration, Instant, SystemTime}, @@ -1636,7 +1637,7 @@ async fn upgrade_outlier_to_timeline_pdu( db, &incoming_pdu, val, - extremities, + extremities.iter().map(Deref::deref), state_ids_compressed, soft_fail, &state_lock, @@ -1821,7 +1822,7 @@ async fn upgrade_outlier_to_timeline_pdu( db, &incoming_pdu, val, - extremities, + extremities.iter().map(Deref::deref), state_ids_compressed, soft_fail, &state_lock, @@ -2114,11 +2115,11 @@ pub(crate) async fn fetch_signing_keys( /// Append the incoming event setting the state snapshot to the state from the /// server that sent the event. #[tracing::instrument(skip(db, pdu, pdu_json, new_room_leaves, state_ids_compressed, _mutex_lock))] -fn append_incoming_pdu( +fn append_incoming_pdu<'a>( db: &Database, pdu: &PduEvent, pdu_json: CanonicalJsonObject, - new_room_leaves: HashSet>, + new_room_leaves: impl IntoIterator + Clone + Debug, state_ids_compressed: HashSet, soft_fail: bool, _mutex_lock: &MutexGuard<'_, ()>, // Take mutex guard to make sure users get the room mutex @@ -2135,19 +2136,12 @@ fn append_incoming_pdu( if soft_fail { db.rooms .mark_as_referenced(&pdu.room_id, &pdu.prev_events)?; - db.rooms.replace_pdu_leaves( - &pdu.room_id, - &new_room_leaves.into_iter().collect::>(), - )?; + db.rooms + .replace_pdu_leaves(&pdu.room_id, new_room_leaves.clone())?; return Ok(None); } - let pdu_id = db.rooms.append_pdu( - pdu, - pdu_json, - &new_room_leaves.into_iter().collect::>(), - db, - )?; + let pdu_id = db.rooms.append_pdu(pdu, pdu_json, new_room_leaves, db)?; for appservice in db.appservice.all()? { if db.rooms.appservice_in_room(&pdu.room_id, &appservice, db)? { From 0183d003d0bfd864eab08499fb7385b4c8e9df0a Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Wed, 15 Dec 2021 13:58:25 +0100 Subject: [PATCH 052/201] Revert rename of Ruma<_> parameters --- src/client_server/membership.rs | 12 ++++++------ src/client_server/push.rs | 10 +++++----- src/client_server/sync.rs | 13 ++++++------- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index e6c9d4b6..e28f9a31 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -93,15 +93,15 @@ pub async fn join_room_by_id_route( /// - If the server does not know about the room: asks other servers over federation #[cfg_attr( feature = "conduit_bin", - post("/_matrix/client/r0/join/<_>", data = "") + post("/_matrix/client/r0/join/<_>", data = "") )] -#[tracing::instrument(skip(db, req))] +#[tracing::instrument(skip(db, body))] pub async fn join_room_by_id_or_alias_route( db: DatabaseGuard, - req: Ruma>, + body: Ruma>, ) -> ConduitResult { - let body = req.body; - let sender_user = req.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user.as_deref().expect("user is authenticated"); + let body = body.body; let (servers, room_id) = match Box::::try_from(body.room_id_or_alias) { Ok(room_id) => { @@ -129,7 +129,7 @@ pub async fn join_room_by_id_or_alias_route( let join_room_response = join_room_by_id_helper( &db, - req.sender_user.as_deref(), + Some(sender_user), &room_id, &servers, body.third_party_signed.as_ref(), diff --git a/src/client_server/push.rs b/src/client_server/push.rs index 64f27f1c..a8ba1a2a 100644 --- a/src/client_server/push.rs +++ b/src/client_server/push.rs @@ -105,15 +105,15 @@ pub async fn get_pushrule_route( /// Creates a single specified push rule for this user. #[cfg_attr( feature = "conduit_bin", - put("/_matrix/client/r0/pushrules/<_>/<_>/<_>", data = "") + put("/_matrix/client/r0/pushrules/<_>/<_>/<_>", data = "") )] -#[tracing::instrument(skip(db, req))] +#[tracing::instrument(skip(db, body))] pub async fn set_pushrule_route( db: DatabaseGuard, - req: Ruma>, + body: Ruma>, ) -> ConduitResult { - let sender_user = req.sender_user.as_ref().expect("user is authenticated"); - let body = req.body; + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let body = body.body; if body.scope != "global" { return Err(Error::BadRequest( diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 2e372f91..9ba3b7fb 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -54,17 +54,16 @@ use rocket::{get, tokio}; /// `since` will be cached #[cfg_attr( feature = "conduit_bin", - get("/_matrix/client/r0/sync", data = "") + get("/_matrix/client/r0/sync", data = "") )] -#[tracing::instrument(skip(db, req))] +#[tracing::instrument(skip(db, body))] pub async fn sync_events_route( db: DatabaseGuard, - req: Ruma>, + body: Ruma>, ) -> Result, RumaResponse> { - let body = req.body; - - let sender_user = req.sender_user.expect("user is authenticated"); - let sender_device = req.sender_device.expect("user is authenticated"); + let sender_user = body.sender_user.expect("user is authenticated"); + let sender_device = body.sender_device.expect("user is authenticated"); + let body = body.body; let arc_db = Arc::new(db); From 34d3f74f363719ab60263da62477cf0cd56bbbb0 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Sat, 27 Nov 2021 17:44:52 +0100 Subject: [PATCH 053/201] Use Arc for EventIds in PDUs Upgrades Ruma again to make this work. --- Cargo.lock | 36 ++++++++++++++-------------- Cargo.toml | 4 ++-- src/client_server/account.rs | 4 ++++ src/client_server/membership.rs | 7 +++++- src/client_server/message.rs | 2 +- src/client_server/redact.rs | 4 +++- src/client_server/room.rs | 14 +++++------ src/client_server/state.rs | 4 +++- src/database/rooms.rs | 12 +++++----- src/pdu.rs | 28 ++++++++++++---------- src/server_server.rs | 42 +++++++++++++-------------------- 11 files changed, 81 insertions(+), 76 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8b25b478..fbf4b3f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1984,7 +1984,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" +source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" dependencies = [ "assign", "js_int", @@ -2005,7 +2005,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.18.5" -source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" +source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" dependencies = [ "bytes", "http", @@ -2021,7 +2021,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.18.5" -source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" +source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2032,7 +2032,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" +source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" dependencies = [ "ruma-api", "ruma-common", @@ -2046,7 +2046,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.12.3" -source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" +source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" dependencies = [ "assign", "bytes", @@ -2066,7 +2066,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.6.0" -source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" +source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" dependencies = [ "indexmap", "js_int", @@ -2081,7 +2081,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.24.6" -source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" +source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" dependencies = [ "indoc", "js_int", @@ -2097,7 +2097,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.24.6" -source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" +source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2108,7 +2108,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.3.1" -source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" +source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" dependencies = [ "js_int", "ruma-api", @@ -2123,7 +2123,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.20.0" -source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" +source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" dependencies = [ "percent-encoding", "rand 0.8.4", @@ -2137,7 +2137,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.20.0" -source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" +source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" dependencies = [ "quote", "ruma-identifiers-validation", @@ -2147,7 +2147,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" +source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" dependencies = [ "thiserror", ] @@ -2155,7 +2155,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" +source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" dependencies = [ "js_int", "ruma-api", @@ -2168,7 +2168,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" +source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" dependencies = [ "js_int", "ruma-api", @@ -2183,7 +2183,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" +source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" dependencies = [ "bytes", "form_urlencoded", @@ -2197,7 +2197,7 @@ dependencies = [ [[package]] name = "ruma-serde-macros" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" +source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2208,7 +2208,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.9.0" -source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" +source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" dependencies = [ "base64 0.13.0", "ed25519-dalek", @@ -2225,7 +2225,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.4.1" -source = "git+https://github.com/ruma/ruma?rev=bba7d624425da2c65a834bbd0e633b7577488cdf#bba7d624425da2c65a834bbd0e633b7577488cdf" +source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" dependencies = [ "itertools 0.10.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index b24afb5c..02159e31 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle request # Used for matrix spec type definitions and helpers #ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } -ruma = { git = "https://github.com/ruma/ruma", rev = "bba7d624425da2c65a834bbd0e633b7577488cdf", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +ruma = { git = "https://github.com/ruma/ruma", rev = "16f031fabb7871fcd738b0f25391193ee4ca28a9", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { git = "https://github.com/timokoesters/ruma", rev = "50c1db7e0a3a21fc794b0cce3b64285a4c750c71", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } @@ -40,7 +40,7 @@ serde_json = { version = "1.0.67", features = ["raw_value"] } # Used for appservice registration files serde_yaml = "0.8.20" # Used for pdu definition -serde = "1.0.130" +serde = { version = "1.0.130", features = ["rc"] } # Used for secure identifiers rand = "0.8.4" # Used to hash passwords diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 3149187f..c4e118c9 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -306,6 +306,7 @@ pub async fn register_route( third_party_invite: None, blurhash: None, reason: None, + join_authorized_via_users_server: None, }) .expect("event is valid, we just created it"), unsigned: None, @@ -463,6 +464,7 @@ pub async fn register_route( third_party_invite: None, blurhash: None, reason: None, + join_authorized_via_users_server: None, }) .expect("event is valid, we just created it"), unsigned: None, @@ -485,6 +487,7 @@ pub async fn register_route( third_party_invite: None, blurhash: None, reason: None, + join_authorized_via_users_server: None, }) .expect("event is valid, we just created it"), unsigned: None, @@ -701,6 +704,7 @@ pub async fn deactivate_route( third_party_invite: None, blurhash: None, reason: None, + join_authorized_via_users_server: None, }; let mutex_state = Arc::clone( diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index e28f9a31..cede51f0 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -286,6 +286,7 @@ pub async fn ban_user_route( third_party_invite: None, blurhash: db.users.blurhash(&body.user_id)?, reason: None, + join_authorized_via_users_server: None, }), |event| { serde_json::from_str(event.content.get()) @@ -604,6 +605,7 @@ async fn join_room_by_id_helper( third_party_invite: None, blurhash: db.users.blurhash(sender_user)?, reason: None, + join_authorized_via_users_server: None, }) .expect("event is valid, we just created it"), ); @@ -757,6 +759,7 @@ async fn join_room_by_id_helper( third_party_invite: None, blurhash: db.users.blurhash(sender_user)?, reason: None, + join_authorized_via_users_server: None, }; db.rooms.build_and_append_pdu( @@ -906,6 +909,7 @@ pub(crate) async fn invite_helper<'a>( third_party_invite: None, blurhash: None, reason: None, + join_authorized_via_users_server: None, }) .expect("member event is valid value"); @@ -939,7 +943,7 @@ pub(crate) async fn invite_helper<'a>( } let pdu = PduEvent { - event_id: ruma::event_id!("$thiswillbefilledinlater").to_owned(), + event_id: ruma::event_id!("$thiswillbefilledinlater").into(), room_id: room_id.to_owned(), sender: sender_user.to_owned(), origin_server_ts: utils::millis_since_unix_epoch() @@ -1117,6 +1121,7 @@ pub(crate) async fn invite_helper<'a>( third_party_invite: None, blurhash: db.users.blurhash(user_id)?, reason: None, + join_authorized_via_users_server: None, }) .expect("event is valid, we just created it"), unsigned: None, diff --git a/src/client_server/message.rs b/src/client_server/message.rs index e5219433..60c756a3 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -98,7 +98,7 @@ pub async fn send_message_event_route( db.flush()?; - Ok(send_message_event::Response::new(event_id).into()) + Ok(send_message_event::Response::new((*event_id).to_owned()).into()) } /// # `GET /_matrix/client/r0/rooms/{roomId}/messages` diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index 7435c5c5..85de2330 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -25,6 +25,7 @@ pub async fn redact_event_route( body: Ruma>, ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let body = body.body; let mutex_state = Arc::clone( db.globals @@ -45,7 +46,7 @@ pub async fn redact_event_route( .expect("event is valid, we just created it"), unsigned: None, state_key: None, - redacts: Some(body.event_id.clone()), + redacts: Some(body.event_id.into()), }, sender_user, &body.room_id, @@ -57,5 +58,6 @@ pub async fn redact_event_route( db.flush()?; + let event_id = (*event_id).to_owned(); Ok(redact_event::Response { event_id }.into()) } diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 83571f1d..52d25425 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -22,6 +22,7 @@ use ruma::{ }, EventType, }, + int, serde::{CanonicalJsonObject, JsonObject}, RoomAliasId, RoomId, RoomVersionId, }; @@ -195,6 +196,7 @@ pub async fn create_room_route( third_party_invite: None, blurhash: db.users.blurhash(sender_user)?, reason: None, + join_authorized_via_users_server: None, }) .expect("event is valid, we just created it"), unsigned: None, @@ -220,11 +222,11 @@ pub async fn create_room_route( }); let mut users = BTreeMap::new(); - users.insert(sender_user.clone(), 100.into()); + users.insert(sender_user.clone(), int!(100)); if preset == create_room::RoomPreset::TrustedPrivateChat { for invite_ in &body.invite { - users.insert(invite_.clone(), 100.into()); + users.insert(invite_.clone(), int!(100)); } } @@ -569,7 +571,7 @@ pub async fn upgrade_room_route( // Use the m.room.tombstone event as the predecessor let predecessor = Some(ruma::events::room::create::PreviousRoom::new( body.room_id.clone(), - tombstone_event_id, + (*tombstone_event_id).to_owned(), )); // Send a m.room.create event containing a predecessor field and the applicable room_version @@ -633,6 +635,7 @@ pub async fn upgrade_room_route( third_party_invite: None, blurhash: db.users.blurhash(sender_user)?, reason: None, + join_authorized_via_users_server: None, }) .expect("event is valid, we just created it"), unsigned: None, @@ -697,10 +700,7 @@ pub async fn upgrade_room_route( .map_err(|_| Error::bad_database("Invalid room event in database."))?; // Setting events_default and invite to the greater of 50 and users_default + 1 - let new_level = max( - 50.into(), - power_levels_event_content.users_default + 1.into(), - ); + let new_level = max(int!(50), power_levels_event_content.users_default + int!(1)); power_levels_event_content.events_default = new_level; power_levels_event_content.invite = new_level; diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 0ba20620..e42694ae 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -52,6 +52,7 @@ pub async fn send_state_event_for_key_route( db.flush()?; + let event_id = (*event_id).to_owned(); Ok(send_state_event::Response { event_id }.into()) } @@ -93,6 +94,7 @@ pub async fn send_state_event_for_empty_key_route( db.flush()?; + let event_id = (*event_id).to_owned(); Ok(send_state_event::Response { event_id }.into()) } @@ -267,7 +269,7 @@ async fn send_state_event_for_key_helper( event_type: EventType, json: &Raw, state_key: String, -) -> Result> { +) -> Result> { let sender_user = sender; // TODO: Review this check, error if event is unparsable, use event type, allow alias if it diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 4c092bf7..fb9ecbf0 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -1162,14 +1162,14 @@ impl Rooms { /// Returns the leaf pdus of a room. #[tracing::instrument(skip(self))] - pub fn get_pdu_leaves(&self, room_id: &RoomId) -> Result>> { + pub fn get_pdu_leaves(&self, room_id: &RoomId) -> Result>> { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); self.roomid_pduleaves .scan_prefix(prefix) .map(|(_, bytes)| { - EventId::parse(utils::string_from_bytes(&bytes).map_err(|_| { + EventId::parse_arc(utils::string_from_bytes(&bytes).map_err(|_| { Error::bad_database("EventID in roomid_pduleaves is invalid unicode.") })?) .map_err(|_| Error::bad_database("EventId in roomid_pduleaves is invalid.")) @@ -1178,7 +1178,7 @@ impl Rooms { } #[tracing::instrument(skip(self, room_id, event_ids))] - pub fn mark_as_referenced(&self, room_id: &RoomId, event_ids: &[Box]) -> Result<()> { + pub fn mark_as_referenced(&self, room_id: &RoomId, event_ids: &[Arc]) -> Result<()> { for prev in event_ids { let mut key = room_id.as_bytes().to_vec(); key.extend_from_slice(prev.as_bytes()); @@ -1953,7 +1953,7 @@ impl Rooms { room_id: &RoomId, db: &Database, _mutex_lock: &MutexGuard<'_, ()>, // Take mutex guard to make sure users get the room mutex - ) -> Result> { + ) -> Result> { let PduBuilder { event_type, content, @@ -2019,7 +2019,7 @@ impl Rooms { } let mut pdu = PduEvent { - event_id: ruma::event_id!("$thiswillbefilledinlater").to_owned(), + event_id: ruma::event_id!("$thiswillbefilledinlater").into(), room_id: room_id.to_owned(), sender: sender.to_owned(), origin_server_ts: utils::millis_since_unix_epoch() @@ -2086,7 +2086,7 @@ impl Rooms { .expect("event is valid, we just created it"); // Generate event id - pdu.event_id = EventId::parse(format!( + pdu.event_id = EventId::parse_arc(format!( "${}", ruma::signatures::reference_hash(&pdu_json, &room_version_id) .expect("ruma can calculate reference hashes") diff --git a/src/pdu.rs b/src/pdu.rs index c1f3d27d..db9375e4 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -13,7 +13,7 @@ use serde_json::{ json, value::{to_raw_value, RawValue as RawJsonValue}, }; -use std::{cmp::Ordering, collections::BTreeMap, convert::TryInto, ops::Deref}; +use std::{cmp::Ordering, collections::BTreeMap, convert::TryInto, sync::Arc}; use tracing::warn; /// Content hashes of a PDU. @@ -25,7 +25,7 @@ pub struct EventHash { #[derive(Clone, Deserialize, Serialize, Debug)] pub struct PduEvent { - pub event_id: Box, + pub event_id: Arc, pub room_id: Box, pub sender: Box, pub origin_server_ts: UInt, @@ -34,11 +34,11 @@ pub struct PduEvent { pub content: Box, #[serde(skip_serializing_if = "Option::is_none")] pub state_key: Option, - pub prev_events: Vec>, + pub prev_events: Vec>, pub depth: UInt, - pub auth_events: Vec>, + pub auth_events: Vec>, #[serde(skip_serializing_if = "Option::is_none")] - pub redacts: Option>, + pub redacts: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub unsigned: Option>, pub hashes: EventHash, @@ -266,7 +266,9 @@ impl PduEvent { } impl state_res::Event for PduEvent { - fn event_id(&self) -> &EventId { + type Id = Arc; + + fn event_id(&self) -> &Self::Id { &self.event_id } @@ -294,16 +296,16 @@ impl state_res::Event for PduEvent { self.state_key.as_deref() } - fn prev_events(&self) -> Box + '_> { - Box::new(self.prev_events.iter().map(Deref::deref)) + fn prev_events(&self) -> Box + '_> { + Box::new(self.prev_events.iter()) } - fn auth_events(&self) -> Box + '_> { - Box::new(self.auth_events.iter().map(Deref::deref)) + fn auth_events(&self) -> Box + '_> { + Box::new(self.auth_events.iter()) } - fn redacts(&self) -> Option<&EventId> { - self.redacts.as_deref() + fn redacts(&self) -> Option<&Self::Id> { + self.redacts.as_ref() } } @@ -357,7 +359,7 @@ pub struct PduBuilder { pub content: Box, pub unsigned: Option>, pub state_key: Option, - pub redacts: Option>, + pub redacts: Option>, } /// Direct conversion prevents loss of the empty `state_key` that ruma requires. diff --git a/src/server_server.rs b/src/server_server.rs index ca6bb3fd..594152ae 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -995,13 +995,9 @@ pub(crate) async fn handle_incoming_pdu<'a>( } // 9. Fetch any missing prev events doing all checks listed here starting at 1. These are timeline events - let mut graph = HashMap::new(); + let mut graph: HashMap, _> = HashMap::new(); let mut eventid_info = HashMap::new(); - let mut todo_outlier_stack: Vec> = incoming_pdu - .prev_events - .iter() - .map(|x| Arc::from(&**x)) - .collect(); + let mut todo_outlier_stack: Vec> = incoming_pdu.prev_events.clone(); let mut amount = 0; @@ -1020,7 +1016,7 @@ pub(crate) async fn handle_incoming_pdu<'a>( if amount > 100 { // Max limit reached warn!("Max prev event limit reached!"); - graph.insert((*prev_event_id).to_owned(), HashSet::new()); + graph.insert(prev_event_id.clone(), HashSet::new()); continue; } @@ -1031,27 +1027,27 @@ pub(crate) async fn handle_incoming_pdu<'a>( amount += 1; for prev_prev in &pdu.prev_events { if !graph.contains_key(prev_prev) { - todo_outlier_stack.push(dbg!(Arc::from(&**prev_prev))); + todo_outlier_stack.push(dbg!(prev_prev.clone())); } } graph.insert( - (*prev_event_id).to_owned(), + prev_event_id.clone(), pdu.prev_events.iter().cloned().collect(), ); } else { // Time based check failed - graph.insert((*prev_event_id).to_owned(), HashSet::new()); + graph.insert(prev_event_id.clone(), HashSet::new()); } eventid_info.insert(prev_event_id.clone(), (pdu, json)); } else { // Get json failed - graph.insert((*prev_event_id).to_owned(), HashSet::new()); + graph.insert(prev_event_id.clone(), HashSet::new()); } } else { // Fetch and handle failed - graph.insert((*prev_event_id).to_owned(), HashSet::new()); + graph.insert(prev_event_id.clone(), HashSet::new()); } } @@ -1401,14 +1397,13 @@ async fn upgrade_outlier_to_timeline_pdu( .get_statekey_from_short(k) .map_err(|_| "Failed to get_statekey_from_short.".to_owned())?; - state.insert(k, (*id).to_owned()); + state.insert(k, id.clone()); starting_events.push(id); } auth_chain_sets.push( get_auth_chain(room_id, starting_events, db) .map_err(|_| "Failed to load auth chain.".to_owned())? - .map(|event_id| (*event_id).to_owned()) .collect(), ); @@ -1435,7 +1430,7 @@ async fn upgrade_outlier_to_timeline_pdu( .rooms .get_or_create_shortstatekey(&event_type, &state_key, &db.globals) .map_err(|_| "Failed to get_or_create_shortstatekey".to_owned())?; - Ok((shortstatekey, Arc::from(event_id))) + Ok((shortstatekey, event_id)) }) .collect::>()?, ), @@ -1752,7 +1747,6 @@ async fn upgrade_outlier_to_timeline_pdu( db, ) .map_err(|_| "Failed to load auth chain.".to_owned())? - .map(|event_id| (*event_id).to_owned()) .collect(), ); } @@ -1761,11 +1755,7 @@ async fn upgrade_outlier_to_timeline_pdu( .into_iter() .map(|map| { map.into_iter() - .map(|(k, id)| { - db.rooms - .get_statekey_from_short(k) - .map(|k| (k, (*id).to_owned())) - }) + .map(|(k, id)| db.rooms.get_statekey_from_short(k).map(|k| (k, id))) .collect::>>() }) .collect::>() @@ -2136,8 +2126,7 @@ fn append_incoming_pdu<'a>( if soft_fail { db.rooms .mark_as_referenced(&pdu.room_id, &pdu.prev_events)?; - db.rooms - .replace_pdu_leaves(&pdu.room_id, new_room_leaves.clone())?; + db.rooms.replace_pdu_leaves(&pdu.room_id, new_room_leaves)?; return Ok(None); } @@ -2282,7 +2271,7 @@ fn get_auth_chain_inner( event_id: &EventId, db: &Database, ) -> Result> { - let mut todo = vec![event_id.to_owned()]; + let mut todo = vec![Arc::from(event_id)]; let mut found = HashSet::new(); while let Some(event_id) = todo.pop() { @@ -2298,7 +2287,7 @@ fn get_auth_chain_inner( if !found.contains(&sauthevent) { found.insert(sauthevent); - todo.push(auth_event.to_owned()); + todo.push(auth_event.clone()); } } } @@ -2676,6 +2665,7 @@ pub fn create_join_event_template_route( membership: MembershipState::Join, third_party_invite: None, reason: None, + join_authorized_via_users_server: None, }) .expect("member event is valid value"); @@ -2709,7 +2699,7 @@ pub fn create_join_event_template_route( } let pdu = PduEvent { - event_id: ruma::event_id!("$thiswillbefilledinlater").to_owned(), + event_id: ruma::event_id!("$thiswillbefilledinlater").into(), room_id: body.room_id.clone(), sender: body.user_id.clone(), origin_server_ts: utils::millis_since_unix_epoch() From 3d25d46dc5b14c506692ea8a82151b6e4f39fafd Mon Sep 17 00:00:00 2001 From: Moritz Bitsch Date: Wed, 20 Oct 2021 06:20:34 +0200 Subject: [PATCH 054/201] Use simple BTreeMap to store uiaa requests some uiaa requests contain plaintext passwords which should never be persisted to disk. Currently there is no cleanup implemented (you have to restart conduit) --- src/database.rs | 3 +-- src/database/uiaa.rs | 16 +++++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/database.rs b/src/database.rs index 84ca68dc..83b0fd5e 100644 --- a/src/database.rs +++ b/src/database.rs @@ -250,8 +250,7 @@ impl Database { }, uiaa: uiaa::Uiaa { userdevicesessionid_uiaainfo: builder.open_tree("userdevicesessionid_uiaainfo")?, - userdevicesessionid_uiaarequest: builder - .open_tree("userdevicesessionid_uiaarequest")?, + userdevicesessionid_uiaarequest: RwLock::new(BTreeMap::new()), }, rooms: rooms::Rooms { edus: rooms::RoomEdus { diff --git a/src/database/uiaa.rs b/src/database/uiaa.rs index 1c0fb566..2ecca93d 100644 --- a/src/database/uiaa.rs +++ b/src/database/uiaa.rs @@ -1,4 +1,6 @@ use std::sync::Arc; +use std::sync::RwLock; +use std::collections::BTreeMap; use crate::{client_server::SESSION_ID_LENGTH, utils, Error, Result}; use ruma::{ @@ -18,7 +20,7 @@ use super::abstraction::Tree; pub struct Uiaa { pub(super) userdevicesessionid_uiaainfo: Arc, // User-interactive authentication - pub(super) userdevicesessionid_uiaarequest: Arc, // UiaaRequest = canonical json value + pub(super) userdevicesessionid_uiaarequest: RwLock, Vec>>, // UiaaRequest = canonical json value } impl Uiaa { @@ -153,10 +155,10 @@ impl Uiaa { userdevicesessionid.push(0xff); userdevicesessionid.extend_from_slice(session.as_bytes()); - self.userdevicesessionid_uiaarequest.insert( - &userdevicesessionid, - &serde_json::to_vec(request).expect("json value to vec always works"), - )?; + self.userdevicesessionid_uiaarequest.write().unwrap().insert( + userdevicesessionid, + serde_json::to_vec(request).expect("json value to vec always works"), + ); Ok(()) } @@ -173,8 +175,8 @@ impl Uiaa { userdevicesessionid.push(0xff); userdevicesessionid.extend_from_slice(session.as_bytes()); - self.userdevicesessionid_uiaarequest - .get(&userdevicesessionid)? + self.userdevicesessionid_uiaarequest.read().unwrap() + .get(&userdevicesessionid) .map(|bytes| { serde_json::from_str::( &utils::string_from_bytes(&bytes) From fe8cfe05569e667b03ee855a2463964a5a029661 Mon Sep 17 00:00:00 2001 From: Moritz Bitsch Date: Tue, 14 Dec 2021 17:55:28 +0100 Subject: [PATCH 055/201] Add database migration to remove stored passwords uiaarequests can contain plaintext passwords, which were stored on disk --- src/database.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/database.rs b/src/database.rs index 83b0fd5e..8b29b221 100644 --- a/src/database.rs +++ b/src/database.rs @@ -754,6 +754,15 @@ impl Database { println!("Migration: 9 -> 10 finished"); } + + if db.globals.database_version()? < 11 { + db._db + .open_tree("userdevicesessionid_uiaarequest")? + .clear()?; + db.globals.bump_database_version(11)?; + + println!("Migration: 10 -> 11 finished"); + } } let guard = db.read().await; From 0725b69abb7453df534a764947b6015ffe8293c4 Mon Sep 17 00:00:00 2001 From: Moritz Bitsch Date: Sat, 18 Dec 2021 18:46:38 +0100 Subject: [PATCH 056/201] Clean up userdevicesessionid_uiaarequest BTreeMap There is no need to encode or decode anything as we are not saving to disk --- src/database/uiaa.rs | 52 ++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/src/database/uiaa.rs b/src/database/uiaa.rs index 2ecca93d..461a3e27 100644 --- a/src/database/uiaa.rs +++ b/src/database/uiaa.rs @@ -1,6 +1,6 @@ +use std::collections::BTreeMap; use std::sync::Arc; use std::sync::RwLock; -use std::collections::BTreeMap; use crate::{client_server::SESSION_ID_LENGTH, utils, Error, Result}; use ruma::{ @@ -20,7 +20,8 @@ use super::abstraction::Tree; pub struct Uiaa { pub(super) userdevicesessionid_uiaainfo: Arc, // User-interactive authentication - pub(super) userdevicesessionid_uiaarequest: RwLock, Vec>>, // UiaaRequest = canonical json value + pub(super) userdevicesessionid_uiaarequest: + RwLock>, } impl Uiaa { @@ -149,16 +150,17 @@ impl Uiaa { session: &str, request: &CanonicalJsonValue, ) -> Result<()> { - let mut userdevicesessionid = user_id.as_bytes().to_vec(); - userdevicesessionid.push(0xff); - userdevicesessionid.extend_from_slice(device_id.as_bytes()); - userdevicesessionid.push(0xff); - userdevicesessionid.extend_from_slice(session.as_bytes()); - - self.userdevicesessionid_uiaarequest.write().unwrap().insert( - userdevicesessionid, - serde_json::to_vec(request).expect("json value to vec always works"), - ); + self.userdevicesessionid_uiaarequest + .write() + .unwrap() + .insert( + ( + user_id.to_owned(), + device_id.to_string(), + session.to_string(), + ), + request.to_owned(), + ); Ok(()) } @@ -169,22 +171,16 @@ impl Uiaa { device_id: &DeviceId, session: &str, ) -> Result> { - let mut userdevicesessionid = user_id.as_bytes().to_vec(); - userdevicesessionid.push(0xff); - userdevicesessionid.extend_from_slice(device_id.as_bytes()); - userdevicesessionid.push(0xff); - userdevicesessionid.extend_from_slice(session.as_bytes()); - - self.userdevicesessionid_uiaarequest.read().unwrap() - .get(&userdevicesessionid) - .map(|bytes| { - serde_json::from_str::( - &utils::string_from_bytes(&bytes) - .map_err(|_| Error::bad_database("Invalid uiaa request bytes in db."))?, - ) - .map_err(|_| Error::bad_database("Invalid uiaa request in db.")) - }) - .transpose() + Ok(self + .userdevicesessionid_uiaarequest + .read() + .unwrap() + .get(&( + user_id.to_owned(), + device_id.to_string(), + session.to_string(), + )) + .map(|j| j.to_owned())) } fn update_uiaa_session( From 720a54b3bb74301eaf08f54edd163995bf5ef7fa Mon Sep 17 00:00:00 2001 From: Moritz Bitsch Date: Sat, 18 Dec 2021 19:05:18 +0100 Subject: [PATCH 057/201] Use String to store UserId for uiaa request Fixes compilation error after ruma upgrade --- src/database/uiaa.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/database/uiaa.rs b/src/database/uiaa.rs index 461a3e27..6a5f7a33 100644 --- a/src/database/uiaa.rs +++ b/src/database/uiaa.rs @@ -21,7 +21,7 @@ use super::abstraction::Tree; pub struct Uiaa { pub(super) userdevicesessionid_uiaainfo: Arc, // User-interactive authentication pub(super) userdevicesessionid_uiaarequest: - RwLock>, + RwLock>, } impl Uiaa { @@ -155,7 +155,7 @@ impl Uiaa { .unwrap() .insert( ( - user_id.to_owned(), + user_id.to_string(), device_id.to_string(), session.to_string(), ), @@ -176,7 +176,7 @@ impl Uiaa { .read() .unwrap() .get(&( - user_id.to_owned(), + user_id.to_string(), device_id.to_string(), session.to_string(), )) From 7857da8a0b6322618b12e4b41c6945bcd7dee9ef Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Mon, 20 Dec 2021 15:46:36 +0100 Subject: [PATCH 058/201] Add ability to remove an appservice --- APPSERVICES.md | 8 ++++++++ src/database/admin.rs | 4 ++++ src/database/appservice.rs | 9 +++++++++ src/database/rooms.rs | 9 +++++++++ 4 files changed, 30 insertions(+) diff --git a/APPSERVICES.md b/APPSERVICES.md index 26c34cc4..894bc6f4 100644 --- a/APPSERVICES.md +++ b/APPSERVICES.md @@ -42,6 +42,14 @@ could help. ## Appservice-specific instructions +### Remove an appservice + +To remove an appservice go to your admin room and execute + +```@conduit:your.server.name: unregister_appservice ``` + +where `` one of the output of `list_appservices`. + ### Tested appservices These appservices have been tested and work with Conduit without any extra steps: diff --git a/src/database/admin.rs b/src/database/admin.rs index 1e5c47c9..0702bcdd 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -12,6 +12,7 @@ use tracing::warn; pub enum AdminCommand { RegisterAppservice(serde_yaml::Value), + UnregisterAppservice(String), ListAppservices, SendMessage(RoomMessageEventContent), } @@ -96,6 +97,9 @@ impl Admin { AdminCommand::RegisterAppservice(yaml) => { guard.appservice.register_appservice(yaml).unwrap(); // TODO handle error } + AdminCommand::UnregisterAppservice(service_name) => { + guard.appservice.unregister_appservice(&service_name).unwrap(); // TODO: see above + } AdminCommand::ListAppservices => { if let Ok(appservices) = guard.appservice.iter_ids().map(|ids| ids.collect::>()) { let count = appservices.len(); diff --git a/src/database/appservice.rs b/src/database/appservice.rs index 7cc91372..caa48ad0 100644 --- a/src/database/appservice.rs +++ b/src/database/appservice.rs @@ -27,6 +27,15 @@ impl Appservice { Ok(()) } + /** + * Remove an appservice registration + * service_name is the name you send to register the service + */ + pub fn unregister_appservice(&self, service_name: &str) -> Result<()> { + self.id_appserviceregistrations.remove(service_name.as_bytes())?; + Ok(()) + } + pub fn get_registration(&self, id: &str) -> Result> { self.cached_registrations .read() diff --git a/src/database/rooms.rs b/src/database/rooms.rs index fb9ecbf0..612bd51d 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -1528,6 +1528,15 @@ impl Rooms { )); } } + "unregister_appservice" => { + if args.len() == 1 { + db.admin.send(AdminCommand::UnregisterAppservice(args[0].to_owned())); + } else { + db.admin.send(AdminCommand::SendMessage( + RoomMessageEventContent::text_plain("Missing appservice identifier"), + )); + } + } "list_appservices" => { db.admin.send(AdminCommand::ListAppservices); } From b6c9582cf4e9255e0610a63849bb3c5113be16e2 Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Wed, 22 Dec 2021 13:09:56 +0100 Subject: [PATCH 059/201] Fix doc style comment according to Rust; VSCode added line breaks --- src/database/appservice.rs | 12 +++++++----- src/database/rooms.rs | 8 ++++++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/database/appservice.rs b/src/database/appservice.rs index caa48ad0..910964a4 100644 --- a/src/database/appservice.rs +++ b/src/database/appservice.rs @@ -27,12 +27,14 @@ impl Appservice { Ok(()) } - /** - * Remove an appservice registration - * service_name is the name you send to register the service - */ + /// Remove an appservice registration + /// + /// # Arguments + /// + /// * `service_name` - the name you send to register the service previously pub fn unregister_appservice(&self, service_name: &str) -> Result<()> { - self.id_appserviceregistrations.remove(service_name.as_bytes())?; + self.id_appserviceregistrations + .remove(service_name.as_bytes())?; Ok(()) } diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 612bd51d..775e2f8d 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -1530,10 +1530,14 @@ impl Rooms { } "unregister_appservice" => { if args.len() == 1 { - db.admin.send(AdminCommand::UnregisterAppservice(args[0].to_owned())); + db.admin.send(AdminCommand::UnregisterAppservice( + args[0].to_owned(), + )); } else { db.admin.send(AdminCommand::SendMessage( - RoomMessageEventContent::text_plain("Missing appservice identifier"), + RoomMessageEventContent::text_plain( + "Missing appservice identifier", + ), )); } } From 7f2445be6ca7798ec25458e5447b23e7aeea1f7f Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Wed, 22 Dec 2021 16:48:27 +0100 Subject: [PATCH 060/201] On unregister_appservice(service_name), remove the appservice service_name from cache too --- src/database/appservice.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/database/appservice.rs b/src/database/appservice.rs index 910964a4..847d7479 100644 --- a/src/database/appservice.rs +++ b/src/database/appservice.rs @@ -35,6 +35,10 @@ impl Appservice { pub fn unregister_appservice(&self, service_name: &str) -> Result<()> { self.id_appserviceregistrations .remove(service_name.as_bytes())?; + self.cached_registrations. + write(). + unwrap(). + remove(service_name); Ok(()) } From c4a438460e0537e465f5b93514fd05b66a03ad37 Mon Sep 17 00:00:00 2001 From: Moritz Bitsch Date: Wed, 22 Dec 2021 19:26:23 +0100 Subject: [PATCH 061/201] Use Box to store UserID and DeviceID Userid and DeviceID are of unknown size, use Box to be able to store them into the userdevicesessionid_uiaarequest BTreeMap --- src/database/uiaa.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/database/uiaa.rs b/src/database/uiaa.rs index 6a5f7a33..772dab9e 100644 --- a/src/database/uiaa.rs +++ b/src/database/uiaa.rs @@ -21,7 +21,7 @@ use super::abstraction::Tree; pub struct Uiaa { pub(super) userdevicesessionid_uiaainfo: Arc, // User-interactive authentication pub(super) userdevicesessionid_uiaarequest: - RwLock>, + RwLock, Box, String), CanonicalJsonValue>>, } impl Uiaa { @@ -155,8 +155,8 @@ impl Uiaa { .unwrap() .insert( ( - user_id.to_string(), - device_id.to_string(), + user_id.to_owned(), + device_id.to_owned(), session.to_string(), ), request.to_owned(), @@ -176,8 +176,8 @@ impl Uiaa { .read() .unwrap() .get(&( - user_id.to_string(), - device_id.to_string(), + user_id.to_owned(), + device_id.to_owned(), session.to_string(), )) .map(|j| j.to_owned())) From aba95b20f3b3c252e72ac87312b10df8068f7419 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Wed, 22 Dec 2021 19:41:33 +0100 Subject: [PATCH 062/201] Upgrade Ruma --- Cargo.lock | 56 +++++++++------- Cargo.toml | 4 +- src/client_server/keys.rs | 45 +++++++------ src/client_server/sync.rs | 2 + src/database/key_backups.rs | 65 +++++++++++------- src/database/users.rs | 129 ++++++++++++++++++++++-------------- 6 files changed, 182 insertions(+), 119 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fbf4b3f2..69a026b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -938,7 +938,7 @@ checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11" dependencies = [ "bytes", "fnv", - "itoa", + "itoa 0.4.8", ] [[package]] @@ -979,7 +979,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa", + "itoa 0.4.8", "pin-project-lite", "socket2 0.4.1", "tokio", @@ -1114,6 +1114,12 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" +[[package]] +name = "itoa" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" + [[package]] name = "jobserver" version = "0.1.24" @@ -1984,7 +1990,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" +source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" dependencies = [ "assign", "js_int", @@ -2005,7 +2011,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.18.5" -source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" +source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" dependencies = [ "bytes", "http", @@ -2021,7 +2027,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.18.5" -source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" +source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2032,7 +2038,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" +source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" dependencies = [ "ruma-api", "ruma-common", @@ -2046,7 +2052,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.12.3" -source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" +source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" dependencies = [ "assign", "bytes", @@ -2066,7 +2072,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.6.0" -source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" +source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" dependencies = [ "indexmap", "js_int", @@ -2081,7 +2087,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.24.6" -source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" +source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" dependencies = [ "indoc", "js_int", @@ -2097,7 +2103,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.24.6" -source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" +source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2108,7 +2114,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.3.1" -source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" +source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" dependencies = [ "js_int", "ruma-api", @@ -2123,7 +2129,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.20.0" -source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" +source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" dependencies = [ "percent-encoding", "rand 0.8.4", @@ -2137,7 +2143,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.20.0" -source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" +source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" dependencies = [ "quote", "ruma-identifiers-validation", @@ -2147,7 +2153,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" +source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" dependencies = [ "thiserror", ] @@ -2155,7 +2161,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" +source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" dependencies = [ "js_int", "ruma-api", @@ -2168,7 +2174,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" +source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" dependencies = [ "js_int", "ruma-api", @@ -2183,11 +2189,11 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" +source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" dependencies = [ "bytes", "form_urlencoded", - "itoa", + "itoa 0.4.8", "js_int", "ruma-serde-macros", "serde", @@ -2197,7 +2203,7 @@ dependencies = [ [[package]] name = "ruma-serde-macros" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" +source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2208,7 +2214,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.9.0" -source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" +source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" dependencies = [ "base64 0.13.0", "ed25519-dalek", @@ -2225,7 +2231,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.4.1" -source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9" +source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" dependencies = [ "itertools 0.10.1", "js_int", @@ -2404,11 +2410,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.67" +version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7f9e390c27c3c0ce8bc5d725f6e4d30a29d26659494aa4b17535f7522c5c950" +checksum = "bcbd0344bc6533bc7ec56df11d42fb70f1b912351c0825ccb7211b59d8af7cf5" dependencies = [ - "itoa", + "itoa 1.0.1", "ryu", "serde", ] @@ -2420,7 +2426,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" dependencies = [ "form_urlencoded", - "itoa", + "itoa 0.4.8", "ryu", "serde", ] diff --git a/Cargo.toml b/Cargo.toml index 02159e31..e64e2751 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle request # Used for matrix spec type definitions and helpers #ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } -ruma = { git = "https://github.com/ruma/ruma", rev = "16f031fabb7871fcd738b0f25391193ee4ca28a9", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +ruma = { git = "https://github.com/ruma/ruma", rev = "7cf3abbaf02995b03db74429090ca5af1cd71edc", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { git = "https://github.com/timokoesters/ruma", rev = "50c1db7e0a3a21fc794b0cce3b64285a4c750c71", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } @@ -36,7 +36,7 @@ http = "0.2.4" # Used to find data directory for default db path directories = "3.0.2" # Used for ruma wrapper -serde_json = { version = "1.0.67", features = ["raw_value"] } +serde_json = { version = "1.0.70", features = ["raw_value"] } # Used for appservice registration files serde_yaml = "0.8.20" # Used for pdu definition diff --git a/src/client_server/keys.rs b/src/client_server/keys.rs index 08ea6e76..be0675d8 100644 --- a/src/client_server/keys.rs +++ b/src/client_server/keys.rs @@ -15,7 +15,7 @@ use ruma::{ }, federation, }, - encryption::UnsignedDeviceInfo, + serde::Raw, DeviceId, DeviceKeyAlgorithm, UserId, }; use serde_json::json; @@ -42,16 +42,9 @@ pub async fn upload_keys_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); - if let Some(one_time_keys) = &body.one_time_keys { - for (key_key, key_value) in one_time_keys { - db.users.add_one_time_key( - sender_user, - sender_device, - key_key, - key_value, - &db.globals, - )?; - } + for (key_key, key_value) in &body.one_time_keys { + db.users + .add_one_time_key(sender_user, sender_device, key_key, key_value, &db.globals)?; } if let Some(device_keys) = &body.device_keys { @@ -350,10 +343,8 @@ pub(crate) async fn get_keys_helper bool>( Error::bad_database("all_device_keys contained nonexistent device.") })?; - keys.unsigned = UnsignedDeviceInfo { - device_display_name: metadata.display_name, - }; - + add_unsigned_device_display_name(&mut keys, metadata) + .map_err(|_| Error::bad_database("invalid device keys in database"))?; container.insert(device_id, keys); } } @@ -369,10 +360,8 @@ pub(crate) async fn get_keys_helper bool>( ), )?; - keys.unsigned = UnsignedDeviceInfo { - device_display_name: metadata.display_name, - }; - + add_unsigned_device_display_name(&mut keys, metadata) + .map_err(|_| Error::bad_database("invalid device keys in database"))?; container.insert(device_id.to_owned(), keys); } device_keys.insert(user_id.to_owned(), container); @@ -441,6 +430,24 @@ pub(crate) async fn get_keys_helper bool>( }) } +fn add_unsigned_device_display_name( + keys: &mut Raw, + metadata: ruma::api::client::r0::device::Device, +) -> serde_json::Result<()> { + if let Some(display_name) = metadata.display_name { + let mut object = keys.deserialize_as::>()?; + + let unsigned = object.entry("unsigned").or_insert_with(|| json!({})); + if let serde_json::Value::Object(unsigned_object) = unsigned { + unsigned_object.insert("device_display_name".to_owned(), display_name.into()); + } + + *keys = Raw::from_json(serde_json::value::to_raw_value(&object)?); + } + + Ok(()) +} + pub(crate) async fn claim_keys_helper( one_time_keys_input: &BTreeMap, BTreeMap, DeviceKeyAlgorithm>>, db: &Database, diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 9ba3b7fb..64588a2c 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -762,6 +762,8 @@ async fn sync_helper( .users .get_to_device_events(&sender_user, &sender_device)?, }, + // Fallback keys are not yet supported + device_unused_fallback_key_types: None, }; // TODO: Retry the endpoint instead of returning (waiting for #118) diff --git a/src/database/key_backups.rs b/src/database/key_backups.rs index 56963c08..b74bc408 100644 --- a/src/database/key_backups.rs +++ b/src/database/key_backups.rs @@ -4,8 +4,10 @@ use ruma::{ error::ErrorKind, r0::backup::{BackupAlgorithm, KeyBackupData, RoomKeyBackup}, }, + serde::Raw, RoomId, UserId, }; +use serde_json::json; use std::{collections::BTreeMap, sync::Arc}; use super::abstraction::Tree; @@ -20,7 +22,7 @@ impl KeyBackups { pub fn create_backup( &self, user_id: &UserId, - backup_metadata: &BackupAlgorithm, + backup_metadata: &Raw, globals: &super::globals::Globals, ) -> Result { let version = globals.next_count()?.to_string(); @@ -59,7 +61,7 @@ impl KeyBackups { &self, user_id: &UserId, version: &str, - backup_metadata: &BackupAlgorithm, + backup_metadata: &Raw, globals: &super::globals::Globals, ) -> Result { let mut key = user_id.as_bytes().to_vec(); @@ -73,12 +75,8 @@ impl KeyBackups { )); } - self.backupid_algorithm.insert( - &key, - serde_json::to_string(backup_metadata) - .expect("BackupAlgorithm::to_string always works") - .as_bytes(), - )?; + self.backupid_algorithm + .insert(&key, backup_metadata.json().get().as_bytes())?; self.backupid_etag .insert(&key, &globals.next_count()?.to_be_bytes())?; Ok(version.to_owned()) @@ -105,7 +103,10 @@ impl KeyBackups { .transpose() } - pub fn get_latest_backup(&self, user_id: &UserId) -> Result> { + pub fn get_latest_backup( + &self, + user_id: &UserId, + ) -> Result)>> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xff); let mut last_possible_key = prefix.clone(); @@ -133,7 +134,11 @@ impl KeyBackups { .transpose() } - pub fn get_backup(&self, user_id: &UserId, version: &str) -> Result> { + pub fn get_backup( + &self, + user_id: &UserId, + version: &str, + ) -> Result>> { let mut key = user_id.as_bytes().to_vec(); key.push(0xff); key.extend_from_slice(version.as_bytes()); @@ -152,7 +157,7 @@ impl KeyBackups { version: &str, room_id: &RoomId, session_id: &str, - key_data: &KeyBackupData, + key_data: &Raw, globals: &super::globals::Globals, ) -> Result<()> { let mut key = user_id.as_bytes().to_vec(); @@ -174,10 +179,8 @@ impl KeyBackups { key.push(0xff); key.extend_from_slice(session_id.as_bytes()); - self.backupkeyid_backup.insert( - &key, - &serde_json::to_vec(&key_data).expect("KeyBackupData::to_vec always works"), - )?; + self.backupkeyid_backup + .insert(&key, key_data.json().get().as_bytes())?; Ok(()) } @@ -209,13 +212,13 @@ impl KeyBackups { &self, user_id: &UserId, version: &str, - ) -> Result, RoomKeyBackup>> { + ) -> Result, Raw>> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xff); prefix.extend_from_slice(version.as_bytes()); prefix.push(0xff); - let mut rooms = BTreeMap::, RoomKeyBackup>::new(); + let mut rooms = BTreeMap::, Raw>::new(); for result in self .backupkeyid_backup @@ -241,7 +244,7 @@ impl KeyBackups { Error::bad_database("backupkeyid_backup room_id is invalid room id.") })?; - let key_data = serde_json::from_slice(&value).map_err(|_| { + let key_data: serde_json::Value = serde_json::from_slice(&value).map_err(|_| { Error::bad_database("KeyBackupData in backupkeyid_backup is invalid.") })?; @@ -249,13 +252,25 @@ impl KeyBackups { }) { let (room_id, session_id, key_data) = result?; - rooms - .entry(room_id) - .or_insert_with(|| RoomKeyBackup { + let room_key_backup = rooms.entry(room_id).or_insert_with(|| { + Raw::new(&RoomKeyBackup { sessions: BTreeMap::new(), }) - .sessions - .insert(session_id, key_data); + .expect("RoomKeyBackup serialization") + }); + + let mut object = room_key_backup + .deserialize_as::>() + .map_err(|_| Error::bad_database("RoomKeyBackup is not an object"))?; + + let sessions = object.entry("session").or_insert_with(|| json!({})); + if let serde_json::Value::Object(unsigned_object) = sessions { + unsigned_object.insert(session_id, key_data); + } + + *room_key_backup = Raw::from_json( + serde_json::value::to_raw_value(&object).expect("Value => RawValue serialization"), + ); } Ok(rooms) @@ -266,7 +281,7 @@ impl KeyBackups { user_id: &UserId, version: &str, room_id: &RoomId, - ) -> Result> { + ) -> Result>> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xff); prefix.extend_from_slice(version.as_bytes()); @@ -304,7 +319,7 @@ impl KeyBackups { version: &str, room_id: &RoomId, session_id: &str, - ) -> Result> { + ) -> Result>> { let mut key = user_id.as_bytes().to_vec(); key.push(0xff); key.extend_from_slice(version.as_bytes()); diff --git a/src/database/users.rs b/src/database/users.rs index d4bf4890..63a63f00 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -8,7 +8,12 @@ use ruma::{ DeviceId, DeviceKeyAlgorithm, DeviceKeyId, MilliSecondsSinceUnixEpoch, RoomAliasId, UInt, UserId, }; -use std::{collections::BTreeMap, convert::TryInto, mem, sync::Arc}; +use std::{ + collections::BTreeMap, + convert::{TryFrom, TryInto}, + mem, + sync::Arc, +}; use tracing::warn; use super::abstraction::Tree; @@ -359,7 +364,7 @@ impl Users { user_id: &UserId, device_id: &DeviceId, one_time_key_key: &DeviceKeyId, - one_time_key_value: &OneTimeKey, + one_time_key_value: &Raw, globals: &super::globals::Globals, ) -> Result<()> { let mut key = user_id.as_bytes().to_vec(); @@ -409,7 +414,7 @@ impl Users { device_id: &DeviceId, key_algorithm: &DeviceKeyAlgorithm, globals: &super::globals::Globals, - ) -> Result, OneTimeKey)>> { + ) -> Result, Raw)>> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xff); prefix.extend_from_slice(device_id.as_bytes()); @@ -480,7 +485,7 @@ impl Users { &self, user_id: &UserId, device_id: &DeviceId, - device_keys: &DeviceKeys, + device_keys: &Raw, rooms: &super::rooms::Rooms, globals: &super::globals::Globals, ) -> Result<()> { @@ -509,9 +514,9 @@ impl Users { pub fn add_cross_signing_keys( &self, user_id: &UserId, - master_key: &CrossSigningKey, - self_signing_key: &Option, - user_signing_key: &Option, + master_key: &Raw, + self_signing_key: &Option>, + user_signing_key: &Option>, rooms: &super::rooms::Rooms, globals: &super::globals::Globals, ) -> Result<()> { @@ -521,7 +526,12 @@ impl Users { prefix.push(0xff); // Master key - let mut master_key_ids = master_key.keys.values(); + let master_key_map = master_key + .deserialize() + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid master key"))? + .keys; + let mut master_key_ids = master_key_map.values(); + let master_key_id = master_key_ids.next().ok_or(Error::BadRequest( ErrorKind::InvalidParam, "Master key contained no key.", @@ -537,17 +547,21 @@ impl Users { let mut master_key_key = prefix.clone(); master_key_key.extend_from_slice(master_key_id.as_bytes()); - self.keyid_key.insert( - &master_key_key, - &serde_json::to_vec(&master_key).expect("CrossSigningKey::to_vec always works"), - )?; + self.keyid_key + .insert(&master_key_key, master_key.json().get().as_bytes())?; self.userid_masterkeyid .insert(user_id.as_bytes(), &master_key_key)?; // Self-signing key if let Some(self_signing_key) = self_signing_key { - let mut self_signing_key_ids = self_signing_key.keys.values(); + let self_signing_key_map = self_signing_key + .deserialize() + .map_err(|_| { + Error::BadRequest(ErrorKind::InvalidParam, "Invalid self signing key") + })? + .keys; + let mut self_signing_key_ids = self_signing_key_map.values(); let self_signing_key_id = self_signing_key_ids.next().ok_or(Error::BadRequest( ErrorKind::InvalidParam, "Self signing key contained no key.", @@ -565,8 +579,7 @@ impl Users { self.keyid_key.insert( &self_signing_key_key, - &serde_json::to_vec(&self_signing_key) - .expect("CrossSigningKey::to_vec always works"), + self_signing_key.json().get().as_bytes(), )?; self.userid_selfsigningkeyid @@ -575,7 +588,13 @@ impl Users { // User-signing key if let Some(user_signing_key) = user_signing_key { - let mut user_signing_key_ids = user_signing_key.keys.values(); + let user_signing_key_map = user_signing_key + .deserialize() + .map_err(|_| { + Error::BadRequest(ErrorKind::InvalidParam, "Invalid user signing key") + })? + .keys; + let mut user_signing_key_ids = user_signing_key_map.values(); let user_signing_key_id = user_signing_key_ids.next().ok_or(Error::BadRequest( ErrorKind::InvalidParam, "User signing key contained no key.", @@ -593,8 +612,7 @@ impl Users { self.keyid_key.insert( &user_signing_key_key, - &serde_json::to_vec(&user_signing_key) - .expect("CrossSigningKey::to_vec always works"), + user_signing_key.json().get().as_bytes(), )?; self.userid_usersigningkeyid @@ -727,7 +745,7 @@ impl Users { &self, user_id: &UserId, device_id: &DeviceId, - ) -> Result> { + ) -> Result>> { let mut key = user_id.as_bytes().to_vec(); key.push(0xff); key.extend_from_slice(device_id.as_bytes()); @@ -744,25 +762,19 @@ impl Users { &self, user_id: &UserId, allowed_signatures: F, - ) -> Result> { + ) -> Result>> { self.userid_masterkeyid .get(user_id.as_bytes())? .map_or(Ok(None), |key| { self.keyid_key.get(&key)?.map_or(Ok(None), |bytes| { - let mut cross_signing_key = serde_json::from_slice::(&bytes) - .map_err(|_| { - Error::bad_database("CrossSigningKey in db is invalid.") - })?; + let mut cross_signing_key = serde_json::from_slice::(&bytes) + .map_err(|_| Error::bad_database("CrossSigningKey in db is invalid."))?; + clean_signatures(&mut cross_signing_key, user_id, allowed_signatures)?; - // A user is not allowed to see signatures from users other than himself and - // the target user - cross_signing_key.signatures = cross_signing_key - .signatures - .into_iter() - .filter(|(user, _)| allowed_signatures(user)) - .collect(); - - Ok(Some(cross_signing_key)) + Ok(Some(Raw::from_json( + serde_json::value::to_raw_value(&cross_signing_key) + .expect("Value to RawValue serialization"), + ))) }) }) } @@ -772,31 +784,25 @@ impl Users { &self, user_id: &UserId, allowed_signatures: F, - ) -> Result> { + ) -> Result>> { self.userid_selfsigningkeyid .get(user_id.as_bytes())? .map_or(Ok(None), |key| { self.keyid_key.get(&key)?.map_or(Ok(None), |bytes| { - let mut cross_signing_key = serde_json::from_slice::(&bytes) - .map_err(|_| { - Error::bad_database("CrossSigningKey in db is invalid.") - })?; + let mut cross_signing_key = serde_json::from_slice::(&bytes) + .map_err(|_| Error::bad_database("CrossSigningKey in db is invalid."))?; + clean_signatures(&mut cross_signing_key, user_id, allowed_signatures)?; - // A user is not allowed to see signatures from users other than himself and - // the target user - cross_signing_key.signatures = cross_signing_key - .signatures - .into_iter() - .filter(|(user, _)| user == user_id || allowed_signatures(user)) - .collect(); - - Ok(Some(cross_signing_key)) + Ok(Some(Raw::from_json( + serde_json::value::to_raw_value(&cross_signing_key) + .expect("Value to RawValue serialization"), + ))) }) }) } #[tracing::instrument(skip(self, user_id))] - pub fn get_user_signing_key(&self, user_id: &UserId) -> Result> { + pub fn get_user_signing_key(&self, user_id: &UserId) -> Result>> { self.userid_usersigningkeyid .get(user_id.as_bytes())? .map_or(Ok(None), |key| { @@ -991,3 +997,30 @@ impl Users { Ok(()) } } + +/// Ensure that a user only sees signatures from themselves and the target user +fn clean_signatures bool>( + cross_signing_key: &mut serde_json::Value, + user_id: &UserId, + allowed_signatures: F, +) -> Result<(), Error> { + if let Some(signatures) = cross_signing_key + .get_mut("signatures") + .and_then(|v| v.as_object_mut()) + { + // Don't allocate for the full size of the current signatures, but require + // at most one resize if nothing is dropped + let new_capacity = signatures.len() / 2; + for (user, signature) in + mem::replace(signatures, serde_json::Map::with_capacity(new_capacity)) + { + let id = <&UserId>::try_from(user.as_str()) + .map_err(|_| Error::bad_database("Invalid user ID in database."))?; + if id == user_id || allowed_signatures(id) { + signatures.insert(user, signature); + } + } + } + + Ok(()) +} From a889e884e684aa433772d8d61ee965c062a38790 Mon Sep 17 00:00:00 2001 From: Tglman Date: Thu, 23 Dec 2021 22:16:40 +0000 Subject: [PATCH 063/201] refactor:moved key watch wake logic to specific module --- Cargo.toml | 2 +- src/database/abstraction.rs | 3 ++ src/database/abstraction/heed.rs | 48 ++++--------------------- src/database/abstraction/sqlite.rs | 46 ++++-------------------- src/database/abstraction/watchers.rs | 54 ++++++++++++++++++++++++++++ 5 files changed, 70 insertions(+), 83 deletions(-) create mode 100644 src/database/abstraction/watchers.rs diff --git a/Cargo.toml b/Cargo.toml index 02159e31..ceae6ae9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,7 +87,7 @@ sha-1 = "0.9.8" default = ["conduit_bin", "backend_sqlite"] backend_sled = ["sled"] backend_sqlite = ["sqlite"] -backend_heed = ["heed", "crossbeam"] +backend_heed = ["heed", "crossbeam", "parking_lot"] sqlite = ["rusqlite", "parking_lot", "crossbeam", "tokio/signal"] conduit_bin = [] # TODO: add rocket to this when it is optional diff --git a/src/database/abstraction.rs b/src/database/abstraction.rs index 11bbc3b1..67b80d1a 100644 --- a/src/database/abstraction.rs +++ b/src/database/abstraction.rs @@ -12,6 +12,9 @@ pub mod sqlite; #[cfg(feature = "heed")] pub mod heed; +#[cfg(any(feature = "sqlite", feature = "heed"))] +pub mod watchers; + pub trait DatabaseEngine: Sized { fn open(config: &Config) -> Result>; fn open_tree(self: &Arc, name: &'static str) -> Result>; diff --git a/src/database/abstraction/heed.rs b/src/database/abstraction/heed.rs index e767e22b..83dafc57 100644 --- a/src/database/abstraction/heed.rs +++ b/src/database/abstraction/heed.rs @@ -1,15 +1,13 @@ -use super::super::Config; +use super::{super::Config, watchers::Watchers}; use crossbeam::channel::{bounded, Sender as ChannelSender}; use threadpool::ThreadPool; use crate::{Error, Result}; use std::{ - collections::HashMap, future::Future, pin::Pin, - sync::{Arc, Mutex, RwLock}, + sync::{Arc, Mutex}, }; -use tokio::sync::oneshot::Sender; use super::{DatabaseEngine, Tree}; @@ -23,7 +21,7 @@ pub struct Engine { pub struct EngineTree { engine: Arc, tree: Arc, - watchers: RwLock, Vec>>>, + watchers: Watchers, } fn convert_error(error: heed::Error) -> Error { @@ -60,7 +58,7 @@ impl DatabaseEngine for Engine { .create_database(Some(name)) .map_err(convert_error)?, ), - watchers: RwLock::new(HashMap::new()), + watchers: Default::default(), })) } @@ -145,29 +143,7 @@ impl Tree for EngineTree { .put(&mut txn, &key, &value) .map_err(convert_error)?; txn.commit().map_err(convert_error)?; - - let watchers = self.watchers.read().unwrap(); - let mut triggered = Vec::new(); - - for length in 0..=key.len() { - if watchers.contains_key(&key[..length]) { - triggered.push(&key[..length]); - } - } - - drop(watchers); - - if !triggered.is_empty() { - let mut watchers = self.watchers.write().unwrap(); - for prefix in triggered { - if let Some(txs) = watchers.remove(prefix) { - for tx in txs { - let _ = tx.send(()); - } - } - } - }; - + self.watchers.wake(key); Ok(()) } @@ -223,18 +199,6 @@ impl Tree for EngineTree { #[tracing::instrument(skip(self, prefix))] fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin + Send + 'a>> { - let (tx, rx) = tokio::sync::oneshot::channel(); - - self.watchers - .write() - .unwrap() - .entry(prefix.to_vec()) - .or_default() - .push(tx); - - Box::pin(async move { - // Tx is never destroyed - rx.await.unwrap(); - }) + self.watchers.watch(prefix) } } diff --git a/src/database/abstraction/sqlite.rs b/src/database/abstraction/sqlite.rs index 1d2038c5..1e6a2d89 100644 --- a/src/database/abstraction/sqlite.rs +++ b/src/database/abstraction/sqlite.rs @@ -1,17 +1,15 @@ -use super::{DatabaseEngine, Tree}; +use super::{watchers::Watchers, DatabaseEngine, Tree}; use crate::{database::Config, Result}; -use parking_lot::{Mutex, MutexGuard, RwLock}; +use parking_lot::{Mutex, MutexGuard}; use rusqlite::{Connection, DatabaseName::Main, OptionalExtension}; use std::{ cell::RefCell, - collections::{hash_map, HashMap}, future::Future, path::{Path, PathBuf}, pin::Pin, sync::Arc, }; use thread_local::ThreadLocal; -use tokio::sync::watch; use tracing::debug; thread_local! { @@ -113,7 +111,7 @@ impl DatabaseEngine for Engine { Ok(Arc::new(SqliteTable { engine: Arc::clone(self), name: name.to_owned(), - watchers: RwLock::new(HashMap::new()), + watchers: Watchers::default(), })) } @@ -126,7 +124,7 @@ impl DatabaseEngine for Engine { pub struct SqliteTable { engine: Arc, name: String, - watchers: RwLock, (watch::Sender<()>, watch::Receiver<()>)>>, + watchers: Watchers, } type TupleOfBytes = (Vec, Vec); @@ -200,27 +198,7 @@ impl Tree for SqliteTable { let guard = self.engine.write_lock(); self.insert_with_guard(&guard, key, value)?; drop(guard); - - let watchers = self.watchers.read(); - let mut triggered = Vec::new(); - - for length in 0..=key.len() { - if watchers.contains_key(&key[..length]) { - triggered.push(&key[..length]); - } - } - - drop(watchers); - - if !triggered.is_empty() { - let mut watchers = self.watchers.write(); - for prefix in triggered { - if let Some(tx) = watchers.remove(prefix) { - let _ = tx.0.send(()); - } - } - }; - + self.watchers.wake(key); Ok(()) } @@ -365,19 +343,7 @@ impl Tree for SqliteTable { #[tracing::instrument(skip(self, prefix))] fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin + Send + 'a>> { - let mut rx = match self.watchers.write().entry(prefix.to_vec()) { - hash_map::Entry::Occupied(o) => o.get().1.clone(), - hash_map::Entry::Vacant(v) => { - let (tx, rx) = tokio::sync::watch::channel(()); - v.insert((tx, rx.clone())); - rx - } - }; - - Box::pin(async move { - // Tx is never destroyed - rx.changed().await.unwrap(); - }) + self.watchers.watch(prefix) } #[tracing::instrument(skip(self))] diff --git a/src/database/abstraction/watchers.rs b/src/database/abstraction/watchers.rs new file mode 100644 index 00000000..404f3f06 --- /dev/null +++ b/src/database/abstraction/watchers.rs @@ -0,0 +1,54 @@ +use parking_lot::RwLock; +use std::{ + collections::{hash_map, HashMap}, + future::Future, + pin::Pin, +}; +use tokio::sync::watch; + +#[derive(Default)] +pub(super) struct Watchers { + watchers: RwLock, (watch::Sender<()>, watch::Receiver<()>)>>, +} + +impl Watchers { + pub(super) fn watch<'a>( + &'a self, + prefix: &[u8], + ) -> Pin + Send + 'a>> { + let mut rx = match self.watchers.write().entry(prefix.to_vec()) { + hash_map::Entry::Occupied(o) => o.get().1.clone(), + hash_map::Entry::Vacant(v) => { + let (tx, rx) = tokio::sync::watch::channel(()); + v.insert((tx, rx.clone())); + rx + } + }; + + Box::pin(async move { + // Tx is never destroyed + rx.changed().await.unwrap(); + }) + } + pub(super) fn wake(&self, key: &[u8]) { + let watchers = self.watchers.read(); + let mut triggered = Vec::new(); + + for length in 0..=key.len() { + if watchers.contains_key(&key[..length]) { + triggered.push(&key[..length]); + } + } + + drop(watchers); + + if !triggered.is_empty() { + let mut watchers = self.watchers.write(); + for prefix in triggered { + if let Some(tx) = watchers.remove(prefix) { + let _ = tx.0.send(()); + } + } + }; + } +} From 7c1b2625cf8f315bced5e560574c0c64eedd368f Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Fri, 24 Dec 2021 23:06:54 +0100 Subject: [PATCH 064/201] Prepare to add an option to list local user accounts from your homeserver --- src/database/admin.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/database/admin.rs b/src/database/admin.rs index 1e5c47c9..5ea872e8 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -13,6 +13,7 @@ use tracing::warn; pub enum AdminCommand { RegisterAppservice(serde_yaml::Value), ListAppservices, + CountUsers, SendMessage(RoomMessageEventContent), } @@ -93,6 +94,16 @@ impl Admin { let state_lock = mutex_state.lock().await; match event { + AdminCommand::CountUsers => { + // count() does not return an error on failure... + if let Ok(usercount) = guard.users.count() { + let message = format!("Found {} total user accounts", usercount); + send_message(RoomMessageEventContent::text_plain(message), guard, &state_lock); + } else { + // ... so we simply spit out a generic non-explaining-info in case count() did not return Ok() + send_message(RoomMessageEventContent::text_plain("Unable to count users"), guard, &state_lock); + } + } AdminCommand::RegisterAppservice(yaml) => { guard.appservice.register_appservice(yaml).unwrap(); // TODO handle error } From 567cf6dbe970ee5422cd38439498f7e5a86b89ac Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Sat, 25 Dec 2021 20:51:22 +0100 Subject: [PATCH 065/201] Add command count_local_users to database/rooms.rs --- src/database/rooms.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/database/rooms.rs b/src/database/rooms.rs index fb9ecbf0..0236c839 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -1531,6 +1531,9 @@ impl Rooms { "list_appservices" => { db.admin.send(AdminCommand::ListAppservices); } + "count_local_users" => { + db.admin.send(AdminCommand::CountUsers); + } "get_auth_chain" => { if args.len() == 1 { if let Ok(event_id) = EventId::parse_arc(args[0]) { From d21030566c174509c4030d2f6428ffe1109e6c1d Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Sat, 25 Dec 2021 21:29:03 +0100 Subject: [PATCH 066/201] Rename/Add count methods to count_local_users --- src/database/admin.rs | 2 +- src/database/users.rs | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index 5ea872e8..b18e50c3 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -96,7 +96,7 @@ impl Admin { match event { AdminCommand::CountUsers => { // count() does not return an error on failure... - if let Ok(usercount) = guard.users.count() { + if let Ok(usercount) = guard.users.count_local_users() { let message = format!("Found {} total user accounts", usercount); send_message(RoomMessageEventContent::text_plain(message), guard, &state_lock); } else { diff --git a/src/database/users.rs b/src/database/users.rs index d4bf4890..5a32f16a 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -77,11 +77,23 @@ impl Users { } /// Returns the number of users registered on this server. + /// It really returns all users, not only real ones with a + /// password to login but also bridge puppets... #[tracing::instrument(skip(self))] pub fn count(&self) -> Result { Ok(self.userid_password.iter().count()) } + /// This method will only count those local user accounts with + /// a password thus returning only real accounts on this instance. + #[tracing::instrument(skip(self))] + pub fn count_local_users(&self) -> Result { + self.userid_password.iter().map(|(key, value)| { + + }); + Ok(1) + } + /// Find out which user an access token belongs to. #[tracing::instrument(skip(self, token))] pub fn find_from_token(&self, token: &str) -> Result, String)>> { From 2281bcefc631e02c83800297a4838e127ded7973 Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Sun, 26 Dec 2021 11:06:28 +0100 Subject: [PATCH 067/201] Finalize count_local_users function --- src/database/users.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/database/users.rs b/src/database/users.rs index 5a32f16a..1e103fa2 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -88,10 +88,8 @@ impl Users { /// a password thus returning only real accounts on this instance. #[tracing::instrument(skip(self))] pub fn count_local_users(&self) -> Result { - self.userid_password.iter().map(|(key, value)| { - - }); - Ok(1) + let n = self.userid_password.iter().filter(|(_, bytes)| bytes.len() > 0).count(); + Ok(n) } /// Find out which user an access token belongs to. From 39787b41cb341ca3d270cc00c9ac46b8f4bd384d Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Sun, 26 Dec 2021 12:04:38 +0100 Subject: [PATCH 068/201] Rename admin command CountUsers -> CountLocalUsers; Update comments --- src/database/admin.rs | 12 ++++++------ src/database/rooms.rs | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index b18e50c3..330fecb1 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -13,7 +13,7 @@ use tracing::warn; pub enum AdminCommand { RegisterAppservice(serde_yaml::Value), ListAppservices, - CountUsers, + CountLocalUsers, SendMessage(RoomMessageEventContent), } @@ -94,14 +94,14 @@ impl Admin { let state_lock = mutex_state.lock().await; match event { - AdminCommand::CountUsers => { - // count() does not return an error on failure... + AdminCommand::CountLocalUsers => { + // count_local_users() only returns with OK(x) where x is the number of found accounts if let Ok(usercount) = guard.users.count_local_users() { - let message = format!("Found {} total user accounts", usercount); + let message = format!("Found {} local user account(s)", usercount); send_message(RoomMessageEventContent::text_plain(message), guard, &state_lock); } else { - // ... so we simply spit out a generic non-explaining-info in case count() did not return Ok() - send_message(RoomMessageEventContent::text_plain("Unable to count users"), guard, &state_lock); + // if count_local_users() only returns with OK(x), then why is this? ;-) + send_message(RoomMessageEventContent::text_plain("Unable to count local users"), guard, &state_lock); } } AdminCommand::RegisterAppservice(yaml) => { diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 0236c839..b1dd103c 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -1532,7 +1532,7 @@ impl Rooms { db.admin.send(AdminCommand::ListAppservices); } "count_local_users" => { - db.admin.send(AdminCommand::CountUsers); + db.admin.send(AdminCommand::CountLocalUsers); } "get_auth_chain" => { if args.len() == 1 { From a69eb277d46d074d2bb4fef82f4111f70845f874 Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Sun, 26 Dec 2021 20:00:31 +0100 Subject: [PATCH 069/201] Update count users: It's now list_local_users and contains the number and the usernames --- src/database/admin.rs | 22 ++++++++++++---------- src/database/rooms.rs | 4 ++-- src/database/users.rs | 14 ++++++++++++++ 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index 330fecb1..58d9e83e 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -13,7 +13,7 @@ use tracing::warn; pub enum AdminCommand { RegisterAppservice(serde_yaml::Value), ListAppservices, - CountLocalUsers, + ListLocalUsers, SendMessage(RoomMessageEventContent), } @@ -94,15 +94,17 @@ impl Admin { let state_lock = mutex_state.lock().await; match event { - AdminCommand::CountLocalUsers => { - // count_local_users() only returns with OK(x) where x is the number of found accounts - if let Ok(usercount) = guard.users.count_local_users() { - let message = format!("Found {} local user account(s)", usercount); - send_message(RoomMessageEventContent::text_plain(message), guard, &state_lock); - } else { - // if count_local_users() only returns with OK(x), then why is this? ;-) - send_message(RoomMessageEventContent::text_plain("Unable to count local users"), guard, &state_lock); - } + AdminCommand::ListLocalUsers => { + // collect all local users + let users = guard.users.iter_locals(); + + let mut msg: String = format!("Found {} local user account(s):\n", users.len()); + msg += &users.join("\n"); + + // send number of local users as plain text: + // TODO: send as Markdown + send_message(RoomMessageEventContent::text_plain(&msg), guard, &state_lock); + } AdminCommand::RegisterAppservice(yaml) => { guard.appservice.register_appservice(yaml).unwrap(); // TODO handle error diff --git a/src/database/rooms.rs b/src/database/rooms.rs index b1dd103c..4d839d38 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -1531,8 +1531,8 @@ impl Rooms { "list_appservices" => { db.admin.send(AdminCommand::ListAppservices); } - "count_local_users" => { - db.admin.send(AdminCommand::CountLocalUsers); + "list_local_users" => { + db.admin.send(AdminCommand::ListLocalUsers); } "get_auth_chain" => { if args.len() == 1 { diff --git a/src/database/users.rs b/src/database/users.rs index 1e103fa2..d3e1fe43 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -84,6 +84,8 @@ impl Users { Ok(self.userid_password.iter().count()) } + /// The method is DEPRECATED and was replaced by iter_locals() + /// /// This method will only count those local user accounts with /// a password thus returning only real accounts on this instance. #[tracing::instrument(skip(self))] @@ -92,6 +94,7 @@ impl Users { Ok(n) } + /// Find out which user an access token belongs to. #[tracing::instrument(skip(self, token))] pub fn find_from_token(&self, token: &str) -> Result, String)>> { @@ -131,6 +134,17 @@ impl Users { }) } + /// Returns a vector of local usernames + #[tracing::instrument(skip(self))] + pub fn iter_locals(&self) -> Vec { + self.userid_password.iter().filter(|(_, pw)| pw.len() > 0).map(|(username, _)| { + match utils::string_from_bytes(&username) { + Ok(s) => s, + Err(e) => e.to_string() + } + }).collect::>() + } + /// Returns the password hash for the given user. #[tracing::instrument(skip(self, user_id))] pub fn password_hash(&self, user_id: &UserId) -> Result> { From 8d51359668199585175e7e0095de66e96bc1a3e1 Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Sun, 26 Dec 2021 20:49:19 +0100 Subject: [PATCH 070/201] Fix typo and remove unneeded newline --- src/database/admin.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index 58d9e83e..5418f53a 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -95,7 +95,7 @@ impl Admin { match event { AdminCommand::ListLocalUsers => { - // collect all local users + // collect local users only let users = guard.users.iter_locals(); let mut msg: String = format!("Found {} local user account(s):\n", users.len()); @@ -104,7 +104,6 @@ impl Admin { // send number of local users as plain text: // TODO: send as Markdown send_message(RoomMessageEventContent::text_plain(&msg), guard, &state_lock); - } AdminCommand::RegisterAppservice(yaml) => { guard.appservice.register_appservice(yaml).unwrap(); // TODO handle error From b746f17e562ba02d9471d23c42c9bb8c9f4ee070 Mon Sep 17 00:00:00 2001 From: Ticho 34782694 Date: Fri, 7 Jan 2022 13:06:21 +0000 Subject: [PATCH 071/201] Make traefik+nginx config more self-contained The nginx instance which is serving the .well-known endpoints can serve the simple JSON replies directly from memory, instead of having them as external files on disk. --- docker/README.md | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/docker/README.md b/docker/README.md index 19d9dca6..1f38d66a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -94,26 +94,20 @@ So...step by step: server_name .; listen 80 default_server; - location /.well-known/matrix/ { - root /var/www; - default_type application/json; - add_header Access-Control-Allow-Origin *; + location /.well-known/matrix/server { + return 200 '{"m.server": ".:443"}'; + add_header Content-Type application/json; } - } - ``` - - `./nginx/www/.well-known/matrix/client` (relative to the compose file, you can change this, but then also need to change the volume mapping) - ```json - { - "m.homeserver": { - "base_url": "https://." - } - } - ``` - - `./nginx/www/.well-known/matrix/server` (relative to the compose file, you can change this, but then also need to change the volume mapping) - ```json - { - "m.server": ".:443" + location /.well-known/matrix/client { + return 200 '{"m.homeserver": {"base_url": "https://."}}'; + add_header Content-Type application/json; + add_header "Access-Control-Allow-Origin" *; + } + + location / { + return 404; + } } ``` From 349865d3ccb9ee78c9410de28e0d5d8c043ae0c8 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Thu, 13 Jan 2022 11:44:23 +0100 Subject: [PATCH 072/201] Upgrade Ruma --- Cargo.lock | 36 ++++++++++++++++++------------------ Cargo.toml | 2 +- src/client_server/message.rs | 4 ++-- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 69a026b2..07cae94e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1990,7 +1990,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" +source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" dependencies = [ "assign", "js_int", @@ -2011,7 +2011,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.18.5" -source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" +source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" dependencies = [ "bytes", "http", @@ -2027,7 +2027,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.18.5" -source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" +source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2038,7 +2038,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" +source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" dependencies = [ "ruma-api", "ruma-common", @@ -2052,7 +2052,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.12.3" -source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" +source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" dependencies = [ "assign", "bytes", @@ -2072,7 +2072,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.6.0" -source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" +source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" dependencies = [ "indexmap", "js_int", @@ -2087,7 +2087,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.24.6" -source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" +source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" dependencies = [ "indoc", "js_int", @@ -2103,7 +2103,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.24.6" -source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" +source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2114,7 +2114,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.3.1" -source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" +source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" dependencies = [ "js_int", "ruma-api", @@ -2129,7 +2129,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.20.0" -source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" +source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" dependencies = [ "percent-encoding", "rand 0.8.4", @@ -2143,7 +2143,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.20.0" -source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" +source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" dependencies = [ "quote", "ruma-identifiers-validation", @@ -2153,7 +2153,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" +source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" dependencies = [ "thiserror", ] @@ -2161,7 +2161,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" +source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" dependencies = [ "js_int", "ruma-api", @@ -2174,7 +2174,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" +source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" dependencies = [ "js_int", "ruma-api", @@ -2189,7 +2189,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" +source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" dependencies = [ "bytes", "form_urlencoded", @@ -2203,7 +2203,7 @@ dependencies = [ [[package]] name = "ruma-serde-macros" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" +source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2214,7 +2214,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.9.0" -source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" +source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" dependencies = [ "base64 0.13.0", "ed25519-dalek", @@ -2231,7 +2231,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.4.1" -source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc" +source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" dependencies = [ "itertools 0.10.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 7b3432c2..5e09dee0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle request # Used for matrix spec type definitions and helpers #ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } -ruma = { git = "https://github.com/ruma/ruma", rev = "7cf3abbaf02995b03db74429090ca5af1cd71edc", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +ruma = { git = "https://github.com/ruma/ruma", rev = "f8ba7f795765bf4aeb4db06849f9fdde9c162ac3", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { git = "https://github.com/timokoesters/ruma", rev = "50c1db7e0a3a21fc794b0cce3b64285a4c750c71", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 60c756a3..da6ae875 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -160,7 +160,7 @@ pub async fn get_message_events_route( .collect(); let mut resp = get_message_events::Response::new(); - resp.start = Some(body.from.to_owned()); + resp.start = body.from.to_owned(); resp.end = end_token; resp.chunk = events_after; resp.state = Vec::new(); @@ -190,7 +190,7 @@ pub async fn get_message_events_route( .collect(); let mut resp = get_message_events::Response::new(); - resp.start = Some(body.from.to_owned()); + resp.start = body.from.to_owned(); resp.end = start_token; resp.chunk = events_before; resp.state = Vec::new(); From cf54185a1cfe6b7cbed4c8c472198360aa705663 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Thu, 13 Jan 2022 11:48:18 +0100 Subject: [PATCH 073/201] Use struct literals for consistency --- src/client_server/context.rs | 25 +++++++++++++------------ src/client_server/message.rs | 22 ++++++++++++---------- src/client_server/unversioned.rs | 9 ++++----- 3 files changed, 29 insertions(+), 27 deletions(-) diff --git a/src/client_server/context.rs b/src/client_server/context.rs index 97fc4fd8..9bfec9e1 100644 --- a/src/client_server/context.rs +++ b/src/client_server/context.rs @@ -92,18 +92,19 @@ pub async fn get_context_route( .map(|(_, pdu)| pdu.to_room_event()) .collect(); - let mut resp = get_context::Response::new(); - resp.start = start_token; - resp.end = end_token; - resp.events_before = events_before; - resp.event = Some(base_event); - resp.events_after = events_after; - resp.state = db // TODO: State at event - .rooms - .room_state_full(&body.room_id)? - .values() - .map(|pdu| pdu.to_state_event()) - .collect(); + let resp = get_context::Response { + start: start_token, + end: end_token, + events_before, + event: Some(base_event), + events_after, + state: db // TODO: State at event + .rooms + .room_state_full(&body.room_id)? + .values() + .map(|pdu| pdu.to_state_event()) + .collect(), + }; Ok(resp.into()) } diff --git a/src/client_server/message.rs b/src/client_server/message.rs index da6ae875..cbce019e 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -159,11 +159,12 @@ pub async fn get_message_events_route( .map(|(_, pdu)| pdu.to_room_event()) .collect(); - let mut resp = get_message_events::Response::new(); - resp.start = body.from.to_owned(); - resp.end = end_token; - resp.chunk = events_after; - resp.state = Vec::new(); + let resp = get_message_events::Response { + start: body.from.to_owned(), + end: end_token, + chunk: events_after, + state: Vec::new(), + }; Ok(resp.into()) } @@ -189,11 +190,12 @@ pub async fn get_message_events_route( .map(|(_, pdu)| pdu.to_room_event()) .collect(); - let mut resp = get_message_events::Response::new(); - resp.start = body.from.to_owned(); - resp.end = start_token; - resp.chunk = events_before; - resp.state = Vec::new(); + let resp = get_message_events::Response { + start: body.from.to_owned(), + end: start_token, + chunk: events_before, + state: Vec::new(), + }; Ok(resp.into()) } diff --git a/src/client_server/unversioned.rs b/src/client_server/unversioned.rs index f2624bbc..f17d8cf3 100644 --- a/src/client_server/unversioned.rs +++ b/src/client_server/unversioned.rs @@ -17,11 +17,10 @@ use rocket::get; #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/versions"))] #[tracing::instrument] pub async fn get_supported_versions_route() -> ConduitResult { - let mut resp = - get_supported_versions::Response::new(vec!["r0.5.0".to_owned(), "r0.6.0".to_owned()]); - - resp.unstable_features - .insert("org.matrix.e2e_cross_signing".to_owned(), true); + let resp = get_supported_versions::Response { + versions: vec!["r0.5.0".to_owned(), "r0.6.0".to_owned()], + unstable_features: [("org.matrix.e2e_cross_signing".to_owned(), true)].into(), + }; Ok(resp.into()) } From 84862352bacd7172602f1b8200a774d668a9f087 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Thu, 13 Jan 2022 11:48:40 +0100 Subject: [PATCH 074/201] Replace to_string calls on string literals with to_owned --- src/database/uiaa.rs | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/database/uiaa.rs b/src/database/uiaa.rs index 772dab9e..5e11467e 100644 --- a/src/database/uiaa.rs +++ b/src/database/uiaa.rs @@ -154,11 +154,7 @@ impl Uiaa { .write() .unwrap() .insert( - ( - user_id.to_owned(), - device_id.to_owned(), - session.to_string(), - ), + (user_id.to_owned(), device_id.to_owned(), session.to_owned()), request.to_owned(), ); @@ -175,11 +171,7 @@ impl Uiaa { .userdevicesessionid_uiaarequest .read() .unwrap() - .get(&( - user_id.to_owned(), - device_id.to_owned(), - session.to_string(), - )) + .get(&(user_id.to_owned(), device_id.to_owned(), session.to_owned())) .map(|j| j.to_owned())) } From bcf4ede0bc356efb4bd8b8909ca3db0ab157f97e Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Thu, 13 Jan 2022 12:06:20 +0100 Subject: [PATCH 075/201] Restore compatibility with Rust 1.53 --- src/client_server/unversioned.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/client_server/unversioned.rs b/src/client_server/unversioned.rs index f17d8cf3..ea685b4b 100644 --- a/src/client_server/unversioned.rs +++ b/src/client_server/unversioned.rs @@ -1,3 +1,5 @@ +use std::{collections::BTreeMap, iter::FromIterator}; + use crate::ConduitResult; use ruma::api::client::unversioned::get_supported_versions; @@ -19,7 +21,7 @@ use rocket::get; pub async fn get_supported_versions_route() -> ConduitResult { let resp = get_supported_versions::Response { versions: vec!["r0.5.0".to_owned(), "r0.6.0".to_owned()], - unstable_features: [("org.matrix.e2e_cross_signing".to_owned(), true)].into(), + unstable_features: BTreeMap::from_iter([("org.matrix.e2e_cross_signing".to_owned(), true)]), }; Ok(resp.into()) From eecd664c43c652f7fe4afc06154b346fc6a45b58 Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Thu, 13 Jan 2022 12:26:23 +0100 Subject: [PATCH 076/201] Reformat code --- src/database/appservice.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/database/appservice.rs b/src/database/appservice.rs index 847d7479..88de1f33 100644 --- a/src/database/appservice.rs +++ b/src/database/appservice.rs @@ -28,17 +28,17 @@ impl Appservice { } /// Remove an appservice registration - /// + /// /// # Arguments - /// + /// /// * `service_name` - the name you send to register the service previously pub fn unregister_appservice(&self, service_name: &str) -> Result<()> { self.id_appserviceregistrations .remove(service_name.as_bytes())?; - self.cached_registrations. - write(). - unwrap(). - remove(service_name); + self.cached_registrations + .write() + .unwrap() + .remove(service_name); Ok(()) } From 1d647a1a9a0a3075ee1bdbe2a039d22ee73baa2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sat, 16 Oct 2021 15:19:25 +0200 Subject: [PATCH 077/201] improvement: allow rocksdb again --- Cargo.lock | 727 +++++++++++++++++----------- Cargo.toml | 4 +- src/database.rs | 11 +- src/database/abstraction.rs | 5 +- src/database/abstraction/rocksdb.rs | 183 +++++++ src/database/abstraction/sqlite.rs | 14 +- src/error.rs | 6 + src/utils.rs | 11 + 8 files changed, 664 insertions(+), 297 deletions(-) create mode 100644 src/database/abstraction/rocksdb.rs diff --git a/Cargo.lock b/Cargo.lock index 07cae94e..794445f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,9 +10,9 @@ checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" [[package]] name = "ahash" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43bb833f0bf979d8475d38fbf09ed3b8a55e1885fe93ad3f93239fc6a4f17b98" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ "getrandom 0.2.3", "once_cell", @@ -78,9 +78,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.51" +version = "0.1.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44318e776df68115a881de9a8fd1b9e53368d7a4a5ce4cc48517da3393233a5e" +checksum = "061a7acccaa286c011ddc30970520b98fa40e00c9d644633fb26b5fc63a265e3" dependencies = [ "proc-macro2", "quote", @@ -89,9 +89,9 @@ dependencies = [ [[package]] name = "atomic" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3410529e8288c463bedb5930f82833bc0c90e5d2fe639a56582a4d09220b281" +checksum = "b88d82667eca772c4aa12f0f1348b3ae643424c8876448f3f7bd5787032e234c" dependencies = [ "autocfg", ] @@ -146,6 +146,25 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.59.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "peeking_take_while", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -174,15 +193,15 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.7.0" +version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" +checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" [[package]] name = "bytemuck" -version = "1.7.2" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72957246c41db82b8ef88a5486143830adeb8227ef9837740bdec67724cf2c5b" +checksum = "439989e6b8c38d1b6570a384ef1e49c8848128f5a97f3914baef02920842712f" [[package]] name = "byteorder" @@ -198,13 +217,22 @@ checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" [[package]] name = "cc" -version = "1.0.70" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26a6ce4b6a484fa3edb70f7efa6fc430fd2b87285fe8b84304fd0936faa0dc0" +checksum = "22a9137b95ea06864e018375b72adfb7db6e6f68cfc8df5a04d00288050485ee" dependencies = [ "jobserver", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "0.1.10" @@ -230,6 +258,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "clang-sys" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa66045b9cb23c2e9c1520732030608b02ee07e5cfaa5a521ec15ded7fa24c90" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "color_quant" version = "1.1.0" @@ -259,11 +298,12 @@ dependencies = [ "reqwest", "ring", "rocket", + "rocksdb", "ruma", "rusqlite", "rust-argon2", - "rustls", - "rustls-native-certs", + "rustls 0.19.1", + "rustls-native-certs 0.5.0", "serde", "serde_json", "serde_yaml", @@ -275,22 +315,22 @@ dependencies = [ "tokio", "tracing", "tracing-flame", - "tracing-subscriber", + "tracing-subscriber 0.2.25", "trust-dns-resolver", "webpki 0.22.0", ] [[package]] name = "const-oid" -version = "0.6.0" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c32f031ea41b4291d695026c023b95d59db2d8a2c7640800ed56bc8f510f22" +checksum = "9d6f2aa4d0537bcc1c74df8755072bd31c1ef1a3a1b85a68e8404a8c353b7b8b" [[package]] name = "const_fn" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92cfa0fd5690b3cf8c1ef2cabbd9b7ef22fa53cf5e1f92b05103f6d5d1cf6e7" +checksum = "fbdcdcb6d86f71c5e97409ad45898af11cbc995b4ee8112d59095a28d376c935" [[package]] name = "constant_time_eq" @@ -311,9 +351,9 @@ dependencies = [ [[package]] name = "core-foundation" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +checksum = "6888e10551bb93e424d8df1d07f1a8b4fceb0001a3a4b048bfc47554946f47b3" dependencies = [ "core-foundation-sys", "libc", @@ -321,9 +361,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "cpufeatures" @@ -336,9 +376,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +checksum = "738c290dfaea84fc1ca15ad9c168d083b05a714e1efddd8edaab678dc28d2836" dependencies = [ "cfg-if 1.0.0", ] @@ -353,18 +393,18 @@ dependencies = [ "crossbeam-channel", "crossbeam-deque", "crossbeam-epoch", - "crossbeam-queue 0.3.2", - "crossbeam-utils 0.8.5", + "crossbeam-queue 0.3.3", + "crossbeam-utils 0.8.6", ] [[package]] name = "crossbeam-channel" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" +checksum = "e54ea8bc3fb1ee042f5aace6e3c6e025d3874866da222930f70ce62aceba0bfa" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.5", + "crossbeam-utils 0.8.6", ] [[package]] @@ -375,17 +415,17 @@ checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" dependencies = [ "cfg-if 1.0.0", "crossbeam-epoch", - "crossbeam-utils 0.8.5", + "crossbeam-utils 0.8.6", ] [[package]] name = "crossbeam-epoch" -version = "0.9.5" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" +checksum = "97242a70df9b89a65d0b6df3c4bf5b9ce03c5b7309019777fbde37e7537f8762" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.5", + "crossbeam-utils 0.8.6", "lazy_static", "memoffset", "scopeguard", @@ -402,12 +442,12 @@ dependencies = [ [[package]] name = "crossbeam-queue" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b10ddc024425c88c2ad148c1b0fd53f4c6d38db9697c9f1588381212fa657c9" +checksum = "b979d76c9fcb84dffc80a73f7290da0f83e4c95773494674cb44b76d13a7a110" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.5", + "crossbeam-utils 0.8.6", ] [[package]] @@ -422,9 +462,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db" +checksum = "cfcae03edb34f947e64acdb1c33ec169824e20657e9ecb61cef6c8c74dcb8120" dependencies = [ "cfg-if 1.0.0", "lazy_static", @@ -471,9 +511,9 @@ dependencies = [ [[package]] name = "der" -version = "0.4.1" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e21d2d0f22cde6e88694108429775c0219760a07779bf96503b434a03d7412" +checksum = "79b71cca7d95d7681a4b3b9cdf63c8dbc3730d0584c2c74e31416d64a90493f4" dependencies = [ "const-oid", ] @@ -546,17 +586,11 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" -[[package]] -name = "dtoa" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" - [[package]] name = "ed25519" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4620d40f6d2601794401d6dd95a5cf69b6c157852539470eeda433a99b3c0efc" +checksum = "74e1069e39f1454367eb2de793ed062fac4c35c2934b76a81d90dd9abcd28816" dependencies = [ "signature", ] @@ -583,9 +617,9 @@ checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" [[package]] name = "encoding_rs" -version = "0.8.28" +version = "0.8.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" +checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df" dependencies = [ "cfg-if 1.0.0", ] @@ -614,6 +648,15 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +[[package]] +name = "fastrand" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "779d043b6a0b90cc4c0ed7ee380a6504394cee7efd7db050e3774eee387324b2" +dependencies = [ + "instant", +] + [[package]] name = "figment" version = "0.10.6" @@ -656,9 +699,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.17" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12aa0eb539080d55c3f2d45a67c3b58b6b0773c1a3ca2dfec66d58c97fd66ca" +checksum = "28560757fe2bb34e79f907794bb6b22ae8b0e5c669b638a1132f2592b19035b4" dependencies = [ "futures-channel", "futures-core", @@ -671,9 +714,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.17" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5da6ba8c3bb3c165d3c7319fc1cc8304facf1fb8db99c5de877183c08a273888" +checksum = "ba3dda0b6588335f360afc675d0564c17a77a2bda81ca178a4b6081bd86c7f0b" dependencies = [ "futures-core", "futures-sink", @@ -681,15 +724,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.17" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d1c26957f23603395cd326b0ffe64124b818f4449552f960d815cfba83a53d" +checksum = "d0c8ff0461b82559810cdccfde3215c3f373807f5e5232b71479bff7bb2583d7" [[package]] name = "futures-executor" -version = "0.3.17" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45025be030969d763025784f7f355043dc6bc74093e4ecc5000ca4dc50d8745c" +checksum = "29d6d2ff5bb10fb95c85b8ce46538a2e5f5e7fdc755623a7d4529ab8a4ed9d2a" dependencies = [ "futures-core", "futures-task", @@ -698,18 +741,16 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.17" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377" +checksum = "b1f9d34af5a1aac6fb380f735fe510746c38067c5bf16c7fd250280503c971b2" [[package]] name = "futures-macro" -version = "0.3.17" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18e4a4b95cea4b4ccbcf1c5675ca7c4ee4e9e75eb79944d07defde18068f79bb" +checksum = "6dbd947adfffb0efc70599b3ddcf7b5597bb5fa9e245eb99f62b3a5f7bb8bd3c" dependencies = [ - "autocfg", - "proc-macro-hack", "proc-macro2", "quote", "syn", @@ -717,23 +758,22 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.17" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36ea153c13024fe480590b3e3d4cad89a0cfacecc24577b68f86c6ced9c2bc11" +checksum = "e3055baccb68d74ff6480350f8d6eb8fcfa3aa11bdc1a1ae3afdd0514617d508" [[package]] name = "futures-task" -version = "0.3.17" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d3d00f4eddb73e498a54394f228cd55853bdf059259e8e7bc6e69d408892e99" +checksum = "6ee7c6485c30167ce4dfb83ac568a849fe53274c831081476ee13e0dce1aad72" [[package]] name = "futures-util" -version = "0.3.17" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36568465210a3a6ee45e1f165136d68671471a501e632e9a98d96872222b5481" +checksum = "d9b5cf40b47a271f77a8b1bec03ca09044d99d2372c0de244e66430761127164" dependencies = [ - "autocfg", "futures-channel", "futures-core", "futures-io", @@ -743,8 +783,6 @@ dependencies = [ "memchr", "pin-project-lite", "pin-utils", - "proc-macro-hack", - "proc-macro-nested", "slab", ] @@ -772,9 +810,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.4" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803" dependencies = [ "typenum", "version_check", @@ -804,9 +842,9 @@ dependencies = [ [[package]] name = "gif" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a668f699973d0f573d15749b7002a9ac9e1f9c6b220e7b165601334c173d8de" +checksum = "c3a7187e78088aead22ceedeee99779455b23fc231fe13ec443f99bb71694e5b" dependencies = [ "color_quant", "weezl", @@ -820,9 +858,9 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" [[package]] name = "h2" -version = "0.3.4" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f3675cfef6a30c8031cf9e6493ebdc3bb3272a3fea3923c4210d1830e6a472" +checksum = "0c9de88456263e249e241fcd211d3954e2c9b0ef7ccfc235a444eb367cae3689" dependencies = [ "bytes", "fnv", @@ -932,20 +970,20 @@ dependencies = [ [[package]] name = "http" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11" +checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" dependencies = [ "bytes", "fnv", - "itoa 0.4.8", + "itoa 1.0.1", ] [[package]] name = "http-body" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "399c583b2979440c60be0821a6199eca73bc3c8dcd9d070d75ac726e2c6186e5" +checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" dependencies = [ "bytes", "http", @@ -960,15 +998,15 @@ checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503" [[package]] name = "httpdate" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.12" +version = "0.14.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13f67199e765030fa08fe0bd581af683f0d5bc04ea09c2b1102012c5fb90e7fd" +checksum = "b7ec3e62bdc98a2f0393a5048e4c30ef659440ea6e0e572965103e72bd836f55" dependencies = [ "bytes", "futures-channel", @@ -981,7 +1019,7 @@ dependencies = [ "httpdate", "itoa 0.4.8", "pin-project-lite", - "socket2 0.4.1", + "socket2 0.4.2", "tokio", "tower-service", "tracing", @@ -990,17 +1028,15 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.22.1" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" +checksum = "d87c48c02e0dc5e3b849a2041db3029fd066650f8f717c07bf8ed78ccb895cac" dependencies = [ - "futures-util", + "http", "hyper", - "log", - "rustls", + "rustls 0.20.2", "tokio", - "tokio-rustls", - "webpki 0.21.4", + "tokio-rustls 0.23.2", ] [[package]] @@ -1033,9 +1069,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" +checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" dependencies = [ "autocfg", "hashbrown", @@ -1053,15 +1089,15 @@ dependencies = [ [[package]] name = "inlinable_string" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3094308123a0e9fd59659ce45e22de9f53fc1d2ac6e1feb9fef988e4f76cad77" +checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" [[package]] name = "instant" -version = "0.1.10" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if 1.0.0", ] @@ -1092,18 +1128,9 @@ checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" [[package]] name = "itertools" -version = "0.9.0" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69ddb889f9d0d08a67338271fa9b62996bc788c7796a5c18cf057420aaed5eaf" +checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" dependencies = [ "either", ] @@ -1137,9 +1164,9 @@ checksum = "229d53d58899083193af11e15917b5640cd40b29ff475a1fe4ef725deb02d0f2" [[package]] name = "js-sys" -version = "0.3.53" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4bf49d50e2961077d9c99f4b7997d770a1114f087c3c2e0069b36c13fc2979d" +checksum = "7cc9ffccd38c451a86bf13657df244e9c3f37493cce8e5e21e940963777acc84" dependencies = [ "wasm-bindgen", ] @@ -1174,10 +1201,38 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] -name = "libc" -version = "0.2.101" +name = "lazycell" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cb00336871be5ed2c8ed44b60ae9959dc5b9f08539422ed43f09e34ecaeba21" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.112" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125" + +[[package]] +name = "libloading" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afe203d669ec979b7128619bae5a63b7b42e9203c1b29146079ee05e2f604b52" +dependencies = [ + "cfg-if 1.0.0", + "winapi", +] + +[[package]] +name = "librocksdb-sys" +version = "6.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c309a9d2470844aceb9a4a098cf5286154d20596868b75a6b36357d2bb9ca25d" +dependencies = [ + "bindgen", + "cc", + "glob", + "libc", +] [[package]] name = "libsqlite3-sys" @@ -1227,15 +1282,17 @@ dependencies = [ [[package]] name = "loom" -version = "0.5.1" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2111607c723d7857e0d8299d5ce7a0bf4b844d3e44f8de136b13da513eaf8fc4" +checksum = "edc5c7d328e32cc4954e8e01193d7f0ef5ab257b5090b70a964e099a36034309" dependencies = [ "cfg-if 1.0.0", "generator", "scoped-tls", "serde", "serde_json", + "tracing", + "tracing-subscriber 0.3.5", ] [[package]] @@ -1268,6 +1325,15 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata", +] + [[package]] name = "matches" version = "0.1.9" @@ -1282,9 +1348,9 @@ checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" [[package]] name = "memoffset" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" dependencies = [ "autocfg", ] @@ -1295,6 +1361,12 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.3.7" @@ -1306,9 +1378,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.7.13" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16" +checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc" dependencies = [ "libc", "log", @@ -1328,9 +1400,9 @@ dependencies = [ [[package]] name = "multer" -version = "2.0.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "408327e2999b839cd1af003fc01b2019a6c10a1361769542203f6fedc5179680" +checksum = "5f8f35e687561d5c1667590911e6698a8cb714a134a7505718a182e7bc9d3836" dependencies = [ "bytes", "encoding_rs", @@ -1338,11 +1410,22 @@ dependencies = [ "http", "httparse", "log", + "memchr", "mime", "spin 0.9.2", "tokio", "tokio-util", - "twoway", + "version_check", +] + +[[package]] +name = "nom" +version = "7.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109" +dependencies = [ + "memchr", + "minimal-lexical", "version_check", ] @@ -1409,9 +1492,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" dependencies = [ "hermit-abi", "libc", @@ -1419,9 +1502,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" +checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5" [[package]] name = "opaque-debug" @@ -1431,9 +1514,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl-probe" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "opentelemetry" @@ -1545,6 +1628,12 @@ dependencies = [ "syn", ] +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "pem" version = "0.8.3" @@ -1564,18 +1653,18 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pin-project" -version = "1.0.8" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "576bc800220cc65dac09e99e97b08b358cfab6e17078de8dc5fee223bd2d0c08" +checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.8" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e8fe8163d14ce7f0cdac2e040116f22eac817edabff0be91e8aff7e9accf389" +checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb" dependencies = [ "proc-macro2", "quote", @@ -1584,9 +1673,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443" +checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c" [[package]] name = "pin-utils" @@ -1596,9 +1685,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkcs8" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbee84ed13e44dd82689fa18348a49934fa79cc774a344c42fc9b301c71b140a" +checksum = "ee3ef9b64d26bad0536099c816c6734379e45bbd5f14798def6809e5cc350447" dependencies = [ "der", "spki", @@ -1607,9 +1696,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.19" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" +checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" [[package]] name = "png" @@ -1625,15 +1714,15 @@ dependencies = [ [[package]] name = "ppv-lite86" -version = "0.2.10" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" [[package]] name = "proc-macro-crate" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fdbd1df62156fbc5945f4762632564d7d038153091c3fcf1067f6aef7cff92" +checksum = "1ebace6889caf889b4d3f76becee12e90353f2b8c7d875534a71e5742f8f6f83" dependencies = [ "thiserror", "toml", @@ -1645,17 +1734,11 @@ version = "0.5.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" -[[package]] -name = "proc-macro-nested" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" - [[package]] name = "proc-macro2" -version = "1.0.29" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f5105d4fdaab20335ca9565e106a5d9b82b6219b5ba735731124ac6711d23d" +checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" dependencies = [ "unicode-xid", ] @@ -1681,9 +1764,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.9" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +checksum = "47aa80447ce4daf1717500037052af176af5d38cc3e571d9ec1c7353fc10c87d" dependencies = [ "proc-macro2", ] @@ -1845,15 +1928,16 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.4" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "246e9f61b9bb77df069a947682be06e31ac43ea37862e244a69f177694ea6d22" +checksum = "87f242f1488a539a79bac6dbe7c8609ae43b7914b7736210f239a37cccb32525" dependencies = [ "base64 0.13.0", "bytes", "encoding_rs", "futures-core", "futures-util", + "h2", "http", "http-body", "hyper", @@ -1865,12 +1949,14 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite", - "rustls", - "rustls-native-certs", + "rustls 0.20.2", + "rustls-native-certs 0.6.1", + "rustls-pemfile", "serde", + "serde_json", "serde_urlencoded", "tokio", - "tokio-rustls", + "tokio-rustls 0.23.2", "tokio-socks", "url", "wasm-bindgen", @@ -1983,10 +2069,20 @@ dependencies = [ "state", "time 0.2.27", "tokio", - "tokio-rustls", + "tokio-rustls 0.22.0", "uncased", ] +[[package]] +name = "rocksdb" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c749134fda8bfc90d0de643d59bfc841dcb3ac8a1062e12b6754bd60235c48b3" +dependencies = [ + "libc", + "librocksdb-sys", +] + [[package]] name = "ruma" version = "0.4.0" @@ -2233,7 +2329,7 @@ name = "ruma-state-res" version = "0.4.1" source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" dependencies = [ - "itertools 0.10.1", + "itertools", "js_int", "ruma-common", "ruma-events", @@ -2247,9 +2343,9 @@ dependencies = [ [[package]] name = "rusqlite" -version = "0.25.3" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57adcf67c8faaf96f3248c2a7b419a0dbc52ebe36ba83dd57fe83827c1ea4eb3" +checksum = "5c4b1eaf239b47034fb450ee9cdedd7d0226571689d8823030c4b6c2cb407152" dependencies = [ "bitflags", "fallible-iterator", @@ -2269,9 +2365,15 @@ dependencies = [ "base64 0.13.0", "blake2b_simd", "constant_time_eq", - "crossbeam-utils 0.8.5", + "crossbeam-utils 0.8.6", ] +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.2.3" @@ -2290,10 +2392,22 @@ dependencies = [ "base64 0.13.0", "log", "ring", - "sct", + "sct 0.6.1", "webpki 0.21.4", ] +[[package]] +name = "rustls" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d37e5e2290f3e040b594b1a9e04377c2c671f1a1cfd9bfdef82106ac1c113f84" +dependencies = [ + "log", + "ring", + "sct 0.7.0", + "webpki 0.22.0", +] + [[package]] name = "rustls-native-certs" version = "0.5.0" @@ -2301,22 +2415,43 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" dependencies = [ "openssl-probe", - "rustls", + "rustls 0.19.1", "schannel", "security-framework", ] [[package]] -name = "rustversion" -version = "1.0.5" +name = "rustls-native-certs" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61b3909d758bb75c79f23d4736fac9433868679d3ad2ea7a61e3c25cfda9a088" +checksum = "5ca9ebdfa27d3fc180e42879037b5338ab1c040c06affd00d8338598e7800943" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eebeaeb360c87bfb72e84abdb3447159c0eaececf1bef2aecd65a8be949d1c9" +dependencies = [ + "base64 0.13.0", +] + +[[package]] +name = "rustversion" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" [[package]] name = "ryu" -version = "1.0.5" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" [[package]] name = "schannel" @@ -2350,6 +2485,16 @@ dependencies = [ "untrusted", ] +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "security-framework" version = "2.4.2" @@ -2390,18 +2535,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.130" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913" +checksum = "97565067517b60e2d1ea8b268e59ce036de907ac523ad83a0475da04e818989a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.130" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b" +checksum = "ed201699328568d8d08208fdd080e3ff594e6c422e438b6705905da01005d537" dependencies = [ "proc-macro2", "quote", @@ -2410,9 +2555,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.73" +version = "1.0.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcbd0344bc6533bc7ec56df11d42fb70f1b912351c0825ccb7211b59d8af7cf5" +checksum = "ee2bb9cd061c5865d345bb02ca49fcef1391741b672b54a0bf7b679badec3142" dependencies = [ "itoa 1.0.1", "ryu", @@ -2433,12 +2578,12 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.8.20" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad104641f3c958dab30eb3010e834c2622d1f3f4c530fef1dee20ad9485f3c09" +checksum = "a4a521f2940385c165a24ee286aa8599633d162077a54bdcae2a6fd5a7bfa7a0" dependencies = [ - "dtoa", "indexmap", + "ryu", "serde", "yaml-rust", ] @@ -2464,9 +2609,9 @@ checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" [[package]] name = "sha2" -version = "0.9.6" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9204c41a1597a8c5af23c82d1c921cb01ec0a4c59e07a9c7306062829a3903f3" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer", "cfg-if 1.0.0", @@ -2477,13 +2622,19 @@ dependencies = [ [[package]] name = "sharded-slab" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "740223c51853f3145fe7c90360d2d4232f2b62e3449489c207eccde818979982" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" + [[package]] name = "signal-hook-registry" version = "1.4.0" @@ -2495,9 +2646,9 @@ dependencies = [ [[package]] name = "signature" -version = "1.3.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c19772be3c4dd2ceaacf03cb41d5885f2a02c4d8804884918e3a258480803335" +checksum = "f054c6c1a6e95179d6f23ed974060dcefb2d9388bb7256900badad682c499de4" [[package]] name = "simple_asn1" @@ -2512,19 +2663,19 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c307a32c1c5c437f38c7fd45d753050587732ba8628319fbdf12a7e289ccc590" +checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" [[package]] name = "sled" -version = "0.34.6" +version = "0.34.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d0132f3e393bcb7390c60bb45769498cf4550bcb7a21d7f95c02b69f6362cdc" +checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" dependencies = [ "crc32fast", "crossbeam-epoch", - "crossbeam-utils 0.8.5", + "crossbeam-utils 0.8.6", "fs2", "fxhash", "libc", @@ -2535,9 +2686,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" +checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309" [[package]] name = "socket2" @@ -2552,9 +2703,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "765f090f0e423d2b55843402a07915add955e7d60657db13707a159727326cad" +checksum = "5dc90fe6c7be1a323296982db1836d1ea9e47b6839496dde9a541bc496df3516" dependencies = [ "libc", "winapi", @@ -2574,9 +2725,9 @@ checksum = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5" [[package]] name = "spki" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "987637c5ae6b3121aba9d513f869bd2bff11c4cc086c22473befd6649c0bd521" +checksum = "5c01a0c15da1b0b0e1494112e7af814a678fec9bd157881b49beac661e9b6f32" dependencies = [ "der", ] @@ -2665,9 +2816,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.75" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7f58f7e8eaa0009c5fec437aabf511bd9933e4b2d7407bd05273c01a8906ea7" +checksum = "a684ac3dcd8913827e18cd09a68384ee66c1de24157e3c556c9ab16d85695fb7" dependencies = [ "proc-macro2", "quote", @@ -2685,9 +2836,9 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.12.5" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "474aaa926faa1603c40b7885a9eaea29b444d1cb2850cb7c0e37bb1a4182f4fa" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ "proc-macro2", "quote", @@ -2697,13 +2848,13 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" dependencies = [ "cfg-if 1.0.0", + "fastrand", "libc", - "rand 0.8.4", "redox_syscall", "remove_dir_all", "winapi", @@ -2711,18 +2862,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.28" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "283d5230e63df9608ac7d9691adc1dfb6e701225436eb64d0b9a7f0a5a04f6ec" +checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.28" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa3884228611f5cd3608e2d409bf7dce832e4eb3135e3f11addbd7e41bd68e71" +checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" dependencies = [ "proc-macro2", "quote", @@ -2810,9 +2961,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.3.1" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "848a1e1181b9f6753b5e96a092749e29b11d19ede67dfbbd6c7dc7e0f49b5338" +checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2" dependencies = [ "tinyvec_macros", ] @@ -2825,11 +2976,10 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.11.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4efe6fc2395938c8155973d7be49fe8d03a843726e285e100a8a383cc0154ce" +checksum = "fbbf1c778ec206785635ce8ad57fe52b3009ae9e0c9f574a728f3049d3e55838" dependencies = [ - "autocfg", "bytes", "libc", "memchr", @@ -2844,9 +2994,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.3.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54473be61f4ebe4efd09cec9bd5d16fa51d70ea0192213d754d2d500457db110" +checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" dependencies = [ "proc-macro2", "quote", @@ -2859,11 +3009,22 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" dependencies = [ - "rustls", + "rustls 0.19.1", "tokio", "webpki 0.21.4", ] +[[package]] +name = "tokio-rustls" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a27d5f2b839802bd8267fa19b0530f5a08b9c08cd417976be2a65d130fe1c11b" +dependencies = [ + "rustls 0.20.2", + "tokio", + "webpki 0.22.0", +] + [[package]] name = "tokio-socks" version = "0.5.1" @@ -2878,9 +3039,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2f3f698253f03119ac0102beaa64f67a67e08074d03a22d18784104543727f" +checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3" dependencies = [ "futures-core", "pin-project-lite", @@ -2889,9 +3050,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.6.7" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" +checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" dependencies = [ "bytes", "futures-core", @@ -2918,9 +3079,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.26" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" +checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105" dependencies = [ "cfg-if 1.0.0", "pin-project-lite", @@ -2930,9 +3091,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.15" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42e6fa53307c8a17e4ccd4dc81cf5ec38db9209f59b222210375b54ee40d1e2" +checksum = "f4f480b8f81512e825f337ad51e94c1eb5d3bbdf2b363dcd01e2b19a9ffe3f8e" dependencies = [ "proc-macro2", "quote", @@ -2941,9 +3102,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.19" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ca517f43f0fb96e0c3072ed5c275fe5eece87e8cb52f4a77b69226d3b1c9df8" +checksum = "1f4ed65637b8390770814083d20756f87bfa2c21bf2f110babdc5438351746e4" dependencies = [ "lazy_static", ] @@ -2956,7 +3117,7 @@ checksum = "bd520fe41c667b437952383f3a1ec14f1fa45d653f719a77eedd6e6a02d8fa54" dependencies = [ "lazy_static", "tracing", - "tracing-subscriber", + "tracing-subscriber 0.2.25", ] [[package]] @@ -2982,14 +3143,14 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.2.20" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9cbe87a2fa7e35900ce5de20220a582a9483a7063811defce79d7cbd59d4cfe" +checksum = "0e0d2eaa99c3c2e41547cfa109e910a68ea03823cccad4a0525dcbc9b01e8c71" dependencies = [ "ansi_term", "chrono", "lazy_static", - "matchers", + "matchers 0.0.1", "regex", "serde", "serde_json", @@ -3002,6 +3163,24 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "tracing-subscriber" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d81bfa81424cc98cb034b837c985b7a290f592e5b4322f353f94a0ab0f9f594" +dependencies = [ + "ansi_term", + "lazy_static", + "matchers 0.1.0", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + [[package]] name = "trust-dns-proto" version = "0.20.3" @@ -3053,21 +3232,11 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" -[[package]] -name = "twoway" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c57ffb460d7c24cd6eda43694110189030a3d1dfe418416d9468fd1c1d290b47" -dependencies = [ - "memchr", - "unchecked-index", -] - [[package]] name = "typenum" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" [[package]] name = "ubyte" @@ -3088,17 +3257,11 @@ dependencies = [ "version_check", ] -[[package]] -name = "unchecked-index" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeba86d422ce181a719445e51872fa30f1f7413b62becb52e95ec91aa262d85c" - [[package]] name = "unicode-bidi" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "246f4c42e67e7a4e3c6106ff716a5d067d4132a642840b242e357e468a2a0085" +checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" [[package]] name = "unicode-normalization" @@ -3153,9 +3316,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "version_check" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "want" @@ -3181,21 +3344,19 @@ checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" [[package]] name = "wasm-bindgen" -version = "0.2.76" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce9b1b516211d33767048e5d47fa2a381ed8b76fc48d2ce4aa39877f9f183e0" +checksum = "632f73e236b219150ea279196e54e610f5dbafa5d61786303d4da54f84e47fce" dependencies = [ "cfg-if 1.0.0", - "serde", - "serde_json", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.76" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe8dc78e2326ba5f845f4b5bf548401604fa20b1dd1d365fb73b6c1d6364041" +checksum = "a317bf8f9fba2476b4b2c85ef4c4af8ff39c3c7f0cdfeed4f82c34a880aa837b" dependencies = [ "bumpalo", "lazy_static", @@ -3208,9 +3369,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.26" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95fded345a6559c2cfee778d562300c581f7d4ff3edb9b0d230d69800d213972" +checksum = "8e8d7523cb1f2a4c96c1317ca690031b714a51cc14e05f712446691f413f5d39" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -3220,9 +3381,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.76" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44468aa53335841d9d6b6c023eaab07c0cd4bddbcfdee3e2bb1e8d2cb8069fef" +checksum = "d56146e7c495528bf6587663bea13a8eb588d39b36b679d83972e1a2dbbdacf9" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3230,9 +3391,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.76" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0195807922713af1e67dc66132c7328206ed9766af3858164fb583eedc25fbad" +checksum = "7803e0eea25835f8abdc585cd3021b3deb11543c6fe226dcd30b228857c5c5ab" dependencies = [ "proc-macro2", "quote", @@ -3243,15 +3404,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.76" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdb075a845574a1fa5f09fd77e43f7747599301ea3417a9fbffdeedfc1f4a29" +checksum = "0237232789cf037d5480773fe568aac745bfe2afbc11a863e97901780a6b47cc" [[package]] name = "web-sys" -version = "0.3.53" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224b2f6b67919060055ef1a67807367c2066ed520c3862cc013d26cf893a783c" +checksum = "38eb105f1c59d9eaa6b5cdc92b859d85b926e82cb2e0945cd0c9259faa6fe9fb" dependencies = [ "js-sys", "wasm-bindgen", @@ -3352,18 +3513,18 @@ checksum = "9fc79f4a1e39857fc00c3f662cbf2651c771f00e9c15fe2abc341806bd46bd71" [[package]] name = "zeroize" -version = "1.4.1" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "377db0846015f7ae377174787dd452e1c5f5a9050bc6f954911d01f116daa0cd" +checksum = "d68d9dcec5f9b43a30d38c49f91dfedfaac384cb8f085faca366c26207dd1619" dependencies = [ "zeroize_derive", ] [[package]] name = "zeroize_derive" -version = "1.1.0" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2c1e130bebaeab2f23886bf9acbaca14b092408c452543c857f66399cd6dab1" +checksum = "65f1a51723ec88c66d5d1fe80c841f17f63587d6691901d66be9bec6c3b51f73" dependencies = [ "proc-macro2", "quote", @@ -3373,18 +3534,18 @@ dependencies = [ [[package]] name = "zstd" -version = "0.5.4+zstd.1.4.7" +version = "0.9.2+zstd.1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69996ebdb1ba8b1517f61387a883857818a66c8a295f487b1ffd8fd9d2c82910" +checksum = "2390ea1bf6c038c39674f22d95f0564725fc06034a47129179810b2fc58caa54" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "2.0.6+zstd.1.4.7" +version = "4.1.3+zstd.1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98aa931fb69ecee256d44589d19754e61851ae4769bf963b385119b1cc37a49e" +checksum = "e99d81b99fb3c2c2c794e3fe56c305c63d5173a16a46b5850b07c935ffc7db79" dependencies = [ "libc", "zstd-sys", @@ -3392,12 +3553,10 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "1.4.18+zstd.1.4.7" +version = "1.6.2+zstd.1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e6e8778706838f43f771d80d37787cb2fe06dafe89dd3aebaf6721b9eaec81" +checksum = "2daf2f248d9ea44454bfcb2516534e8b8ad2fc91bf818a1885495fc42bc8ac9f" dependencies = [ "cc", - "glob", - "itertools 0.9.0", "libc", ] diff --git a/Cargo.toml b/Cargo.toml index 5e09dee0..5cc6a83c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -78,6 +78,7 @@ crossbeam = { version = "0.8.1", optional = true } num_cpus = "1.13.0" threadpool = "1.8.1" heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c758867e05ad973ef800a6fe1d5d", optional = true } +rocksdb = { version = "0.16.0", features = ["multi-threaded-cf"], optional = true } thread_local = "1.1.3" # used for TURN server authentication hmac = "0.11.0" @@ -87,7 +88,8 @@ sha-1 = "0.9.8" default = ["conduit_bin", "backend_sqlite"] backend_sled = ["sled"] backend_sqlite = ["sqlite"] -backend_heed = ["heed", "crossbeam", "parking_lot"] +backend_heed = ["heed", "crossbeam"] +backend_rocksdb = ["rocksdb"] sqlite = ["rusqlite", "parking_lot", "crossbeam", "tokio/signal"] conduit_bin = [] # TODO: add rocket to this when it is optional diff --git a/src/database.rs b/src/database.rs index 8b29b221..4c377f06 100644 --- a/src/database.rs +++ b/src/database.rs @@ -154,6 +154,9 @@ pub type Engine = abstraction::sqlite::Engine; #[cfg(feature = "heed")] pub type Engine = abstraction::heed::Engine; +#[cfg(feature = "rocksdb")] +pub type Engine = abstraction::rocksdb::Engine; + pub struct Database { _db: Arc, pub globals: globals::Globals, @@ -314,10 +317,10 @@ impl Database { .expect("pdu cache capacity fits into usize"), )), auth_chain_cache: Mutex::new(LruCache::new(1_000_000)), - shorteventid_cache: Mutex::new(LruCache::new(1_000_000)), - eventidshort_cache: Mutex::new(LruCache::new(1_000_000)), - shortstatekey_cache: Mutex::new(LruCache::new(1_000_000)), - statekeyshort_cache: Mutex::new(LruCache::new(1_000_000)), + shorteventid_cache: Mutex::new(LruCache::new(100_000_000)), + eventidshort_cache: Mutex::new(LruCache::new(100_000_000)), + shortstatekey_cache: Mutex::new(LruCache::new(100_000_000)), + statekeyshort_cache: Mutex::new(LruCache::new(100_000_000)), our_real_users_cache: RwLock::new(HashMap::new()), appservice_in_room_cache: RwLock::new(HashMap::new()), stateinfo_cache: Mutex::new(LruCache::new(1000)), diff --git a/src/database/abstraction.rs b/src/database/abstraction.rs index 67b80d1a..a347f831 100644 --- a/src/database/abstraction.rs +++ b/src/database/abstraction.rs @@ -12,7 +12,10 @@ pub mod sqlite; #[cfg(feature = "heed")] pub mod heed; -#[cfg(any(feature = "sqlite", feature = "heed"))] +#[cfg(feature = "rocksdb")] +pub mod rocksdb; + +#[cfg(any(feature = "sqlite", feature = "rocksdb", feature = "heed"))] pub mod watchers; pub trait DatabaseEngine: Sized { diff --git a/src/database/abstraction/rocksdb.rs b/src/database/abstraction/rocksdb.rs new file mode 100644 index 00000000..3ff6ab86 --- /dev/null +++ b/src/database/abstraction/rocksdb.rs @@ -0,0 +1,183 @@ +use super::super::Config; +use crate::{utils, Result}; + +use std::{future::Future, pin::Pin, sync::Arc}; + +use super::{DatabaseEngine, Tree}; + +use std::{collections::HashMap, sync::RwLock}; + +pub struct Engine { + rocks: rocksdb::DBWithThreadMode, + old_cfs: Vec, +} + +pub struct RocksDbEngineTree<'a> { + db: Arc, + name: &'a str, + watchers: Watchers, +} + +impl DatabaseEngine for Engine { + fn open(config: &Config) -> Result> { + let mut db_opts = rocksdb::Options::default(); + db_opts.create_if_missing(true); + db_opts.set_max_open_files(16); + db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level); + db_opts.set_compression_type(rocksdb::DBCompressionType::Snappy); + db_opts.set_target_file_size_base(256 << 20); + db_opts.set_write_buffer_size(256 << 20); + + let mut block_based_options = rocksdb::BlockBasedOptions::default(); + block_based_options.set_block_size(512 << 10); + db_opts.set_block_based_table_factory(&block_based_options); + + let cfs = rocksdb::DBWithThreadMode::::list_cf( + &db_opts, + &config.database_path, + ) + .unwrap_or_default(); + + let db = rocksdb::DBWithThreadMode::::open_cf_descriptors( + &db_opts, + &config.database_path, + cfs.iter().map(|name| { + let mut options = rocksdb::Options::default(); + let prefix_extractor = rocksdb::SliceTransform::create_fixed_prefix(1); + options.set_prefix_extractor(prefix_extractor); + options.set_merge_operator_associative("increment", utils::increment_rocksdb); + + rocksdb::ColumnFamilyDescriptor::new(name, options) + }), + )?; + + Ok(Arc::new(Engine { + rocks: db, + old_cfs: cfs, + })) + } + + fn open_tree(self: &Arc, name: &'static str) -> Result> { + if !self.old_cfs.contains(&name.to_owned()) { + // Create if it didn't exist + let mut options = rocksdb::Options::default(); + let prefix_extractor = rocksdb::SliceTransform::create_fixed_prefix(1); + options.set_prefix_extractor(prefix_extractor); + options.set_merge_operator_associative("increment", utils::increment_rocksdb); + + let _ = self.rocks.create_cf(name, &options); + println!("created cf"); + } + + Ok(Arc::new(RocksDbEngineTree { + name, + db: Arc::clone(self), + watchers: Watchers::default(), + })) + } + + fn flush(self: &Arc) -> Result<()> { + // TODO? + Ok(()) + } +} + +impl RocksDbEngineTree<'_> { + fn cf(&self) -> rocksdb::BoundColumnFamily<'_> { + self.db.rocks.cf_handle(self.name).unwrap() + } +} + +impl Tree for RocksDbEngineTree<'_> { + fn get(&self, key: &[u8]) -> Result>> { + Ok(self.db.rocks.get_cf(self.cf(), key)?) + } + + fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> { + self.db.rocks.put_cf(self.cf(), key, value)?; + self.watchers.wake(key); + Ok(()) + } + + fn insert_batch<'a>(&self, iter: &mut dyn Iterator, Vec)>) -> Result<()> { + for (key, value) in iter { + self.db.rocks.put_cf(self.cf(), key, value)?; + } + + Ok(()) + } + + fn remove(&self, key: &[u8]) -> Result<()> { + Ok(self.db.rocks.delete_cf(self.cf(), key)?) + } + + fn iter<'a>(&'a self) -> Box, Vec)> + 'a> { + Box::new( + self.db + .rocks + .iterator_cf(self.cf(), rocksdb::IteratorMode::Start) + .map(|(k, v)| (Vec::from(k), Vec::from(v))), + ) + } + + fn iter_from<'a>( + &'a self, + from: &[u8], + backwards: bool, + ) -> Box, Vec)> + 'a> { + Box::new( + self.db + .rocks + .iterator_cf( + self.cf(), + rocksdb::IteratorMode::From( + from, + if backwards { + rocksdb::Direction::Reverse + } else { + rocksdb::Direction::Forward + }, + ), + ) + .map(|(k, v)| (Vec::from(k), Vec::from(v))), + ) + } + + fn increment(&self, key: &[u8]) -> Result> { + // TODO: make atomic + let old = self.db.rocks.get_cf(self.cf(), &key)?; + let new = utils::increment(old.as_deref()).unwrap(); + self.db.rocks.put_cf(self.cf(), key, &new)?; + Ok(new) + } + + fn increment_batch<'a>(&self, iter: &mut dyn Iterator>) -> Result<()> { + for key in iter { + let old = self.db.rocks.get_cf(self.cf(), &key)?; + let new = utils::increment(old.as_deref()).unwrap(); + self.db.rocks.put_cf(self.cf(), key, new)?; + } + + Ok(()) + } + + fn scan_prefix<'a>( + &'a self, + prefix: Vec, + ) -> Box, Vec)> + 'a> { + Box::new( + self.db + .rocks + .iterator_cf( + self.cf(), + rocksdb::IteratorMode::From(&prefix, rocksdb::Direction::Forward), + ) + .map(|(k, v)| (Vec::from(k), Vec::from(v))) + .take_while(move |(k, _)| k.starts_with(&prefix)), + ) + } + + fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin + Send + 'a>> { + self.watchers.watch(prefix) + } +} diff --git a/src/database/abstraction/sqlite.rs b/src/database/abstraction/sqlite.rs index 1e6a2d89..31875667 100644 --- a/src/database/abstraction/sqlite.rs +++ b/src/database/abstraction/sqlite.rs @@ -132,7 +132,7 @@ type TupleOfBytes = (Vec, Vec); impl SqliteTable { #[tracing::instrument(skip(self, guard, key))] fn get_with_guard(&self, guard: &Connection, key: &[u8]) -> Result>> { - //dbg!(&self.name); + dbg!(&self.name); Ok(guard .prepare(format!("SELECT value FROM {} WHERE key = ?", self.name).as_str())? .query_row([key], |row| row.get(0)) @@ -141,7 +141,7 @@ impl SqliteTable { #[tracing::instrument(skip(self, guard, key, value))] fn insert_with_guard(&self, guard: &Connection, key: &[u8], value: &[u8]) -> Result<()> { - //dbg!(&self.name); + dbg!(&self.name); guard.execute( format!( "INSERT OR REPLACE INTO {} (key, value) VALUES (?, ?)", @@ -168,14 +168,14 @@ impl SqliteTable { let statement_ref = NonAliasingBox(statement); - //let name = self.name.clone(); + let name = self.name.clone(); let iterator = Box::new( statement .query_map([], |row| Ok((row.get_unwrap(0), row.get_unwrap(1)))) .unwrap() .map(move |r| { - //dbg!(&name); + dbg!(&name); r.unwrap() }), ); @@ -263,7 +263,7 @@ impl Tree for SqliteTable { let guard = self.engine.read_lock_iterator(); let from = from.to_vec(); // TODO change interface? - //let name = self.name.clone(); + let name = self.name.clone(); if backwards { let statement = Box::leak(Box::new( @@ -282,7 +282,7 @@ impl Tree for SqliteTable { .query_map([from], |row| Ok((row.get_unwrap(0), row.get_unwrap(1)))) .unwrap() .map(move |r| { - //dbg!(&name); + dbg!(&name); r.unwrap() }), ); @@ -307,7 +307,7 @@ impl Tree for SqliteTable { .query_map([from], |row| Ok((row.get_unwrap(0), row.get_unwrap(1)))) .unwrap() .map(move |r| { - //dbg!(&name); + dbg!(&name); r.unwrap() }), ); diff --git a/src/error.rs b/src/error.rs index 7faddc91..4d427da4 100644 --- a/src/error.rs +++ b/src/error.rs @@ -39,6 +39,12 @@ pub enum Error { #[cfg(feature = "heed")] #[error("There was a problem with the connection to the heed database: {error}")] HeedError { error: String }, + #[cfg(feature = "rocksdb")] + #[error("There was a problem with the connection to the rocksdb database: {source}")] + RocksDbError { + #[from] + source: rocksdb::Error, + }, #[error("Could not generate an image.")] ImageError { #[from] diff --git a/src/utils.rs b/src/utils.rs index 26d71a8c..4702d051 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -29,6 +29,17 @@ pub fn increment(old: Option<&[u8]>) -> Option> { Some(number.to_be_bytes().to_vec()) } +#[cfg(feature = "rocksdb")] +pub fn increment_rocksdb( + _new_key: &[u8], + old: Option<&[u8]>, + _operands: &mut rocksdb::MergeOperands, +) -> Option> { + dbg!(_new_key); + dbg!(old); + increment(old) +} + pub fn generate_keypair() -> Vec { let mut value = random_string(8).as_bytes().to_vec(); value.push(0xff); From a30b588ede6135642946afd575a2411c6d0d21e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Fri, 10 Dec 2021 21:34:45 +0100 Subject: [PATCH 078/201] rocksdb as default --- Cargo.toml | 2 +- src/database.rs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5cc6a83c..0a2b4459 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -85,7 +85,7 @@ hmac = "0.11.0" sha-1 = "0.9.8" [features] -default = ["conduit_bin", "backend_sqlite"] +default = ["conduit_bin", "backend_rocksdb"] backend_sled = ["sled"] backend_sqlite = ["sqlite"] backend_heed = ["heed", "crossbeam"] diff --git a/src/database.rs b/src/database.rs index 4c377f06..af6136b3 100644 --- a/src/database.rs +++ b/src/database.rs @@ -317,10 +317,10 @@ impl Database { .expect("pdu cache capacity fits into usize"), )), auth_chain_cache: Mutex::new(LruCache::new(1_000_000)), - shorteventid_cache: Mutex::new(LruCache::new(100_000_000)), - eventidshort_cache: Mutex::new(LruCache::new(100_000_000)), - shortstatekey_cache: Mutex::new(LruCache::new(100_000_000)), - statekeyshort_cache: Mutex::new(LruCache::new(100_000_000)), + shorteventid_cache: Mutex::new(LruCache::new(1_000_000)), + eventidshort_cache: Mutex::new(LruCache::new(1_000_000)), + shortstatekey_cache: Mutex::new(LruCache::new(1_000_000)), + statekeyshort_cache: Mutex::new(LruCache::new(1_000_000)), our_real_users_cache: RwLock::new(HashMap::new()), appservice_in_room_cache: RwLock::new(HashMap::new()), stateinfo_cache: Mutex::new(LruCache::new(1000)), From c9c99746412155fcdce6a6430bd5ef9c567cc3fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Thu, 16 Dec 2021 14:52:19 +0100 Subject: [PATCH 079/201] fix: stack overflows when fetching auth events --- src/database/abstraction/rocksdb.rs | 18 ++-- src/server_server.rs | 157 +++++++++++++++------------- 2 files changed, 94 insertions(+), 81 deletions(-) diff --git a/src/database/abstraction/rocksdb.rs b/src/database/abstraction/rocksdb.rs index 3ff6ab86..825c02e0 100644 --- a/src/database/abstraction/rocksdb.rs +++ b/src/database/abstraction/rocksdb.rs @@ -22,14 +22,20 @@ impl DatabaseEngine for Engine { fn open(config: &Config) -> Result> { let mut db_opts = rocksdb::Options::default(); db_opts.create_if_missing(true); - db_opts.set_max_open_files(16); + db_opts.set_max_open_files(512); db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level); - db_opts.set_compression_type(rocksdb::DBCompressionType::Snappy); - db_opts.set_target_file_size_base(256 << 20); - db_opts.set_write_buffer_size(256 << 20); + db_opts.set_compression_type(rocksdb::DBCompressionType::Zstd); + db_opts.set_target_file_size_base(2 << 22); + db_opts.set_max_bytes_for_level_base(2 << 24); + db_opts.set_max_bytes_for_level_multiplier(2.0); + db_opts.set_num_levels(8); + db_opts.set_write_buffer_size(2 << 27); + + let rocksdb_cache = rocksdb::Cache::new_lru_cache((config.db_cache_capacity_mb * 1024.0 * 1024.0) as usize).unwrap(); let mut block_based_options = rocksdb::BlockBasedOptions::default(); - block_based_options.set_block_size(512 << 10); + block_based_options.set_block_size(2 << 19); + block_based_options.set_block_cache(&rocksdb_cache); db_opts.set_block_based_table_factory(&block_based_options); let cfs = rocksdb::DBWithThreadMode::::list_cf( @@ -45,7 +51,6 @@ impl DatabaseEngine for Engine { let mut options = rocksdb::Options::default(); let prefix_extractor = rocksdb::SliceTransform::create_fixed_prefix(1); options.set_prefix_extractor(prefix_extractor); - options.set_merge_operator_associative("increment", utils::increment_rocksdb); rocksdb::ColumnFamilyDescriptor::new(name, options) }), @@ -63,7 +68,6 @@ impl DatabaseEngine for Engine { let mut options = rocksdb::Options::default(); let prefix_extractor = rocksdb::SliceTransform::create_fixed_prefix(1); options.set_prefix_extractor(prefix_extractor); - options.set_merge_operator_associative("increment", utils::increment_rocksdb); let _ = self.rocks.create_cf(name, &options); println!("created cf"); diff --git a/src/server_server.rs b/src/server_server.rs index 594152ae..d6bc9b91 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1392,12 +1392,11 @@ async fn upgrade_outlier_to_timeline_pdu( let mut starting_events = Vec::with_capacity(leaf_state.len()); for (k, id) in leaf_state { - let k = db - .rooms - .get_statekey_from_short(k) - .map_err(|_| "Failed to get_statekey_from_short.".to_owned())?; - - state.insert(k, id.clone()); + if let Ok(k) = db.rooms.get_statekey_from_short(k) { + state.insert(k, id.clone()); + } else { + warn!("Failed to get_statekey_from_short."); + } starting_events.push(id); } @@ -1755,11 +1754,16 @@ async fn upgrade_outlier_to_timeline_pdu( .into_iter() .map(|map| { map.into_iter() - .map(|(k, id)| db.rooms.get_statekey_from_short(k).map(|k| (k, id))) - .collect::>>() + .filter_map(|(k, id)| { + db.rooms + .get_statekey_from_short(k) + .map(|k| (k, id)) + .map_err(|e| warn!("Failed to get_statekey_from_short: {}", e)) + .ok() + }) + .collect::>() }) - .collect::>() - .map_err(|_| "Failed to get_statekey_from_short.".to_owned())?; + .collect(); let state = match state_res::resolve( room_version_id, @@ -1871,73 +1875,78 @@ pub(crate) fn fetch_and_handle_outliers<'a>( // a. Look in the main timeline (pduid_pdu tree) // b. Look at outlier pdu tree // (get_pdu_json checks both) - let local_pdu = db.rooms.get_pdu(id); - let pdu = match local_pdu { - Ok(Some(pdu)) => { + if let Ok(Some(local_pdu)) = db.rooms.get_pdu(id) { + trace!("Found {} in db", id); + pdus.push((local_pdu, None)); + } + + // c. Ask origin server over federation + // We also handle its auth chain here so we don't get a stack overflow in + // handle_outlier_pdu. + let mut todo_auth_events = vec![id]; + let mut events_in_reverse_order = Vec::new(); + while let Some(next_id) = todo_auth_events.pop() { + if let Ok(Some(_)) = db.rooms.get_pdu(next_id) { trace!("Found {} in db", id); - (pdu, None) - } - Ok(None) => { - // c. Ask origin server over federation - warn!("Fetching {} over federation.", id); - match db - .sending - .send_federation_request( - &db.globals, - origin, - get_event::v1::Request { event_id: id }, - ) - .await - { - Ok(res) => { - warn!("Got {} over federation", id); - let (calculated_event_id, value) = - match crate::pdu::gen_event_id_canonical_json(&res.pdu) { - Ok(t) => t, - Err(_) => { - back_off((**id).to_owned()); - continue; - } - }; - - if calculated_event_id != **id { - warn!("Server didn't return event id we requested: requested: {}, we got {}. Event: {:?}", - id, calculated_event_id, &res.pdu); - } - - // This will also fetch the auth chain - match handle_outlier_pdu( - origin, - create_event, - id, - room_id, - value.clone(), - db, - pub_key_map, - ) - .await - { - Ok((pdu, json)) => (pdu, Some(json)), - Err(e) => { - warn!("Authentication of event {} failed: {:?}", id, e); - back_off((**id).to_owned()); - continue; - } - } - } - Err(_) => { - warn!("Failed to fetch event: {}", id); - back_off((**id).to_owned()); - continue; - } - } - } - Err(e) => { - warn!("Error loading {}: {}", id, e); continue; } - }; - pdus.push(pdu); + + warn!("Fetching {} over federation.", next_id); + match db + .sending + .send_federation_request( + &db.globals, + origin, + get_event::v1::Request { event_id: next_id }, + ) + .await + { + Ok(res) => { + warn!("Got {} over federation", next_id); + let (calculated_event_id, value) = + match crate::pdu::gen_event_id_canonical_json(&res.pdu) { + Ok(t) => t, + Err(_) => { + back_off((**next_id).to_owned()); + continue; + } + }; + + if calculated_event_id != **next_id { + warn!("Server didn't return event id we requested: requested: {}, we got {}. Event: {:?}", + next_id, calculated_event_id, &res.pdu); + } + + events_in_reverse_order.push((next_id, value)); + } + Err(_) => { + warn!("Failed to fetch event: {}", next_id); + back_off((**next_id).to_owned()); + } + } + } + + while let Some((next_id, value)) = events_in_reverse_order.pop() { + match handle_outlier_pdu( + origin, + create_event, + next_id, + room_id, + value.clone(), + db, + pub_key_map, + ) + .await + { + Ok((pdu, json)) => { + pdus.push((pdu, Some(json))); + } + Err(e) => { + warn!("Authentication of event {} failed: {:?}", next_id, e); + back_off((**next_id).to_owned()); + } + } + } } pdus }) From 4b4afea2abb4289d6fa31e02bd2be2799f51e0ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Thu, 16 Dec 2021 15:54:42 +0100 Subject: [PATCH 080/201] fix auth event fetching --- src/server_server.rs | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/server_server.rs b/src/server_server.rs index d6bc9b91..28c3ea07 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1878,15 +1878,16 @@ pub(crate) fn fetch_and_handle_outliers<'a>( if let Ok(Some(local_pdu)) = db.rooms.get_pdu(id) { trace!("Found {} in db", id); pdus.push((local_pdu, None)); + continue; } // c. Ask origin server over federation // We also handle its auth chain here so we don't get a stack overflow in // handle_outlier_pdu. - let mut todo_auth_events = vec![id]; + let mut todo_auth_events = vec![Arc::clone(id)]; let mut events_in_reverse_order = Vec::new(); while let Some(next_id) = todo_auth_events.pop() { - if let Ok(Some(_)) = db.rooms.get_pdu(next_id) { + if let Ok(Some(_)) = db.rooms.get_pdu(&next_id) { trace!("Found {} in db", id); continue; } @@ -1897,7 +1898,7 @@ pub(crate) fn fetch_and_handle_outliers<'a>( .send_federation_request( &db.globals, origin, - get_event::v1::Request { event_id: next_id }, + get_event::v1::Request { event_id: &next_id }, ) .await { @@ -1907,21 +1908,35 @@ pub(crate) fn fetch_and_handle_outliers<'a>( match crate::pdu::gen_event_id_canonical_json(&res.pdu) { Ok(t) => t, Err(_) => { - back_off((**next_id).to_owned()); + back_off((*next_id).to_owned()); continue; } }; - if calculated_event_id != **next_id { + if calculated_event_id != *next_id { warn!("Server didn't return event id we requested: requested: {}, we got {}. Event: {:?}", next_id, calculated_event_id, &res.pdu); } + + if let Some(auth_events) = value.get("auth_events").and_then(|c| c.as_array()) { + for auth_event in auth_events { + if let Some(Ok(auth_event)) = auth_event.as_str() + .map(|e| serde_json::from_str(e)) { + todo_auth_events.push(auth_event); + } else { + warn!("Auth event id is not valid"); + } + } + } else { + warn!("Auth event list invalid"); + } + events_in_reverse_order.push((next_id, value)); } Err(_) => { warn!("Failed to fetch event: {}", next_id); - back_off((**next_id).to_owned()); + back_off((*next_id).to_owned()); } } } @@ -1930,7 +1945,7 @@ pub(crate) fn fetch_and_handle_outliers<'a>( match handle_outlier_pdu( origin, create_event, - next_id, + &next_id, room_id, value.clone(), db, @@ -1943,7 +1958,7 @@ pub(crate) fn fetch_and_handle_outliers<'a>( } Err(e) => { warn!("Authentication of event {} failed: {:?}", next_id, e); - back_off((**next_id).to_owned()); + back_off((*next_id).to_owned()); } } } From 74951cb239b5ec7ef41ba080729bc93df046fb66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Thu, 16 Dec 2021 21:42:53 +0100 Subject: [PATCH 081/201] dbg --- src/server_server.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/server_server.rs b/src/server_server.rs index 28c3ea07..b6bea0c5 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1922,8 +1922,9 @@ pub(crate) fn fetch_and_handle_outliers<'a>( if let Some(auth_events) = value.get("auth_events").and_then(|c| c.as_array()) { for auth_event in auth_events { if let Some(Ok(auth_event)) = auth_event.as_str() - .map(|e| serde_json::from_str(e)) { - todo_auth_events.push(auth_event); + .map(|e| {let ev: std::result::Result, _> = dbg!(serde_json::from_str(dbg!(e))); ev}) { + let a: Arc = auth_event; + todo_auth_events.push(a); } else { warn!("Auth event id is not valid"); } From 83a9095cdc3febd617d9bfd2d8cacf0fe3e89990 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Thu, 16 Dec 2021 22:25:24 +0100 Subject: [PATCH 082/201] fix? --- src/server_server.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/server_server.rs b/src/server_server.rs index b6bea0c5..8c5c09f2 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1921,8 +1921,7 @@ pub(crate) fn fetch_and_handle_outliers<'a>( if let Some(auth_events) = value.get("auth_events").and_then(|c| c.as_array()) { for auth_event in auth_events { - if let Some(Ok(auth_event)) = auth_event.as_str() - .map(|e| {let ev: std::result::Result, _> = dbg!(serde_json::from_str(dbg!(e))); ev}) { + if let Ok(auth_event) = serde_json::from_value(auth_event.clone().into()) { let a: Arc = auth_event; todo_auth_events.push(a); } else { From ee3d2db8e061bcdac43674aa050bcd3aad79d4a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sun, 19 Dec 2021 10:48:28 +0100 Subject: [PATCH 083/201] improvement, maybe not safe --- src/server_server.rs | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/src/server_server.rs b/src/server_server.rs index 8c5c09f2..57f55867 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1686,25 +1686,6 @@ async fn upgrade_outlier_to_timeline_pdu( // We do this by adding the current state to the list of fork states extremity_sstatehashes.remove(¤t_sstatehash); fork_states.push(current_state_ids); - dbg!(&extremity_sstatehashes); - - for (sstatehash, leaf_pdu) in extremity_sstatehashes { - let mut leaf_state = db - .rooms - .state_full_ids(sstatehash) - .map_err(|_| "Failed to ask db for room state.".to_owned())?; - - if let Some(state_key) = &leaf_pdu.state_key { - let shortstatekey = db - .rooms - .get_or_create_shortstatekey(&leaf_pdu.kind, state_key, &db.globals) - .map_err(|_| "Failed to create shortstatekey.".to_owned())?; - leaf_state.insert(shortstatekey, Arc::from(&*leaf_pdu.event_id)); - // Now it's the state after the pdu - } - - fork_states.push(leaf_state); - } // We also add state after incoming event to the fork states let mut state_after = state_at_incoming_event.clone(); @@ -1941,7 +1922,7 @@ pub(crate) fn fetch_and_handle_outliers<'a>( } } - while let Some((next_id, value)) = events_in_reverse_order.pop() { + for (next_id, value) in events_in_reverse_order { match handle_outlier_pdu( origin, create_event, @@ -1954,7 +1935,9 @@ pub(crate) fn fetch_and_handle_outliers<'a>( .await { Ok((pdu, json)) => { - pdus.push((pdu, Some(json))); + if next_id == *id { + pdus.push((pdu, Some(json))); + } } Err(e) => { warn!("Authentication of event {} failed: {:?}", next_id, e); From b1d9ec3efccafaf887da1b54e4b3ef2bfa4d84a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 20 Dec 2021 10:16:22 +0100 Subject: [PATCH 084/201] fix: atomic increment --- Cargo.toml | 2 +- src/database/abstraction/rocksdb.rs | 24 ++++++++++++++++-------- src/database/abstraction/watchers.rs | 8 ++++---- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0a2b4459..6241b6a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,7 +90,7 @@ backend_sled = ["sled"] backend_sqlite = ["sqlite"] backend_heed = ["heed", "crossbeam"] backend_rocksdb = ["rocksdb"] -sqlite = ["rusqlite", "parking_lot", "crossbeam", "tokio/signal"] +sqlite = ["rusqlite", "parking_lot", "tokio/signal"] conduit_bin = [] # TODO: add rocket to this when it is optional [[bin]] diff --git a/src/database/abstraction/rocksdb.rs b/src/database/abstraction/rocksdb.rs index 825c02e0..b2142dfe 100644 --- a/src/database/abstraction/rocksdb.rs +++ b/src/database/abstraction/rocksdb.rs @@ -1,11 +1,6 @@ -use super::super::Config; +use super::{super::Config, watchers::Watchers, DatabaseEngine, Tree}; use crate::{utils, Result}; - -use std::{future::Future, pin::Pin, sync::Arc}; - -use super::{DatabaseEngine, Tree}; - -use std::{collections::HashMap, sync::RwLock}; +use std::{future::Future, pin::Pin, sync::Arc, collections::HashMap, sync::RwLock}; pub struct Engine { rocks: rocksdb::DBWithThreadMode, @@ -16,6 +11,7 @@ pub struct RocksDbEngineTree<'a> { db: Arc, name: &'a str, watchers: Watchers, + write_lock: RwLock<()> } impl DatabaseEngine for Engine { @@ -77,6 +73,7 @@ impl DatabaseEngine for Engine { name, db: Arc::clone(self), watchers: Watchers::default(), + write_lock: RwLock::new(()), })) } @@ -98,8 +95,12 @@ impl Tree for RocksDbEngineTree<'_> { } fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> { + let lock = self.write_lock.read().unwrap(); self.db.rocks.put_cf(self.cf(), key, value)?; + drop(lock); + self.watchers.wake(key); + Ok(()) } @@ -148,20 +149,27 @@ impl Tree for RocksDbEngineTree<'_> { } fn increment(&self, key: &[u8]) -> Result> { - // TODO: make atomic + let lock = self.write_lock.write().unwrap(); + let old = self.db.rocks.get_cf(self.cf(), &key)?; let new = utils::increment(old.as_deref()).unwrap(); self.db.rocks.put_cf(self.cf(), key, &new)?; + + drop(lock); Ok(new) } fn increment_batch<'a>(&self, iter: &mut dyn Iterator>) -> Result<()> { + let lock = self.write_lock.write().unwrap(); + for key in iter { let old = self.db.rocks.get_cf(self.cf(), &key)?; let new = utils::increment(old.as_deref()).unwrap(); self.db.rocks.put_cf(self.cf(), key, new)?; } + drop(lock); + Ok(()) } diff --git a/src/database/abstraction/watchers.rs b/src/database/abstraction/watchers.rs index 404f3f06..fec1f27a 100644 --- a/src/database/abstraction/watchers.rs +++ b/src/database/abstraction/watchers.rs @@ -1,6 +1,6 @@ -use parking_lot::RwLock; use std::{ collections::{hash_map, HashMap}, + sync::RwLock, future::Future, pin::Pin, }; @@ -16,7 +16,7 @@ impl Watchers { &'a self, prefix: &[u8], ) -> Pin + Send + 'a>> { - let mut rx = match self.watchers.write().entry(prefix.to_vec()) { + let mut rx = match self.watchers.write().unwrap().entry(prefix.to_vec()) { hash_map::Entry::Occupied(o) => o.get().1.clone(), hash_map::Entry::Vacant(v) => { let (tx, rx) = tokio::sync::watch::channel(()); @@ -31,7 +31,7 @@ impl Watchers { }) } pub(super) fn wake(&self, key: &[u8]) { - let watchers = self.watchers.read(); + let watchers = self.watchers.read().unwrap(); let mut triggered = Vec::new(); for length in 0..=key.len() { @@ -43,7 +43,7 @@ impl Watchers { drop(watchers); if !triggered.is_empty() { - let mut watchers = self.watchers.write(); + let mut watchers = self.watchers.write().unwrap(); for prefix in triggered { if let Some(tx) = watchers.remove(prefix) { let _ = tx.0.send(()); From 54f4d39e3ed92106ec3a902de22d2366cfd8e8be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 21 Dec 2021 16:02:12 +0100 Subject: [PATCH 085/201] improvement: don't fetch event multiple times --- src/database/abstraction/rocksdb.rs | 4 +++- src/server_server.rs | 17 +++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/database/abstraction/rocksdb.rs b/src/database/abstraction/rocksdb.rs index b2142dfe..397047bd 100644 --- a/src/database/abstraction/rocksdb.rs +++ b/src/database/abstraction/rocksdb.rs @@ -27,7 +27,9 @@ impl DatabaseEngine for Engine { db_opts.set_num_levels(8); db_opts.set_write_buffer_size(2 << 27); - let rocksdb_cache = rocksdb::Cache::new_lru_cache((config.db_cache_capacity_mb * 1024.0 * 1024.0) as usize).unwrap(); + let rocksdb_cache = + rocksdb::Cache::new_lru_cache((config.db_cache_capacity_mb * 1024.0 * 1024.0) as usize) + .unwrap(); let mut block_based_options = rocksdb::BlockBasedOptions::default(); block_based_options.set_block_size(2 << 19); diff --git a/src/server_server.rs b/src/server_server.rs index 57f55867..6e8ebf38 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1867,7 +1867,12 @@ pub(crate) fn fetch_and_handle_outliers<'a>( // handle_outlier_pdu. let mut todo_auth_events = vec![Arc::clone(id)]; let mut events_in_reverse_order = Vec::new(); + let mut events_all = HashSet::new(); while let Some(next_id) = todo_auth_events.pop() { + if events_all.contains(&next_id) { + continue; + } + if let Ok(Some(_)) = db.rooms.get_pdu(&next_id) { trace!("Found {} in db", id); continue; @@ -1899,10 +1904,13 @@ pub(crate) fn fetch_and_handle_outliers<'a>( next_id, calculated_event_id, &res.pdu); } - - if let Some(auth_events) = value.get("auth_events").and_then(|c| c.as_array()) { + if let Some(auth_events) = + value.get("auth_events").and_then(|c| c.as_array()) + { for auth_event in auth_events { - if let Ok(auth_event) = serde_json::from_value(auth_event.clone().into()) { + if let Ok(auth_event) = + serde_json::from_value(auth_event.clone().into()) + { let a: Arc = auth_event; todo_auth_events.push(a); } else { @@ -1913,7 +1921,8 @@ pub(crate) fn fetch_and_handle_outliers<'a>( warn!("Auth event list invalid"); } - events_in_reverse_order.push((next_id, value)); + events_in_reverse_order.push((next_id.clone(), value)); + events_all.insert(next_id); } Err(_) => { warn!("Failed to fetch event: {}", next_id); From 5bcc1324ed3936444ba189c399e906482cc67d3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 21 Dec 2021 22:10:31 +0100 Subject: [PATCH 086/201] fix: auth event fetch order --- src/server_server.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server_server.rs b/src/server_server.rs index 6e8ebf38..c76afd34 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1931,7 +1931,7 @@ pub(crate) fn fetch_and_handle_outliers<'a>( } } - for (next_id, value) in events_in_reverse_order { + for (next_id, value) in events_in_reverse_order.iter().rev() { match handle_outlier_pdu( origin, create_event, @@ -1944,13 +1944,13 @@ pub(crate) fn fetch_and_handle_outliers<'a>( .await { Ok((pdu, json)) => { - if next_id == *id { + if next_id == id { pdus.push((pdu, Some(json))); } } Err(e) => { warn!("Authentication of event {} failed: {:?}", next_id, e); - back_off((*next_id).to_owned()); + back_off((**next_id).to_owned()); } } } From 68e910bb77f7bbc93269dd1dfd0f70a26f1e8ef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 4 Jan 2022 14:30:13 +0100 Subject: [PATCH 087/201] feat: lazy loading --- src/client_server/context.rs | 62 ++++++++++++++++--- src/client_server/message.rs | 85 +++++++++++++++++++------ src/client_server/sync.rs | 117 +++++++++++++++++++++++++++++++---- src/database.rs | 3 + src/database/rooms.rs | 96 +++++++++++++++++++++++++++- 5 files changed, 321 insertions(+), 42 deletions(-) diff --git a/src/client_server/context.rs b/src/client_server/context.rs index 9bfec9e1..94a44e39 100644 --- a/src/client_server/context.rs +++ b/src/client_server/context.rs @@ -1,5 +1,9 @@ use crate::{database::DatabaseGuard, ConduitResult, Error, Ruma}; -use ruma::api::client::{error::ErrorKind, r0::context::get_context}; +use ruma::{ + api::client::{error::ErrorKind, r0::context::get_context}, + events::EventType, +}; +use std::collections::HashSet; use std::convert::TryFrom; #[cfg(feature = "conduit_bin")] @@ -21,6 +25,7 @@ pub async fn get_context_route( body: Ruma>, ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_device = body.sender_device.as_ref().expect("user is authenticated"); if !db.rooms.is_joined(sender_user, &body.room_id)? { return Err(Error::BadRequest( @@ -29,6 +34,8 @@ pub async fn get_context_route( )); } + let mut lazy_loaded = HashSet::new(); + let base_pdu_id = db .rooms .get_pdu_id(&body.event_id)? @@ -45,8 +52,18 @@ pub async fn get_context_route( .ok_or(Error::BadRequest( ErrorKind::NotFound, "Base event not found.", - ))? - .to_room_event(); + ))?; + + if !db.rooms.lazy_load_was_sent_before( + &sender_user, + &sender_device, + &body.room_id, + &base_event.sender, + )? { + lazy_loaded.insert(base_event.sender.clone()); + } + + let base_event = base_event.to_room_event(); let events_before: Vec<_> = db .rooms @@ -60,6 +77,17 @@ pub async fn get_context_route( .filter_map(|r| r.ok()) // Remove buggy events .collect(); + for (_, event) in &events_before { + if !db.rooms.lazy_load_was_sent_before( + &sender_user, + &sender_device, + &body.room_id, + &event.sender, + )? { + lazy_loaded.insert(event.sender.clone()); + } + } + let start_token = events_before .last() .and_then(|(pdu_id, _)| db.rooms.pdu_count(pdu_id).ok()) @@ -82,6 +110,17 @@ pub async fn get_context_route( .filter_map(|r| r.ok()) // Remove buggy events .collect(); + for (_, event) in &events_after { + if !db.rooms.lazy_load_was_sent_before( + &sender_user, + &sender_device, + &body.room_id, + &event.sender, + )? { + lazy_loaded.insert(event.sender.clone()); + } + } + let end_token = events_after .last() .and_then(|(pdu_id, _)| db.rooms.pdu_count(pdu_id).ok()) @@ -92,18 +131,23 @@ pub async fn get_context_route( .map(|(_, pdu)| pdu.to_room_event()) .collect(); + let mut state = Vec::new(); + for ll_id in &lazy_loaded { + if let Some(member_event) = + db.rooms + .room_state_get(&body.room_id, &EventType::RoomMember, ll_id.as_str())? + { + state.push(member_event.to_state_event()); + } + } + let resp = get_context::Response { start: start_token, end: end_token, events_before, event: Some(base_event), events_after, - state: db // TODO: State at event - .rooms - .room_state_full(&body.room_id)? - .values() - .map(|pdu| pdu.to_state_event()) - .collect(), + state, }; Ok(resp.into()) diff --git a/src/client_server/message.rs b/src/client_server/message.rs index cbce019e..48ca4ae8 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -6,7 +6,11 @@ use ruma::{ }, events::EventType, }; -use std::{collections::BTreeMap, convert::TryInto, sync::Arc}; +use std::{ + collections::{BTreeMap, HashSet}, + convert::TryInto, + sync::Arc, +}; #[cfg(feature = "conduit_bin")] use rocket::{get, put}; @@ -117,6 +121,7 @@ pub async fn get_message_events_route( body: Ruma>, ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_device = body.sender_device.as_ref().expect("user is authenticated"); if !db.rooms.is_joined(sender_user, &body.room_id)? { return Err(Error::BadRequest( @@ -136,6 +141,12 @@ pub async fn get_message_events_route( // Use limit or else 10 let limit = body.limit.try_into().map_or(10_usize, |l: u32| l as usize); + let next_token; + + let mut resp = get_message_events::Response::new(); + + let mut lazy_loaded = HashSet::new(); + match body.dir { get_message_events::Direction::Forward => { let events_after: Vec<_> = db @@ -152,21 +163,27 @@ pub async fn get_message_events_route( .take_while(|&(k, _)| Some(Ok(k)) != to) // Stop at `to` .collect(); - let end_token = events_after.last().map(|(count, _)| count.to_string()); + for (_, event) in &events_after { + if !db.rooms.lazy_load_was_sent_before( + &sender_user, + &sender_device, + &body.room_id, + &event.sender, + )? { + lazy_loaded.insert(event.sender.clone()); + } + } + + next_token = events_after.last().map(|(count, _)| count).copied(); let events_after: Vec<_> = events_after .into_iter() .map(|(_, pdu)| pdu.to_room_event()) .collect(); - let resp = get_message_events::Response { - start: body.from.to_owned(), - end: end_token, - chunk: events_after, - state: Vec::new(), - }; - - Ok(resp.into()) + resp.start = body.from.to_owned(); + resp.end = next_token.map(|count| count.to_string()); + resp.chunk = events_after; } get_message_events::Direction::Backward => { let events_before: Vec<_> = db @@ -183,21 +200,51 @@ pub async fn get_message_events_route( .take_while(|&(k, _)| Some(Ok(k)) != to) // Stop at `to` .collect(); - let start_token = events_before.last().map(|(count, _)| count.to_string()); + for (_, event) in &events_before { + if !db.rooms.lazy_load_was_sent_before( + &sender_user, + &sender_device, + &body.room_id, + &event.sender, + )? { + lazy_loaded.insert(event.sender.clone()); + } + } + + next_token = events_before.last().map(|(count, _)| count).copied(); let events_before: Vec<_> = events_before .into_iter() .map(|(_, pdu)| pdu.to_room_event()) .collect(); - let resp = get_message_events::Response { - start: body.from.to_owned(), - end: start_token, - chunk: events_before, - state: Vec::new(), - }; - - Ok(resp.into()) + resp.start = body.from.to_owned(); + resp.end = next_token.map(|count| count.to_string()); + resp.chunk = events_before; } } + + db.rooms + .lazy_load_confirm_delivery(&sender_user, &sender_device, &body.room_id, from)?; + resp.state = Vec::new(); + for ll_id in &lazy_loaded { + if let Some(member_event) = + db.rooms + .room_state_get(&body.room_id, &EventType::RoomMember, ll_id.as_str())? + { + resp.state.push(member_event.to_state_event()); + } + } + + if let Some(next_token) = next_token { + db.rooms.lazy_load_mark_sent( + &sender_user, + &sender_device, + &body.room_id, + lazy_loaded.into_iter().collect(), + next_token, + ); + } + + Ok(resp.into()) } diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 64588a2c..88bf8614 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -264,6 +264,14 @@ async fn sync_helper( // limited unless there are events in non_timeline_pdus let limited = non_timeline_pdus.next().is_some(); + let mut timeline_users = HashSet::new(); + for (_, event) in &timeline_pdus { + timeline_users.insert(event.sender.as_str().to_owned()); + } + + db.rooms + .lazy_load_confirm_delivery(&sender_user, &sender_device, &room_id, since)?; + // Database queries: let current_shortstatehash = db @@ -344,14 +352,51 @@ async fn sync_helper( state_events, ) = if since_shortstatehash.is_none() { // Probably since = 0, we will do an initial sync + let (joined_member_count, invited_member_count, heroes) = calculate_counts()?; let current_state_ids = db.rooms.state_full_ids(current_shortstatehash)?; - let state_events: Vec<_> = current_state_ids - .iter() - .map(|(_, id)| db.rooms.get_pdu(id)) - .filter_map(|r| r.ok().flatten()) - .collect(); + + let mut state_events = Vec::new(); + let mut lazy_loaded = Vec::new(); + + for (_, id) in current_state_ids { + let pdu = match db.rooms.get_pdu(&id)? { + Some(pdu) => pdu, + None => { + error!("Pdu in state not found: {}", id); + continue; + } + }; + let state_key = pdu + .state_key + .as_ref() + .expect("state events have state keys"); + if pdu.kind != EventType::RoomMember { + state_events.push(pdu); + } else if full_state || timeline_users.contains(state_key) { + // TODO: check filter: is ll enabled? + lazy_loaded.push( + UserId::parse(state_key.as_ref()) + .expect("they are in timeline_users, so they should be correct"), + ); + state_events.push(pdu); + } + } + + // Reset lazy loading because this is an initial sync + db.rooms + .lazy_load_reset(&sender_user, &sender_device, &room_id)?; + + // The state_events above should contain all timeline_users, let's mark them as lazy + // loaded. + db.rooms.lazy_load_mark_sent( + &sender_user, + &sender_device, + &room_id, + lazy_loaded, + next_batch, + ); ( heroes, @@ -387,20 +432,66 @@ async fn sync_helper( let since_state_ids = db.rooms.state_full_ids(since_shortstatehash)?; - let state_events = if joined_since_last_sync { + /* + let state_events = if joined_since_last_sync || full_state { current_state_ids .iter() .map(|(_, id)| db.rooms.get_pdu(id)) .filter_map(|r| r.ok().flatten()) .collect::>() } else { - current_state_ids - .iter() - .filter(|(key, id)| since_state_ids.get(key) != Some(id)) - .map(|(_, id)| db.rooms.get_pdu(id)) - .filter_map(|r| r.ok().flatten()) - .collect() - }; + */ + let mut state_events = Vec::new(); + let mut lazy_loaded = Vec::new(); + + for (key, id) in current_state_ids { + let pdu = match db.rooms.get_pdu(&id)? { + Some(pdu) => pdu, + None => { + error!("Pdu in state not found: {}", id); + continue; + } + }; + + let state_key = pdu + .state_key + .as_ref() + .expect("state events have state keys"); + + if pdu.kind != EventType::RoomMember { + if full_state || since_state_ids.get(&key) != Some(&id) { + state_events.push(pdu); + } + continue; + } + + // Pdu has to be a member event + let state_key_userid = UserId::parse(state_key.as_ref()) + .expect("they are in timeline_users, so they should be correct"); + + if full_state || since_state_ids.get(&key) != Some(&id) { + lazy_loaded.push(state_key_userid); + state_events.push(pdu); + } else if timeline_users.contains(state_key) + && !db.rooms.lazy_load_was_sent_before( + &sender_user, + &sender_device, + &room_id, + &state_key_userid, + )? + { + lazy_loaded.push(state_key_userid); + state_events.push(pdu); + } + } + + db.rooms.lazy_load_mark_sent( + &sender_user, + &sender_device, + &room_id, + lazy_loaded, + next_batch, + ); let encrypted_room = db .rooms diff --git a/src/database.rs b/src/database.rs index af6136b3..9e020198 100644 --- a/src/database.rs +++ b/src/database.rs @@ -288,6 +288,8 @@ impl Database { userroomid_leftstate: builder.open_tree("userroomid_leftstate")?, roomuserid_leftcount: builder.open_tree("roomuserid_leftcount")?, + lazyloadedids: builder.open_tree("lazyloadedids")?, + userroomid_notificationcount: builder.open_tree("userroomid_notificationcount")?, userroomid_highlightcount: builder.open_tree("userroomid_highlightcount")?, @@ -323,6 +325,7 @@ impl Database { statekeyshort_cache: Mutex::new(LruCache::new(1_000_000)), our_real_users_cache: RwLock::new(HashMap::new()), appservice_in_room_cache: RwLock::new(HashMap::new()), + lazy_load_waiting: Mutex::new(HashMap::new()), stateinfo_cache: Mutex::new(LruCache::new(1000)), }, account_data: account_data::AccountData { diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 775e2f8d..b957b55d 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -28,7 +28,7 @@ use ruma::{ push::{Action, Ruleset, Tweak}, serde::{CanonicalJsonObject, CanonicalJsonValue, Raw}, state_res::{self, RoomVersion, StateMap}, - uint, EventId, RoomAliasId, RoomId, RoomVersionId, ServerName, UserId, + uint, DeviceId, EventId, RoomAliasId, RoomId, RoomVersionId, ServerName, UserId, }; use serde::Deserialize; use serde_json::value::to_raw_value; @@ -79,6 +79,8 @@ pub struct Rooms { pub(super) userroomid_leftstate: Arc, pub(super) roomuserid_leftcount: Arc, + pub(super) lazyloadedids: Arc, // LazyLoadedIds = UserId + DeviceId + RoomId + LazyLoadedUserId + pub(super) userroomid_notificationcount: Arc, // NotifyCount = u64 pub(super) userroomid_highlightcount: Arc, // HightlightCount = u64 @@ -117,6 +119,8 @@ pub struct Rooms { pub(super) shortstatekey_cache: Mutex>, pub(super) our_real_users_cache: RwLock, Arc>>>>, pub(super) appservice_in_room_cache: RwLock, HashMap>>, + pub(super) lazy_load_waiting: + Mutex, Box, Box, u64), Vec>>>, pub(super) stateinfo_cache: Mutex< LruCache< u64, @@ -3466,4 +3470,94 @@ impl Rooms { Ok(()) } + + #[tracing::instrument(skip(self))] + pub fn lazy_load_was_sent_before( + &self, + user_id: &UserId, + device_id: &DeviceId, + room_id: &RoomId, + ll_user: &UserId, + ) -> Result { + let mut key = user_id.as_bytes().to_vec(); + key.push(0xff); + key.extend_from_slice(&device_id.as_bytes()); + key.push(0xff); + key.extend_from_slice(&room_id.as_bytes()); + key.push(0xff); + key.extend_from_slice(&ll_user.as_bytes()); + Ok(self.lazyloadedids.get(&key)?.is_some()) + } + + #[tracing::instrument(skip(self))] + pub fn lazy_load_mark_sent( + &self, + user_id: &UserId, + device_id: &DeviceId, + room_id: &RoomId, + lazy_load: Vec>, + count: u64, + ) { + self.lazy_load_waiting.lock().unwrap().insert( + ( + user_id.to_owned(), + device_id.to_owned(), + room_id.to_owned(), + count, + ), + lazy_load, + ); + } + + #[tracing::instrument(skip(self))] + pub fn lazy_load_confirm_delivery( + &self, + user_id: &UserId, + device_id: &DeviceId, + room_id: &RoomId, + since: u64, + ) -> Result<()> { + if let Some(user_ids) = self.lazy_load_waiting.lock().unwrap().remove(&( + user_id.to_owned(), + device_id.to_owned(), + room_id.to_owned(), + since, + )) { + let mut prefix = user_id.as_bytes().to_vec(); + prefix.push(0xff); + prefix.extend_from_slice(&device_id.as_bytes()); + prefix.push(0xff); + prefix.extend_from_slice(&room_id.as_bytes()); + prefix.push(0xff); + + for ll_id in user_ids { + let mut key = prefix.clone(); + key.extend_from_slice(&ll_id.as_bytes()); + self.lazyloadedids.insert(&key, &[])?; + } + } + + Ok(()) + } + + #[tracing::instrument(skip(self))] + pub fn lazy_load_reset( + &self, + user_id: &Box, + device_id: &Box, + room_id: &Box, + ) -> Result<()> { + let mut prefix = user_id.as_bytes().to_vec(); + prefix.push(0xff); + prefix.extend_from_slice(&device_id.as_bytes()); + prefix.push(0xff); + prefix.extend_from_slice(&room_id.as_bytes()); + prefix.push(0xff); + + for (key, _) in self.lazyloadedids.scan_prefix(prefix) { + self.lazyloadedids.remove(&key)?; + } + + Ok(()) + } } From 1bd9fd74b31383105526ea27b6df8d461aacc223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 5 Jan 2022 18:15:00 +0100 Subject: [PATCH 088/201] feat: partially support sync filters --- src/client_server/filter.rs | 57 ++++++++++++++++--------- src/client_server/message.rs | 5 ++- src/client_server/sync.rs | 83 ++++++++++++++++++++---------------- src/database.rs | 1 + src/database/users.rs | 48 ++++++++++++++++++++- 5 files changed, 133 insertions(+), 61 deletions(-) diff --git a/src/client_server/filter.rs b/src/client_server/filter.rs index dfb53770..f8845f1e 100644 --- a/src/client_server/filter.rs +++ b/src/client_server/filter.rs @@ -1,32 +1,47 @@ -use crate::{utils, ConduitResult}; -use ruma::api::client::r0::filter::{self, create_filter, get_filter}; +use crate::{database::DatabaseGuard, ConduitResult, Error, Ruma}; +use ruma::api::client::{ + error::ErrorKind, + r0::filter::{create_filter, get_filter}, +}; #[cfg(feature = "conduit_bin")] use rocket::{get, post}; /// # `GET /_matrix/client/r0/user/{userId}/filter/{filterId}` /// -/// TODO: Loads a filter that was previously created. -#[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/user/<_>/filter/<_>"))] -#[tracing::instrument] -pub async fn get_filter_route() -> ConduitResult { - // TODO - Ok(get_filter::Response::new(filter::IncomingFilterDefinition { - event_fields: None, - event_format: filter::EventFormat::default(), - account_data: filter::IncomingFilter::default(), - room: filter::IncomingRoomFilter::default(), - presence: filter::IncomingFilter::default(), - }) - .into()) +/// Loads a filter that was previously created. +/// +/// - A user can only access their own filters +#[cfg_attr( + feature = "conduit_bin", + get("/_matrix/client/r0/user/<_>/filter/<_>", data = "") +)] +#[tracing::instrument(skip(db, body))] +pub async fn get_filter_route( + db: DatabaseGuard, + body: Ruma>, +) -> ConduitResult { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let filter = match db.users.get_filter(sender_user, &body.filter_id)? { + Some(filter) => filter, + None => return Err(Error::BadRequest(ErrorKind::NotFound, "Filter not found.")), + }; + + Ok(get_filter::Response::new(filter).into()) } /// # `PUT /_matrix/client/r0/user/{userId}/filter` /// -/// TODO: Creates a new filter to be used by other endpoints. -#[cfg_attr(feature = "conduit_bin", post("/_matrix/client/r0/user/<_>/filter"))] -#[tracing::instrument] -pub async fn create_filter_route() -> ConduitResult { - // TODO - Ok(create_filter::Response::new(utils::random_string(10)).into()) +/// Creates a new filter to be used by other endpoints. +#[cfg_attr( + feature = "conduit_bin", + post("/_matrix/client/r0/user/<_>/filter", data = "") +)] +#[tracing::instrument(skip(db, body))] +pub async fn create_filter_route( + db: DatabaseGuard, + body: Ruma>, +) -> ConduitResult { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + Ok(create_filter::Response::new(db.users.create_filter(sender_user, &body.filter)?).into()) } diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 48ca4ae8..899c45a2 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -138,6 +138,9 @@ pub async fn get_message_events_route( let to = body.to.as_ref().map(|t| t.parse()); + db.rooms + .lazy_load_confirm_delivery(&sender_user, &sender_device, &body.room_id, from)?; + // Use limit or else 10 let limit = body.limit.try_into().map_or(10_usize, |l: u32| l as usize); @@ -224,8 +227,6 @@ pub async fn get_message_events_route( } } - db.rooms - .lazy_load_confirm_delivery(&sender_user, &sender_device, &body.room_id, from)?; resp.state = Vec::new(); for ll_id in &lazy_loaded { if let Some(member_event) = diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 88bf8614..6d8ac28d 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -1,6 +1,10 @@ use crate::{database::DatabaseGuard, ConduitResult, Database, Error, Result, Ruma, RumaResponse}; use ruma::{ - api::client::r0::{sync::sync_events, uiaa::UiaaResponse}, + api::client::r0::{ + filter::{IncomingFilterDefinition, LazyLoadOptions}, + sync::sync_events, + uiaa::UiaaResponse, + }, events::{ room::member::{MembershipState, RoomMemberEventContent}, AnySyncEphemeralRoomEvent, EventType, @@ -77,34 +81,32 @@ pub async fn sync_events_route( Entry::Vacant(v) => { let (tx, rx) = tokio::sync::watch::channel(None); + v.insert((body.since.clone(), rx.clone())); + tokio::spawn(sync_helper_wrapper( Arc::clone(&arc_db), sender_user.clone(), sender_device.clone(), - body.since.clone(), - body.full_state, - body.timeout, + body, tx, )); - v.insert((body.since.clone(), rx)).1.clone() + rx } Entry::Occupied(mut o) => { if o.get().0 != body.since { let (tx, rx) = tokio::sync::watch::channel(None); + o.insert((body.since.clone(), rx.clone())); + tokio::spawn(sync_helper_wrapper( Arc::clone(&arc_db), sender_user.clone(), sender_device.clone(), - body.since.clone(), - body.full_state, - body.timeout, + body, tx, )); - o.insert((body.since.clone(), rx.clone())); - rx } else { o.get().1.clone() @@ -135,18 +137,16 @@ async fn sync_helper_wrapper( db: Arc, sender_user: Box, sender_device: Box, - since: Option, - full_state: bool, - timeout: Option, + body: sync_events::IncomingRequest, tx: Sender>>, ) { + let since = body.since.clone(); + let r = sync_helper( Arc::clone(&db), sender_user.clone(), sender_device.clone(), - since.clone(), - full_state, - timeout, + body, ) .await; @@ -179,9 +179,7 @@ async fn sync_helper( db: Arc, sender_user: Box, sender_device: Box, - since: Option, - full_state: bool, - timeout: Option, + body: sync_events::IncomingRequest, // bool = caching allowed ) -> Result<(sync_events::Response, bool), Error> { // TODO: match body.set_presence { @@ -193,8 +191,26 @@ async fn sync_helper( let next_batch = db.globals.current_count()?; let next_batch_string = next_batch.to_string(); + // Load filter + let filter = match body.filter { + None => IncomingFilterDefinition::default(), + Some(sync_events::IncomingFilter::FilterDefinition(filter)) => filter, + Some(sync_events::IncomingFilter::FilterId(filter_id)) => db + .users + .get_filter(&sender_user, &filter_id)? + .unwrap_or_default(), + }; + + let (lazy_load_enabled, lazy_load_send_redundant) = match filter.room.state.lazy_load_options { + LazyLoadOptions::Enabled { + include_redundant_members: redundant, + } => (true, redundant), + _ => (false, false), + }; + let mut joined_rooms = BTreeMap::new(); - let since = since + let since = body + .since .clone() .and_then(|string| string.parse().ok()) .unwrap_or(0); @@ -374,8 +390,10 @@ async fn sync_helper( .expect("state events have state keys"); if pdu.kind != EventType::RoomMember { state_events.push(pdu); - } else if full_state || timeline_users.contains(state_key) { - // TODO: check filter: is ll enabled? + } else if !lazy_load_enabled + || body.full_state + || timeline_users.contains(state_key) + { lazy_loaded.push( UserId::parse(state_key.as_ref()) .expect("they are in timeline_users, so they should be correct"), @@ -432,15 +450,6 @@ async fn sync_helper( let since_state_ids = db.rooms.state_full_ids(since_shortstatehash)?; - /* - let state_events = if joined_since_last_sync || full_state { - current_state_ids - .iter() - .map(|(_, id)| db.rooms.get_pdu(id)) - .filter_map(|r| r.ok().flatten()) - .collect::>() - } else { - */ let mut state_events = Vec::new(); let mut lazy_loaded = Vec::new(); @@ -459,7 +468,7 @@ async fn sync_helper( .expect("state events have state keys"); if pdu.kind != EventType::RoomMember { - if full_state || since_state_ids.get(&key) != Some(&id) { + if body.full_state || since_state_ids.get(&key) != Some(&id) { state_events.push(pdu); } continue; @@ -469,16 +478,16 @@ async fn sync_helper( let state_key_userid = UserId::parse(state_key.as_ref()) .expect("they are in timeline_users, so they should be correct"); - if full_state || since_state_ids.get(&key) != Some(&id) { + if body.full_state || since_state_ids.get(&key) != Some(&id) { lazy_loaded.push(state_key_userid); state_events.push(pdu); } else if timeline_users.contains(state_key) - && !db.rooms.lazy_load_was_sent_before( + && (!db.rooms.lazy_load_was_sent_before( &sender_user, &sender_device, &room_id, &state_key_userid, - )? + )? || lazy_load_send_redundant) { lazy_loaded.push(state_key_userid); state_events.push(pdu); @@ -858,7 +867,7 @@ async fn sync_helper( }; // TODO: Retry the endpoint instead of returning (waiting for #118) - if !full_state + if !body.full_state && response.rooms.is_empty() && response.presence.is_empty() && response.account_data.is_empty() @@ -867,7 +876,7 @@ async fn sync_helper( { // Hang a few seconds so requests are not spammed // Stop hanging if new info arrives - let mut duration = timeout.unwrap_or_default(); + let mut duration = body.timeout.unwrap_or_default(); if duration.as_secs() > 30 { duration = Duration::from_secs(30); } diff --git a/src/database.rs b/src/database.rs index 9e020198..ddf701bb 100644 --- a/src/database.rs +++ b/src/database.rs @@ -249,6 +249,7 @@ impl Database { userid_masterkeyid: builder.open_tree("userid_masterkeyid")?, userid_selfsigningkeyid: builder.open_tree("userid_selfsigningkeyid")?, userid_usersigningkeyid: builder.open_tree("userid_usersigningkeyid")?, + userfilterid_filter: builder.open_tree("userfilterid_filter")?, todeviceid_events: builder.open_tree("todeviceid_events")?, }, uiaa: uiaa::Uiaa { diff --git a/src/database/users.rs b/src/database/users.rs index 63a63f00..c4fcee3d 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -1,6 +1,9 @@ use crate::{utils, Error, Result}; use ruma::{ - api::client::{error::ErrorKind, r0::device::Device}, + api::client::{ + error::ErrorKind, + r0::{device::Device, filter::IncomingFilterDefinition}, + }, encryption::{CrossSigningKey, DeviceKeys, OneTimeKey}, events::{AnyToDeviceEvent, EventType}, identifiers::MxcUri, @@ -36,6 +39,8 @@ pub struct Users { pub(super) userid_selfsigningkeyid: Arc, pub(super) userid_usersigningkeyid: Arc, + pub(super) userfilterid_filter: Arc, // UserFilterId = UserId + FilterId + pub(super) todeviceid_events: Arc, // ToDeviceId = UserId + DeviceId + Count } @@ -996,6 +1001,47 @@ impl Users { // TODO: Unhook 3PID Ok(()) } + + /// Creates a new sync filter. Returns the filter id. + #[tracing::instrument(skip(self))] + pub fn create_filter( + &self, + user_id: &UserId, + filter: &IncomingFilterDefinition, + ) -> Result { + let filter_id = utils::random_string(4); + + let mut key = user_id.as_bytes().to_vec(); + key.push(0xff); + key.extend_from_slice(filter_id.as_bytes()); + + self.userfilterid_filter.insert( + &key, + &serde_json::to_vec(&filter).expect("filter is valid json"), + )?; + + Ok(filter_id) + } + + #[tracing::instrument(skip(self))] + pub fn get_filter( + &self, + user_id: &UserId, + filter_id: &str, + ) -> Result> { + let mut key = user_id.as_bytes().to_vec(); + key.push(0xff); + key.extend_from_slice(filter_id.as_bytes()); + + let raw = self.userfilterid_filter.get(&key)?; + + if let Some(raw) = raw { + serde_json::from_slice(&raw) + .map_err(|_| Error::bad_database("Invalid filter event in db.")) + } else { + Ok(None) + } + } } /// Ensure that a user only sees signatures from themselves and the target user From 93d225fd1ec186d1957c670b0e6f7f161888dd21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 5 Jan 2022 20:31:20 +0100 Subject: [PATCH 089/201] improvement: faster way to load required state --- src/client_server/sync.rs | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 6d8ac28d..a41e728e 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -376,24 +376,29 @@ async fn sync_helper( let mut state_events = Vec::new(); let mut lazy_loaded = Vec::new(); - for (_, id) in current_state_ids { - let pdu = match db.rooms.get_pdu(&id)? { - Some(pdu) => pdu, - None => { - error!("Pdu in state not found: {}", id); - continue; - } - }; - let state_key = pdu - .state_key - .as_ref() - .expect("state events have state keys"); - if pdu.kind != EventType::RoomMember { + for (shortstatekey, id) in current_state_ids { + let (event_type, state_key) = db.rooms.get_statekey_from_short(shortstatekey)?; + + if event_type != EventType::RoomMember { + let pdu = match db.rooms.get_pdu(&id)? { + Some(pdu) => pdu, + None => { + error!("Pdu in state not found: {}", id); + continue; + } + }; state_events.push(pdu); } else if !lazy_load_enabled || body.full_state - || timeline_users.contains(state_key) + || timeline_users.contains(&state_key) { + let pdu = match db.rooms.get_pdu(&id)? { + Some(pdu) => pdu, + None => { + error!("Pdu in state not found: {}", id); + continue; + } + }; lazy_loaded.push( UserId::parse(state_key.as_ref()) .expect("they are in timeline_users, so they should be correct"), From f285c89006e48b0644f421ae399f1a8eb47e37f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Thu, 6 Jan 2022 00:15:34 +0100 Subject: [PATCH 090/201] fix: make incremental sync efficient again --- src/client_server/message.rs | 2 +- src/client_server/sync.rs | 79 ++++++++++++++++++++---------------- src/database/rooms.rs | 4 +- 3 files changed, 48 insertions(+), 37 deletions(-) diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 899c45a2..9705e4c0 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -242,7 +242,7 @@ pub async fn get_message_events_route( &sender_user, &sender_device, &body.room_id, - lazy_loaded.into_iter().collect(), + lazy_loaded, next_token, ); } diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index a41e728e..c2014403 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -374,7 +374,7 @@ async fn sync_helper( let current_state_ids = db.rooms.state_full_ids(current_shortstatehash)?; let mut state_events = Vec::new(); - let mut lazy_loaded = Vec::new(); + let mut lazy_loaded = HashSet::new(); for (shortstatekey, id) in current_state_ids { let (event_type, state_key) = db.rooms.get_statekey_from_short(shortstatekey)?; @@ -399,7 +399,7 @@ async fn sync_helper( continue; } }; - lazy_loaded.push( + lazy_loaded.insert( UserId::parse(state_key.as_ref()) .expect("they are in timeline_users, so they should be correct"), ); @@ -456,46 +456,57 @@ async fn sync_helper( let since_state_ids = db.rooms.state_full_ids(since_shortstatehash)?; let mut state_events = Vec::new(); - let mut lazy_loaded = Vec::new(); + let mut lazy_loaded = HashSet::new(); for (key, id) in current_state_ids { - let pdu = match db.rooms.get_pdu(&id)? { - Some(pdu) => pdu, - None => { - error!("Pdu in state not found: {}", id); + if body.full_state || since_state_ids.get(&key) != Some(&id) { + let pdu = match db.rooms.get_pdu(&id)? { + Some(pdu) => pdu, + None => { + error!("Pdu in state not found: {}", id); + continue; + } + }; + + if pdu.kind == EventType::RoomMember { + match UserId::parse( + pdu.state_key + .as_ref() + .expect("State event has state key") + .clone(), + ) { + Ok(state_key_userid) => { + lazy_loaded.insert(state_key_userid); + } + Err(e) => error!("Invalid state key for member event: {}", e), + } + } + + state_events.push(pdu); + } + for (_, event) in &timeline_pdus { + if lazy_loaded.contains(&event.sender) { continue; } - }; - let state_key = pdu - .state_key - .as_ref() - .expect("state events have state keys"); - - if pdu.kind != EventType::RoomMember { - if body.full_state || since_state_ids.get(&key) != Some(&id) { - state_events.push(pdu); - } - continue; - } - - // Pdu has to be a member event - let state_key_userid = UserId::parse(state_key.as_ref()) - .expect("they are in timeline_users, so they should be correct"); - - if body.full_state || since_state_ids.get(&key) != Some(&id) { - lazy_loaded.push(state_key_userid); - state_events.push(pdu); - } else if timeline_users.contains(state_key) - && (!db.rooms.lazy_load_was_sent_before( + if !db.rooms.lazy_load_was_sent_before( &sender_user, &sender_device, &room_id, - &state_key_userid, - )? || lazy_load_send_redundant) - { - lazy_loaded.push(state_key_userid); - state_events.push(pdu); + &event.sender, + )? || lazy_load_send_redundant + { + let pdu = match db.rooms.get_pdu(&id)? { + Some(pdu) => pdu, + None => { + error!("Pdu in state not found: {}", id); + continue; + } + }; + + lazy_loaded.insert(event.sender.clone()); + state_events.push(pdu); + } } } diff --git a/src/database/rooms.rs b/src/database/rooms.rs index b957b55d..600f46df 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -120,7 +120,7 @@ pub struct Rooms { pub(super) our_real_users_cache: RwLock, Arc>>>>, pub(super) appservice_in_room_cache: RwLock, HashMap>>, pub(super) lazy_load_waiting: - Mutex, Box, Box, u64), Vec>>>, + Mutex, Box, Box, u64), HashSet>>>, pub(super) stateinfo_cache: Mutex< LruCache< u64, @@ -3495,7 +3495,7 @@ impl Rooms { user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, - lazy_load: Vec>, + lazy_load: HashSet>, count: u64, ) { self.lazy_load_waiting.lock().unwrap().insert( From c6d88359d7aae985f9688dddb321d07ef2043708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Fri, 7 Jan 2022 09:56:09 +0100 Subject: [PATCH 091/201] fix: incremental lazy loading --- src/client_server/sync.rs | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index c2014403..a6122893 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -484,28 +484,27 @@ async fn sync_helper( state_events.push(pdu); } - for (_, event) in &timeline_pdus { - if lazy_loaded.contains(&event.sender) { - continue; - } + } - if !db.rooms.lazy_load_was_sent_before( - &sender_user, - &sender_device, + for (_, event) in &timeline_pdus { + if lazy_loaded.contains(&event.sender) { + continue; + } + + if !db.rooms.lazy_load_was_sent_before( + &sender_user, + &sender_device, + &room_id, + &event.sender, + )? || lazy_load_send_redundant + { + if let Some(member_event) = db.rooms.room_state_get( &room_id, - &event.sender, - )? || lazy_load_send_redundant - { - let pdu = match db.rooms.get_pdu(&id)? { - Some(pdu) => pdu, - None => { - error!("Pdu in state not found: {}", id); - continue; - } - }; - + &EventType::RoomMember, + event.sender.as_str(), + )? { lazy_loaded.insert(event.sender.clone()); - state_events.push(pdu); + state_events.push(member_event); } } } From 4f39d36e980d8f4e6fcc7ae7c9a292db52d915e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sun, 9 Jan 2022 13:42:25 +0100 Subject: [PATCH 092/201] docs: lazy loading --- src/client_server/sync.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index a6122893..bd2f48a3 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -40,13 +40,15 @@ use rocket::{get, tokio}; /// Calling this endpoint with a `since` parameter from a previous `next_batch` returns: /// For joined rooms: /// - Some of the most recent events of each timeline that happened after since -/// - If user joined the room after since: All state events and device list updates in that room +/// - If user joined the room after since: All state events (unless lazy loading is activated) and +/// all device list updates in that room /// - If the user was already in the room: A list of all events that are in the state now, but were /// not in the state at `since` /// - If the state we send contains a member event: Joined and invited member counts, heroes /// - Device list updates that happened after `since` /// - If there are events in the timeline we send or the user send updated his read mark: Notification counts /// - EDUs that are active now (read receipts, typing updates, presence) +/// - TODO: Allow multiple sync streams to support Pantalaimon /// /// For invited rooms: /// - If the user was invited after `since`: A subset of the state of the room at the point of the invite From fa6d7f7ccd14426f1fc2d802fff021b06f39bf02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sun, 9 Jan 2022 16:44:44 +0100 Subject: [PATCH 093/201] feat: database backend selection at runtime --- Cargo.toml | 2 +- conduit-example.toml | 15 ++- src/database.rs | 140 +++++++++++++++++----------- src/database/abstraction.rs | 13 ++- src/database/abstraction/rocksdb.rs | 9 +- src/database/abstraction/sqlite.rs | 14 ++- src/utils.rs | 11 --- 7 files changed, 116 insertions(+), 88 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6241b6a8..c898d4d6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -85,7 +85,7 @@ hmac = "0.11.0" sha-1 = "0.9.8" [features] -default = ["conduit_bin", "backend_rocksdb"] +default = ["conduit_bin", "backend_sqlite", "backend_rocksdb"] backend_sled = ["sled"] backend_sqlite = ["sqlite"] backend_heed = ["heed", "crossbeam"] diff --git a/conduit-example.toml b/conduit-example.toml index 4275f528..c0274a4d 100644 --- a/conduit-example.toml +++ b/conduit-example.toml @@ -1,11 +1,15 @@ [global] -# The server_name is the name of this server. It is used as a suffix for user +# The server_name is the pretty name of this server. It is used as a suffix for user # and room ids. Examples: matrix.org, conduit.rs -# The Conduit server needs to be reachable at https://your.server.name/ on port -# 443 (client-server) and 8448 (federation) OR you can create /.well-known -# files to redirect requests. See + +# The Conduit server needs all /_matrix/ requests to be reachable at +# https://your.server.name/ on port 443 (client-server) and 8448 (federation). + +# If that's not possible for you, you can create /.well-known files to redirect +# requests. See # https://matrix.org/docs/spec/client_server/latest#get-well-known-matrix-client -# and https://matrix.org/docs/spec/server_server/r0.1.4#get-well-known-matrix-server +# and +# https://matrix.org/docs/spec/server_server/r0.1.4#get-well-known-matrix-server # for more information # YOU NEED TO EDIT THIS @@ -13,6 +17,7 @@ # This is the only directory where Conduit will save its data database_path = "/var/lib/conduit/" +database_backend = "rocksdb" # The port Conduit will be running on. You need to set up a reverse proxy in # your web server (e.g. apache or nginx), so all requests to /_matrix on port diff --git a/src/database.rs b/src/database.rs index ddf701bb..c2b3e2b9 100644 --- a/src/database.rs +++ b/src/database.rs @@ -44,13 +44,15 @@ use self::proxy::ProxyConfig; #[derive(Clone, Debug, Deserialize)] pub struct Config { server_name: Box, + #[serde(default = "default_database_backend")] + database_backend: String, database_path: String, #[serde(default = "default_db_cache_capacity_mb")] db_cache_capacity_mb: f64, #[serde(default = "default_pdu_cache_capacity")] pdu_cache_capacity: u32, - #[serde(default = "default_sqlite_wal_clean_second_interval")] - sqlite_wal_clean_second_interval: u32, + #[serde(default = "default_cleanup_second_interval")] + cleanup_second_interval: u32, #[serde(default = "default_max_request_size")] max_request_size: u32, #[serde(default = "default_max_concurrent_requests")] @@ -117,6 +119,10 @@ fn true_fn() -> bool { true } +fn default_database_backend() -> String { + "sqlite".to_owned() +} + fn default_db_cache_capacity_mb() -> f64 { 200.0 } @@ -125,7 +131,7 @@ fn default_pdu_cache_capacity() -> u32 { 100_000 } -fn default_sqlite_wal_clean_second_interval() -> u32 { +fn default_cleanup_second_interval() -> u32 { 1 * 60 // every minute } @@ -145,20 +151,8 @@ fn default_turn_ttl() -> u64 { 60 * 60 * 24 } -#[cfg(feature = "sled")] -pub type Engine = abstraction::sled::Engine; - -#[cfg(feature = "sqlite")] -pub type Engine = abstraction::sqlite::Engine; - -#[cfg(feature = "heed")] -pub type Engine = abstraction::heed::Engine; - -#[cfg(feature = "rocksdb")] -pub type Engine = abstraction::rocksdb::Engine; - pub struct Database { - _db: Arc, + _db: Arc, pub globals: globals::Globals, pub users: users::Users, pub uiaa: uiaa::Uiaa, @@ -186,27 +180,53 @@ impl Database { Ok(()) } - fn check_sled_or_sqlite_db(config: &Config) -> Result<()> { - #[cfg(feature = "backend_sqlite")] - { - let path = Path::new(&config.database_path); + fn check_db_setup(config: &Config) -> Result<()> { + let path = Path::new(&config.database_path); - let sled_exists = path.join("db").exists(); - let sqlite_exists = path.join("conduit.db").exists(); - if sled_exists { - if sqlite_exists { - // most likely an in-place directory, only warn - warn!("Both sled and sqlite databases are detected in database directory"); - warn!("Currently running from the sqlite database, but consider removing sled database files to free up space") - } else { - error!( - "Sled database detected, conduit now uses sqlite for database operations" - ); - error!("This database must be converted to sqlite, go to https://github.com/ShadowJonathan/conduit_toolbox#conduit_sled_to_sqlite"); - return Err(Error::bad_config( - "sled database detected, migrate to sqlite", - )); - } + let sled_exists = path.join("db").exists(); + let sqlite_exists = path.join("conduit.db").exists(); + let rocksdb_exists = path.join("IDENTITY").exists(); + + let mut count = 0; + + if sled_exists { + count += 1; + } + + if sqlite_exists { + count += 1; + } + + if rocksdb_exists { + count += 1; + } + + if count > 1 { + warn!("Multiple databases at database_path detected"); + return Ok(()); + } + + if sled_exists { + if config.database_backend != "sled" { + return Err(Error::bad_config( + "Found sled at database_path, but is not specified in config.", + )); + } + } + + if sqlite_exists { + if config.database_backend != "sqlite" { + return Err(Error::bad_config( + "Found sqlite at database_path, but is not specified in config.", + )); + } + } + + if rocksdb_exists { + if config.database_backend != "rocksdb" { + return Err(Error::bad_config( + "Found rocksdb at database_path, but is not specified in config.", + )); } } @@ -215,14 +235,30 @@ impl Database { /// Load an existing database or create a new one. pub async fn load_or_create(config: &Config) -> Result>> { - Self::check_sled_or_sqlite_db(config)?; + Self::check_db_setup(config)?; if !Path::new(&config.database_path).exists() { std::fs::create_dir_all(&config.database_path) .map_err(|_| Error::BadConfig("Database folder doesn't exists and couldn't be created (e.g. due to missing permissions). Please create the database folder yourself."))?; } - let builder = Engine::open(config)?; + let builder: Arc = match &*config.database_backend { + "sqlite" => { + #[cfg(not(feature = "sqlite"))] + return Err(Error::BadConfig("Database backend not found.")); + #[cfg(feature = "sqlite")] + Arc::new(Arc::::open(config)?) + } + "rocksdb" => { + #[cfg(not(feature = "rocksdb"))] + return Err(Error::BadConfig("Database backend not found.")); + #[cfg(feature = "rocksdb")] + Arc::new(Arc::::open(config)?) + } + _ => { + return Err(Error::BadConfig("Database backend not found.")); + } + }; if config.max_request_size < 1024 { eprintln!("ERROR: Max request size is less than 1KB. Please increase it."); @@ -784,10 +820,7 @@ impl Database { drop(guard); - #[cfg(feature = "sqlite")] - { - Self::start_wal_clean_task(Arc::clone(&db), config).await; - } + Self::start_cleanup_task(Arc::clone(&db), config).await; Ok(db) } @@ -925,15 +958,8 @@ impl Database { res } - #[cfg(feature = "sqlite")] - #[tracing::instrument(skip(self))] - pub fn flush_wal(&self) -> Result<()> { - self._db.flush_wal() - } - - #[cfg(feature = "sqlite")] #[tracing::instrument(skip(db, config))] - pub async fn start_wal_clean_task(db: Arc>, config: &Config) { + pub async fn start_cleanup_task(db: Arc>, config: &Config) { use tokio::time::interval; #[cfg(unix)] @@ -942,7 +968,7 @@ impl Database { use std::time::{Duration, Instant}; - let timer_interval = Duration::from_secs(config.sqlite_wal_clean_second_interval as u64); + let timer_interval = Duration::from_secs(config.cleanup_second_interval as u64); tokio::spawn(async move { let mut i = interval(timer_interval); @@ -953,23 +979,23 @@ impl Database { #[cfg(unix)] tokio::select! { _ = i.tick() => { - info!("wal-trunc: Timer ticked"); + info!("cleanup: Timer ticked"); } _ = s.recv() => { - info!("wal-trunc: Received SIGHUP"); + info!("cleanup: Received SIGHUP"); } }; #[cfg(not(unix))] { i.tick().await; - info!("wal-trunc: Timer ticked") + info!("cleanup: Timer ticked") } let start = Instant::now(); - if let Err(e) = db.read().await.flush_wal() { - error!("wal-trunc: Errored: {}", e); + if let Err(e) = db.read().await._db.cleanup() { + error!("cleanup: Errored: {}", e); } else { - info!("wal-trunc: Flushed in {:?}", start.elapsed()); + info!("cleanup: Finished in {:?}", start.elapsed()); } } }); diff --git a/src/database/abstraction.rs b/src/database/abstraction.rs index a347f831..45627bbc 100644 --- a/src/database/abstraction.rs +++ b/src/database/abstraction.rs @@ -18,10 +18,15 @@ pub mod rocksdb; #[cfg(any(feature = "sqlite", feature = "rocksdb", feature = "heed"))] pub mod watchers; -pub trait DatabaseEngine: Sized { - fn open(config: &Config) -> Result>; - fn open_tree(self: &Arc, name: &'static str) -> Result>; - fn flush(self: &Arc) -> Result<()>; +pub trait DatabaseEngine: Send + Sync { + fn open(config: &Config) -> Result + where + Self: Sized; + fn open_tree(&self, name: &'static str) -> Result>; + fn flush(self: &Self) -> Result<()>; + fn cleanup(self: &Self) -> Result<()> { + Ok(()) + } } pub trait Tree: Send + Sync { diff --git a/src/database/abstraction/rocksdb.rs b/src/database/abstraction/rocksdb.rs index 397047bd..a41ed1fb 100644 --- a/src/database/abstraction/rocksdb.rs +++ b/src/database/abstraction/rocksdb.rs @@ -14,8 +14,8 @@ pub struct RocksDbEngineTree<'a> { write_lock: RwLock<()> } -impl DatabaseEngine for Engine { - fn open(config: &Config) -> Result> { +impl DatabaseEngine for Arc { + fn open(config: &Config) -> Result { let mut db_opts = rocksdb::Options::default(); db_opts.create_if_missing(true); db_opts.set_max_open_files(512); @@ -60,7 +60,7 @@ impl DatabaseEngine for Engine { })) } - fn open_tree(self: &Arc, name: &'static str) -> Result> { + fn open_tree(&self, name: &'static str) -> Result> { if !self.old_cfs.contains(&name.to_owned()) { // Create if it didn't exist let mut options = rocksdb::Options::default(); @@ -68,7 +68,6 @@ impl DatabaseEngine for Engine { options.set_prefix_extractor(prefix_extractor); let _ = self.rocks.create_cf(name, &options); - println!("created cf"); } Ok(Arc::new(RocksDbEngineTree { @@ -79,7 +78,7 @@ impl DatabaseEngine for Engine { })) } - fn flush(self: &Arc) -> Result<()> { + fn flush(&self) -> Result<()> { // TODO? Ok(()) } diff --git a/src/database/abstraction/sqlite.rs b/src/database/abstraction/sqlite.rs index 31875667..d4fd0bdd 100644 --- a/src/database/abstraction/sqlite.rs +++ b/src/database/abstraction/sqlite.rs @@ -80,8 +80,8 @@ impl Engine { } } -impl DatabaseEngine for Engine { - fn open(config: &Config) -> Result> { +impl DatabaseEngine for Arc { + fn open(config: &Config) -> Result { let path = Path::new(&config.database_path).join("conduit.db"); // calculates cache-size per permanent connection @@ -92,7 +92,7 @@ impl DatabaseEngine for Engine { / ((num_cpus::get().max(1) * 2) + 1) as f64) as u32; - let writer = Mutex::new(Self::prepare_conn(&path, cache_size_per_thread)?); + let writer = Mutex::new(Engine::prepare_conn(&path, cache_size_per_thread)?); let arc = Arc::new(Engine { writer, @@ -105,7 +105,7 @@ impl DatabaseEngine for Engine { Ok(arc) } - fn open_tree(self: &Arc, name: &str) -> Result> { + fn open_tree(&self, name: &str) -> Result> { self.write_lock().execute(&format!("CREATE TABLE IF NOT EXISTS {} ( \"key\" BLOB PRIMARY KEY, \"value\" BLOB NOT NULL )", name), [])?; Ok(Arc::new(SqliteTable { @@ -115,10 +115,14 @@ impl DatabaseEngine for Engine { })) } - fn flush(self: &Arc) -> Result<()> { + fn flush(&self) -> Result<()> { // we enabled PRAGMA synchronous=normal, so this should not be necessary Ok(()) } + + fn cleanup(&self) -> Result<()> { + self.flush_wal() + } } pub struct SqliteTable { diff --git a/src/utils.rs b/src/utils.rs index 4702d051..26d71a8c 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -29,17 +29,6 @@ pub fn increment(old: Option<&[u8]>) -> Option> { Some(number.to_be_bytes().to_vec()) } -#[cfg(feature = "rocksdb")] -pub fn increment_rocksdb( - _new_key: &[u8], - old: Option<&[u8]>, - _operands: &mut rocksdb::MergeOperands, -) -> Option> { - dbg!(_new_key); - dbg!(old); - increment(old) -} - pub fn generate_keypair() -> Vec { let mut value = random_string(8).as_bytes().to_vec(); value.push(0xff); From 71431f330aadb1ee92cd63a36351af834aa65215 Mon Sep 17 00:00:00 2001 From: Andrej Kacian Date: Sun, 9 Jan 2022 20:07:03 +0100 Subject: [PATCH 094/201] Add memory_usage() to DatabaseEngine trait --- src/database/abstraction.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/database/abstraction.rs b/src/database/abstraction.rs index 45627bbc..17bd971f 100644 --- a/src/database/abstraction.rs +++ b/src/database/abstraction.rs @@ -27,6 +27,9 @@ pub trait DatabaseEngine: Send + Sync { fn cleanup(self: &Self) -> Result<()> { Ok(()) } + fn memory_usage(self: &Self) -> Result { + Ok("Current database engine does not support memory usage reporting.".to_string()) + } } pub trait Tree: Send + Sync { From ff243870f850c07907a6944151fd909c234da662 Mon Sep 17 00:00:00 2001 From: Andrej Kacian Date: Sun, 9 Jan 2022 20:07:50 +0100 Subject: [PATCH 095/201] Add "database_memory_usage" AdminCommand --- src/database/admin.rs | 8 ++++++++ src/database/rooms.rs | 3 +++ 2 files changed, 11 insertions(+) diff --git a/src/database/admin.rs b/src/database/admin.rs index 0702bcdd..c3083309 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -14,6 +14,7 @@ pub enum AdminCommand { RegisterAppservice(serde_yaml::Value), UnregisterAppservice(String), ListAppservices, + ShowMemoryUsage, SendMessage(RoomMessageEventContent), } @@ -113,6 +114,13 @@ impl Admin { send_message(RoomMessageEventContent::text_plain("Failed to get appservices."), guard, &state_lock); } } + AdminCommand::ShowMemoryUsage => { + if let Ok(response) = guard._db.memory_usage() { + send_message(RoomMessageEventContent::text_plain(response), guard, &state_lock); + } else { + send_message(RoomMessageEventContent::text_plain("Failed to get database memory usage".to_string()), guard, &state_lock); + } + } AdminCommand::SendMessage(message) => { send_message(message, guard, &state_lock); } diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 600f46df..0ba6c9ba 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -1693,6 +1693,9 @@ impl Rooms { )); } } + "database_memory_usage" => { + db.admin.send(AdminCommand::ShowMemoryUsage); + } _ => { db.admin.send(AdminCommand::SendMessage( RoomMessageEventContent::text_plain(format!( From 68ee1a5408595804625a6dd0ebab5f333e7f0fe6 Mon Sep 17 00:00:00 2001 From: Andrej Kacian Date: Sun, 9 Jan 2022 20:08:15 +0100 Subject: [PATCH 096/201] Add rocksdb implementation of memory_usage() --- src/database/abstraction/rocksdb.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/database/abstraction/rocksdb.rs b/src/database/abstraction/rocksdb.rs index a41ed1fb..f0affd32 100644 --- a/src/database/abstraction/rocksdb.rs +++ b/src/database/abstraction/rocksdb.rs @@ -82,6 +82,19 @@ impl DatabaseEngine for Arc { // TODO? Ok(()) } + + fn memory_usage(&self) -> Result { + let stats = rocksdb::perf::get_memory_usage_stats(Some(&[&self.rocks]), None)?; + Ok(format!("Approximate memory usage of all the mem-tables: {:.3} MB\n\ + Approximate memory usage of un-flushed mem-tables: {:.3} MB\n\ + Approximate memory usage of all the table readers: {:.3} MB\n\ + Approximate memory usage by cache: {:.3} MB", + stats.mem_table_total as f64 / 1024.0 / 1024.0, + stats.mem_table_unflushed as f64 / 1024.0 / 1024.0, + stats.mem_table_readers_total as f64 / 1024.0 / 1024.0, + stats.cache_total as f64 / 1024.0 / 1024.0 + )) + } } impl RocksDbEngineTree<'_> { From 077e9ad4380715688a8ad5a2f40afd7331157bd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 10 Jan 2022 15:53:28 +0100 Subject: [PATCH 097/201] improvement: memory usage for caches --- Cargo.lock | 4 +-- Cargo.toml | 2 +- src/database/abstraction/rocksdb.rs | 40 ++++++++++++++++------------- src/database/admin.rs | 2 +- 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 794445f9..d297102c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2075,9 +2075,9 @@ dependencies = [ [[package]] name = "rocksdb" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c749134fda8bfc90d0de643d59bfc841dcb3ac8a1062e12b6754bd60235c48b3" +checksum = "7a62eca5cacf2c8261128631bed9f045598d40bfbe4b29f5163f0f802f8f44a7" dependencies = [ "libc", "librocksdb-sys", diff --git a/Cargo.toml b/Cargo.toml index c898d4d6..c87d949c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -78,7 +78,7 @@ crossbeam = { version = "0.8.1", optional = true } num_cpus = "1.13.0" threadpool = "1.8.1" heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c758867e05ad973ef800a6fe1d5d", optional = true } -rocksdb = { version = "0.16.0", features = ["multi-threaded-cf"], optional = true } +rocksdb = { version = "0.17.0", features = ["multi-threaded-cf"], optional = true } thread_local = "1.1.3" # used for TURN server authentication hmac = "0.11.0" diff --git a/src/database/abstraction/rocksdb.rs b/src/database/abstraction/rocksdb.rs index f0affd32..a7dd6e16 100644 --- a/src/database/abstraction/rocksdb.rs +++ b/src/database/abstraction/rocksdb.rs @@ -4,6 +4,7 @@ use std::{future::Future, pin::Pin, sync::Arc, collections::HashMap, sync::RwLoc pub struct Engine { rocks: rocksdb::DBWithThreadMode, + cache: rocksdb::Cache, old_cfs: Vec, } @@ -56,6 +57,7 @@ impl DatabaseEngine for Arc { Ok(Arc::new(Engine { rocks: db, + cache: rocksdb_cache, old_cfs: cfs, })) } @@ -84,33 +86,35 @@ impl DatabaseEngine for Arc { } fn memory_usage(&self) -> Result { - let stats = rocksdb::perf::get_memory_usage_stats(Some(&[&self.rocks]), None)?; - Ok(format!("Approximate memory usage of all the mem-tables: {:.3} MB\n\ + let stats = + rocksdb::perf::get_memory_usage_stats(Some(&[&self.rocks]), Some(&[&self.cache]))?; + Ok(format!( + "Approximate memory usage of all the mem-tables: {:.3} MB\n\ Approximate memory usage of un-flushed mem-tables: {:.3} MB\n\ Approximate memory usage of all the table readers: {:.3} MB\n\ Approximate memory usage by cache: {:.3} MB", - stats.mem_table_total as f64 / 1024.0 / 1024.0, - stats.mem_table_unflushed as f64 / 1024.0 / 1024.0, - stats.mem_table_readers_total as f64 / 1024.0 / 1024.0, - stats.cache_total as f64 / 1024.0 / 1024.0 + stats.mem_table_total as f64 / 1024.0 / 1024.0, + stats.mem_table_unflushed as f64 / 1024.0 / 1024.0, + stats.mem_table_readers_total as f64 / 1024.0 / 1024.0, + stats.cache_total as f64 / 1024.0 / 1024.0 )) } } impl RocksDbEngineTree<'_> { - fn cf(&self) -> rocksdb::BoundColumnFamily<'_> { + fn cf(&self) -> Arc> { self.db.rocks.cf_handle(self.name).unwrap() } } impl Tree for RocksDbEngineTree<'_> { fn get(&self, key: &[u8]) -> Result>> { - Ok(self.db.rocks.get_cf(self.cf(), key)?) + Ok(self.db.rocks.get_cf(&self.cf(), key)?) } fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> { let lock = self.write_lock.read().unwrap(); - self.db.rocks.put_cf(self.cf(), key, value)?; + self.db.rocks.put_cf(&self.cf(), key, value)?; drop(lock); self.watchers.wake(key); @@ -120,21 +124,21 @@ impl Tree for RocksDbEngineTree<'_> { fn insert_batch<'a>(&self, iter: &mut dyn Iterator, Vec)>) -> Result<()> { for (key, value) in iter { - self.db.rocks.put_cf(self.cf(), key, value)?; + self.db.rocks.put_cf(&self.cf(), key, value)?; } Ok(()) } fn remove(&self, key: &[u8]) -> Result<()> { - Ok(self.db.rocks.delete_cf(self.cf(), key)?) + Ok(self.db.rocks.delete_cf(&self.cf(), key)?) } fn iter<'a>(&'a self) -> Box, Vec)> + 'a> { Box::new( self.db .rocks - .iterator_cf(self.cf(), rocksdb::IteratorMode::Start) + .iterator_cf(&self.cf(), rocksdb::IteratorMode::Start) .map(|(k, v)| (Vec::from(k), Vec::from(v))), ) } @@ -148,7 +152,7 @@ impl Tree for RocksDbEngineTree<'_> { self.db .rocks .iterator_cf( - self.cf(), + &self.cf(), rocksdb::IteratorMode::From( from, if backwards { @@ -165,9 +169,9 @@ impl Tree for RocksDbEngineTree<'_> { fn increment(&self, key: &[u8]) -> Result> { let lock = self.write_lock.write().unwrap(); - let old = self.db.rocks.get_cf(self.cf(), &key)?; + let old = self.db.rocks.get_cf(&self.cf(), &key)?; let new = utils::increment(old.as_deref()).unwrap(); - self.db.rocks.put_cf(self.cf(), key, &new)?; + self.db.rocks.put_cf(&self.cf(), key, &new)?; drop(lock); Ok(new) @@ -177,9 +181,9 @@ impl Tree for RocksDbEngineTree<'_> { let lock = self.write_lock.write().unwrap(); for key in iter { - let old = self.db.rocks.get_cf(self.cf(), &key)?; + let old = self.db.rocks.get_cf(&self.cf(), &key)?; let new = utils::increment(old.as_deref()).unwrap(); - self.db.rocks.put_cf(self.cf(), key, new)?; + self.db.rocks.put_cf(&self.cf(), key, new)?; } drop(lock); @@ -195,7 +199,7 @@ impl Tree for RocksDbEngineTree<'_> { self.db .rocks .iterator_cf( - self.cf(), + &self.cf(), rocksdb::IteratorMode::From(&prefix, rocksdb::Direction::Forward), ) .map(|(k, v)| (Vec::from(k), Vec::from(v))) diff --git a/src/database/admin.rs b/src/database/admin.rs index c3083309..7d2301d9 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -118,7 +118,7 @@ impl Admin { if let Ok(response) = guard._db.memory_usage() { send_message(RoomMessageEventContent::text_plain(response), guard, &state_lock); } else { - send_message(RoomMessageEventContent::text_plain("Failed to get database memory usage".to_string()), guard, &state_lock); + send_message(RoomMessageEventContent::text_plain("Failed to get database memory usage.".to_string()), guard, &state_lock); } } AdminCommand::SendMessage(message) => { From 0bb7d76dec4b3f54b1cbb37e57ddbe54e1dbd38f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 10 Jan 2022 20:20:45 +0100 Subject: [PATCH 098/201] improvement: rocksdb configuration --- src/database/abstraction/rocksdb.rs | 33 +++++++++++++++-------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/database/abstraction/rocksdb.rs b/src/database/abstraction/rocksdb.rs index a7dd6e16..3f1793a2 100644 --- a/src/database/abstraction/rocksdb.rs +++ b/src/database/abstraction/rocksdb.rs @@ -17,25 +17,21 @@ pub struct RocksDbEngineTree<'a> { impl DatabaseEngine for Arc { fn open(config: &Config) -> Result { - let mut db_opts = rocksdb::Options::default(); - db_opts.create_if_missing(true); - db_opts.set_max_open_files(512); - db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level); - db_opts.set_compression_type(rocksdb::DBCompressionType::Zstd); - db_opts.set_target_file_size_base(2 << 22); - db_opts.set_max_bytes_for_level_base(2 << 24); - db_opts.set_max_bytes_for_level_multiplier(2.0); - db_opts.set_num_levels(8); - db_opts.set_write_buffer_size(2 << 27); - let rocksdb_cache = rocksdb::Cache::new_lru_cache((config.db_cache_capacity_mb * 1024.0 * 1024.0) as usize) .unwrap(); let mut block_based_options = rocksdb::BlockBasedOptions::default(); - block_based_options.set_block_size(2 << 19); block_based_options.set_block_cache(&rocksdb_cache); + + let mut db_opts = rocksdb::Options::default(); db_opts.set_block_based_table_factory(&block_based_options); + db_opts.create_if_missing(true); + db_opts.increase_parallelism(num_cpus::get() as i32); + db_opts.set_max_open_files(512); + db_opts.set_compression_type(rocksdb::DBCompressionType::Zstd); + db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level); + db_opts.optimize_level_style_compaction((config.db_cache_capacity_mb * 1024.0 * 1024.0) as usize); let cfs = rocksdb::DBWithThreadMode::::list_cf( &db_opts, @@ -90,13 +86,18 @@ impl DatabaseEngine for Arc { rocksdb::perf::get_memory_usage_stats(Some(&[&self.rocks]), Some(&[&self.cache]))?; Ok(format!( "Approximate memory usage of all the mem-tables: {:.3} MB\n\ - Approximate memory usage of un-flushed mem-tables: {:.3} MB\n\ - Approximate memory usage of all the table readers: {:.3} MB\n\ - Approximate memory usage by cache: {:.3} MB", + Approximate memory usage of un-flushed mem-tables: {:.3} MB\n\ + Approximate memory usage of all the table readers: {:.3} MB\n\ + Approximate memory usage by cache: {:.3} MB\n\ + self.cache.get_usage(): {:.3} MB\n\ + self.cache.get_pinned_usage(): {:.3} MB\n\ + ", stats.mem_table_total as f64 / 1024.0 / 1024.0, stats.mem_table_unflushed as f64 / 1024.0 / 1024.0, stats.mem_table_readers_total as f64 / 1024.0 / 1024.0, - stats.cache_total as f64 / 1024.0 / 1024.0 + stats.cache_total as f64 / 1024.0 / 1024.0, + self.cache.get_usage() as f64 / 1024.0 / 1024.0, + self.cache.get_pinned_usage() as f64 / 1024.0 / 1024.0, )) } } From b96822b6174de4d404bf0b9013a39f8fd2a06f87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 10 Jan 2022 21:20:29 +0100 Subject: [PATCH 099/201] fix: use db options for column families too --- src/database/abstraction/rocksdb.rs | 55 ++++++++++++++++------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/src/database/abstraction/rocksdb.rs b/src/database/abstraction/rocksdb.rs index 3f1793a2..c82e4bc8 100644 --- a/src/database/abstraction/rocksdb.rs +++ b/src/database/abstraction/rocksdb.rs @@ -4,6 +4,7 @@ use std::{future::Future, pin::Pin, sync::Arc, collections::HashMap, sync::RwLoc pub struct Engine { rocks: rocksdb::DBWithThreadMode, + cache_capacity_bytes: usize, cache: rocksdb::Cache, old_cfs: Vec, } @@ -15,23 +16,31 @@ pub struct RocksDbEngineTree<'a> { write_lock: RwLock<()> } +fn db_options(cache_capacity_bytes: usize, rocksdb_cache: &rocksdb::Cache) -> rocksdb::Options { + let mut block_based_options = rocksdb::BlockBasedOptions::default(); + block_based_options.set_block_cache(rocksdb_cache); + + let mut db_opts = rocksdb::Options::default(); + db_opts.set_block_based_table_factory(&block_based_options); + db_opts.create_if_missing(true); + db_opts.increase_parallelism(num_cpus::get() as i32); + db_opts.set_max_open_files(512); + db_opts.set_compression_type(rocksdb::DBCompressionType::Zstd); + db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level); + db_opts.optimize_level_style_compaction(cache_capacity_bytes); + + let prefix_extractor = rocksdb::SliceTransform::create_fixed_prefix(1); + db_opts.set_prefix_extractor(prefix_extractor); + + db_opts +} + impl DatabaseEngine for Arc { fn open(config: &Config) -> Result { - let rocksdb_cache = - rocksdb::Cache::new_lru_cache((config.db_cache_capacity_mb * 1024.0 * 1024.0) as usize) - .unwrap(); + let cache_capacity_bytes = (config.db_cache_capacity_mb * 1024.0 * 1024.0) as usize; + let rocksdb_cache = rocksdb::Cache::new_lru_cache(cache_capacity_bytes).unwrap(); - let mut block_based_options = rocksdb::BlockBasedOptions::default(); - block_based_options.set_block_cache(&rocksdb_cache); - - let mut db_opts = rocksdb::Options::default(); - db_opts.set_block_based_table_factory(&block_based_options); - db_opts.create_if_missing(true); - db_opts.increase_parallelism(num_cpus::get() as i32); - db_opts.set_max_open_files(512); - db_opts.set_compression_type(rocksdb::DBCompressionType::Zstd); - db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level); - db_opts.optimize_level_style_compaction((config.db_cache_capacity_mb * 1024.0 * 1024.0) as usize); + let db_opts = db_options(cache_capacity_bytes, &rocksdb_cache); let cfs = rocksdb::DBWithThreadMode::::list_cf( &db_opts, @@ -43,16 +52,16 @@ impl DatabaseEngine for Arc { &db_opts, &config.database_path, cfs.iter().map(|name| { - let mut options = rocksdb::Options::default(); - let prefix_extractor = rocksdb::SliceTransform::create_fixed_prefix(1); - options.set_prefix_extractor(prefix_extractor); - - rocksdb::ColumnFamilyDescriptor::new(name, options) + rocksdb::ColumnFamilyDescriptor::new( + name, + db_options(cache_capacity_bytes, &rocksdb_cache), + ) }), )?; Ok(Arc::new(Engine { rocks: db, + cache_capacity_bytes, cache: rocksdb_cache, old_cfs: cfs, })) @@ -61,11 +70,9 @@ impl DatabaseEngine for Arc { fn open_tree(&self, name: &'static str) -> Result> { if !self.old_cfs.contains(&name.to_owned()) { // Create if it didn't exist - let mut options = rocksdb::Options::default(); - let prefix_extractor = rocksdb::SliceTransform::create_fixed_prefix(1); - options.set_prefix_extractor(prefix_extractor); - - let _ = self.rocks.create_cf(name, &options); + let _ = self + .rocks + .create_cf(name, &db_options(self.cache_capacity_bytes, &self.cache)); } Ok(Arc::new(RocksDbEngineTree { From 7f27af032b7d0cb79248607decd1bb5f2a818507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 12 Jan 2022 10:07:10 +0100 Subject: [PATCH 100/201] improvement: optimize rocksdb for spinning disks --- src/database/abstraction/rocksdb.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/database/abstraction/rocksdb.rs b/src/database/abstraction/rocksdb.rs index c82e4bc8..32095566 100644 --- a/src/database/abstraction/rocksdb.rs +++ b/src/database/abstraction/rocksdb.rs @@ -20,8 +20,20 @@ fn db_options(cache_capacity_bytes: usize, rocksdb_cache: &rocksdb::Cache) -> ro let mut block_based_options = rocksdb::BlockBasedOptions::default(); block_based_options.set_block_cache(rocksdb_cache); + // "Difference of spinning disk" + // https://zhangyuchi.gitbooks.io/rocksdbbook/content/RocksDB-Tuning-Guide.html + block_based_options.set_block_size(64 * 1024); + block_based_options.set_cache_index_and_filter_blocks(true); + let mut db_opts = rocksdb::Options::default(); db_opts.set_block_based_table_factory(&block_based_options); + db_opts.set_optimize_filters_for_hits(true); + db_opts.set_skip_stats_update_on_db_open(true); + db_opts.set_level_compaction_dynamic_level_bytes(true); + db_opts.set_target_file_size_base(256 * 1024 * 1024); + db_opts.set_compaction_readahead_size(2 * 1024 * 1024); + db_opts.set_use_direct_reads(true); + db_opts.set_use_direct_io_for_flush_and_compaction(true); db_opts.create_if_missing(true); db_opts.increase_parallelism(num_cpus::get() as i32); db_opts.set_max_open_files(512); From 9e77f7617cfcdc8d1c0e1b3146cbef6566ed0dc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 12 Jan 2022 12:27:02 +0100 Subject: [PATCH 101/201] fix: disable direct IO again --- src/database/abstraction/rocksdb.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/database/abstraction/rocksdb.rs b/src/database/abstraction/rocksdb.rs index 32095566..b7f6d3b6 100644 --- a/src/database/abstraction/rocksdb.rs +++ b/src/database/abstraction/rocksdb.rs @@ -22,7 +22,7 @@ fn db_options(cache_capacity_bytes: usize, rocksdb_cache: &rocksdb::Cache) -> ro // "Difference of spinning disk" // https://zhangyuchi.gitbooks.io/rocksdbbook/content/RocksDB-Tuning-Guide.html - block_based_options.set_block_size(64 * 1024); + block_based_options.set_block_size(4 * 1024); block_based_options.set_cache_index_and_filter_blocks(true); let mut db_opts = rocksdb::Options::default(); @@ -31,9 +31,9 @@ fn db_options(cache_capacity_bytes: usize, rocksdb_cache: &rocksdb::Cache) -> ro db_opts.set_skip_stats_update_on_db_open(true); db_opts.set_level_compaction_dynamic_level_bytes(true); db_opts.set_target_file_size_base(256 * 1024 * 1024); - db_opts.set_compaction_readahead_size(2 * 1024 * 1024); - db_opts.set_use_direct_reads(true); - db_opts.set_use_direct_io_for_flush_and_compaction(true); + //db_opts.set_compaction_readahead_size(2 * 1024 * 1024); + //db_opts.set_use_direct_reads(true); + //db_opts.set_use_direct_io_for_flush_and_compaction(true); db_opts.create_if_missing(true); db_opts.increase_parallelism(num_cpus::get() as i32); db_opts.set_max_open_files(512); From 447639054e21523aed76e408667c5263ccde85ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Thu, 13 Jan 2022 21:03:53 +0100 Subject: [PATCH 102/201] improvement: higher default pdu capacity --- src/client_server/device.rs | 2 +- src/database.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client_server/device.rs b/src/client_server/device.rs index 03a3004b..f240f2e7 100644 --- a/src/client_server/device.rs +++ b/src/client_server/device.rs @@ -85,7 +85,7 @@ pub async fn update_device_route( Ok(update_device::Response {}.into()) } -/// # `PUT /_matrix/client/r0/devices/{deviceId}` +/// # `DELETE /_matrix/client/r0/devices/{deviceId}` /// /// Deletes the given device. /// diff --git a/src/database.rs b/src/database.rs index c2b3e2b9..9a71e737 100644 --- a/src/database.rs +++ b/src/database.rs @@ -128,7 +128,7 @@ fn default_db_cache_capacity_mb() -> f64 { } fn default_pdu_cache_capacity() -> u32 { - 100_000 + 1_000_000 } fn default_cleanup_second_interval() -> u32 { From a336027b0e45b512c55e4c0b68e095d40ebd01ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Thu, 13 Jan 2022 21:11:45 +0100 Subject: [PATCH 103/201] fix: better memory usage message --- src/database/abstraction/rocksdb.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/database/abstraction/rocksdb.rs b/src/database/abstraction/rocksdb.rs index b7f6d3b6..d1706d45 100644 --- a/src/database/abstraction/rocksdb.rs +++ b/src/database/abstraction/rocksdb.rs @@ -108,14 +108,12 @@ impl DatabaseEngine for Arc { Approximate memory usage of un-flushed mem-tables: {:.3} MB\n\ Approximate memory usage of all the table readers: {:.3} MB\n\ Approximate memory usage by cache: {:.3} MB\n\ - self.cache.get_usage(): {:.3} MB\n\ - self.cache.get_pinned_usage(): {:.3} MB\n\ + Approximate memory usage by cache pinned: {:.3} MB\n\ ", stats.mem_table_total as f64 / 1024.0 / 1024.0, stats.mem_table_unflushed as f64 / 1024.0 / 1024.0, stats.mem_table_readers_total as f64 / 1024.0 / 1024.0, stats.cache_total as f64 / 1024.0 / 1024.0, - self.cache.get_usage() as f64 / 1024.0 / 1024.0, self.cache.get_pinned_usage() as f64 / 1024.0 / 1024.0, )) } From 6fa01aa9826c2a4f7643289e0b86aee40efc59d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Thu, 13 Jan 2022 21:46:20 +0100 Subject: [PATCH 104/201] fix: remove dbg --- src/database/abstraction/sqlite.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/database/abstraction/sqlite.rs b/src/database/abstraction/sqlite.rs index d4fd0bdd..f80f50e4 100644 --- a/src/database/abstraction/sqlite.rs +++ b/src/database/abstraction/sqlite.rs @@ -136,7 +136,7 @@ type TupleOfBytes = (Vec, Vec); impl SqliteTable { #[tracing::instrument(skip(self, guard, key))] fn get_with_guard(&self, guard: &Connection, key: &[u8]) -> Result>> { - dbg!(&self.name); + //dbg!(&self.name); Ok(guard .prepare(format!("SELECT value FROM {} WHERE key = ?", self.name).as_str())? .query_row([key], |row| row.get(0)) @@ -145,7 +145,7 @@ impl SqliteTable { #[tracing::instrument(skip(self, guard, key, value))] fn insert_with_guard(&self, guard: &Connection, key: &[u8], value: &[u8]) -> Result<()> { - dbg!(&self.name); + //dbg!(&self.name); guard.execute( format!( "INSERT OR REPLACE INTO {} (key, value) VALUES (?, ?)", @@ -179,7 +179,7 @@ impl SqliteTable { .query_map([], |row| Ok((row.get_unwrap(0), row.get_unwrap(1)))) .unwrap() .map(move |r| { - dbg!(&name); + //dbg!(&name); r.unwrap() }), ); @@ -286,7 +286,7 @@ impl Tree for SqliteTable { .query_map([from], |row| Ok((row.get_unwrap(0), row.get_unwrap(1)))) .unwrap() .map(move |r| { - dbg!(&name); + //dbg!(&name); r.unwrap() }), ); @@ -311,7 +311,7 @@ impl Tree for SqliteTable { .query_map([from], |row| Ok((row.get_unwrap(0), row.get_unwrap(1)))) .unwrap() .map(move |r| { - dbg!(&name); + //dbg!(&name); r.unwrap() }), ); From 16f826773bc26dd388f04e3e862bef7d1be9cdeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Thu, 13 Jan 2022 22:47:30 +0100 Subject: [PATCH 105/201] refactor: fix warnings --- src/database/abstraction/rocksdb.rs | 4 ++-- src/database/abstraction/sqlite.rs | 4 ++-- src/database/abstraction/watchers.rs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/database/abstraction/rocksdb.rs b/src/database/abstraction/rocksdb.rs index d1706d45..79a3d82a 100644 --- a/src/database/abstraction/rocksdb.rs +++ b/src/database/abstraction/rocksdb.rs @@ -1,6 +1,6 @@ use super::{super::Config, watchers::Watchers, DatabaseEngine, Tree}; use crate::{utils, Result}; -use std::{future::Future, pin::Pin, sync::Arc, collections::HashMap, sync::RwLock}; +use std::{future::Future, pin::Pin, sync::Arc, sync::RwLock}; pub struct Engine { rocks: rocksdb::DBWithThreadMode, @@ -13,7 +13,7 @@ pub struct RocksDbEngineTree<'a> { db: Arc, name: &'a str, watchers: Watchers, - write_lock: RwLock<()> + write_lock: RwLock<()>, } fn db_options(cache_capacity_bytes: usize, rocksdb_cache: &rocksdb::Cache) -> rocksdb::Options { diff --git a/src/database/abstraction/sqlite.rs b/src/database/abstraction/sqlite.rs index f80f50e4..d4aab7dd 100644 --- a/src/database/abstraction/sqlite.rs +++ b/src/database/abstraction/sqlite.rs @@ -172,7 +172,7 @@ impl SqliteTable { let statement_ref = NonAliasingBox(statement); - let name = self.name.clone(); + //let name = self.name.clone(); let iterator = Box::new( statement @@ -267,7 +267,7 @@ impl Tree for SqliteTable { let guard = self.engine.read_lock_iterator(); let from = from.to_vec(); // TODO change interface? - let name = self.name.clone(); + //let name = self.name.clone(); if backwards { let statement = Box::leak(Box::new( diff --git a/src/database/abstraction/watchers.rs b/src/database/abstraction/watchers.rs index fec1f27a..55cb60b3 100644 --- a/src/database/abstraction/watchers.rs +++ b/src/database/abstraction/watchers.rs @@ -1,8 +1,8 @@ use std::{ collections::{hash_map, HashMap}, - sync::RwLock, future::Future, pin::Pin, + sync::RwLock, }; use tokio::sync::watch; From f67785caaf6a4be5c7d330df0f7a89781aa21f91 Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Thu, 13 Jan 2022 22:24:47 +0000 Subject: [PATCH 106/201] Fix(ci): Disable CARGO_HOME caching --- .gitlab-ci.yml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1dedd8ff..f47327b8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,18 +23,12 @@ variables: interruptible: true image: "rust:latest" tags: ["docker"] - cache: - paths: - - cargohome - key: "build_cache--$TARGET--$CI_COMMIT_BRANCH" variables: CARGO_PROFILE_RELEASE_LTO: "true" CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1" CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow - CARGO_HOME: $CI_PROJECT_DIR/cargohome before_script: - 'echo "Building for target $TARGET"' - - "mkdir -p $CARGO_HOME" - "rustc --version && cargo --version && rustup show" # Print version info for debugging - "rustup target add $TARGET" # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: @@ -219,15 +213,10 @@ test:cargo: image: "rust:latest" tags: ["docker"] variables: - CARGO_HOME: "$CI_PROJECT_DIR/cargohome" CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow - cache: - paths: - - cargohome - key: "test_cache--$CI_COMMIT_BRANCH" interruptible: true before_script: - - mkdir -p $CARGO_HOME + # - mkdir -p $CARGO_HOME - apt-get update -yqq - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config - rustup component add clippy rustfmt From 80e51986c42ea449a3f1d7860c16722431f4fcaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Fri, 14 Jan 2022 11:08:31 +0100 Subject: [PATCH 107/201] improvement: better default cache capacity --- src/database.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database.rs b/src/database.rs index 9a71e737..d688ff9f 100644 --- a/src/database.rs +++ b/src/database.rs @@ -124,7 +124,7 @@ fn default_database_backend() -> String { } fn default_db_cache_capacity_mb() -> f64 { - 200.0 + 10.0 } fn default_pdu_cache_capacity() -> u32 { From d434dfb3a56afde239023685ca0a8d191355314b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Fri, 14 Jan 2022 11:40:49 +0100 Subject: [PATCH 108/201] feat: config option for rocksdb max open files --- src/database.rs | 6 ++++++ src/database/abstraction/rocksdb.rs | 29 ++++++++++++++++++++++------- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/database.rs b/src/database.rs index d688ff9f..fd7a1451 100644 --- a/src/database.rs +++ b/src/database.rs @@ -49,6 +49,8 @@ pub struct Config { database_path: String, #[serde(default = "default_db_cache_capacity_mb")] db_cache_capacity_mb: f64, + #[serde(default = "default_rocksdb_max_open_files")] + rocksdb_max_open_files: i32, #[serde(default = "default_pdu_cache_capacity")] pdu_cache_capacity: u32, #[serde(default = "default_cleanup_second_interval")] @@ -127,6 +129,10 @@ fn default_db_cache_capacity_mb() -> f64 { 10.0 } +fn default_rocksdb_max_open_files() -> i32 { + 512 +} + fn default_pdu_cache_capacity() -> u32 { 1_000_000 } diff --git a/src/database/abstraction/rocksdb.rs b/src/database/abstraction/rocksdb.rs index 79a3d82a..adda6787 100644 --- a/src/database/abstraction/rocksdb.rs +++ b/src/database/abstraction/rocksdb.rs @@ -5,6 +5,7 @@ use std::{future::Future, pin::Pin, sync::Arc, sync::RwLock}; pub struct Engine { rocks: rocksdb::DBWithThreadMode, cache_capacity_bytes: usize, + max_open_files: i32, cache: rocksdb::Cache, old_cfs: Vec, } @@ -16,7 +17,11 @@ pub struct RocksDbEngineTree<'a> { write_lock: RwLock<()>, } -fn db_options(cache_capacity_bytes: usize, rocksdb_cache: &rocksdb::Cache) -> rocksdb::Options { +fn db_options( + cache_capacity_bytes: usize, + max_open_files: i32, + rocksdb_cache: &rocksdb::Cache, +) -> rocksdb::Options { let mut block_based_options = rocksdb::BlockBasedOptions::default(); block_based_options.set_block_cache(rocksdb_cache); @@ -36,7 +41,7 @@ fn db_options(cache_capacity_bytes: usize, rocksdb_cache: &rocksdb::Cache) -> ro //db_opts.set_use_direct_io_for_flush_and_compaction(true); db_opts.create_if_missing(true); db_opts.increase_parallelism(num_cpus::get() as i32); - db_opts.set_max_open_files(512); + db_opts.set_max_open_files(max_open_files); db_opts.set_compression_type(rocksdb::DBCompressionType::Zstd); db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level); db_opts.optimize_level_style_compaction(cache_capacity_bytes); @@ -52,7 +57,11 @@ impl DatabaseEngine for Arc { let cache_capacity_bytes = (config.db_cache_capacity_mb * 1024.0 * 1024.0) as usize; let rocksdb_cache = rocksdb::Cache::new_lru_cache(cache_capacity_bytes).unwrap(); - let db_opts = db_options(cache_capacity_bytes, &rocksdb_cache); + let db_opts = db_options( + cache_capacity_bytes, + config.rocksdb_max_open_files, + &rocksdb_cache, + ); let cfs = rocksdb::DBWithThreadMode::::list_cf( &db_opts, @@ -66,7 +75,11 @@ impl DatabaseEngine for Arc { cfs.iter().map(|name| { rocksdb::ColumnFamilyDescriptor::new( name, - db_options(cache_capacity_bytes, &rocksdb_cache), + db_options( + cache_capacity_bytes, + config.rocksdb_max_open_files, + &rocksdb_cache, + ), ) }), )?; @@ -74,6 +87,7 @@ impl DatabaseEngine for Arc { Ok(Arc::new(Engine { rocks: db, cache_capacity_bytes, + max_open_files: config.rocksdb_max_open_files, cache: rocksdb_cache, old_cfs: cfs, })) @@ -82,9 +96,10 @@ impl DatabaseEngine for Arc { fn open_tree(&self, name: &'static str) -> Result> { if !self.old_cfs.contains(&name.to_owned()) { // Create if it didn't exist - let _ = self - .rocks - .create_cf(name, &db_options(self.cache_capacity_bytes, &self.cache)); + let _ = self.rocks.create_cf( + name, + &db_options(self.cache_capacity_bytes, self.max_open_files, &self.cache), + ); } Ok(Arc::new(RocksDbEngineTree { From ab15ec6c32f2e5463369fe7a29f5ea2e6d9c4f2d Mon Sep 17 00:00:00 2001 From: Tglman Date: Fri, 18 Jun 2021 00:38:32 +0100 Subject: [PATCH 109/201] feat: Integration with persy using background ops --- Cargo.toml | 5 + src/database.rs | 6 + src/database/abstraction.rs | 5 +- src/database/abstraction/persy.rs | 245 ++++++++++++++++++++++++++++++ src/error.rs | 15 ++ 5 files changed, 275 insertions(+), 1 deletion(-) create mode 100644 src/database/abstraction/persy.rs diff --git a/Cargo.toml b/Cargo.toml index c87d949c..2dbd3fd3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,10 @@ tokio = "1.11.0" # Used for storing data permanently sled = { version = "0.34.6", features = ["compression", "no_metrics"], optional = true } #sled = { git = "https://github.com/spacejam/sled.git", rev = "e4640e0773595229f398438886f19bca6f7326a2", features = ["compression"] } +persy = { git = "https://gitlab.com/tglman/persy.git", branch="master" , optional = true, features=["background_ops"] } +# Used by the persy write cache for background flush +timer = "0.2" +chrono = "0.4" # Used for the http request / response body type for Ruma endpoints used with reqwest bytes = "1.1.0" @@ -87,6 +91,7 @@ sha-1 = "0.9.8" [features] default = ["conduit_bin", "backend_sqlite", "backend_rocksdb"] backend_sled = ["sled"] +backend_persy = ["persy"] backend_sqlite = ["sqlite"] backend_heed = ["heed", "crossbeam"] backend_rocksdb = ["rocksdb"] diff --git a/src/database.rs b/src/database.rs index d688ff9f..c2cd9f29 100644 --- a/src/database.rs +++ b/src/database.rs @@ -255,6 +255,12 @@ impl Database { #[cfg(feature = "rocksdb")] Arc::new(Arc::::open(config)?) } + "persy" => { + #[cfg(not(feature = "persy"))] + return Err(Error::BadConfig("Database backend not found.")); + #[cfg(feature = "persy")] + Arc::new(Arc::::open(config)?) + } _ => { return Err(Error::BadConfig("Database backend not found.")); } diff --git a/src/database/abstraction.rs b/src/database/abstraction.rs index 17bd971f..9a3771f3 100644 --- a/src/database/abstraction.rs +++ b/src/database/abstraction.rs @@ -15,7 +15,10 @@ pub mod heed; #[cfg(feature = "rocksdb")] pub mod rocksdb; -#[cfg(any(feature = "sqlite", feature = "rocksdb", feature = "heed"))] +#[cfg(feature = "persy")] +pub mod persy; + +#[cfg(any(feature = "sqlite", feature = "rocksdb", feature = "heed", feature="persy"))] pub mod watchers; pub trait DatabaseEngine: Send + Sync { diff --git a/src/database/abstraction/persy.rs b/src/database/abstraction/persy.rs new file mode 100644 index 00000000..5d633ab4 --- /dev/null +++ b/src/database/abstraction/persy.rs @@ -0,0 +1,245 @@ +use crate::{ + database::{ + abstraction::{DatabaseEngine, Tree}, + Config, + }, + Result, +}; +use persy::{ByteVec, OpenOptions, Persy, Transaction, TransactionConfig, ValueMode}; + +use std::{ + collections::HashMap, + future::Future, + pin::Pin, + sync::{Arc, RwLock}, +}; + +use tokio::sync::oneshot::Sender; +use tracing::warn; + +pub struct PersyEngine { + persy: Persy, +} + +impl DatabaseEngine for PersyEngine { + fn open(config: &Config) -> Result> { + let mut cfg = persy::Config::new(); + cfg.change_cache_size((config.db_cache_capacity_mb * 1024.0 * 1024.0) as u64); + + let persy = OpenOptions::new() + .create(true) + .config(cfg) + .open(&format!("{}/db.persy", config.database_path))?; + Ok(Arc::new(PersyEngine { persy })) + } + + fn open_tree(self: &Arc, name: &'static str) -> Result> { + // Create if it doesn't exist + if !self.persy.exists_index(name)? { + let mut tx = self.persy.begin()?; + tx.create_index::(name, ValueMode::Replace)?; + tx.prepare()?.commit()?; + } + + Ok(Arc::new(PersyTree { + persy: self.persy.clone(), + name: name.to_owned(), + watchers: RwLock::new(HashMap::new()), + })) + } + + fn flush(self: &Arc) -> Result<()> { + Ok(()) + } +} + +pub struct PersyTree { + persy: Persy, + name: String, + watchers: RwLock, Vec>>>, +} + +impl PersyTree { + fn begin(&self) -> Result { + Ok(self + .persy + .begin_with(TransactionConfig::new().set_background_sync(true))?) + } +} + +impl Tree for PersyTree { + #[tracing::instrument(skip(self, key))] + fn get(&self, key: &[u8]) -> Result>> { + let result = self + .persy + .get::(&self.name, &ByteVec::from(key))? + .next() + .map(|v| (*v).to_owned()); + Ok(result) + } + + #[tracing::instrument(skip(self, key, value))] + fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> { + self.insert_batch(&mut Some((key.to_owned(), value.to_owned())).into_iter())?; + let watchers = self.watchers.read().unwrap(); + let mut triggered = Vec::new(); + + for length in 0..=key.len() { + if watchers.contains_key(&key[..length]) { + triggered.push(&key[..length]); + } + } + + drop(watchers); + + if !triggered.is_empty() { + let mut watchers = self.watchers.write().unwrap(); + for prefix in triggered { + if let Some(txs) = watchers.remove(prefix) { + for tx in txs { + let _ = tx.send(()); + } + } + } + } + Ok(()) + } + + #[tracing::instrument(skip(self, iter))] + fn insert_batch<'a>(&self, iter: &mut dyn Iterator, Vec)>) -> Result<()> { + let mut tx = self.begin()?; + for (key, value) in iter { + tx.put::( + &self.name, + ByteVec::from(key.clone()), + ByteVec::from(value), + )?; + } + tx.prepare()?.commit()?; + Ok(()) + } + + #[tracing::instrument(skip(self, iter))] + fn increment_batch<'a>(&self, iter: &mut dyn Iterator>) -> Result<()> { + let mut tx = self.begin()?; + for key in iter { + let old = tx + .get::(&self.name, &ByteVec::from(key.clone()))? + .next() + .map(|v| (*v).to_owned()); + let new = crate::utils::increment(old.as_deref()).unwrap(); + tx.put::(&self.name, ByteVec::from(key), ByteVec::from(new))?; + } + tx.prepare()?.commit()?; + Ok(()) + } + + #[tracing::instrument(skip(self, key))] + fn remove(&self, key: &[u8]) -> Result<()> { + let mut tx = self.begin()?; + tx.remove::(&self.name, ByteVec::from(key), None)?; + tx.prepare()?.commit()?; + Ok(()) + } + + #[tracing::instrument(skip(self))] + fn iter<'a>(&'a self) -> Box, Vec)> + 'a> { + let iter = self.persy.range::(&self.name, ..); + match iter { + Ok(iter) => Box::new(iter.filter_map(|(k, v)| { + v.into_iter() + .map(|val| ((*k).to_owned().into(), (*val).to_owned().into())) + .next() + })), + Err(e) => { + warn!("error iterating {:?}", e); + Box::new(std::iter::empty()) + } + } + } + + #[tracing::instrument(skip(self, from, backwards))] + fn iter_from<'a>( + &'a self, + from: &[u8], + backwards: bool, + ) -> Box, Vec)> + 'a> { + let range = if backwards { + self.persy + .range::(&self.name, ..=ByteVec::from(from)) + } else { + self.persy + .range::(&self.name, ByteVec::from(from)..) + }; + match range { + Ok(iter) => { + let map = iter.filter_map(|(k, v)| { + v.into_iter() + .map(|val| ((*k).to_owned().into(), (*val).to_owned().into())) + .next() + }); + if backwards { + Box::new(map.rev()) + } else { + Box::new(map) + } + } + Err(e) => { + warn!("error iterating with prefix {:?}", e); + Box::new(std::iter::empty()) + } + } + } + + #[tracing::instrument(skip(self, key))] + fn increment(&self, key: &[u8]) -> Result> { + self.increment_batch(&mut Some(key.to_owned()).into_iter())?; + Ok(self.get(key)?.unwrap()) + } + + #[tracing::instrument(skip(self, prefix))] + fn scan_prefix<'a>( + &'a self, + prefix: Vec, + ) -> Box, Vec)> + 'a> { + let range_prefix = ByteVec::from(prefix.clone()); + let range = self + .persy + .range::(&self.name, range_prefix..); + + match range { + Ok(iter) => { + let owned_prefix = prefix.clone(); + Box::new( + iter.take_while(move |(k, _)| (*k).starts_with(&owned_prefix)) + .filter_map(|(k, v)| { + v.into_iter() + .map(|val| ((*k).to_owned().into(), (*val).to_owned().into())) + .next() + }), + ) + } + Err(e) => { + warn!("error scanning prefix {:?}", e); + Box::new(std::iter::empty()) + } + } + } + + #[tracing::instrument(skip(self, prefix))] + fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin + Send + 'a>> { + let (tx, rx) = tokio::sync::oneshot::channel(); + + self.watchers + .write() + .unwrap() + .entry(prefix.to_vec()) + .or_default() + .push(tx); + + Box::pin(async move { + // Tx is never destroyed + rx.await.unwrap(); + }) + } +} diff --git a/src/error.rs b/src/error.rs index 4d427da4..5ffe48c9 100644 --- a/src/error.rs +++ b/src/error.rs @@ -8,6 +8,9 @@ use ruma::{ use thiserror::Error; use tracing::warn; +#[cfg(feature = "persy")] +use persy::PersyError; + #[cfg(feature = "conduit_bin")] use { crate::RumaResponse, @@ -36,6 +39,9 @@ pub enum Error { #[from] source: rusqlite::Error, }, + #[cfg(feature = "persy")] + #[error("There was a problem with the connection to the persy database.")] + PersyError { source: PersyError }, #[cfg(feature = "heed")] #[error("There was a problem with the connection to the heed database: {error}")] HeedError { error: String }, @@ -142,3 +148,12 @@ where self.to_response().respond_to(r) } } + +#[cfg(feature = "persy")] +impl> From> for Error { + fn from(err: persy::PE) -> Self { + Error::PersyError { + source: err.error().into(), + } + } +} From 1cc41937bd9ae7679aa48c10074ac1041d3a94b5 Mon Sep 17 00:00:00 2001 From: Tglman Date: Thu, 23 Dec 2021 22:59:17 +0000 Subject: [PATCH 110/201] refactor:use generic watcher in persy implementation --- Cargo.lock | 46 ++++++++++++++++++++++++++++ Cargo.toml | 5 +--- src/database/abstraction/persy.rs | 50 ++++--------------------------- 3 files changed, 53 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d297102c..df37fd58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -293,6 +293,7 @@ dependencies = [ "opentelemetry", "opentelemetry-jaeger", "parking_lot", + "persy", "rand 0.8.4", "regex", "reqwest", @@ -374,6 +375,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49fc9a695bca7f35f5f4c15cddc84415f66a74ea78eef08e90c5024f2b540e23" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403" + [[package]] name = "crc32fast" version = "1.3.0" @@ -1651,6 +1667,21 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "persy" +version = "1.2.0" +source = "git+https://gitlab.com/tglman/persy.git?branch=master#ff102d6edeaf14d30a846c2e2376a814685d09e7" +dependencies = [ + "crc", + "data-encoding", + "fs2", + "linked-hash-map", + "rand 0.8.4", + "thiserror", + "unsigned-varint", + "zigzag", +] + [[package]] name = "pin-project" version = "1.0.10" @@ -3290,6 +3321,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f14ee04d9415b52b3aeab06258a3f07093182b88ba0f9b8d203f211a7a7d41c7" +[[package]] +name = "unsigned-varint" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d86a8dc7f45e4c1b0d30e43038c38f274e77af056aa5f74b93c2cf9eb3c1c836" + [[package]] name = "untrusted" version = "0.7.1" @@ -3532,6 +3569,15 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zigzag" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70b40401a28d86ce16a330b863b86fd7dbee4d7c940587ab09ab8c019f9e3fdf" +dependencies = [ + "num-traits", +] + [[package]] name = "zstd" version = "0.9.2+zstd.1.5.1" diff --git a/Cargo.toml b/Cargo.toml index 2dbd3fd3..7c94a693 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,9 +29,6 @@ tokio = "1.11.0" sled = { version = "0.34.6", features = ["compression", "no_metrics"], optional = true } #sled = { git = "https://github.com/spacejam/sled.git", rev = "e4640e0773595229f398438886f19bca6f7326a2", features = ["compression"] } persy = { git = "https://gitlab.com/tglman/persy.git", branch="master" , optional = true, features=["background_ops"] } -# Used by the persy write cache for background flush -timer = "0.2" -chrono = "0.4" # Used for the http request / response body type for Ruma endpoints used with reqwest bytes = "1.1.0" @@ -91,7 +88,7 @@ sha-1 = "0.9.8" [features] default = ["conduit_bin", "backend_sqlite", "backend_rocksdb"] backend_sled = ["sled"] -backend_persy = ["persy"] +backend_persy = ["persy", "parking_lot"] backend_sqlite = ["sqlite"] backend_heed = ["heed", "crossbeam"] backend_rocksdb = ["rocksdb"] diff --git a/src/database/abstraction/persy.rs b/src/database/abstraction/persy.rs index 5d633ab4..71efed3b 100644 --- a/src/database/abstraction/persy.rs +++ b/src/database/abstraction/persy.rs @@ -1,20 +1,14 @@ use crate::{ database::{ - abstraction::{DatabaseEngine, Tree}, + abstraction::{watchers::Watchers, DatabaseEngine, Tree}, Config, }, Result, }; use persy::{ByteVec, OpenOptions, Persy, Transaction, TransactionConfig, ValueMode}; -use std::{ - collections::HashMap, - future::Future, - pin::Pin, - sync::{Arc, RwLock}, -}; +use std::{future::Future, pin::Pin, sync::Arc}; -use tokio::sync::oneshot::Sender; use tracing::warn; pub struct PersyEngine { @@ -44,7 +38,7 @@ impl DatabaseEngine for PersyEngine { Ok(Arc::new(PersyTree { persy: self.persy.clone(), name: name.to_owned(), - watchers: RwLock::new(HashMap::new()), + watchers: Watchers::default(), })) } @@ -56,7 +50,7 @@ impl DatabaseEngine for PersyEngine { pub struct PersyTree { persy: Persy, name: String, - watchers: RwLock, Vec>>>, + watchers: Watchers, } impl PersyTree { @@ -81,27 +75,7 @@ impl Tree for PersyTree { #[tracing::instrument(skip(self, key, value))] fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> { self.insert_batch(&mut Some((key.to_owned(), value.to_owned())).into_iter())?; - let watchers = self.watchers.read().unwrap(); - let mut triggered = Vec::new(); - - for length in 0..=key.len() { - if watchers.contains_key(&key[..length]) { - triggered.push(&key[..length]); - } - } - - drop(watchers); - - if !triggered.is_empty() { - let mut watchers = self.watchers.write().unwrap(); - for prefix in triggered { - if let Some(txs) = watchers.remove(prefix) { - for tx in txs { - let _ = tx.send(()); - } - } - } - } + self.watchers.wake(key); Ok(()) } @@ -228,18 +202,6 @@ impl Tree for PersyTree { #[tracing::instrument(skip(self, prefix))] fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin + Send + 'a>> { - let (tx, rx) = tokio::sync::oneshot::channel(); - - self.watchers - .write() - .unwrap() - .entry(prefix.to_vec()) - .or_default() - .push(tx); - - Box::pin(async move { - // Tx is never destroyed - rx.await.unwrap(); - }) + self.watchers.watch(prefix) } } From f9977ca64f84768c0a71535f6038f4a6487ddc17 Mon Sep 17 00:00:00 2001 From: Tglman Date: Thu, 13 Jan 2022 22:37:19 +0000 Subject: [PATCH 111/201] fix: changes to update to the last database engine trait definition --- src/database/abstraction.rs | 7 ++++++- src/database/abstraction/persy.rs | 12 ++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/database/abstraction.rs b/src/database/abstraction.rs index 9a3771f3..09081826 100644 --- a/src/database/abstraction.rs +++ b/src/database/abstraction.rs @@ -18,7 +18,12 @@ pub mod rocksdb; #[cfg(feature = "persy")] pub mod persy; -#[cfg(any(feature = "sqlite", feature = "rocksdb", feature = "heed", feature="persy"))] +#[cfg(any( + feature = "sqlite", + feature = "rocksdb", + feature = "heed", + feature = "persy" +))] pub mod watchers; pub trait DatabaseEngine: Send + Sync { diff --git a/src/database/abstraction/persy.rs b/src/database/abstraction/persy.rs index 71efed3b..628cf32b 100644 --- a/src/database/abstraction/persy.rs +++ b/src/database/abstraction/persy.rs @@ -11,12 +11,12 @@ use std::{future::Future, pin::Pin, sync::Arc}; use tracing::warn; -pub struct PersyEngine { +pub struct Engine { persy: Persy, } -impl DatabaseEngine for PersyEngine { - fn open(config: &Config) -> Result> { +impl DatabaseEngine for Arc { + fn open(config: &Config) -> Result { let mut cfg = persy::Config::new(); cfg.change_cache_size((config.db_cache_capacity_mb * 1024.0 * 1024.0) as u64); @@ -24,10 +24,10 @@ impl DatabaseEngine for PersyEngine { .create(true) .config(cfg) .open(&format!("{}/db.persy", config.database_path))?; - Ok(Arc::new(PersyEngine { persy })) + Ok(Arc::new(Engine { persy })) } - fn open_tree(self: &Arc, name: &'static str) -> Result> { + fn open_tree(&self, name: &'static str) -> Result> { // Create if it doesn't exist if !self.persy.exists_index(name)? { let mut tx = self.persy.begin()?; @@ -42,7 +42,7 @@ impl DatabaseEngine for PersyEngine { })) } - fn flush(self: &Arc) -> Result<()> { + fn flush(&self) -> Result<()> { Ok(()) } } From c1cd4b5e26c68d1c5e91f85df2a65591f774d13c Mon Sep 17 00:00:00 2001 From: Tglman Date: Fri, 14 Jan 2022 21:00:13 +0000 Subject: [PATCH 112/201] chore: set the released version of persy in Cargo.toml --- Cargo.lock | 3 ++- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index df37fd58..469c5663 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1670,7 +1670,8 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "persy" version = "1.2.0" -source = "git+https://gitlab.com/tglman/persy.git?branch=master#ff102d6edeaf14d30a846c2e2376a814685d09e7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c6aa7d7f093620a28b74fcf5f5da73ba17a9e52fcbbdbb4ecc89e61cb2d673" dependencies = [ "crc", "data-encoding", diff --git a/Cargo.toml b/Cargo.toml index 7c94a693..df879c36 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ tokio = "1.11.0" # Used for storing data permanently sled = { version = "0.34.6", features = ["compression", "no_metrics"], optional = true } #sled = { git = "https://github.com/spacejam/sled.git", rev = "e4640e0773595229f398438886f19bca6f7326a2", features = ["compression"] } -persy = { git = "https://gitlab.com/tglman/persy.git", branch="master" , optional = true, features=["background_ops"] } +persy = { version = "1.2" , optional = true, features=["background_ops"] } # Used for the http request / response body type for Ruma endpoints used with reqwest bytes = "1.1.0" From fb19114bd9bbfc9bcc50caaab72074247bbe726b Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Sat, 15 Jan 2022 15:52:47 +0100 Subject: [PATCH 113/201] rename iter_locals to get_local_users; make get_local_users skip on parse errors; remove deprecated function count_local_users --- src/database/admin.rs | 5 +---- src/database/users.rs | 22 +++++++++------------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index 5418f53a..859977e9 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -95,14 +95,11 @@ impl Admin { match event { AdminCommand::ListLocalUsers => { - // collect local users only - let users = guard.users.iter_locals(); + let users = guard.users.get_local_users(); let mut msg: String = format!("Found {} local user account(s):\n", users.len()); msg += &users.join("\n"); - // send number of local users as plain text: - // TODO: send as Markdown send_message(RoomMessageEventContent::text_plain(&msg), guard, &state_lock); } AdminCommand::RegisterAppservice(yaml) => { diff --git a/src/database/users.rs b/src/database/users.rs index d3e1fe43..021c710b 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -84,16 +84,6 @@ impl Users { Ok(self.userid_password.iter().count()) } - /// The method is DEPRECATED and was replaced by iter_locals() - /// - /// This method will only count those local user accounts with - /// a password thus returning only real accounts on this instance. - #[tracing::instrument(skip(self))] - pub fn count_local_users(&self) -> Result { - let n = self.userid_password.iter().filter(|(_, bytes)| bytes.len() > 0).count(); - Ok(n) - } - /// Find out which user an access token belongs to. #[tracing::instrument(skip(self, token))] @@ -134,13 +124,19 @@ impl Users { }) } - /// Returns a vector of local usernames + /// Returns a list of local usernames, that is, a parseable username + /// with a password of length greater then zero bytes. + /// If utils::string_from_bytes returns an error that username will be skipped + /// and the function will log the error #[tracing::instrument(skip(self))] - pub fn iter_locals(&self) -> Vec { + pub fn get_local_users(&self) -> Vec { self.userid_password.iter().filter(|(_, pw)| pw.len() > 0).map(|(username, _)| { match utils::string_from_bytes(&username) { Ok(s) => s, - Err(e) => e.to_string() + Err(e) => { + Error::bad_database(format!("Failed to parse username: {}", e.to_string())); + None + } } }).collect::>() } From 91eb6c4d08c5293f8af6436489923871fb2477a9 Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Sat, 15 Jan 2022 17:10:23 +0100 Subject: [PATCH 114/201] Return a Result instead of a vector --- src/database/admin.rs | 16 ++++++++++------ src/database/users.rs | 18 +++++++++++------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index 859977e9..7799ffa2 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -95,12 +95,16 @@ impl Admin { match event { AdminCommand::ListLocalUsers => { - let users = guard.users.get_local_users(); - - let mut msg: String = format!("Found {} local user account(s):\n", users.len()); - msg += &users.join("\n"); - - send_message(RoomMessageEventContent::text_plain(&msg), guard, &state_lock); + match guard.users.get_local_users() { + Ok(users) => { + let mut msg: String = format!("Found {} local user account(s):\n", users.len()); + msg += &users.join("\n"); + send_message(RoomMessageEventContent::text_plain(&msg), guard, &state_lock); + } + Err(e) => { + send_message(RoomMessageEventContent::text_plain(e.to_string()), guard, &state_lock); + } + } } AdminCommand::RegisterAppservice(yaml) => { guard.appservice.register_appservice(yaml).unwrap(); // TODO handle error diff --git a/src/database/users.rs b/src/database/users.rs index 021c710b..7d14b3e4 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -84,7 +84,6 @@ impl Users { Ok(self.userid_password.iter().count()) } - /// Find out which user an access token belongs to. #[tracing::instrument(skip(self, token))] pub fn find_from_token(&self, token: &str) -> Result, String)>> { @@ -129,16 +128,21 @@ impl Users { /// If utils::string_from_bytes returns an error that username will be skipped /// and the function will log the error #[tracing::instrument(skip(self))] - pub fn get_local_users(&self) -> Vec { - self.userid_password.iter().filter(|(_, pw)| pw.len() > 0).map(|(username, _)| { - match utils::string_from_bytes(&username) { + pub fn get_local_users(&self) -> Result> { + self.userid_password + .iter() + .filter(|(_, pw)| pw.len() > 0) + .map(|(username, _)| match utils::string_from_bytes(&username) { Ok(s) => s, Err(e) => { - Error::bad_database(format!("Failed to parse username: {}", e.to_string())); + Error::bad_database(format!( + "Failed to parse username while calling get_local_users(): {}", + e.to_string() + )); None } - } - }).collect::>() + }) + .collect::>>() } /// Returns the password hash for the given user. From 217e3789929b7a1b227058b3b88664ee5f74ca75 Mon Sep 17 00:00:00 2001 From: Julius de Bruijn Date: Sat, 15 Jan 2022 17:34:13 +0000 Subject: [PATCH 115/201] Add mautrix-signal to tested appservices --- APPSERVICES.md | 36 +----------------------------------- 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/APPSERVICES.md b/APPSERVICES.md index 894bc6f4..f23918b4 100644 --- a/APPSERVICES.md +++ b/APPSERVICES.md @@ -57,38 +57,4 @@ These appservices have been tested and work with Conduit without any extra steps - [matrix-appservice-discord](https://github.com/Half-Shot/matrix-appservice-discord) - [mautrix-hangouts](https://github.com/mautrix/hangouts/) - [mautrix-telegram](https://github.com/mautrix/telegram/) - -### [mautrix-signal](https://github.com/mautrix/signal) - -There are a few things you need to do, in order for the Signal bridge (at least -up to version `0.2.0`) to work. How you do this depends on whether you use -Docker or `virtualenv` to run it. In either case you need to modify -[portal.py](https://github.com/mautrix/signal/blob/master/mautrix_signal/portal.py). -Do this **before** following the bridge installation guide. - -1. **Create a copy of `portal.py`**. Go to - [portal.py](https://github.com/mautrix/signal/blob/master/mautrix_signal/portal.py) -at [mautrix-signal](https://github.com/mautrix/signal) (make sure you change to -the correct commit/version of mautrix-signal you're using) and copy its -content. Create a new `portal.py` on your system and paste the content in. -2. **Patch the copy**. Exact line numbers may be slightly different, look nearby if they don't match: - - [Line 1020](https://github.com/mautrix/signal/blob/4ea831536f154aba6419d13292479eb383ea3308/mautrix_signal/portal.py#L1020) - ```diff - --- levels.users[self.main_intent.mxid] = 9001 if is_initial else 100 - +++ levels.users[self.main_intent.mxid] = 100 if is_initial else 100 - ``` - - [Between lines 1041 and 1042](https://github.com/mautrix/signal/blob/4ea831536f154aba6419d13292479eb383ea3308/mautrix_signal/portal.py#L1041-L1042) add a new line: - ```diff - "type": str(EventType.ROOM_POWER_LEVELS), - +++ "state_key": "", - "content": power_levels.serialize(), - ``` -3. **Deploy the patch**. This is different depending on how you have `mautrix-signal` deployed: - - [*If using virtualenv*] Copy your patched `portal.py` to `./lib/python3.7/site-packages/mautrix_signal/portal.py` (the exact version of Python may be different on your system). - - [*If using Docker*] Map the patched `portal.py` into the `mautrix-signal` container: - - ```yaml - volumes: - - ./your/path/on/host/portal.py:/usr/lib/python3.9/site-packages/mautrix_signal/portal.py - ``` -4. Now continue with the [bridge installation instructions ](https://docs.mau.fi/bridges/index.html) and the general bridge notes above. +- [mautrix-signal](https://github.com/mautrix/signal/) from version `0.2.2` forward. From c03bf6ef11bd88459d6dc1eed75d23879ae2fa1a Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Sat, 15 Jan 2022 22:20:51 +0100 Subject: [PATCH 116/201] name the function after its purpose: iter_locals -> get_local_users --- src/database/users.rs | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/src/database/users.rs b/src/database/users.rs index 7d14b3e4..6f51e1f9 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -129,20 +129,35 @@ impl Users { /// and the function will log the error #[tracing::instrument(skip(self))] pub fn get_local_users(&self) -> Result> { - self.userid_password + let users: Vec = self + .userid_password .iter() - .filter(|(_, pw)| pw.len() > 0) - .map(|(username, _)| match utils::string_from_bytes(&username) { - Ok(s) => s, - Err(e) => { - Error::bad_database(format!( - "Failed to parse username while calling get_local_users(): {}", - e.to_string() - )); + .filter_map(|(username, pw)| self.get_username_on_valid_password(&username, &pw)) + .collect(); + Ok(users) + } + + /// A private helper to avoid double filtering the iterator + fn get_username_on_valid_password(&self, username: &[u8], password: &[u8]) -> Option { + // A valid password is not empty + if password.len() > 0 { + match utils::string_from_bytes(username) { + Ok(u) => Some(u), + Err(_) => { + // TODO: add error cause! + // let msg: String = format!( + // "Failed to parse username while calling get_local_users(): {}", + // e.to_string() + // ); + Error::bad_database( + "Failed to parse username while calling get_username_on_valid_password", + ); None } - }) - .collect::>>() + } + } else { + None + } } /// Returns the password hash for the given user. From 9205c070485a234463119182bd976a0d45c1ace0 Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Sat, 15 Jan 2022 22:37:39 +0100 Subject: [PATCH 117/201] Update get_local_users description --- src/database/users.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/database/users.rs b/src/database/users.rs index 6f51e1f9..e510140d 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -123,10 +123,11 @@ impl Users { }) } - /// Returns a list of local usernames, that is, a parseable username - /// with a password of length greater then zero bytes. + /// Returns a list of local users as list of usernames. + /// + /// A user account is considered `local` if the length of it's password + /// is greater then zero. /// If utils::string_from_bytes returns an error that username will be skipped - /// and the function will log the error #[tracing::instrument(skip(self))] pub fn get_local_users(&self) -> Result> { let users: Vec = self @@ -138,6 +139,7 @@ impl Users { } /// A private helper to avoid double filtering the iterator + #[tracing::instrument(skip(self))] fn get_username_on_valid_password(&self, username: &[u8], password: &[u8]) -> Option { // A valid password is not empty if password.len() > 0 { From 13ae036ca04b4ebd427444252ef9856b3028b7ac Mon Sep 17 00:00:00 2001 From: Andrei Vasiliu Date: Sun, 16 Jan 2022 13:52:23 +0200 Subject: [PATCH 118/201] Move and refactor admin commands into admin module --- src/database/admin.rs | 239 +++++++++++++++++++++++++++++++++++++++++- src/database/rooms.rs | 220 +------------------------------------- 2 files changed, 240 insertions(+), 219 deletions(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index 7d2301d9..518d7587 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -1,10 +1,17 @@ -use std::{convert::TryInto, sync::Arc}; +use std::{convert::TryFrom, convert::TryInto, sync::Arc, time::Instant}; -use crate::{pdu::PduBuilder, Database}; -use rocket::futures::{channel::mpsc, stream::StreamExt}; +use crate::{ + error::{Error, Result}, + pdu::PduBuilder, + server_server, Database, PduEvent, +}; +use rocket::{ + futures::{channel::mpsc, stream::StreamExt}, + http::RawStr, +}; use ruma::{ events::{room::message::RoomMessageEventContent, EventType}, - UserId, + EventId, RoomId, RoomVersionId, UserId, }; use serde_json::value::to_raw_value; use tokio::sync::{MutexGuard, RwLock, RwLockReadGuard}; @@ -137,3 +144,227 @@ impl Admin { self.sender.unbounded_send(command).unwrap(); } } + +pub fn parse_admin_command(db: &Database, command_line: &str, body: Vec<&str>) -> AdminCommand { + let mut parts = command_line.split_whitespace().skip(1); + + let command_name = match parts.next() { + Some(command) => command, + None => { + let message = "No command given. Use help for a list of commands."; + return AdminCommand::SendMessage(RoomMessageEventContent::text_html( + html_to_markdown(message), + message, + )); + } + }; + + let args: Vec<_> = parts.collect(); + + match try_parse_admin_command(db, command_name, args, body) { + Ok(admin_command) => admin_command, + Err(error) => { + let message = format!( + "Encountered error while handling {} command:\n\ + 
      {}
      ", + command_name, error, + ); + + AdminCommand::SendMessage(RoomMessageEventContent::text_html( + html_to_markdown(&message), + message, + )) + } + } +} + +// Helper for `RoomMessageEventContent::text_html`, which needs the content as +// both markdown and HTML. +fn html_to_markdown(text: &str) -> String { + text.replace("

      ", "") + .replace("

      ", "\n") + .replace("
      ", "```\n")
+        .replace("
      ", "\n```") + .replace("", "`") + .replace("", "`") + .replace("
    • ", "* ") + .replace("
      • ", "") + .replace("
        \n", "") + .replace("
      \n", "") +} + +const HELP_TEXT: &'static str = r#" +

      The following commands are available:

      +
        +
      • register_appservice: Register a bridge using its registration YAML
        • +
      • unregister_appservice: Unregister a bridge using its ID
        • +
      • list_appservices: List all the currently registered bridges
        • +
      • get_auth_chain: Get the `auth_chain` of a PDU
        • +
      • parse_pdu: Parse and print a PDU from a JSON
        • +
      • get_pdu: Retrieve and print a PDU by ID from the Conduit database
        • +
      • database_memory_usage: Print database memory usage statistics
        • +
          +"#; + +pub fn try_parse_admin_command( + db: &Database, + command: &str, + args: Vec<&str>, + body: Vec<&str>, +) -> Result { + let command = match command { + "help" => AdminCommand::SendMessage(RoomMessageEventContent::text_html( + html_to_markdown(HELP_TEXT), + HELP_TEXT, + )), + "register_appservice" => { + if body.len() > 2 && body[0].trim() == "```" && body.last().unwrap().trim() == "```" { + let appservice_config = body[1..body.len() - 1].join("\n"); + let parsed_config = serde_yaml::from_str::(&appservice_config); + match parsed_config { + Ok(yaml) => AdminCommand::RegisterAppservice(yaml), + Err(e) => AdminCommand::SendMessage(RoomMessageEventContent::text_plain( + format!("Could not parse appservice config: {}", e), + )), + } + } else { + AdminCommand::SendMessage(RoomMessageEventContent::text_plain( + "Expected code block in command body.", + )) + } + } + "unregister_appservice" => { + if args.len() == 1 { + AdminCommand::UnregisterAppservice(args[0].to_owned()) + } else { + AdminCommand::SendMessage(RoomMessageEventContent::text_plain( + "Missing appservice identifier", + )) + } + } + "list_appservices" => AdminCommand::ListAppservices, + "get_auth_chain" => { + if args.len() == 1 { + if let Ok(event_id) = EventId::parse_arc(args[0]) { + if let Some(event) = db.rooms.get_pdu_json(&event_id)? { + let room_id_str = event + .get("room_id") + .and_then(|val| val.as_str()) + .ok_or_else(|| Error::bad_database("Invalid event in database"))?; + + let room_id = <&RoomId>::try_from(room_id_str).map_err(|_| { + Error::bad_database("Invalid room id field in event in database") + })?; + let start = Instant::now(); + let count = + server_server::get_auth_chain(room_id, vec![event_id], db)?.count(); + let elapsed = start.elapsed(); + return Ok(AdminCommand::SendMessage( + RoomMessageEventContent::text_plain(format!( + "Loaded auth chain with length {} in {:?}", + count, elapsed + )), + )); + } + } + } + + AdminCommand::SendMessage(RoomMessageEventContent::text_plain( + "Usage: get_auth_chain ", + )) + } + "parse_pdu" => { + if body.len() > 2 && body[0].trim() == "```" && body.last().unwrap().trim() == "```" { + let string = body[1..body.len() - 1].join("\n"); + match serde_json::from_str(&string) { + Ok(value) => { + let event_id = EventId::parse(format!( + "${}", + // Anything higher than version3 behaves the same + ruma::signatures::reference_hash(&value, &RoomVersionId::V6) + .expect("ruma can calculate reference hashes") + )) + .expect("ruma's reference hashes are valid event ids"); + + match serde_json::from_value::( + serde_json::to_value(value).expect("value is json"), + ) { + Ok(pdu) => { + AdminCommand::SendMessage(RoomMessageEventContent::text_plain( + format!("EventId: {:?}\n{:#?}", event_id, pdu), + )) + } + Err(e) => AdminCommand::SendMessage( + RoomMessageEventContent::text_plain(format!( + "EventId: {:?}\nCould not parse event: {}", + event_id, e + )), + ), + } + } + Err(e) => AdminCommand::SendMessage(RoomMessageEventContent::text_plain( + format!("Invalid json in command body: {}", e), + )), + } + } else { + AdminCommand::SendMessage(RoomMessageEventContent::text_plain( + "Expected code block in command body.", + )) + } + } + "get_pdu" => { + if args.len() == 1 { + if let Ok(event_id) = EventId::parse(args[0]) { + let mut outlier = false; + let mut pdu_json = db.rooms.get_non_outlier_pdu_json(&event_id)?; + if pdu_json.is_none() { + outlier = true; + pdu_json = db.rooms.get_pdu_json(&event_id)?; + } + match pdu_json { + Some(json) => { + let json_text = serde_json::to_string_pretty(&json) + .expect("canonical json is valid json"); + AdminCommand::SendMessage( + RoomMessageEventContent::text_html( + format!("{}\n```json\n{}\n```", + if outlier { + "PDU is outlier" + } else { "PDU was accepted"}, json_text), + format!("

          {}

          \n
          {}\n
          \n", + if outlier { + "PDU is outlier" + } else { "PDU was accepted"}, RawStr::new(&json_text).html_escape()) + ), + ) + } + None => AdminCommand::SendMessage(RoomMessageEventContent::text_plain( + "PDU not found.", + )), + } + } else { + AdminCommand::SendMessage(RoomMessageEventContent::text_plain( + "Event ID could not be parsed.", + )) + } + } else { + AdminCommand::SendMessage(RoomMessageEventContent::text_plain( + "Usage: get_pdu ", + )) + } + } + "database_memory_usage" => AdminCommand::ShowMemoryUsage, + _ => { + let message = format!( + "Unrecognized command {}, try help for a list of commands.", + command, + ); + AdminCommand::SendMessage(RoomMessageEventContent::text_html( + html_to_markdown(&message), + message, + )) + } + }; + + Ok(command) +} diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 0ba6c9ba..14df8f50 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -3,13 +3,13 @@ mod edus; pub use edus::RoomEdus; use crate::{ + database::admin::parse_admin_command, pdu::{EventHash, PduBuilder}, - server_server, utils, Database, Error, PduEvent, Result, + utils, Database, Error, PduEvent, Result, }; use lru_cache::LruCache; use regex::Regex; use ring::digest; -use rocket::http::RawStr; use ruma::{ api::{client::error::ErrorKind, federation}, events::{ @@ -19,7 +19,6 @@ use ruma::{ room::{ create::RoomCreateEventContent, member::{MembershipState, RoomMemberEventContent}, - message::RoomMessageEventContent, power_levels::RoomPowerLevelsEventContent, }, tag::TagEvent, @@ -40,12 +39,11 @@ use std::{ iter, mem::size_of, sync::{Arc, Mutex, RwLock}, - time::Instant, }; use tokio::sync::MutexGuard; use tracing::{error, warn}; -use super::{abstraction::Tree, admin::AdminCommand, pusher}; +use super::{abstraction::Tree, pusher}; /// The unique identifier of each state group. /// @@ -1496,216 +1494,8 @@ impl Rooms { let command_line = lines.next().expect("each string has at least one line"); let body: Vec<_> = lines.collect(); - let mut parts = command_line.split_whitespace().skip(1); - if let Some(command) = parts.next() { - let args: Vec<_> = parts.collect(); - - match command { - "register_appservice" => { - if body.len() > 2 - && body[0].trim() == "```" - && body.last().unwrap().trim() == "```" - { - let appservice_config = body[1..body.len() - 1].join("\n"); - let parsed_config = serde_yaml::from_str::( - &appservice_config, - ); - match parsed_config { - Ok(yaml) => { - db.admin - .send(AdminCommand::RegisterAppservice(yaml)); - } - Err(e) => { - db.admin.send(AdminCommand::SendMessage( - RoomMessageEventContent::text_plain(format!( - "Could not parse appservice config: {}", - e - )), - )); - } - } - } else { - db.admin.send(AdminCommand::SendMessage( - RoomMessageEventContent::text_plain( - "Expected code block in command body.", - ), - )); - } - } - "unregister_appservice" => { - if args.len() == 1 { - db.admin.send(AdminCommand::UnregisterAppservice( - args[0].to_owned(), - )); - } else { - db.admin.send(AdminCommand::SendMessage( - RoomMessageEventContent::text_plain( - "Missing appservice identifier", - ), - )); - } - } - "list_appservices" => { - db.admin.send(AdminCommand::ListAppservices); - } - "get_auth_chain" => { - if args.len() == 1 { - if let Ok(event_id) = EventId::parse_arc(args[0]) { - if let Some(event) = db.rooms.get_pdu_json(&event_id)? { - let room_id_str = event - .get("room_id") - .and_then(|val| val.as_str()) - .ok_or_else(|| { - Error::bad_database( - "Invalid event in database", - ) - })?; - - let room_id = <&RoomId>::try_from(room_id_str) - .map_err(|_| Error::bad_database("Invalid room id field in event in database"))?; - let start = Instant::now(); - let count = server_server::get_auth_chain( - room_id, - vec![event_id], - db, - )? - .count(); - let elapsed = start.elapsed(); - db.admin.send(AdminCommand::SendMessage( - RoomMessageEventContent::text_plain(format!( - "Loaded auth chain with length {} in {:?}", - count, elapsed - )), - )); - } - } - } - } - "parse_pdu" => { - if body.len() > 2 - && body[0].trim() == "```" - && body.last().unwrap().trim() == "```" - { - let string = body[1..body.len() - 1].join("\n"); - match serde_json::from_str(&string) { - Ok(value) => { - let event_id = EventId::parse(format!( - "${}", - // Anything higher than version3 behaves the same - ruma::signatures::reference_hash( - &value, - &RoomVersionId::V6 - ) - .expect("ruma can calculate reference hashes") - )) - .expect( - "ruma's reference hashes are valid event ids", - ); - - match serde_json::from_value::( - serde_json::to_value(value) - .expect("value is json"), - ) { - Ok(pdu) => { - db.admin.send(AdminCommand::SendMessage( - RoomMessageEventContent::text_plain( - format!( - "EventId: {:?}\n{:#?}", - event_id, pdu - ), - ), - )); - } - Err(e) => { - db.admin.send(AdminCommand::SendMessage( - RoomMessageEventContent::text_plain( - format!("EventId: {:?}\nCould not parse event: {}", event_id, e), - ), - )); - } - } - } - Err(e) => { - db.admin.send(AdminCommand::SendMessage( - RoomMessageEventContent::text_plain(format!( - "Invalid json in command body: {}", - e - )), - )); - } - } - } else { - db.admin.send(AdminCommand::SendMessage( - RoomMessageEventContent::text_plain( - "Expected code block in command body.", - ), - )); - } - } - "get_pdu" => { - if args.len() == 1 { - if let Ok(event_id) = EventId::parse(args[0]) { - let mut outlier = false; - let mut pdu_json = - db.rooms.get_non_outlier_pdu_json(&event_id)?; - if pdu_json.is_none() { - outlier = true; - pdu_json = db.rooms.get_pdu_json(&event_id)?; - } - match pdu_json { - Some(json) => { - let json_text = - serde_json::to_string_pretty(&json) - .expect("canonical json is valid json"); - db.admin.send(AdminCommand::SendMessage( - RoomMessageEventContent::text_html( - format!("{}\n```json\n{}\n```", - if outlier { - "PDU is outlier" - } else { "PDU was accepted"}, json_text), - format!("

          {}

          \n
          {}\n
          \n", - if outlier { - "PDU is outlier" - } else { "PDU was accepted"}, RawStr::new(&json_text).html_escape()) - ), - )); - } - None => { - db.admin.send(AdminCommand::SendMessage( - RoomMessageEventContent::text_plain( - "PDU not found.", - ), - )); - } - } - } else { - db.admin.send(AdminCommand::SendMessage( - RoomMessageEventContent::text_plain( - "Event ID could not be parsed.", - ), - )); - } - } else { - db.admin.send(AdminCommand::SendMessage( - RoomMessageEventContent::text_plain( - "Usage: get_pdu ", - ), - )); - } - } - "database_memory_usage" => { - db.admin.send(AdminCommand::ShowMemoryUsage); - } - _ => { - db.admin.send(AdminCommand::SendMessage( - RoomMessageEventContent::text_plain(format!( - "Unrecognized command: {}", - command - )), - )); - } - } - } + let command = parse_admin_command(db, command_line, body); + db.admin.send(command); } } } From 3e79d154957211b98343f18077282b6ab5e6d36e Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Sun, 16 Jan 2022 20:15:53 +0100 Subject: [PATCH 119/201] Updated function documentation --- src/database/users.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/database/users.rs b/src/database/users.rs index e510140d..9fe4a4ee 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -125,28 +125,27 @@ impl Users { /// Returns a list of local users as list of usernames. /// - /// A user account is considered `local` if the length of it's password - /// is greater then zero. - /// If utils::string_from_bytes returns an error that username will be skipped + /// A user account is considered `local` if the length of it's password is greater then zero. #[tracing::instrument(skip(self))] pub fn get_local_users(&self) -> Result> { let users: Vec = self .userid_password .iter() - .filter_map(|(username, pw)| self.get_username_on_valid_password(&username, &pw)) + .filter_map(|(username, pw)| self.get_username_with_valid_password(&username, &pw)) .collect(); Ok(users) } - /// A private helper to avoid double filtering the iterator + /// A private helper to avoid double filtering the iterator in get_local_users(). + /// If utils::string_from_bytes(...) returns an error that username will be skipped + /// and the error will be logged. TODO: add error cause. #[tracing::instrument(skip(self))] - fn get_username_on_valid_password(&self, username: &[u8], password: &[u8]) -> Option { + fn get_username_with_valid_password(&self, username: &[u8], password: &[u8]) -> Option { // A valid password is not empty if password.len() > 0 { match utils::string_from_bytes(username) { Ok(u) => Some(u), Err(_) => { - // TODO: add error cause! // let msg: String = format!( // "Failed to parse username while calling get_local_users(): {}", // e.to_string() From 52284ef9e2de88798408913b73d6b783768e13f3 Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Sun, 16 Jan 2022 20:25:16 +0100 Subject: [PATCH 120/201] Add some debug/info if user was found --- src/database/users.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/database/users.rs b/src/database/users.rs index 9fe4a4ee..f73c1c8b 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -10,6 +10,7 @@ use ruma::{ }; use std::{collections::BTreeMap, convert::TryInto, mem, sync::Arc}; use tracing::warn; +use tracing::info; use super::abstraction::Tree; @@ -144,7 +145,10 @@ impl Users { // A valid password is not empty if password.len() > 0 { match utils::string_from_bytes(username) { - Ok(u) => Some(u), + Ok(u) => { + info!("list_local_users_test: found user {}", u); + Some(u) + }, Err(_) => { // let msg: String = format!( // "Failed to parse username while calling get_local_users(): {}", From 50430cf4ab8c742b3942b2735f0d264b17be936e Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Sun, 16 Jan 2022 21:22:57 +0100 Subject: [PATCH 121/201] Name function after command: list_local_users --- src/database/admin.rs | 2 +- src/database/users.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index 7799ffa2..3b347b10 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -95,7 +95,7 @@ impl Admin { match event { AdminCommand::ListLocalUsers => { - match guard.users.get_local_users() { + match guard.users.list_local_users() { Ok(users) => { let mut msg: String = format!("Found {} local user account(s):\n", users.len()); msg += &users.join("\n"); diff --git a/src/database/users.rs b/src/database/users.rs index 9fe4a4ee..645e54c0 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -127,7 +127,7 @@ impl Users { /// /// A user account is considered `local` if the length of it's password is greater then zero. #[tracing::instrument(skip(self))] - pub fn get_local_users(&self) -> Result> { + pub fn list_local_users(&self) -> Result> { let users: Vec = self .userid_password .iter() From 10f1da12bfa17c05ae219913c411fd3c27dc3a29 Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Sun, 16 Jan 2022 20:57:23 +0000 Subject: [PATCH 122/201] CI: Fix cargo-test --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f47327b8..73a1a928 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -218,7 +218,7 @@ test:cargo: before_script: # - mkdir -p $CARGO_HOME - apt-get update -yqq - - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config + - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config libclang-dev - rustup component add clippy rustfmt - curl "https://faulty-storage.de/gitlab-report" --output ./gitlab-report && chmod +x ./gitlab-report # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: From ee8e72f7a809cfbe58697ad69aff437d35e1404f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 17 Jan 2022 14:35:38 +0100 Subject: [PATCH 123/201] feat: implement server ACLs --- Cargo.lock | 48 +++++++---- Cargo.toml | 2 +- src/client_server/membership.rs | 4 +- src/client_server/message.rs | 4 +- src/client_server/state.rs | 4 +- src/client_server/to_device.rs | 4 +- src/database/abstraction/rocksdb.rs | 2 +- src/database/sending.rs | 8 +- src/database/transaction_ids.rs | 6 +- src/server_server.rs | 126 +++++++++++++++++++++++----- 10 files changed, 150 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d297102c..5be10f14 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2086,7 +2086,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" +source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" dependencies = [ "assign", "js_int", @@ -2107,7 +2107,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.18.5" -source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" +source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" dependencies = [ "bytes", "http", @@ -2123,7 +2123,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.18.5" -source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" +source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2134,7 +2134,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" +source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" dependencies = [ "ruma-api", "ruma-common", @@ -2148,7 +2148,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.12.3" -source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" +source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" dependencies = [ "assign", "bytes", @@ -2168,7 +2168,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.6.0" -source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" +source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" dependencies = [ "indexmap", "js_int", @@ -2183,7 +2183,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.24.6" -source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" +source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" dependencies = [ "indoc", "js_int", @@ -2194,12 +2194,13 @@ dependencies = [ "serde", "serde_json", "thiserror", + "wildmatch", ] [[package]] name = "ruma-events-macros" version = "0.24.6" -source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" +source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2210,7 +2211,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.3.1" -source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" +source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" dependencies = [ "js_int", "ruma-api", @@ -2225,7 +2226,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.20.0" -source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" +source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" dependencies = [ "percent-encoding", "rand 0.8.4", @@ -2234,12 +2235,13 @@ dependencies = [ "ruma-serde", "ruma-serde-macros", "serde", + "uuid", ] [[package]] name = "ruma-identifiers-macros" version = "0.20.0" -source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" +source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" dependencies = [ "quote", "ruma-identifiers-validation", @@ -2249,7 +2251,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" +source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" dependencies = [ "thiserror", ] @@ -2257,7 +2259,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" +source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" dependencies = [ "js_int", "ruma-api", @@ -2270,7 +2272,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" +source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" dependencies = [ "js_int", "ruma-api", @@ -2285,8 +2287,9 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" +source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" dependencies = [ + "base64 0.13.0", "bytes", "form_urlencoded", "itoa 0.4.8", @@ -2299,7 +2302,7 @@ dependencies = [ [[package]] name = "ruma-serde-macros" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" +source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2310,7 +2313,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.9.0" -source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" +source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" dependencies = [ "base64 0.13.0", "ed25519-dalek", @@ -2327,7 +2330,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.4.1" -source = "git+https://github.com/ruma/ruma?rev=f8ba7f795765bf4aeb4db06849f9fdde9c162ac3#f8ba7f795765bf4aeb4db06849f9fdde9c162ac3" +source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" dependencies = [ "itertools", "js_int", @@ -3308,6 +3311,15 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom 0.2.3", +] + [[package]] name = "vcpkg" version = "0.2.15" diff --git a/Cargo.toml b/Cargo.toml index c87d949c..29a090c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle request # Used for matrix spec type definitions and helpers #ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } -ruma = { git = "https://github.com/ruma/ruma", rev = "f8ba7f795765bf4aeb4db06849f9fdde9c162ac3", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +ruma = { git = "https://github.com/ruma/ruma", rev = "08d60b3d376b63462f769d4b9bd3bbfb560d501a", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { git = "https://github.com/timokoesters/ruma", rev = "50c1db7e0a3a21fc794b0cce3b64285a4c750c71", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index cede51f0..70352784 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -23,7 +23,7 @@ use ruma::{ }, EventType, }, - serde::{to_canonical_value, CanonicalJsonObject, CanonicalJsonValue}, + serde::{to_canonical_value, Base64, CanonicalJsonObject, CanonicalJsonValue}, state_res::{self, RoomVersion}, uint, EventId, RoomId, RoomVersionId, ServerName, UserId, }; @@ -787,7 +787,7 @@ async fn join_room_by_id_helper( fn validate_and_add_event_id( pdu: &RawJsonValue, room_version: &RoomVersionId, - pub_key_map: &RwLock>>, + pub_key_map: &RwLock>>, db: &Database, ) -> Result<(Box, CanonicalJsonObject)> { let mut value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| { diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 9705e4c0..36653fab 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -74,11 +74,11 @@ pub async fn send_message_event_route( } let mut unsigned = BTreeMap::new(); - unsigned.insert("transaction_id".to_owned(), body.txn_id.clone().into()); + unsigned.insert("transaction_id".to_owned(), body.txn_id.to_string().into()); let event_id = db.rooms.build_and_append_pdu( PduBuilder { - event_type: EventType::from(&body.event_type), + event_type: EventType::from(&*body.event_type), content: serde_json::from_str(body.body.body.json().get()) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?, unsigned: Some(unsigned), diff --git a/src/client_server/state.rs b/src/client_server/state.rs index e42694ae..c07d4825 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -44,7 +44,7 @@ pub async fn send_state_event_for_key_route( &db, sender_user, &body.room_id, - EventType::from(&body.event_type), + EventType::from(&*body.event_type), &body.body.body, // Yes, I hate it too body.state_key.to_owned(), ) @@ -86,7 +86,7 @@ pub async fn send_state_event_for_empty_key_route( &db, sender_user, &body.room_id, - EventType::from(&body.event_type), + EventType::from(&*body.event_type), &body.body.body, body.state_key.to_owned(), ) diff --git a/src/client_server/to_device.rs b/src/client_server/to_device.rs index 177b1234..6e764deb 100644 --- a/src/client_server/to_device.rs +++ b/src/client_server/to_device.rs @@ -53,8 +53,8 @@ pub async fn send_event_to_device_route( serde_json::to_vec(&federation::transactions::edu::Edu::DirectToDevice( DirectDeviceContent { sender: sender_user.clone(), - ev_type: EventType::from(&body.event_type), - message_id: body.txn_id.clone(), + ev_type: EventType::from(&*body.event_type), + message_id: body.txn_id.to_string(), messages, }, )) diff --git a/src/database/abstraction/rocksdb.rs b/src/database/abstraction/rocksdb.rs index adda6787..15ea9f73 100644 --- a/src/database/abstraction/rocksdb.rs +++ b/src/database/abstraction/rocksdb.rs @@ -44,7 +44,7 @@ fn db_options( db_opts.set_max_open_files(max_open_files); db_opts.set_compression_type(rocksdb::DBCompressionType::Zstd); db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level); - db_opts.optimize_level_style_compaction(cache_capacity_bytes); + db_opts.optimize_level_style_compaction(10 * 1024 * 1024); let prefix_extractor = rocksdb::SliceTransform::create_fixed_prefix(1); db_opts.set_prefix_extractor(prefix_extractor); diff --git a/src/database/sending.rs b/src/database/sending.rs index 1e180d43..65284a4f 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -524,7 +524,7 @@ impl Sending { .unwrap(), // TODO: handle error appservice::event::push_events::v1::Request { events: &pdu_jsons, - txn_id: &base64::encode_config( + txn_id: (&*base64::encode_config( Self::calculate_hash( &events .iter() @@ -534,7 +534,7 @@ impl Sending { .collect::>(), ), base64::URL_SAFE_NO_PAD, - ), + )).into(), }, ) .await @@ -682,7 +682,7 @@ impl Sending { pdus: &pdu_jsons, edus: &edu_jsons, origin_server_ts: MilliSecondsSinceUnixEpoch::now(), - transaction_id: &base64::encode_config( + transaction_id: (&*base64::encode_config( Self::calculate_hash( &events .iter() @@ -692,7 +692,7 @@ impl Sending { .collect::>(), ), base64::URL_SAFE_NO_PAD, - ), + )).into(), }, ) .await diff --git a/src/database/transaction_ids.rs b/src/database/transaction_ids.rs index f3467572..d576083a 100644 --- a/src/database/transaction_ids.rs +++ b/src/database/transaction_ids.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use crate::Result; -use ruma::{DeviceId, UserId}; +use ruma::{DeviceId, UserId, identifiers::TransactionId}; use super::abstraction::Tree; @@ -14,7 +14,7 @@ impl TransactionIds { &self, user_id: &UserId, device_id: Option<&DeviceId>, - txn_id: &str, + txn_id: &TransactionId, data: &[u8], ) -> Result<()> { let mut key = user_id.as_bytes().to_vec(); @@ -32,7 +32,7 @@ impl TransactionIds { &self, user_id: &UserId, device_id: Option<&DeviceId>, - txn_id: &str, + txn_id: &TransactionId, ) -> Result>> { let mut key = user_id.as_bytes().to_vec(); key.push(0xff); diff --git a/src/server_server.rs b/src/server_server.rs index c76afd34..5cd43d81 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -42,6 +42,7 @@ use ruma::{ events::{ receipt::{ReceiptEvent, ReceiptEventContent}, room::{ + server_acl::RoomServerAclEventContent, create::RoomCreateEventContent, member::{MembershipState, RoomMemberEventContent}, }, @@ -49,7 +50,7 @@ use ruma::{ }, int, receipt::ReceiptType, - serde::JsonObject, + serde::{Base64, JsonObject}, signatures::{CanonicalJsonObject, CanonicalJsonValue}, state_res::{self, RoomVersion, StateMap}, to_device::DeviceIdOrAllDevices, @@ -551,7 +552,7 @@ pub fn get_server_keys_route(db: DatabaseGuard) -> Json { .try_into() .expect("found invalid server signing keys in DB"), VerifyKey { - key: base64::encode_config(db.globals.keypair().public_key(), base64::STANDARD_NO_PAD), + key: Base64::new(db.globals.keypair().public_key().to_vec()), }, ); let mut response = serde_json::from_slice( @@ -740,6 +741,8 @@ pub async fn send_transaction_message_route( } }; + acl_check(&body.origin, &room_id, &db)?; + let mutex = Arc::clone( db.globals .roomid_mutex_federation @@ -854,7 +857,7 @@ pub async fn send_transaction_message_route( // Check if this is a new transaction id if db .transaction_ids - .existing_txnid(&sender, None, &message_id)? + .existing_txnid(&sender, None, (&*message_id).into())? .is_some() { continue; @@ -902,7 +905,7 @@ pub async fn send_transaction_message_route( // Save transaction id with empty data db.transaction_ids - .add_txnid(&sender, None, &message_id, &[])?; + .add_txnid(&sender, None, (&*message_id).into(), &[])?; } Edu::_Custom(_) => {} } @@ -948,7 +951,7 @@ pub(crate) async fn handle_incoming_pdu<'a>( value: BTreeMap, is_timeline_event: bool, db: &'a Database, - pub_key_map: &'a RwLock>>, + pub_key_map: &'a RwLock>>, ) -> Result>, String> { match db.rooms.exists(room_id) { Ok(true) => {} @@ -1123,7 +1126,7 @@ fn handle_outlier_pdu<'a>( room_id: &'a RoomId, value: BTreeMap, db: &'a Database, - pub_key_map: &'a RwLock>>, + pub_key_map: &'a RwLock>>, ) -> AsyncRecursiveType<'a, Result<(Arc, BTreeMap), String>> { Box::pin(async move { // TODO: For RoomVersion6 we must check that Raw<..> is canonical do we anywhere?: https://matrix.org/docs/spec/rooms/v6#canonical-json @@ -1285,7 +1288,7 @@ async fn upgrade_outlier_to_timeline_pdu( origin: &ServerName, db: &Database, room_id: &RoomId, - pub_key_map: &RwLock>>, + pub_key_map: &RwLock>>, ) -> Result>, String> { if let Ok(Some(pduid)) = db.rooms.get_pdu_id(&incoming_pdu.event_id) { return Ok(Some(pduid)); @@ -1827,7 +1830,7 @@ pub(crate) fn fetch_and_handle_outliers<'a>( events: &'a [Arc], create_event: &'a PduEvent, room_id: &'a RoomId, - pub_key_map: &'a RwLock>>, + pub_key_map: &'a RwLock>>, ) -> AsyncRecursiveType<'a, Vec<(Arc, Option>)>> { Box::pin(async move { let back_off = |id| match db.globals.bad_event_ratelimiter.write().unwrap().entry(id) { @@ -1966,9 +1969,9 @@ pub(crate) async fn fetch_signing_keys( db: &Database, origin: &ServerName, signature_ids: Vec, -) -> Result> { +) -> Result> { let contains_all_ids = - |keys: &BTreeMap| signature_ids.iter().all(|id| keys.contains_key(id)); + |keys: &BTreeMap| signature_ids.iter().all(|id| keys.contains_key(id)); let permit = db .globals @@ -2355,8 +2358,11 @@ pub fn get_event_route( let room_id = <&RoomId>::try_from(room_id_str) .map_err(|_| Error::bad_database("Invalid room id field in event in database"))?; - if !db.rooms.server_in_room(sender_servername, room_id)? { - return Err(Error::BadRequest(ErrorKind::NotFound, "Event not found.")); + if !db.rooms.server_in_room(sender_servername, &room_id)? { + return Err(Error::BadRequest( + ErrorKind::Forbidden, + "Server is not in room", + )); } Ok(get_event::v1::Response { @@ -2395,6 +2401,8 @@ pub fn get_missing_events_route( )); } + acl_check(sender_servername, &body.room_id, &db)?; + let mut queued_events = body.latest_events.clone(); let mut events = Vec::new(); @@ -2464,6 +2472,15 @@ pub fn get_event_authorization_route( .as_ref() .expect("server is authenticated"); + if !db.rooms.server_in_room(sender_servername, &body.room_id)? { + return Err(Error::BadRequest( + ErrorKind::Forbidden, + "Server is not in room.", + )); + } + + acl_check(sender_servername, &body.room_id, &db)?; + let event = db .rooms .get_pdu_json(&body.event_id)? @@ -2477,10 +2494,6 @@ pub fn get_event_authorization_route( let room_id = <&RoomId>::try_from(room_id_str) .map_err(|_| Error::bad_database("Invalid room id field in event in database"))?; - if !db.rooms.server_in_room(sender_servername, room_id)? { - return Err(Error::BadRequest(ErrorKind::NotFound, "Event not found.")); - } - let auth_chain_ids = get_auth_chain(room_id, vec![Arc::from(&*body.event_id)], &db)?; Ok(get_event_authorization::v1::Response { @@ -2520,6 +2533,8 @@ pub fn get_room_state_route( )); } + acl_check(sender_servername, &body.room_id, &db)?; + let shortstatehash = db .rooms .pdu_shortstatehash(&body.event_id)? @@ -2583,6 +2598,8 @@ pub fn get_room_state_ids_route( )); } + acl_check(sender_servername, &body.room_id, &db)?; + let shortstatehash = db .rooms .pdu_shortstatehash(&body.event_id)? @@ -2626,10 +2643,17 @@ pub fn create_join_event_template_route( if !db.rooms.exists(&body.room_id)? { return Err(Error::BadRequest( ErrorKind::NotFound, - "Server is not in room.", + "Room is unknown to this server.", )); } + let sender_servername = body + .sender_servername + .as_ref() + .expect("server is authenticated"); + + acl_check(sender_servername, &body.room_id, &db)?; + let prev_events: Vec<_> = db .rooms .get_pdu_leaves(&body.room_id)? @@ -2782,6 +2806,7 @@ pub fn create_join_event_template_route( async fn create_join_event( db: &DatabaseGuard, + sender_servername: &ServerName, room_id: &RoomId, pdu: &RawJsonValue, ) -> Result { @@ -2789,6 +2814,15 @@ async fn create_join_event( return Err(Error::bad_config("Federation is disabled.")); } + if !db.rooms.exists(room_id)? { + return Err(Error::BadRequest( + ErrorKind::NotFound, + "Room is unknown to this server.", + )); + } + + acl_check(sender_servername, room_id, &db)?; + // We need to return the state prior to joining, let's keep a reference to that here let shortstatehash = db .rooms @@ -2888,7 +2922,12 @@ pub async fn create_join_event_v1_route( db: DatabaseGuard, body: Ruma>, ) -> ConduitResult { - let room_state = create_join_event(&db, &body.room_id, &body.pdu).await?; + let sender_servername = body + .sender_servername + .as_ref() + .expect("server is authenticated"); + + let room_state = create_join_event(&db, sender_servername, &body.room_id, &body.pdu).await?; Ok(create_join_event::v1::Response { room_state }.into()) } @@ -2905,7 +2944,12 @@ pub async fn create_join_event_v2_route( db: DatabaseGuard, body: Ruma>, ) -> ConduitResult { - let room_state = create_join_event(&db, &body.room_id, &body.pdu).await?; + let sender_servername = body + .sender_servername + .as_ref() + .expect("server is authenticated"); + + let room_state = create_join_event(&db, sender_servername, &body.room_id, &body.pdu).await?; Ok(create_join_event::v2::Response { room_state }.into()) } @@ -2926,6 +2970,13 @@ pub async fn create_invite_route( return Err(Error::bad_config("Federation is disabled.")); } + let sender_servername = body + .sender_servername + .as_ref() + .expect("server is authenticated"); + + acl_check(sender_servername, &body.room_id, &db)?; + if body.room_version != RoomVersionId::V5 && body.room_version != RoomVersionId::V6 { return Err(Error::BadRequest( ErrorKind::IncompatibleRoomVersion { @@ -3199,7 +3250,7 @@ pub async fn claim_keys_route( #[tracing::instrument(skip(event, pub_key_map, db))] pub(crate) async fn fetch_required_signing_keys( event: &BTreeMap, - pub_key_map: &RwLock>>, + pub_key_map: &RwLock>>, db: &Database, ) -> Result<()> { let signatures = event @@ -3253,7 +3304,7 @@ fn get_server_keys_from_cache( pdu: &RawJsonValue, servers: &mut BTreeMap, BTreeMap, QueryCriteria>>, room_version: &RoomVersionId, - pub_key_map: &mut RwLockWriteGuard<'_, BTreeMap>>, + pub_key_map: &mut RwLockWriteGuard<'_, BTreeMap>>, db: &Database, ) -> Result<()> { let value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| { @@ -3306,7 +3357,7 @@ fn get_server_keys_from_cache( let signature_ids = signature_object.keys().cloned().collect::>(); let contains_all_ids = - |keys: &BTreeMap| signature_ids.iter().all(|id| keys.contains_key(id)); + |keys: &BTreeMap| signature_ids.iter().all(|id| keys.contains_key(id)); let origin = <&ServerName>::try_from(signature_server.as_str()).map_err(|_| { Error::BadServerResponse("Invalid servername in signatures of server response pdu.") @@ -3339,7 +3390,7 @@ fn get_server_keys_from_cache( pub(crate) async fn fetch_join_signing_keys( event: &create_join_event::v2::Response, room_version: &RoomVersionId, - pub_key_map: &RwLock>>, + pub_key_map: &RwLock>>, db: &Database, ) -> Result<()> { let mut servers: BTreeMap, BTreeMap, QueryCriteria>> = @@ -3439,6 +3490,35 @@ pub(crate) async fn fetch_join_signing_keys( Ok(()) } +/// Returns Ok if the acl allows the server +fn acl_check( + server_name: &ServerName, + room_id: &RoomId, + db: &Database, +) -> Result<()> { + let acl_event = match db + .rooms + .room_state_get(room_id, &EventType::RoomServerAcl, "")? { + Some(acl) => acl, + None => return Ok(()), + }; + + let acl_event_content: RoomServerAclEventContent = match + serde_json::from_str(acl_event.content.get()) { + Ok(content) => content, + Err(_) => { + warn!("Invalid ACL event"); + return Ok(()); + } + }; + + if acl_event_content.is_allowed(server_name) { + Ok(()) + } else { + Err(Error::BadRequest(ErrorKind::Forbidden, "Server was denied by ACL")) + } +} + #[cfg(test)] mod tests { use super::{add_port_to_hostname, get_ip_with_port, FedDest}; From 8c90e7adfb0d06164d17921e6e686cdaab0d8f1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 17 Jan 2022 14:39:37 +0100 Subject: [PATCH 124/201] refactor: fix warnings --- src/database/abstraction/rocksdb.rs | 27 ++++++--------------------- src/database/sending.rs | 6 ++++-- src/database/transaction_ids.rs | 2 +- src/server_server.rs | 26 +++++++++++++------------- 4 files changed, 24 insertions(+), 37 deletions(-) diff --git a/src/database/abstraction/rocksdb.rs b/src/database/abstraction/rocksdb.rs index 15ea9f73..d6157135 100644 --- a/src/database/abstraction/rocksdb.rs +++ b/src/database/abstraction/rocksdb.rs @@ -4,7 +4,6 @@ use std::{future::Future, pin::Pin, sync::Arc, sync::RwLock}; pub struct Engine { rocks: rocksdb::DBWithThreadMode, - cache_capacity_bytes: usize, max_open_files: i32, cache: rocksdb::Cache, old_cfs: Vec, @@ -17,11 +16,7 @@ pub struct RocksDbEngineTree<'a> { write_lock: RwLock<()>, } -fn db_options( - cache_capacity_bytes: usize, - max_open_files: i32, - rocksdb_cache: &rocksdb::Cache, -) -> rocksdb::Options { +fn db_options(max_open_files: i32, rocksdb_cache: &rocksdb::Cache) -> rocksdb::Options { let mut block_based_options = rocksdb::BlockBasedOptions::default(); block_based_options.set_block_cache(rocksdb_cache); @@ -57,11 +52,7 @@ impl DatabaseEngine for Arc { let cache_capacity_bytes = (config.db_cache_capacity_mb * 1024.0 * 1024.0) as usize; let rocksdb_cache = rocksdb::Cache::new_lru_cache(cache_capacity_bytes).unwrap(); - let db_opts = db_options( - cache_capacity_bytes, - config.rocksdb_max_open_files, - &rocksdb_cache, - ); + let db_opts = db_options(config.rocksdb_max_open_files, &rocksdb_cache); let cfs = rocksdb::DBWithThreadMode::::list_cf( &db_opts, @@ -75,18 +66,13 @@ impl DatabaseEngine for Arc { cfs.iter().map(|name| { rocksdb::ColumnFamilyDescriptor::new( name, - db_options( - cache_capacity_bytes, - config.rocksdb_max_open_files, - &rocksdb_cache, - ), + db_options(config.rocksdb_max_open_files, &rocksdb_cache), ) }), )?; Ok(Arc::new(Engine { rocks: db, - cache_capacity_bytes, max_open_files: config.rocksdb_max_open_files, cache: rocksdb_cache, old_cfs: cfs, @@ -96,10 +82,9 @@ impl DatabaseEngine for Arc { fn open_tree(&self, name: &'static str) -> Result> { if !self.old_cfs.contains(&name.to_owned()) { // Create if it didn't exist - let _ = self.rocks.create_cf( - name, - &db_options(self.cache_capacity_bytes, self.max_open_files, &self.cache), - ); + let _ = self + .rocks + .create_cf(name, &db_options(self.max_open_files, &self.cache)); } Ok(Arc::new(RocksDbEngineTree { diff --git a/src/database/sending.rs b/src/database/sending.rs index 65284a4f..69f7c444 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -534,7 +534,8 @@ impl Sending { .collect::>(), ), base64::URL_SAFE_NO_PAD, - )).into(), + )) + .into(), }, ) .await @@ -692,7 +693,8 @@ impl Sending { .collect::>(), ), base64::URL_SAFE_NO_PAD, - )).into(), + )) + .into(), }, ) .await diff --git a/src/database/transaction_ids.rs b/src/database/transaction_ids.rs index d576083a..12b838ba 100644 --- a/src/database/transaction_ids.rs +++ b/src/database/transaction_ids.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use crate::Result; -use ruma::{DeviceId, UserId, identifiers::TransactionId}; +use ruma::{identifiers::TransactionId, DeviceId, UserId}; use super::abstraction::Tree; diff --git a/src/server_server.rs b/src/server_server.rs index 5cd43d81..54ae0251 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -42,9 +42,9 @@ use ruma::{ events::{ receipt::{ReceiptEvent, ReceiptEventContent}, room::{ - server_acl::RoomServerAclEventContent, create::RoomCreateEventContent, member::{MembershipState, RoomMemberEventContent}, + server_acl::RoomServerAclEventContent, }, AnyEphemeralRoomEvent, EventType, }, @@ -3491,20 +3491,17 @@ pub(crate) async fn fetch_join_signing_keys( } /// Returns Ok if the acl allows the server -fn acl_check( - server_name: &ServerName, - room_id: &RoomId, - db: &Database, -) -> Result<()> { +fn acl_check(server_name: &ServerName, room_id: &RoomId, db: &Database) -> Result<()> { let acl_event = match db .rooms - .room_state_get(room_id, &EventType::RoomServerAcl, "")? { - Some(acl) => acl, - None => return Ok(()), - }; + .room_state_get(room_id, &EventType::RoomServerAcl, "")? + { + Some(acl) => acl, + None => return Ok(()), + }; - let acl_event_content: RoomServerAclEventContent = match - serde_json::from_str(acl_event.content.get()) { + let acl_event_content: RoomServerAclEventContent = + match serde_json::from_str(acl_event.content.get()) { Ok(content) => content, Err(_) => { warn!("Invalid ACL event"); @@ -3515,7 +3512,10 @@ fn acl_check( if acl_event_content.is_allowed(server_name) { Ok(()) } else { - Err(Error::BadRequest(ErrorKind::Forbidden, "Server was denied by ACL")) + Err(Error::BadRequest( + ErrorKind::Forbidden, + "Server was denied by ACL", + )) } } From 03b174335cfc472c3ecaba7068ead74f0e2268be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 17 Jan 2022 14:46:53 +0100 Subject: [PATCH 125/201] improvement: lower default pdu cache capacity --- src/database.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database.rs b/src/database.rs index fd7a1451..1997dc0a 100644 --- a/src/database.rs +++ b/src/database.rs @@ -134,7 +134,7 @@ fn default_rocksdb_max_open_files() -> i32 { } fn default_pdu_cache_capacity() -> u32 { - 1_000_000 + 150_000 } fn default_cleanup_second_interval() -> u32 { From fc39b3447c5add8b8c8d2b188751969d752d1ee1 Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Mon, 17 Jan 2022 19:43:45 +0100 Subject: [PATCH 126/201] Little bit of refactoring --- src/database/users.rs | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/src/database/users.rs b/src/database/users.rs index 83c1520e..e608673a 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -10,7 +10,6 @@ use ruma::{ }; use std::{collections::BTreeMap, convert::TryInto, mem, sync::Arc}; use tracing::warn; -use tracing::info; use super::abstraction::Tree; @@ -137,31 +136,23 @@ impl Users { Ok(users) } - /// A private helper to avoid double filtering the iterator in get_local_users(). + /// Will only return with Some(username) if the password was not empty and the + /// username could be successfully parsed. /// If utils::string_from_bytes(...) returns an error that username will be skipped - /// and the error will be logged. TODO: add error cause. + /// and the error will be logged. #[tracing::instrument(skip(self))] fn get_username_with_valid_password(&self, username: &[u8], password: &[u8]) -> Option { // A valid password is not empty - if password.len() > 0 { + if password.is_empty() { + None + } else { match utils::string_from_bytes(username) { - Ok(u) => { - info!("list_local_users_test: found user {}", u); - Some(u) - }, - Err(_) => { - // let msg: String = format!( - // "Failed to parse username while calling get_local_users(): {}", - // e.to_string() - // ); - Error::bad_database( - "Failed to parse username while calling get_username_on_valid_password", - ); + Ok(u) => Some(u), + Err(e) => { + warn!("Failed to parse username while calling get_local_users(): {}", e.to_string()); None } } - } else { - None } } From fd6427a83fef08b7f8f6f74d3a4c88af3171aa77 Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Mon, 17 Jan 2022 22:34:34 +0100 Subject: [PATCH 127/201] Update/Revert code comment --- src/database/users.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/database/users.rs b/src/database/users.rs index e608673a..9b986d4e 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -77,8 +77,6 @@ impl Users { } /// Returns the number of users registered on this server. - /// It really returns all users, not only real ones with a - /// password to login but also bridge puppets... #[tracing::instrument(skip(self))] pub fn count(&self) -> Result { Ok(self.userid_password.iter().count()) From 53de3509087f46b6a45ca20d27e8fa2884269535 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 17 Jan 2022 23:24:27 +0100 Subject: [PATCH 128/201] fix: less load when lazy loading --- src/client_server/sync.rs | 53 ++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index bd2f48a3..14aac3a1 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -453,38 +453,39 @@ async fn sync_helper( let joined_since_last_sync = since_sender_member .map_or(true, |member| member.membership != MembershipState::Join); - let current_state_ids = db.rooms.state_full_ids(current_shortstatehash)?; - - let since_state_ids = db.rooms.state_full_ids(since_shortstatehash)?; - let mut state_events = Vec::new(); let mut lazy_loaded = HashSet::new(); - for (key, id) in current_state_ids { - if body.full_state || since_state_ids.get(&key) != Some(&id) { - let pdu = match db.rooms.get_pdu(&id)? { - Some(pdu) => pdu, - None => { - error!("Pdu in state not found: {}", id); - continue; - } - }; + if since_shortstatehash != current_shortstatehash { + let current_state_ids = db.rooms.state_full_ids(current_shortstatehash)?; + let since_state_ids = db.rooms.state_full_ids(since_shortstatehash)?; - if pdu.kind == EventType::RoomMember { - match UserId::parse( - pdu.state_key - .as_ref() - .expect("State event has state key") - .clone(), - ) { - Ok(state_key_userid) => { - lazy_loaded.insert(state_key_userid); + for (key, id) in current_state_ids { + if body.full_state || since_state_ids.get(&key) != Some(&id) { + let pdu = match db.rooms.get_pdu(&id)? { + Some(pdu) => pdu, + None => { + error!("Pdu in state not found: {}", id); + continue; } - Err(e) => error!("Invalid state key for member event: {}", e), - } - } + }; - state_events.push(pdu); + if pdu.kind == EventType::RoomMember { + match UserId::parse( + pdu.state_key + .as_ref() + .expect("State event has state key") + .clone(), + ) { + Ok(state_key_userid) => { + lazy_loaded.insert(state_key_userid); + } + Err(e) => error!("Invalid state key for member event: {}", e), + } + } + + state_events.push(pdu); + } } } From 13a48c45776de19912ecd040a6434c75152802f7 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Tue, 18 Jan 2022 21:04:44 +0100 Subject: [PATCH 129/201] Clean up mod and use statements in lib.rs and main.rs --- src/lib.rs | 10 ++++++---- src/main.rs | 22 ++++------------------ 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 82b8f340..745eb394 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,21 +7,23 @@ #![allow(clippy::suspicious_else_formatting)] #![deny(clippy::dbg_macro)] -pub mod appservice_server; -pub mod client_server; +use std::ops::Deref; + mod database; mod error; mod pdu; mod ruma_wrapper; -pub mod server_server; mod utils; +pub mod appservice_server; +pub mod client_server; +pub mod server_server; + pub use database::{Config, Database}; pub use error::{Error, Result}; pub use pdu::PduEvent; pub use rocket::Config as RocketConfig; pub use ruma_wrapper::{ConduitResult, Ruma, RumaResponse}; -use std::ops::Deref; pub struct State<'r, T: Send + Sync + 'static>(pub &'r T); diff --git a/src/main.rs b/src/main.rs index 56faa3e7..d9bbc240 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,27 +7,9 @@ #![allow(clippy::suspicious_else_formatting)] #![deny(clippy::dbg_macro)] -pub mod appservice_server; -pub mod client_server; -pub mod server_server; - -mod database; -mod error; -mod pdu; -mod ruma_wrapper; -mod utils; - use std::sync::Arc; -use database::Config; -pub use database::Database; -pub use error::{Error, Result}; use opentelemetry::trace::{FutureExt, Tracer}; -pub use pdu::PduEvent; -pub use rocket::State; -use ruma::api::client::error::ErrorKind; -pub use ruma_wrapper::{ConduitResult, Ruma, RumaResponse}; - use rocket::{ catch, catchers, figment::{ @@ -36,9 +18,13 @@ use rocket::{ }, routes, Request, }; +use ruma::api::client::error::ErrorKind; use tokio::sync::RwLock; use tracing_subscriber::{prelude::*, EnvFilter}; +pub use conduit::*; // Re-export everything from the library crate +pub use rocket::State; + fn setup_rocket(config: Figment, data: Arc>) -> rocket::Rocket { rocket::custom(config) .manage(data) From c6277c72a1f75d889b47708769adf376cac9d1ea Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Tue, 18 Jan 2022 21:05:40 +0100 Subject: [PATCH 130/201] Fix warnings in database::abstraction --- src/database/abstraction.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/database/abstraction.rs b/src/database/abstraction.rs index 17bd971f..321b064f 100644 --- a/src/database/abstraction.rs +++ b/src/database/abstraction.rs @@ -23,12 +23,12 @@ pub trait DatabaseEngine: Send + Sync { where Self: Sized; fn open_tree(&self, name: &'static str) -> Result>; - fn flush(self: &Self) -> Result<()>; - fn cleanup(self: &Self) -> Result<()> { + fn flush(&self) -> Result<()>; + fn cleanup(&self) -> Result<()> { Ok(()) } - fn memory_usage(self: &Self) -> Result { - Ok("Current database engine does not support memory usage reporting.".to_string()) + fn memory_usage(&self) -> Result { + Ok("Current database engine does not support memory usage reporting.".to_owned()) } } From d4eb3e3295ee1b0947b66d1d45ef10bb4d152839 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 19 Jan 2022 07:09:25 +0100 Subject: [PATCH 131/201] fix: rocksdb does not use zstd compression unless we disable everything else --- Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 29a090c7..32233305 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -78,7 +78,8 @@ crossbeam = { version = "0.8.1", optional = true } num_cpus = "1.13.0" threadpool = "1.8.1" heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c758867e05ad973ef800a6fe1d5d", optional = true } -rocksdb = { version = "0.17.0", features = ["multi-threaded-cf"], optional = true } +rocksdb = { version = "0.17.0", default-features = false, features = ["multi-threaded-cf", "zstd"], optional = true } + thread_local = "1.1.3" # used for TURN server authentication hmac = "0.11.0" From a0fc5eba72a7b364cfe91d5b188b136fa555b7e1 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Wed, 19 Jan 2022 23:56:55 +0100 Subject: [PATCH 132/201] Remove unnecessary Result --- src/database/uiaa.rs | 7 +++---- src/ruma_wrapper.rs | 13 +++++-------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/database/uiaa.rs b/src/database/uiaa.rs index 5e11467e..b0c8d6dd 100644 --- a/src/database/uiaa.rs +++ b/src/database/uiaa.rs @@ -166,13 +166,12 @@ impl Uiaa { user_id: &UserId, device_id: &DeviceId, session: &str, - ) -> Result> { - Ok(self - .userdevicesessionid_uiaarequest + ) -> Option { + self.userdevicesessionid_uiaarequest .read() .unwrap() .get(&(user_id.to_owned(), device_id.to_owned(), session.to_owned())) - .map(|j| j.to_owned())) + .map(|j| j.to_owned()) } fn update_uiaa_session( diff --git a/src/ruma_wrapper.rs b/src/ruma_wrapper.rs index 4b8d5dea..1bd921d9 100644 --- a/src/ruma_wrapper.rs +++ b/src/ruma_wrapper.rs @@ -296,14 +296,11 @@ where .and_then(|auth| auth.get("session")) .and_then(|session| session.as_str()) .and_then(|session| { - db.uiaa - .get_uiaa_request( - &user_id, - &sender_device.clone().unwrap_or_else(|| "".into()), - session, - ) - .ok() - .flatten() + db.uiaa.get_uiaa_request( + &user_id, + &sender_device.clone().unwrap_or_else(|| "".into()), + session, + ) }) { for (key, value) in initial_request { From 756a41f22d24c89682eea826e138f8c3896433fb Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Thu, 20 Jan 2022 00:10:39 +0100 Subject: [PATCH 133/201] Fix rustc / clippy warnings --- src/client_server/context.rs | 15 +++++++-------- src/client_server/keys.rs | 2 +- src/client_server/message.rs | 14 +++++++------- src/client_server/profile.rs | 4 ++-- src/database.rs | 30 ++++++++++++------------------ src/database/admin.rs | 2 +- src/database/rooms.rs | 27 +++++++++++++-------------- src/server_server.rs | 6 +++--- 8 files changed, 46 insertions(+), 54 deletions(-) diff --git a/src/client_server/context.rs b/src/client_server/context.rs index 94a44e39..e1177661 100644 --- a/src/client_server/context.rs +++ b/src/client_server/context.rs @@ -3,8 +3,7 @@ use ruma::{ api::client::{error::ErrorKind, r0::context::get_context}, events::EventType, }; -use std::collections::HashSet; -use std::convert::TryFrom; +use std::{collections::HashSet, convert::TryFrom}; #[cfg(feature = "conduit_bin")] use rocket::get; @@ -55,8 +54,8 @@ pub async fn get_context_route( ))?; if !db.rooms.lazy_load_was_sent_before( - &sender_user, - &sender_device, + sender_user, + sender_device, &body.room_id, &base_event.sender, )? { @@ -79,8 +78,8 @@ pub async fn get_context_route( for (_, event) in &events_before { if !db.rooms.lazy_load_was_sent_before( - &sender_user, - &sender_device, + sender_user, + sender_device, &body.room_id, &event.sender, )? { @@ -112,8 +111,8 @@ pub async fn get_context_route( for (_, event) in &events_after { if !db.rooms.lazy_load_was_sent_before( - &sender_user, - &sender_device, + sender_user, + sender_device, &body.room_id, &event.sender, )? { diff --git a/src/client_server/keys.rs b/src/client_server/keys.rs index be0675d8..e7aec26b 100644 --- a/src/client_server/keys.rs +++ b/src/client_server/keys.rs @@ -272,7 +272,7 @@ pub async fn get_key_changes_route( device_list_updates.extend( db.users .keys_changed( - &sender_user.to_string(), + sender_user.as_str(), body.from .parse() .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid `from`."))?, diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 36653fab..7d904f90 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -139,7 +139,7 @@ pub async fn get_message_events_route( let to = body.to.as_ref().map(|t| t.parse()); db.rooms - .lazy_load_confirm_delivery(&sender_user, &sender_device, &body.room_id, from)?; + .lazy_load_confirm_delivery(sender_user, sender_device, &body.room_id, from)?; // Use limit or else 10 let limit = body.limit.try_into().map_or(10_usize, |l: u32| l as usize); @@ -168,8 +168,8 @@ pub async fn get_message_events_route( for (_, event) in &events_after { if !db.rooms.lazy_load_was_sent_before( - &sender_user, - &sender_device, + sender_user, + sender_device, &body.room_id, &event.sender, )? { @@ -205,8 +205,8 @@ pub async fn get_message_events_route( for (_, event) in &events_before { if !db.rooms.lazy_load_was_sent_before( - &sender_user, - &sender_device, + sender_user, + sender_device, &body.room_id, &event.sender, )? { @@ -239,8 +239,8 @@ pub async fn get_message_events_route( if let Some(next_token) = next_token { db.rooms.lazy_load_mark_sent( - &sender_user, - &sender_device, + sender_user, + sender_device, &body.room_id, lazy_loaded, next_token, diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index 29b1ae87..71e61da3 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -52,7 +52,7 @@ pub async fn set_displayname_route( .room_state_get( &room_id, &EventType::RoomMember, - &sender_user.to_string(), + sender_user.as_str(), )? .ok_or_else(|| { Error::bad_database( @@ -195,7 +195,7 @@ pub async fn set_avatar_url_route( .room_state_get( &room_id, &EventType::RoomMember, - &sender_user.to_string(), + sender_user.as_str(), )? .ok_or_else(|| { Error::bad_database( diff --git a/src/database.rs b/src/database.rs index 1997dc0a..7a4ddc66 100644 --- a/src/database.rs +++ b/src/database.rs @@ -212,28 +212,22 @@ impl Database { return Ok(()); } - if sled_exists { - if config.database_backend != "sled" { - return Err(Error::bad_config( - "Found sled at database_path, but is not specified in config.", - )); - } + if sled_exists && config.database_backend != "sled" { + return Err(Error::bad_config( + "Found sled at database_path, but is not specified in config.", + )); } - if sqlite_exists { - if config.database_backend != "sqlite" { - return Err(Error::bad_config( - "Found sqlite at database_path, but is not specified in config.", - )); - } + if sqlite_exists && config.database_backend != "sqlite" { + return Err(Error::bad_config( + "Found sqlite at database_path, but is not specified in config.", + )); } - if rocksdb_exists { - if config.database_backend != "rocksdb" { - return Err(Error::bad_config( - "Found rocksdb at database_path, but is not specified in config.", - )); - } + if rocksdb_exists && config.database_backend != "rocksdb" { + return Err(Error::bad_config( + "Found rocksdb at database_path, but is not specified in config.", + )); } Ok(()) diff --git a/src/database/admin.rs b/src/database/admin.rs index 7d2301d9..bf38bd8c 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -118,7 +118,7 @@ impl Admin { if let Ok(response) = guard._db.memory_usage() { send_message(RoomMessageEventContent::text_plain(response), guard, &state_lock); } else { - send_message(RoomMessageEventContent::text_plain("Failed to get database memory usage.".to_string()), guard, &state_lock); + send_message(RoomMessageEventContent::text_plain("Failed to get database memory usage.".to_owned()), guard, &state_lock); } } AdminCommand::SendMessage(message) => { diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 0ba6c9ba..c9a3c202 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -2727,7 +2727,7 @@ impl Rooms { let state_lock = mutex_state.lock().await; let mut event: RoomMemberEventContent = serde_json::from_str( - self.room_state_get(room_id, &EventType::RoomMember, &user_id.to_string())? + self.room_state_get(room_id, &EventType::RoomMember, user_id.as_str())? .ok_or(Error::BadRequest( ErrorKind::BadState, "Cannot leave a room you are not a member of.", @@ -3462,8 +3462,7 @@ impl Rooms { &key[0].to_be_bytes(), &chain .iter() - .map(|s| s.to_be_bytes().to_vec()) - .flatten() + .flat_map(|s| s.to_be_bytes().to_vec()) .collect::>(), )?; } @@ -3484,11 +3483,11 @@ impl Rooms { ) -> Result { let mut key = user_id.as_bytes().to_vec(); key.push(0xff); - key.extend_from_slice(&device_id.as_bytes()); + key.extend_from_slice(device_id.as_bytes()); key.push(0xff); - key.extend_from_slice(&room_id.as_bytes()); + key.extend_from_slice(room_id.as_bytes()); key.push(0xff); - key.extend_from_slice(&ll_user.as_bytes()); + key.extend_from_slice(ll_user.as_bytes()); Ok(self.lazyloadedids.get(&key)?.is_some()) } @@ -3528,14 +3527,14 @@ impl Rooms { )) { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xff); - prefix.extend_from_slice(&device_id.as_bytes()); + prefix.extend_from_slice(device_id.as_bytes()); prefix.push(0xff); - prefix.extend_from_slice(&room_id.as_bytes()); + prefix.extend_from_slice(room_id.as_bytes()); prefix.push(0xff); for ll_id in user_ids { let mut key = prefix.clone(); - key.extend_from_slice(&ll_id.as_bytes()); + key.extend_from_slice(ll_id.as_bytes()); self.lazyloadedids.insert(&key, &[])?; } } @@ -3546,15 +3545,15 @@ impl Rooms { #[tracing::instrument(skip(self))] pub fn lazy_load_reset( &self, - user_id: &Box, - device_id: &Box, - room_id: &Box, + user_id: &UserId, + device_id: &DeviceId, + room_id: &RoomId, ) -> Result<()> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xff); - prefix.extend_from_slice(&device_id.as_bytes()); + prefix.extend_from_slice(device_id.as_bytes()); prefix.push(0xff); - prefix.extend_from_slice(&room_id.as_bytes()); + prefix.extend_from_slice(room_id.as_bytes()); prefix.push(0xff); for (key, _) in self.lazyloadedids.scan_prefix(prefix) { diff --git a/src/server_server.rs b/src/server_server.rs index 54ae0251..9129951b 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1938,7 +1938,7 @@ pub(crate) fn fetch_and_handle_outliers<'a>( match handle_outlier_pdu( origin, create_event, - &next_id, + next_id, room_id, value.clone(), db, @@ -2358,7 +2358,7 @@ pub fn get_event_route( let room_id = <&RoomId>::try_from(room_id_str) .map_err(|_| Error::bad_database("Invalid room id field in event in database"))?; - if !db.rooms.server_in_room(sender_servername, &room_id)? { + if !db.rooms.server_in_room(sender_servername, room_id)? { return Err(Error::BadRequest( ErrorKind::Forbidden, "Server is not in room", @@ -2821,7 +2821,7 @@ async fn create_join_event( )); } - acl_check(sender_servername, room_id, &db)?; + acl_check(sender_servername, room_id, db)?; // We need to return the state prior to joining, let's keep a reference to that here let shortstatehash = db From 6e322716caf6f9181bf21444b552bf05d5f5a774 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Thu, 20 Jan 2022 12:29:10 +0100 Subject: [PATCH 134/201] Delete rust-toolchain file --- rust-toolchain | 1 - 1 file changed, 1 deletion(-) delete mode 100644 rust-toolchain diff --git a/rust-toolchain b/rust-toolchain deleted file mode 100644 index 74df8b16..00000000 --- a/rust-toolchain +++ /dev/null @@ -1 +0,0 @@ -1.53 From 5afb27a5a9ae887dea042e3ca9f0ecef98feff47 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Thu, 20 Jan 2022 12:29:24 +0100 Subject: [PATCH 135/201] Use latest stable for Docker image --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5812fdf9..b629690d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -FROM docker.io/rust:1.53-alpine AS builder +FROM docker.io/rust:1.58-alpine AS builder WORKDIR /usr/src/conduit # Install required packages to build Conduit and it's dependencies @@ -38,7 +38,7 @@ FROM docker.io/alpine:3.15.0 AS runner # You still need to map the port when using the docker command or docker-compose. EXPOSE 6167 -# Note from @jfowl: I would like to remove this in the future and just have the Docker version be configured with envs. +# Note from @jfowl: I would like to remove this in the future and just have the Docker version be configured with envs. ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" # Conduit needs: @@ -78,4 +78,4 @@ WORKDIR /srv/conduit # Run Conduit and print backtraces on panics ENV RUST_BACKTRACE=1 -ENTRYPOINT [ "/srv/conduit/conduit" ] \ No newline at end of file +ENTRYPOINT [ "/srv/conduit/conduit" ] From ff5fec9e74b4ed12c4dae579344a94f1c1f22f29 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Thu, 20 Jan 2022 12:29:52 +0100 Subject: [PATCH 136/201] Raise minimum supported Rust version to 1.56 --- Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 29a090c7..b6a2a2b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,8 @@ homepage = "https://conduit.rs" repository = "https://gitlab.com/famedly/conduit" readme = "README.md" version = "0.2.0" -edition = "2018" +rust-version = "1.56" +edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 6bb1081b7127a38cdc85614e4250f52b557753c8 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Thu, 20 Jan 2022 13:13:14 +0100 Subject: [PATCH 137/201] Use BTreeMap::into_values Stable under new MSRV. --- src/database/users.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/database/users.rs b/src/database/users.rs index c4fcee3d..69a277c6 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -531,11 +531,11 @@ impl Users { prefix.push(0xff); // Master key - let master_key_map = master_key + let mut master_key_ids = master_key .deserialize() .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid master key"))? - .keys; - let mut master_key_ids = master_key_map.values(); + .keys + .into_values(); let master_key_id = master_key_ids.next().ok_or(Error::BadRequest( ErrorKind::InvalidParam, @@ -560,13 +560,14 @@ impl Users { // Self-signing key if let Some(self_signing_key) = self_signing_key { - let self_signing_key_map = self_signing_key + let mut self_signing_key_ids = self_signing_key .deserialize() .map_err(|_| { Error::BadRequest(ErrorKind::InvalidParam, "Invalid self signing key") })? - .keys; - let mut self_signing_key_ids = self_signing_key_map.values(); + .keys + .into_values(); + let self_signing_key_id = self_signing_key_ids.next().ok_or(Error::BadRequest( ErrorKind::InvalidParam, "Self signing key contained no key.", @@ -593,13 +594,14 @@ impl Users { // User-signing key if let Some(user_signing_key) = user_signing_key { - let user_signing_key_map = user_signing_key + let mut user_signing_key_ids = user_signing_key .deserialize() .map_err(|_| { Error::BadRequest(ErrorKind::InvalidParam, "Invalid user signing key") })? - .keys; - let mut user_signing_key_ids = user_signing_key_map.values(); + .keys + .into_values(); + let user_signing_key_id = user_signing_key_ids.next().ok_or(Error::BadRequest( ErrorKind::InvalidParam, "User signing key contained no key.", From 8d81c1c0722ad2f608adea44d7b4ceb1a8f645ae Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Thu, 20 Jan 2022 13:23:58 +0100 Subject: [PATCH 138/201] Use MSRV for build CI jobs The test job will use the latest stable so all stable lints are included. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 73a1a928..cdc1d4cb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,7 +21,7 @@ variables: - if: '$CI_COMMIT_BRANCH == "next"' - if: "$CI_COMMIT_TAG" interruptible: true - image: "rust:latest" + image: "rust:1.56" tags: ["docker"] variables: CARGO_PROFILE_RELEASE_LTO: "true" From e378bc4a2c5590047b42cd4f8e244396125cb428 Mon Sep 17 00:00:00 2001 From: Andrei Vasiliu Date: Tue, 18 Jan 2022 13:53:17 +0200 Subject: [PATCH 139/201] Refactor admin commands to use structopt --- APPSERVICES.md | 8 +- Cargo.toml | 3 + src/database/admin.rs | 302 +++++++++++++++++++++++------------------- 3 files changed, 175 insertions(+), 138 deletions(-) diff --git a/APPSERVICES.md b/APPSERVICES.md index 894bc6f4..257166eb 100644 --- a/APPSERVICES.md +++ b/APPSERVICES.md @@ -18,7 +18,7 @@ First, go into the #admins room of your homeserver. The first person that registered on the homeserver automatically joins it. Then send a message into the room like this: - @conduit:your.server.name: register_appservice + @conduit:your.server.name: register-appservice ``` paste the @@ -31,7 +31,7 @@ the room like this: ``` You can confirm it worked by sending a message like this: -`@conduit:your.server.name: list_appservices` +`@conduit:your.server.name: list-appservices` The @conduit bot should answer with `Appservices (1): your-bridge` @@ -46,9 +46,9 @@ could help. To remove an appservice go to your admin room and execute -```@conduit:your.server.name: unregister_appservice ``` +```@conduit:your.server.name: unregister-appservice ``` -where `` one of the output of `list_appservices`. +where `` one of the output of `list-appservices`. ### Tested appservices diff --git a/Cargo.toml b/Cargo.toml index c87d949c..08afe1f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,6 +83,9 @@ thread_local = "1.1.3" # used for TURN server authentication hmac = "0.11.0" sha-1 = "0.9.8" +# used for conduit's CLI and admin room command parsing +structopt = { version = "0.3.25", default-features = false } +pulldown-cmark = "0.9.1" [features] default = ["conduit_bin", "backend_sqlite", "backend_rocksdb"] diff --git a/src/database/admin.rs b/src/database/admin.rs index 518d7587..55724db5 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -5,6 +5,7 @@ use crate::{ pdu::PduBuilder, server_server, Database, PduEvent, }; +use regex::Regex; use rocket::{ futures::{channel::mpsc, stream::StreamExt}, http::RawStr, @@ -14,6 +15,7 @@ use ruma::{ EventId, RoomId, RoomVersionId, UserId, }; use serde_json::value::to_raw_value; +use structopt::StructOpt; use tokio::sync::{MutexGuard, RwLock, RwLockReadGuard}; use tracing::warn; @@ -146,78 +148,98 @@ impl Admin { } pub fn parse_admin_command(db: &Database, command_line: &str, body: Vec<&str>) -> AdminCommand { - let mut parts = command_line.split_whitespace().skip(1); + let mut argv: Vec<_> = command_line.split_whitespace().skip(1).collect(); - let command_name = match parts.next() { - Some(command) => command, + let command_name = match argv.get(0) { + Some(command) => *command, None => { - let message = "No command given. Use help for a list of commands."; + let markdown_message = "No command given. Use `help` for a list of commands."; + let html_message = markdown_to_html(&markdown_message); + return AdminCommand::SendMessage(RoomMessageEventContent::text_html( - html_to_markdown(message), - message, + markdown_message, + html_message, )); } }; - let args: Vec<_> = parts.collect(); + // Backwards compatibility with `register_appservice`-style commands + let command_with_dashes; + if command_line.contains("_") { + command_with_dashes = command_name.replace("_", "-"); + argv[0] = &command_with_dashes; + } - match try_parse_admin_command(db, command_name, args, body) { + match try_parse_admin_command(db, argv, body) { Ok(admin_command) => admin_command, Err(error) => { - let message = format!( - "Encountered error while handling {} command:\n\ - 
          {}
          ", + let markdown_message = format!( + "Encountered an error while handling the `{}` command:\n\ + ```\n{}\n```", command_name, error, ); + let html_message = markdown_to_html(&markdown_message); AdminCommand::SendMessage(RoomMessageEventContent::text_html( - html_to_markdown(&message), - message, + markdown_message, + html_message, )) } } } -// Helper for `RoomMessageEventContent::text_html`, which needs the content as -// both markdown and HTML. -fn html_to_markdown(text: &str) -> String { - text.replace("

          ", "") - .replace("

          ", "\n") - .replace("
          ", "```\n")
-        .replace("
          ", "\n```") - .replace("", "`") - .replace("", "`") - .replace("
        • ", "* ") - .replace("
          • ", "") - .replace("
            \n", "") - .replace("
          \n", "") +#[derive(StructOpt)] +enum AdminCommands { + #[structopt(verbatim_doc_comment)] + /// Register a bridge using its registration YAML + /// + /// This command needs a YAML generated by an appservice (such as a mautrix + /// bridge), which must be provided in a code-block below the command. + /// + /// Example: + /// ```` + /// @conduit:example.com: register-appservice + /// ``` + /// yaml content here + /// ``` + /// ```` + RegisterAppservice, + /// Unregister a bridge using its ID + UnregisterAppservice { appservice_identifier: String }, + /// List all the currently registered bridges + ListAppservices, + /// Get the auth_chain of a PDU + GetAuthChain { event_id: Box }, + /// Parse and print a PDU from a JSON + ParsePdu, + /// Retrieve and print a PDU by ID from the Conduit database + GetPdu { event_id: Box }, + /// Print database memory usage statistics + DatabaseMemoryUsage, } -const HELP_TEXT: &'static str = r#" -

          The following commands are available:

          -
            -
          • register_appservice: Register a bridge using its registration YAML
            • -
          • unregister_appservice: Unregister a bridge using its ID
            • -
          • list_appservices: List all the currently registered bridges
            • -
          • get_auth_chain: Get the `auth_chain` of a PDU
            • -
          • parse_pdu: Parse and print a PDU from a JSON
            • -
          • get_pdu: Retrieve and print a PDU by ID from the Conduit database
            • -
          • database_memory_usage: Print database memory usage statistics
            • -
              -"#; - pub fn try_parse_admin_command( db: &Database, - command: &str, - args: Vec<&str>, + mut argv: Vec<&str>, body: Vec<&str>, ) -> Result { - let command = match command { - "help" => AdminCommand::SendMessage(RoomMessageEventContent::text_html( - html_to_markdown(HELP_TEXT), - HELP_TEXT, - )), - "register_appservice" => { + argv.insert(0, "@conduit:example.com:"); + let command = match AdminCommands::from_iter_safe(argv) { + Ok(command) => command, + Err(error) => { + println!("Before:\n{}\n", error.to_string()); + let markdown_message = usage_to_markdown(&error.to_string()) + .replace("example.com", db.globals.server_name().as_str()); + let html_message = markdown_to_html(&markdown_message); + + return Ok(AdminCommand::SendMessage( + RoomMessageEventContent::text_html(markdown_message, html_message), + )); + } + }; + + let admin_command = match command { + AdminCommands::RegisterAppservice => { if body.len() > 2 && body[0].trim() == "```" && body.last().unwrap().trim() == "```" { let appservice_config = body[1..body.len() - 1].join("\n"); let parsed_config = serde_yaml::from_str::(&appservice_config); @@ -233,47 +255,35 @@ pub fn try_parse_admin_command( )) } } - "unregister_appservice" => { - if args.len() == 1 { - AdminCommand::UnregisterAppservice(args[0].to_owned()) + AdminCommands::UnregisterAppservice { + appservice_identifier, + } => AdminCommand::UnregisterAppservice(appservice_identifier), + AdminCommands::ListAppservices => AdminCommand::ListAppservices, + AdminCommands::GetAuthChain { event_id } => { + let event_id = Arc::::from(event_id); + if let Some(event) = db.rooms.get_pdu_json(&event_id)? { + let room_id_str = event + .get("room_id") + .and_then(|val| val.as_str()) + .ok_or_else(|| Error::bad_database("Invalid event in database"))?; + + let room_id = <&RoomId>::try_from(room_id_str).map_err(|_| { + Error::bad_database("Invalid room id field in event in database") + })?; + let start = Instant::now(); + let count = server_server::get_auth_chain(room_id, vec![event_id], db)?.count(); + let elapsed = start.elapsed(); + return Ok(AdminCommand::SendMessage( + RoomMessageEventContent::text_plain(format!( + "Loaded auth chain with length {} in {:?}", + count, elapsed + )), + )); } else { - AdminCommand::SendMessage(RoomMessageEventContent::text_plain( - "Missing appservice identifier", - )) + AdminCommand::SendMessage(RoomMessageEventContent::text_plain("Event not found.")) } } - "list_appservices" => AdminCommand::ListAppservices, - "get_auth_chain" => { - if args.len() == 1 { - if let Ok(event_id) = EventId::parse_arc(args[0]) { - if let Some(event) = db.rooms.get_pdu_json(&event_id)? { - let room_id_str = event - .get("room_id") - .and_then(|val| val.as_str()) - .ok_or_else(|| Error::bad_database("Invalid event in database"))?; - - let room_id = <&RoomId>::try_from(room_id_str).map_err(|_| { - Error::bad_database("Invalid room id field in event in database") - })?; - let start = Instant::now(); - let count = - server_server::get_auth_chain(room_id, vec![event_id], db)?.count(); - let elapsed = start.elapsed(); - return Ok(AdminCommand::SendMessage( - RoomMessageEventContent::text_plain(format!( - "Loaded auth chain with length {} in {:?}", - count, elapsed - )), - )); - } - } - } - - AdminCommand::SendMessage(RoomMessageEventContent::text_plain( - "Usage: get_auth_chain ", - )) - } - "parse_pdu" => { + AdminCommands::ParsePdu => { if body.len() > 2 && body[0].trim() == "```" && body.last().unwrap().trim() == "```" { let string = body[1..body.len() - 1].join("\n"); match serde_json::from_str(&string) { @@ -312,59 +322,83 @@ pub fn try_parse_admin_command( )) } } - "get_pdu" => { - if args.len() == 1 { - if let Ok(event_id) = EventId::parse(args[0]) { - let mut outlier = false; - let mut pdu_json = db.rooms.get_non_outlier_pdu_json(&event_id)?; - if pdu_json.is_none() { - outlier = true; - pdu_json = db.rooms.get_pdu_json(&event_id)?; - } - match pdu_json { - Some(json) => { - let json_text = serde_json::to_string_pretty(&json) - .expect("canonical json is valid json"); - AdminCommand::SendMessage( - RoomMessageEventContent::text_html( - format!("{}\n```json\n{}\n```", - if outlier { - "PDU is outlier" - } else { "PDU was accepted"}, json_text), - format!("

              {}

              \n
              {}\n
              \n", - if outlier { - "PDU is outlier" - } else { "PDU was accepted"}, RawStr::new(&json_text).html_escape()) - ), - ) - } - None => AdminCommand::SendMessage(RoomMessageEventContent::text_plain( - "PDU not found.", - )), - } - } else { - AdminCommand::SendMessage(RoomMessageEventContent::text_plain( - "Event ID could not be parsed.", + AdminCommands::GetPdu { event_id } => { + let mut outlier = false; + let mut pdu_json = db.rooms.get_non_outlier_pdu_json(&event_id)?; + if pdu_json.is_none() { + outlier = true; + pdu_json = db.rooms.get_pdu_json(&event_id)?; + } + match pdu_json { + Some(json) => { + let json_text = + serde_json::to_string_pretty(&json).expect("canonical json is valid json"); + AdminCommand::SendMessage(RoomMessageEventContent::text_html( + format!( + "{}\n```json\n{}\n```", + if outlier { + "PDU is outlier" + } else { + "PDU was accepted" + }, + json_text + ), + format!( + "

              {}

              \n
              {}\n
              \n", + if outlier { + "PDU is outlier" + } else { + "PDU was accepted" + }, + RawStr::new(&json_text).html_escape() + ), )) } - } else { - AdminCommand::SendMessage(RoomMessageEventContent::text_plain( - "Usage: get_pdu ", - )) + None => { + AdminCommand::SendMessage(RoomMessageEventContent::text_plain("PDU not found.")) + } } } - "database_memory_usage" => AdminCommand::ShowMemoryUsage, - _ => { - let message = format!( - "Unrecognized command {}, try help for a list of commands.", - command, - ); - AdminCommand::SendMessage(RoomMessageEventContent::text_html( - html_to_markdown(&message), - message, - )) - } + AdminCommands::DatabaseMemoryUsage => AdminCommand::ShowMemoryUsage, }; - Ok(command) + Ok(admin_command) +} + +fn usage_to_markdown(text: &str) -> String { + // For the conduit admin room, subcommands become main commands + let text = text.replace("SUBCOMMAND", "COMMAND"); + let text = text.replace("subcommand", "command"); + + // Put the first line (command name and version text) on its own paragraph + let re = Regex::new("^(.*?)\n").expect("Regex compilation should not fail"); + let text = re.replace_all(&text, "*$1*\n\n"); + + // Wrap command names in backticks + // (?m) enables multi-line mode for ^ and $ + let re = Regex::new("(?m)^ ([a-z-]+) +(.*)$").expect("Regex compilation should not fail"); + let text = re.replace_all(&text, " `$1`: $2"); + + // Add * to list items + let re = Regex::new("(?m)^ (.*)$").expect("Regex compilation should not fail"); + let text = re.replace_all(&text, "* $1"); + + // Turn section names to headings + let re = Regex::new("(?m)^([A-Z-]+):$").expect("Regex compilation should not fail"); + let text = re.replace_all(&text, "#### $1"); + + text.to_string() +} + +fn markdown_to_html(text: &str) -> String { + // CommonMark's spec allows HTML tags; however, CLI required arguments look + // very much like tags so escape them. + let text = text.replace("<", "<").replace(">", ">"); + + let mut html_output = String::new(); + + let parser = pulldown_cmark::Parser::new(&text); + pulldown_cmark::html::push_html(&mut html_output, parser); + + html_output } From cc3ef1a8be08b9212a16957062304d7bd5da1111 Mon Sep 17 00:00:00 2001 From: Andrei Vasiliu Date: Fri, 21 Jan 2022 11:06:16 +0200 Subject: [PATCH 140/201] Improve help text for admin commands --- src/database/admin.rs | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index f690bdf4..362ef294 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -147,6 +147,7 @@ impl Admin { } } +// Parse chat messages from the admin room into an AdminCommand object pub fn parse_admin_command(db: &Database, command_line: &str, body: Vec<&str>) -> AdminCommand { let mut argv: Vec<_> = command_line.split_whitespace().skip(1).collect(); @@ -191,10 +192,13 @@ pub fn parse_admin_command(db: &Database, command_line: &str, body: Vec<&str>) - #[derive(StructOpt)] enum AdminCommands { #[structopt(verbatim_doc_comment)] - /// Register a bridge using its registration YAML + /// Register an appservice using its registration YAML /// - /// This command needs a YAML generated by an appservice (such as a mautrix - /// bridge), which must be provided in a code-block below the command. + /// This command needs a YAML generated by an appservice (such as a bridge), + /// which must be provided in a Markdown code-block below the command. + /// + /// Registering a new bridge using the ID of an existing bridge will replace + /// the old one. /// /// Example: /// ```` @@ -204,16 +208,27 @@ enum AdminCommands { /// ``` /// ```` RegisterAppservice, - /// Unregister a bridge using its ID + + /// Unregister an appservice using its ID + /// + /// You can find the ID using the `list-appservices` command. UnregisterAppservice { appservice_identifier: String }, - /// List all the currently registered bridges + + /// List all the currently registered appservices ListAppservices, + /// Get the auth_chain of a PDU GetAuthChain { event_id: Box }, + /// Parse and print a PDU from a JSON + /// + /// The PDU event is only checked for validity and is not added to the + /// database. ParsePdu, + /// Retrieve and print a PDU by ID from the Conduit database GetPdu { event_id: Box }, + /// Print database memory usage statistics DatabaseMemoryUsage, } @@ -365,6 +380,7 @@ pub fn try_parse_admin_command( Ok(admin_command) } +// Utility to turn structopt's `--help` text to markdown. fn usage_to_markdown(text: &str) -> String { // For the conduit admin room, subcommands become main commands let text = text.replace("SUBCOMMAND", "COMMAND"); @@ -390,6 +406,7 @@ fn usage_to_markdown(text: &str) -> String { text.to_string() } +// Convert markdown to HTML using the CommonMark flavor fn markdown_to_html(text: &str) -> String { // CommonMark's spec allows HTML tags; however, CLI required arguments look // very much like tags so escape them. From ba6d72f3f93aeb96c9ca98daab3c34e969c76008 Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Fri, 21 Jan 2022 14:28:07 +0100 Subject: [PATCH 141/201] Reformatted --- src/database/users.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/database/users.rs b/src/database/users.rs index 9b986d4e..a6b6fabb 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -147,7 +147,10 @@ impl Users { match utils::string_from_bytes(username) { Ok(u) => Some(u), Err(e) => { - warn!("Failed to parse username while calling get_local_users(): {}", e.to_string()); + warn!( + "Failed to parse username while calling get_local_users(): {}", + e.to_string() + ); None } } From 57979da28c0af4bc14787575d94308d5762e7dc6 Mon Sep 17 00:00:00 2001 From: Andrei Vasiliu Date: Fri, 21 Jan 2022 17:34:21 +0200 Subject: [PATCH 142/201] Change structopt to clap, remove markdown dependency --- Cargo.lock | 75 +++++++++++++++++++++++- Cargo.toml | 3 +- src/database/admin.rs | 131 ++++++++++++++++++++++++++---------------- 3 files changed, 156 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5be10f14..ae385fe6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -269,6 +269,33 @@ dependencies = [ "libloading", ] +[[package]] +name = "clap" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a30c3bf9ff12dfe5dae53f0a96e0febcd18420d1c0e7fad77796d9d5c4b5375" +dependencies = [ + "bitflags", + "clap_derive", + "indexmap", + "lazy_static", + "os_str_bytes", + "textwrap", +] + +[[package]] +name = "clap_derive" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "517358c28fcef6607bf6f76108e02afad7e82297d132a6b846dcc1fc3efcd153" +dependencies = [ + "heck 0.4.0", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "color_quant" version = "1.1.0" @@ -281,6 +308,7 @@ version = "0.2.0" dependencies = [ "base64 0.13.0", "bytes", + "clap", "crossbeam", "directories", "heed", @@ -630,7 +658,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c5f0096a91d210159eceb2ff5e1c4da18388a170e1e3ce948aac9c8fdbbf595" dependencies = [ - "heck", + "heck 0.3.3", "proc-macro2", "quote", "syn", @@ -902,6 +930,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + [[package]] name = "heed" version = "0.10.6" @@ -1570,6 +1604,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "os_str_bytes" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64" +dependencies = [ + "memchr", +] + [[package]] name = "page_size" version = "0.4.2" @@ -1728,6 +1771,30 @@ dependencies = [ "toml", ] +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro-hack" version = "0.5.19" @@ -2863,6 +2930,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "textwrap" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0066c8d12af8b5acd21e00547c3797fde4e8677254a7ee429176ccebbe93dd80" + [[package]] name = "thiserror" version = "1.0.30" diff --git a/Cargo.toml b/Cargo.toml index 9a2d2fdb..3f8677d6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -86,8 +86,7 @@ thread_local = "1.1.3" hmac = "0.11.0" sha-1 = "0.9.8" # used for conduit's CLI and admin room command parsing -structopt = { version = "0.3.25", default-features = false } -pulldown-cmark = "0.9.1" +clap = { version = "3.0.10", default-features = false, features = ["std", "derive"] } [features] default = ["conduit_bin", "backend_sqlite", "backend_rocksdb"] diff --git a/src/database/admin.rs b/src/database/admin.rs index 362ef294..59b8acdf 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -5,6 +5,7 @@ use crate::{ pdu::PduBuilder, server_server, Database, PduEvent, }; +use clap::Parser; use regex::Regex; use rocket::{ futures::{channel::mpsc, stream::StreamExt}, @@ -15,7 +16,6 @@ use ruma::{ EventId, RoomId, RoomVersionId, UserId, }; use serde_json::value::to_raw_value; -use structopt::StructOpt; use tokio::sync::{MutexGuard, RwLock, RwLockReadGuard}; use tracing::warn; @@ -155,7 +155,7 @@ pub fn parse_admin_command(db: &Database, command_line: &str, body: Vec<&str>) - Some(command) => *command, None => { let markdown_message = "No command given. Use `help` for a list of commands."; - let html_message = markdown_to_html(&markdown_message); + let html_message = "No command given. Use help for a list of commands."; return AdminCommand::SendMessage(RoomMessageEventContent::text_html( markdown_message, @@ -164,10 +164,17 @@ pub fn parse_admin_command(db: &Database, command_line: &str, body: Vec<&str>) - } }; + // Replace `help command` with `command --help` + // Clap has a help subcommand, but it omits the long help description. + if argv[0] == "help" { + argv.remove(0); + argv.push("--help"); + } + // Backwards compatibility with `register_appservice`-style commands let command_with_dashes; - if command_line.contains("_") { - command_with_dashes = command_name.replace("_", "-"); + if argv[0].contains("_") { + command_with_dashes = argv[0].replace("_", "-"); argv[0] = &command_with_dashes; } @@ -179,7 +186,11 @@ pub fn parse_admin_command(db: &Database, command_line: &str, body: Vec<&str>) - ```\n{}\n```", command_name, error, ); - let html_message = markdown_to_html(&markdown_message); + let html_message = format!( + "Encountered an error while handling the {} command:\n\ + 
              \n{}\n
              ", + command_name, error, + ); AdminCommand::SendMessage(RoomMessageEventContent::text_html( markdown_message, @@ -189,9 +200,10 @@ pub fn parse_admin_command(db: &Database, command_line: &str, body: Vec<&str>) - } } -#[derive(StructOpt)] +#[derive(Parser)] +#[clap(name = "@conduit:example.com", version = env!("CARGO_PKG_VERSION"))] enum AdminCommands { - #[structopt(verbatim_doc_comment)] + #[clap(verbatim_doc_comment)] /// Register an appservice using its registration YAML /// /// This command needs a YAML generated by an appservice (such as a bridge), @@ -200,25 +212,25 @@ enum AdminCommands { /// Registering a new bridge using the ID of an existing bridge will replace /// the old one. /// - /// Example: - /// ```` - /// @conduit:example.com: register-appservice - /// ``` - /// yaml content here - /// ``` - /// ```` + /// [add-yaml-block-to-usage] RegisterAppservice, /// Unregister an appservice using its ID - /// + /// /// You can find the ID using the `list-appservices` command. - UnregisterAppservice { appservice_identifier: String }, + UnregisterAppservice { + /// The appservice to unregister + appservice_identifier: String, + }, /// List all the currently registered appservices ListAppservices, /// Get the auth_chain of a PDU - GetAuthChain { event_id: Box }, + GetAuthChain { + /// An event ID (the $ character followed by the base64 reference hash) + event_id: Box, + }, /// Parse and print a PDU from a JSON /// @@ -227,7 +239,10 @@ enum AdminCommands { ParsePdu, /// Retrieve and print a PDU by ID from the Conduit database - GetPdu { event_id: Box }, + GetPdu { + /// An event ID (a $ followed by the base64 reference hash) + event_id: Box, + }, /// Print database memory usage statistics DatabaseMemoryUsage, @@ -239,16 +254,16 @@ pub fn try_parse_admin_command( body: Vec<&str>, ) -> Result { argv.insert(0, "@conduit:example.com:"); - let command = match AdminCommands::from_iter_safe(argv) { + let command = match AdminCommands::try_parse_from(argv) { Ok(command) => command, Err(error) => { - println!("Before:\n{}\n", error.to_string()); - let markdown_message = usage_to_markdown(&error.to_string()) + let message = error + .to_string() .replace("example.com", db.globals.server_name().as_str()); - let html_message = markdown_to_html(&markdown_message); + let html_message = usage_to_html(&message); return Ok(AdminCommand::SendMessage( - RoomMessageEventContent::text_html(markdown_message, html_message), + RoomMessageEventContent::text_html(message, html_message), )); } }; @@ -380,42 +395,58 @@ pub fn try_parse_admin_command( Ok(admin_command) } -// Utility to turn structopt's `--help` text to markdown. -fn usage_to_markdown(text: &str) -> String { +// Utility to turn clap's `--help` text to HTML. +fn usage_to_html(text: &str) -> String { // For the conduit admin room, subcommands become main commands let text = text.replace("SUBCOMMAND", "COMMAND"); let text = text.replace("subcommand", "command"); - // Put the first line (command name and version text) on its own paragraph + // Escape option names (e.g. ``) since they look like HTML tags + let text = text.replace("<", "<").replace(">", ">"); + + // Italicize the first line (command name and version text) let re = Regex::new("^(.*?)\n").expect("Regex compilation should not fail"); - let text = re.replace_all(&text, "*$1*\n\n"); + let text = re.replace_all(&text, "$1\n"); - // Wrap command names in backticks + // Unmerge wrapped lines + let text = text.replace("\n ", " "); + + // Wrap option names in backticks. The lines look like: + // -V, --version Prints version information + // And are converted to: + // -V, --version: Prints version information // (?m) enables multi-line mode for ^ and $ - let re = Regex::new("(?m)^ ([a-z-]+) +(.*)$").expect("Regex compilation should not fail"); - let text = re.replace_all(&text, " `$1`: $2"); + let re = Regex::new("(?m)^ (([a-zA-Z_&;-]+(, )?)+) +(.*)$") + .expect("Regex compilation should not fail"); + let text = re.replace_all(&text, "$1: $4"); - // Add * to list items - let re = Regex::new("(?m)^ (.*)$").expect("Regex compilation should not fail"); - let text = re.replace_all(&text, "* $1"); + // // Enclose examples in code blocks + // // (?ms) enables multi-line mode and dot-matches-all + // let re = + // Regex::new("(?ms)^Example:\n(.*?)\nUSAGE:$").expect("Regex compilation should not fail"); + // let text = re.replace_all(&text, "EXAMPLE:\n
              $1
              \nUSAGE:"); - // Turn section names to headings - let re = Regex::new("(?m)^([A-Z-]+):$").expect("Regex compilation should not fail"); - let text = re.replace_all(&text, "#### $1"); + let has_yaml_block_marker = text.contains("\n[add-yaml-block-to-usage]\n"); + let text = text.replace("\n[add-yaml-block-to-usage]\n", ""); + + // Add HTML line-breaks + let text = text.replace("\n", "
              \n"); + + let text = if !has_yaml_block_marker { + // Wrap the usage line in code tags + let re = Regex::new("(?m)^USAGE:
              \n (@conduit:.*)
              $") + .expect("Regex compilation should not fail"); + re.replace_all(&text, "USAGE:
              \n$1
              ") + } else { + // Wrap the usage line in a code block, and add a yaml block example + // This makes the usage of e.g. `register-appservice` more accurate + let re = Regex::new("(?m)^USAGE:
              \n (.*?)
              \n
              \n") + .expect("Regex compilation should not fail"); + re.replace_all( + &text, + "USAGE:
              \n
              $1\n```\nyaml content here\n```
              ", + ) + }; text.to_string() } - -// Convert markdown to HTML using the CommonMark flavor -fn markdown_to_html(text: &str) -> String { - // CommonMark's spec allows HTML tags; however, CLI required arguments look - // very much like tags so escape them. - let text = text.replace("<", "<").replace(">", ">"); - - let mut html_output = String::new(); - - let parser = pulldown_cmark::Parser::new(&text); - pulldown_cmark::html::push_html(&mut html_output, parser); - - html_output -} From 97d56af5bd48474efc9aa1ac94ed4295e282d8ca Mon Sep 17 00:00:00 2001 From: Reiner Herrmann Date: Sat, 15 Jan 2022 17:23:14 +0000 Subject: [PATCH 143/201] Add heisenbridge to tested appservices --- APPSERVICES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/APPSERVICES.md b/APPSERVICES.md index f23918b4..5ff223ed 100644 --- a/APPSERVICES.md +++ b/APPSERVICES.md @@ -58,3 +58,4 @@ These appservices have been tested and work with Conduit without any extra steps - [mautrix-hangouts](https://github.com/mautrix/hangouts/) - [mautrix-telegram](https://github.com/mautrix/telegram/) - [mautrix-signal](https://github.com/mautrix/signal/) from version `0.2.2` forward. +- [heisenbridge](https://github.com/hifi/heisenbridge/) From d94f3c1e9aad363aff552933b104944094f7ddc2 Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Fri, 21 Jan 2022 17:06:15 +0100 Subject: [PATCH 144/201] fix: make sure cc-rs and bindgen use the correct paths when cross-compiling --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cdc1d4cb..d0d4f3e3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,6 +33,12 @@ variables: - "rustup target add $TARGET" # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi + # Make sure that cc-rs links the correct libraries when cross-compiling (required for compiling librocksdb-sys) + # See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information + - 'export TARGET_CFLAGS="-L$TARGET_HOME/lib -latomic"' + # Make sure that rust-bindgen uses the correct include path when cross-compiling (required for compiling librocksdb-sys) + # See https://github.com/rust-lang/rust-bindgen#environment-variables for more information + - 'export BINDGEN_EXTRA_CLANG_ARGS="-I$TARGET_C_INCLUDE_PATH"' script: - time cargo build --target $TARGET --release - 'cp "target/$TARGET/release/conduit" "conduit-$TARGET"' From bfef94f5f4f465156317e5a6d60fffc8e1fd9240 Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Fri, 21 Jan 2022 17:26:25 +0100 Subject: [PATCH 145/201] fix: linking against libatomic is no longer required since the library path is fixed --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d0d4f3e3..236ce0ae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,7 +35,7 @@ variables: - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi # Make sure that cc-rs links the correct libraries when cross-compiling (required for compiling librocksdb-sys) # See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information - - 'export TARGET_CFLAGS="-L$TARGET_HOME/lib -latomic"' + - 'export TARGET_CFLAGS="-L$TARGET_HOME/lib"' # Make sure that rust-bindgen uses the correct include path when cross-compiling (required for compiling librocksdb-sys) # See https://github.com/rust-lang/rust-bindgen#environment-variables for more information - 'export BINDGEN_EXTRA_CLANG_ARGS="-I$TARGET_C_INCLUDE_PATH"' From f88523988e23a09ffc5e1b9ab19e435863be3a9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Fri, 21 Jan 2022 09:19:19 +0100 Subject: [PATCH 146/201] improvement: use jemalloc for lower memory usage --- Cargo.lock | 258 ++++++++++++++++++++++++---------------------------- Cargo.toml | 10 +- src/main.rs | 7 ++ 3 files changed, 132 insertions(+), 143 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8fe767e1..493ac082 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,7 +14,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ - "getrandom 0.2.3", + "getrandom 0.2.4", "once_cell", "version_check", ] @@ -303,8 +303,6 @@ dependencies = [ "ruma", "rusqlite", "rust-argon2", - "rustls 0.19.1", - "rustls-native-certs 0.5.0", "serde", "serde_json", "serde_yaml", @@ -313,12 +311,13 @@ dependencies = [ "thiserror", "thread_local", "threadpool", + "tikv-jemalloc-ctl", + "tikv-jemallocator", "tokio", "tracing", "tracing-flame", "tracing-subscriber 0.2.25", "trust-dns-resolver", - "webpki 0.22.0", ] [[package]] @@ -350,22 +349,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "core-foundation" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6888e10551bb93e424d8df1d07f1a8b4fceb0001a3a4b048bfc47554946f47b3" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" - [[package]] name = "cpufeatures" version = "0.2.1" @@ -392,9 +375,9 @@ checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403" [[package]] name = "crc32fast" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "738c290dfaea84fc1ca15ad9c168d083b05a714e1efddd8edaab678dc28d2836" +checksum = "a2209c310e29876f7f0b2721e7e26b84aff178aa3da5d091f9bfbf47669e60e3" dependencies = [ "cfg-if 1.0.0", ] @@ -713,6 +696,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "fs_extra" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" + [[package]] name = "futures" version = "0.3.19" @@ -847,9 +836,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" +checksum = "418d37c8b1d42553c93648be529cb70f920d3baf8ef469b74b9638df426e0b4c" dependencies = [ "cfg-if 1.0.0", "libc", @@ -1035,7 +1024,7 @@ dependencies = [ "httpdate", "itoa 0.4.8", "pin-project-lite", - "socket2 0.4.2", + "socket2 0.4.3", "tokio", "tower-service", "tracing", @@ -1180,9 +1169,9 @@ checksum = "229d53d58899083193af11e15917b5640cd40b29ff475a1fe4ef725deb02d0f2" [[package]] name = "js-sys" -version = "0.3.55" +version = "0.3.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cc9ffccd38c451a86bf13657df244e9c3f37493cce8e5e21e940963777acc84" +checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04" dependencies = [ "wasm-bindgen", ] @@ -1224,15 +1213,15 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.112" +version = "0.2.113" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125" +checksum = "eef78b64d87775463c549fbd80e19249ef436ea3bf1de2a1eb7e717ec7fab1e9" [[package]] name = "libloading" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afe203d669ec979b7128619bae5a63b7b42e9203c1b29146079ee05e2f604b52" +checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd" dependencies = [ "cfg-if 1.0.0", "winapi", @@ -1308,7 +1297,7 @@ dependencies = [ "serde", "serde_json", "tracing", - "tracing-subscriber 0.3.5", + "tracing-subscriber 0.3.6", ] [[package]] @@ -1528,12 +1517,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" -[[package]] -name = "openssl-probe" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" - [[package]] name = "opentelemetry" version = "0.16.0" @@ -1621,6 +1604,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "paste" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0744126afe1a6dd7f394cb50a716dbe086cb06e255e53d8d0185d82828358fb5" + [[package]] name = "pear" version = "0.2.3" @@ -1669,9 +1658,9 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "persy" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c6aa7d7f093620a28b74fcf5f5da73ba17a9e52fcbbdbb4ecc89e61cb2d673" +checksum = "b71907e1dfa6844b657f5ca59e9a076e7d6281efb4885526ba9e235a18e7e3b3" dependencies = [ "crc", "data-encoding", @@ -1863,7 +1852,7 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" dependencies = [ - "getrandom 0.2.3", + "getrandom 0.2.4", ] [[package]] @@ -1899,7 +1888,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" dependencies = [ - "getrandom 0.2.3", + "getrandom 0.2.4", "redox_syscall", ] @@ -1982,7 +1971,6 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rustls 0.20.2", - "rustls-native-certs 0.6.1", "rustls-pemfile", "serde", "serde_json", @@ -1994,6 +1982,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", + "webpki-roots", "winreg 0.7.0", ] @@ -2443,30 +2432,6 @@ dependencies = [ "webpki 0.22.0", ] -[[package]] -name = "rustls-native-certs" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" -dependencies = [ - "openssl-probe", - "rustls 0.19.1", - "schannel", - "security-framework", -] - -[[package]] -name = "rustls-native-certs" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca9ebdfa27d3fc180e42879037b5338ab1c040c06affd00d8338598e7800943" -dependencies = [ - "openssl-probe", - "rustls-pemfile", - "schannel", - "security-framework", -] - [[package]] name = "rustls-pemfile" version = "0.2.1" @@ -2488,16 +2453,6 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" -[[package]] -name = "schannel" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" -dependencies = [ - "lazy_static", - "winapi", -] - [[package]] name = "scoped-tls" version = "1.0.0" @@ -2530,29 +2485,6 @@ dependencies = [ "untrusted", ] -[[package]] -name = "security-framework" -version = "2.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525bc1abfda2e1998d152c45cf13e696f76d0a4972310b22fac1658b05df7c87" -dependencies = [ - "bitflags", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9dd14d83160b528b7bfd66439110573efcfbe281b17fc2ca9f39f550d619c7e" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "semver" version = "0.9.0" @@ -2570,18 +2502,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.133" +version = "1.0.134" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97565067517b60e2d1ea8b268e59ce036de907ac523ad83a0475da04e818989a" +checksum = "96b3c34c1690edf8174f5b289a336ab03f568a4460d8c6df75f2f3a692b3bc6a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.133" +version = "1.0.134" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed201699328568d8d08208fdd080e3ff594e6c422e438b6705905da01005d537" +checksum = "784ed1fbfa13fe191077537b0d70ec8ad1e903cfe04831da608aa36457cb653d" dependencies = [ "proc-macro2", "quote", @@ -2590,9 +2522,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.74" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2bb9cd061c5865d345bb02ca49fcef1391741b672b54a0bf7b679badec3142" +checksum = "c059c05b48c5c0067d4b4b2b4f0732dd65feb52daf7e0ea09cd87e7dadc1af79" dependencies = [ "itoa 1.0.1", "ryu", @@ -2601,12 +2533,12 @@ dependencies = [ [[package]] name = "serde_urlencoded" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 0.4.8", + "itoa 1.0.1", "ryu", "serde", ] @@ -2638,9 +2570,18 @@ dependencies = [ [[package]] name = "sha1" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" +checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" +dependencies = [ + "sha1_smol", +] + +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" [[package]] name = "sha2" @@ -2721,9 +2662,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309" +checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" [[package]] name = "socket2" @@ -2738,9 +2679,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dc90fe6c7be1a323296982db1836d1ea9e47b6839496dde9a541bc496df3516" +checksum = "0f82496b90c36d70af5fcd482edaa2e0bd16fade569de1330405fecbbdac736b" dependencies = [ "libc", "winapi", @@ -2851,9 +2792,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a684ac3dcd8913827e18cd09a68384ee66c1de24157e3c556c9ab16d85695fb7" +checksum = "8a65b3f4ffa0092e9887669db0eae07941f023991ab58ea44da8fe8e2d511c6b" dependencies = [ "proc-macro2", "quote", @@ -2946,6 +2887,38 @@ dependencies = [ "threadpool", ] +[[package]] +name = "tikv-jemalloc-ctl" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb833c46ecbf8b6daeccb347cefcabf9c1beb5c9b0f853e1cec45632d9963e69" +dependencies = [ + "libc", + "paste", + "tikv-jemalloc-sys", +] + +[[package]] +name = "tikv-jemalloc-sys" +version = "0.4.2+5.2.1-patched.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5844e429d797c62945a566f8da4e24c7fe3fbd5d6617fd8bf7a0b7dc1ee0f22e" +dependencies = [ + "cc", + "fs_extra", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c14a5a604eb8715bc5785018a37d00739b180bcf609916ddf4393d33d49ccdf" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "time" version = "0.1.43" @@ -3200,9 +3173,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d81bfa81424cc98cb034b837c985b7a290f592e5b4322f353f94a0ab0f9f594" +checksum = "77be66445c4eeebb934a7340f227bfe7b338173d3f8c00a60a5a58005c9faecf" dependencies = [ "ansi_term", "lazy_static", @@ -3355,7 +3328,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ - "getrandom 0.2.3", + "getrandom 0.2.4", ] [[package]] @@ -3394,9 +3367,9 @@ checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" [[package]] name = "wasm-bindgen" -version = "0.2.78" +version = "0.2.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "632f73e236b219150ea279196e54e610f5dbafa5d61786303d4da54f84e47fce" +checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -3404,9 +3377,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.78" +version = "0.2.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a317bf8f9fba2476b4b2c85ef4c4af8ff39c3c7f0cdfeed4f82c34a880aa837b" +checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" dependencies = [ "bumpalo", "lazy_static", @@ -3419,9 +3392,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.28" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e8d7523cb1f2a4c96c1317ca690031b714a51cc14e05f712446691f413f5d39" +checksum = "2eb6ec270a31b1d3c7e266b999739109abce8b6c87e4b31fcfcd788b65267395" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -3431,9 +3404,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.78" +version = "0.2.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56146e7c495528bf6587663bea13a8eb588d39b36b679d83972e1a2dbbdacf9" +checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3441,9 +3414,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.78" +version = "0.2.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7803e0eea25835f8abdc585cd3021b3deb11543c6fe226dcd30b228857c5c5ab" +checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" dependencies = [ "proc-macro2", "quote", @@ -3454,15 +3427,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.78" +version = "0.2.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0237232789cf037d5480773fe568aac745bfe2afbc11a863e97901780a6b47cc" +checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" [[package]] name = "web-sys" -version = "0.3.55" +version = "0.3.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38eb105f1c59d9eaa6b5cdc92b859d85b926e82cb2e0945cd0c9259faa6fe9fb" +checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb" dependencies = [ "js-sys", "wasm-bindgen", @@ -3488,6 +3461,15 @@ dependencies = [ "untrusted", ] +[[package]] +name = "webpki-roots" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552ceb903e957524388c4d3475725ff2c8b7960922063af6ce53c9a43da07449" +dependencies = [ + "webpki 0.22.0", +] + [[package]] name = "weezl" version = "0.1.5" @@ -3563,18 +3545,18 @@ checksum = "9fc79f4a1e39857fc00c3f662cbf2651c771f00e9c15fe2abc341806bd46bd71" [[package]] name = "zeroize" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d68d9dcec5f9b43a30d38c49f91dfedfaac384cb8f085faca366c26207dd1619" +checksum = "cc222aec311c323c717f56060324f32b82da1ce1dd81d9a09aa6a9030bfe08db" dependencies = [ "zeroize_derive", ] [[package]] name = "zeroize_derive" -version = "1.2.2" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65f1a51723ec88c66d5d1fe80c841f17f63587d6691901d66be9bec6c3b51f73" +checksum = "81e8f13fef10b63c06356d65d416b070798ddabcadc10d3ece0c5be9b3c7eddb" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index e3614ec4..78a4c8ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,11 +48,7 @@ rand = "0.8.4" # Used to hash passwords rust-argon2 = "0.8.3" # Used to send requests -reqwest = { version = "0.11.4", default-features = false, features = ["rustls-tls-native-roots", "socks"] } -# Custom TLS verifier -rustls = { version = "0.19.1", features = ["dangerous_configuration"] } -rustls-native-certs = "0.5.0" -webpki = "0.22.0" +reqwest = { version = "0.11.4", default-features = false, features = ["rustls-tls", "socks"] } # Used for conduit::Error type thiserror = "1.0.28" # Used to generate thumbnails for images @@ -87,6 +83,10 @@ thread_local = "1.1.3" hmac = "0.11.0" sha-1 = "0.9.8" +[target.'cfg(not(target_env = "msvc"))'.dependencies] +tikv-jemalloc-ctl = { version = "0.4.2", features = ['use_std'] } +tikv-jemallocator = { version = "0.4.1", features = ['unprefixed_malloc_on_supported_platforms'] } + [features] default = ["conduit_bin", "backend_sqlite", "backend_rocksdb"] backend_sled = ["sled"] diff --git a/src/main.rs b/src/main.rs index d9bbc240..b18ca803 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,6 +25,13 @@ use tracing_subscriber::{prelude::*, EnvFilter}; pub use conduit::*; // Re-export everything from the library crate pub use rocket::State; +#[cfg(not(target_env = "msvc"))] +use tikv_jemallocator::Jemalloc; + +#[cfg(not(target_env = "msvc"))] +#[global_allocator] +static GLOBAL: Jemalloc = Jemalloc; + fn setup_rocket(config: Figment, data: Arc>) -> rocket::Rocket { rocket::custom(config) .manage(data) From 3e9abfedb43e6f52ebb3f24adaf8bf0871712181 Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Sat, 22 Jan 2022 00:14:19 +0100 Subject: [PATCH 147/201] fix: make sure libstdc++ is linked statically when cross-compiling --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 236ce0ae..5cae7439 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,10 +33,10 @@ variables: - "rustup target add $TARGET" # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi - # Make sure that cc-rs links the correct libraries when cross-compiling (required for compiling librocksdb-sys) + # Make sure that cc-rs links the correct libraries statically when cross-compiling # See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information - - 'export TARGET_CFLAGS="-L$TARGET_HOME/lib"' - # Make sure that rust-bindgen uses the correct include path when cross-compiling (required for compiling librocksdb-sys) + - 'export CARGO_BUILD_RUSTFLAGS="-L$TARGET_HOME/lib" CXXSTDLIB="static=stdc++"' + # Make sure that rust-bindgen uses the correct include path when cross-compiling # See https://github.com/rust-lang/rust-bindgen#environment-variables for more information - 'export BINDGEN_EXTRA_CLANG_ARGS="-I$TARGET_C_INCLUDE_PATH"' script: From a021680591cf581fccd05a9dbb0914163f69e8ba Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Sat, 22 Jan 2022 01:14:36 +0100 Subject: [PATCH 148/201] fix: make sure libatomic is always linked because it's skipped on arm targets --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5cae7439..b863de98 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,7 +35,7 @@ variables: - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi # Make sure that cc-rs links the correct libraries statically when cross-compiling # See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information - - 'export CARGO_BUILD_RUSTFLAGS="-L$TARGET_HOME/lib" CXXSTDLIB="static=stdc++"' + - 'export CARGO_BUILD_RUSTFLAGS="-L$TARGET_HOME/lib -latomic" CXXSTDLIB="static=stdc++"' # Make sure that rust-bindgen uses the correct include path when cross-compiling # See https://github.com/rust-lang/rust-bindgen#environment-variables for more information - 'export BINDGEN_EXTRA_CLANG_ARGS="-I$TARGET_C_INCLUDE_PATH"' From cd9902637ddf3f8e7711f01a5cf044725704e28a Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Sat, 22 Jan 2022 17:34:30 +0100 Subject: [PATCH 149/201] feat: use rustembedded/cross images and use static relocation model to fix cross-compile --- .gitlab-ci.yml | 39 ++++++++++++++++++++++++--------------- CROSS_COMPILE.md | 11 ----------- Cross.toml | 11 +++++++++++ DEPLOY.md | 2 +- cross/README.md | 37 +++++++++++++++++++++++++++++++++++++ cross/build.sh | 31 +++++++++++++++++++++++++++++++ cross/test.sh | 8 ++++++++ 7 files changed, 112 insertions(+), 27 deletions(-) delete mode 100644 CROSS_COMPILE.md create mode 100644 Cross.toml create mode 100644 cross/README.md create mode 100755 cross/build.sh create mode 100755 cross/test.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b863de98..993145a1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,33 +21,46 @@ variables: - if: '$CI_COMMIT_BRANCH == "next"' - if: "$CI_COMMIT_TAG" interruptible: true - image: "rust:1.56" + image: "rust:1.58" tags: ["docker"] + services: ["docker:dind"] variables: + DOCKER_HOST: tcp://docker:2375/ + DOCKER_TLS_CERTDIR: "" + DOCKER_DRIVER: overlay2 + SHARED_PATH: $CI_PROJECT_DIR/shared/ CARGO_PROFILE_RELEASE_LTO: "true" CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1" CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow before_script: - 'echo "Building for target $TARGET"' - - "rustc --version && cargo --version && rustup show" # Print version info for debugging - - "rustup target add $TARGET" + - "rustup show && rustc --version && cargo --version" # Print version info for debugging # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi - # Make sure that cc-rs links the correct libraries statically when cross-compiling - # See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information - - 'export CARGO_BUILD_RUSTFLAGS="-L$TARGET_HOME/lib -latomic" CXXSTDLIB="static=stdc++"' - # Make sure that rust-bindgen uses the correct include path when cross-compiling - # See https://github.com/rust-lang/rust-bindgen#environment-variables for more information - - 'export BINDGEN_EXTRA_CLANG_ARGS="-I$TARGET_C_INCLUDE_PATH"' script: - - time cargo build --target $TARGET --release + # install cross-compiling prerequisites + - 'apt-get update && apt-get install -y docker.io && docker version' # install docker + - 'cargo install cross && cross --version' # install cross + # fix cargo and rustup mounts from this container (https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41227) + - 'mkdir -p $SHARED_PATH/cargo' + - 'cp -r $CARGO_HOME/bin $SHARED_PATH/cargo' + - 'cp -r $RUSTUP_HOME $SHARED_PATH' + - 'export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup' + # cross-compile conduit for target + - 'time ./cross/build.sh --locked --release' - 'cp "target/$TARGET/release/conduit" "conduit-$TARGET"' + cache: + # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci + - key: 'cargo-cache-$TARGET' + paths: + - $SHARED_PATH/cargo/registry/index + - $SHARED_PATH/cargo/registry/cache + - $SHARED_PATH/cargo/git/db artifacts: expire_in: never build:release:cargo:x86_64-unknown-linux-musl-with-debug: extends: .build-cargo-shared-settings - image: messense/rust-musl-cross:x86_64-musl variables: CARGO_PROFILE_RELEASE_DEBUG: 2 # Enable debug info for flamegraph profiling TARGET: "x86_64-unknown-linux-musl" @@ -61,7 +74,6 @@ build:release:cargo:x86_64-unknown-linux-musl-with-debug: build:release:cargo:x86_64-unknown-linux-musl: extends: .build-cargo-shared-settings - image: messense/rust-musl-cross:x86_64-musl variables: TARGET: "x86_64-unknown-linux-musl" artifacts: @@ -72,7 +84,6 @@ build:release:cargo:x86_64-unknown-linux-musl: build:release:cargo:arm-unknown-linux-musleabihf: extends: .build-cargo-shared-settings - image: messense/rust-musl-cross:arm-musleabihf variables: TARGET: "arm-unknown-linux-musleabihf" artifacts: @@ -83,7 +94,6 @@ build:release:cargo:arm-unknown-linux-musleabihf: build:release:cargo:armv7-unknown-linux-musleabihf: extends: .build-cargo-shared-settings - image: messense/rust-musl-cross:armv7-musleabihf variables: TARGET: "armv7-unknown-linux-musleabihf" artifacts: @@ -94,7 +104,6 @@ build:release:cargo:armv7-unknown-linux-musleabihf: build:release:cargo:aarch64-unknown-linux-musl: extends: .build-cargo-shared-settings - image: messense/rust-musl-cross:aarch64-musl variables: TARGET: "aarch64-unknown-linux-musl" artifacts: diff --git a/CROSS_COMPILE.md b/CROSS_COMPILE.md deleted file mode 100644 index e38a6ad7..00000000 --- a/CROSS_COMPILE.md +++ /dev/null @@ -1,11 +0,0 @@ -Install docker: - -``` -$ sudo apt install docker -$ sudo usermod -aG docker $USER -$ exec sudo su -l $USER -$ sudo systemctl start docker -$ cargo install cross -$ cross build --release --target armv7-unknown-linux-musleabihf -``` -The cross-compiled binary is at target/armv7-unknown-linux-musleabihf/release/conduit diff --git a/Cross.toml b/Cross.toml new file mode 100644 index 00000000..491efcb7 --- /dev/null +++ b/Cross.toml @@ -0,0 +1,11 @@ +[target.aarch64-unknown-linux-musl] +image = "rust-cross:aarch64-unknown-linux-musl" + +[target.arm-unknown-linux-musleabihf] +image = "rust-cross:arm-unknown-linux-musleabihf" + +[target.armv7-unknown-linux-musleabihf] +image = "rust-cross:armv7-unknown-linux-musleabihf" + +[target.x86_64-unknown-linux-musl] +image = "rust-cross:x86_64-unknown-linux-musl" diff --git a/DEPLOY.md b/DEPLOY.md index 0058b93d..38e1e286 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -37,7 +37,7 @@ $ cargo build --release Note that this currently requires Rust 1.50. -If you want to cross compile Conduit to another architecture, read the [Cross-Compile Guide](CROSS_COMPILE.md). +If you want to cross compile Conduit to another architecture, read the [Cross-Compile Guide](cross/README.md). ## Adding a Conduit user diff --git a/cross/README.md b/cross/README.md new file mode 100644 index 00000000..2829d239 --- /dev/null +++ b/cross/README.md @@ -0,0 +1,37 @@ +## Cross compilation + +The `cross` folder contains a set of convenience scripts (`build.sh` and `test.sh`) for cross-compiling Conduit. + +Currently supported targets are + +- aarch64-unknown-linux-musl +- arm-unknown-linux-musleabihf +- armv7-unknown-linux-musleabihf +- x86\_64-unknown-linux-musl + +### Install prerequisites +#### Docker +[Installation guide](https://docs.docker.com/get-docker/). +```sh +$ sudo apt install docker +$ sudo systemctl start docker +$ sudo usermod -aG docker $USER +$ newgrp docker +``` + +#### Cross +[Installation guide](https://github.com/rust-embedded/cross/#installation). +```sh +$ cargo install cross +``` + +### Buiding Conduit +```sh +$ TARGET=armv7-unknown-linux-musleabihf ./cross/build.sh --release +``` +The cross-compiled binary is at `target/armv7-unknown-linux-musleabihf/release/conduit` + +### Testing Conduit +```sh +$ TARGET=armv7-unknown-linux-musleabihf ./cross/test.sh --release +``` diff --git a/cross/build.sh b/cross/build.sh new file mode 100755 index 00000000..4a6d4493 --- /dev/null +++ b/cross/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -ex + +# build custom container with libclang and static compilation +tag="rust-cross:${TARGET:?}" +docker build --tag="$tag" - << EOF +FROM rustembedded/cross:$TARGET + +# Install libclang for generating bindings with rust-bindgen +# The architecture is not relevant here since it's not used for compilation +RUN apt-get update && \ + apt-get install --assume-yes libclang-dev + +# Set the target prefix +ENV TARGET_PREFIX="/usr/local/$(echo "${TARGET:?}" | sed -e 's/armv7/arm/' -e 's/-unknown//')" + +# Make sure that cc-rs links libc/libstdc++ statically when cross-compiling +# See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information +ENV RUSTFLAGS="-L\$TARGET_PREFIX/lib" CXXSTDLIB="static=stdc++" +# Forcefully linking against libatomic, libc and libgcc is required for arm32, otherwise symbols are missing +$([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-lgcc -Clink-arg=-latomic -lstatic=c"') +# Strip symbols while compiling in release mode +$([[ $@ =~ -r ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-s"') + +# Make sure that rust-bindgen uses the correct include path when cross-compiling +# See https://github.com/rust-lang/rust-bindgen#environment-variables for more information +ENV BINDGEN_EXTRA_CLANG_ARGS="-I\$TARGET_PREFIX/include" +EOF + +# build conduit for a specific target +cross build --target="$TARGET" $@ diff --git a/cross/test.sh b/cross/test.sh new file mode 100755 index 00000000..0aa0909c --- /dev/null +++ b/cross/test.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env sh +set -ex + +# Build conduit for a specific target +cross/build.sh $@ + +# Test conduit for a specific target +cross test --target="$TARGET" $@ From fd67cd7450e33b97050372bdd13832828fa75458 Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Sun, 23 Jan 2022 15:10:42 +0100 Subject: [PATCH 150/201] feat: support targetting i686 --- .gitlab-ci.yml | 28 +++++++++++++++++++--------- Cross.toml | 3 +++ cross/build.sh | 6 ++++-- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 993145a1..b5a12f3d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,7 +37,6 @@ variables: - "rustup show && rustc --version && cargo --version" # Print version info for debugging # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi - script: # install cross-compiling prerequisites - 'apt-get update && apt-get install -y docker.io && docker version' # install docker - 'cargo install cross && cross --version' # install cross @@ -46,16 +45,17 @@ variables: - 'cp -r $CARGO_HOME/bin $SHARED_PATH/cargo' - 'cp -r $RUSTUP_HOME $SHARED_PATH' - 'export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup' + script: # cross-compile conduit for target - 'time ./cross/build.sh --locked --release' - - 'cp "target/$TARGET/release/conduit" "conduit-$TARGET"' + - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"' cache: # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci - - key: 'cargo-cache-$TARGET' - paths: - - $SHARED_PATH/cargo/registry/index - - $SHARED_PATH/cargo/registry/cache - - $SHARED_PATH/cargo/git/db + key: 'cargo-cache-$TARGET' + paths: + - $SHARED_PATH/cargo/registry/index + - $SHARED_PATH/cargo/registry/cache + - $SHARED_PATH/cargo/git/db artifacts: expire_in: never @@ -82,6 +82,16 @@ build:release:cargo:x86_64-unknown-linux-musl: - "conduit-x86_64-unknown-linux-musl" expose_as: "Conduit for x86_64-unknown-linux-musl" +build:release:cargo:i686-unknown-linux-musl: + extends: .build-cargo-shared-settings + variables: + TARGET: "i686-unknown-linux-musl" + artifacts: + name: "conduit-i686-unknown-linux-musl" + paths: + - "conduit-i686-unknown-linux-musl" + expose_as: "Conduit for i686-unknown-linux-musl" + build:release:cargo:arm-unknown-linux-musleabihf: extends: .build-cargo-shared-settings variables: @@ -119,14 +129,14 @@ build:release:cargo:aarch64-unknown-linux-musl: cache: key: "build_cache--$TARGET--$CI_COMMIT_BRANCH--debug" script: - - "time cargo build --target $TARGET" + # cross-compile conduit for target + - 'time ./cross/build.sh --locked' - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"' artifacts: expire_in: 4 weeks build:debug:cargo:x86_64-unknown-linux-musl: extends: ".cargo-debug-shared-settings" - image: messense/rust-musl-cross:x86_64-musl variables: TARGET: "x86_64-unknown-linux-musl" artifacts: diff --git a/Cross.toml b/Cross.toml index 491efcb7..22c84b97 100644 --- a/Cross.toml +++ b/Cross.toml @@ -7,5 +7,8 @@ image = "rust-cross:arm-unknown-linux-musleabihf" [target.armv7-unknown-linux-musleabihf] image = "rust-cross:armv7-unknown-linux-musleabihf" +[target.i686-unknown-linux-musl] +image = "rust-cross:i686-unknown-linux-musl" + [target.x86_64-unknown-linux-musl] image = "rust-cross:x86_64-unknown-linux-musl" diff --git a/cross/build.sh b/cross/build.sh index 4a6d4493..24a2224b 100755 --- a/cross/build.sh +++ b/cross/build.sh @@ -17,8 +17,10 @@ ENV TARGET_PREFIX="/usr/local/$(echo "${TARGET:?}" | sed -e 's/armv7/arm/' -e 's # Make sure that cc-rs links libc/libstdc++ statically when cross-compiling # See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information ENV RUSTFLAGS="-L\$TARGET_PREFIX/lib" CXXSTDLIB="static=stdc++" -# Forcefully linking against libatomic, libc and libgcc is required for arm32, otherwise symbols are missing -$([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-lgcc -Clink-arg=-latomic -lstatic=c"') +# Forcefully linking against libatomic and libgcc is required for arm32, otherwise symbols are missing +$([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-lgcc -Clink-arg=-latomic"') +# Forcefully linking against libc is required for 32-bit, otherwise symbols are missing +$([[ $TARGET =~ arm|i686 ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -lstatic=c"') # Strip symbols while compiling in release mode $([[ $@ =~ -r ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-s"') From 219dfbabd58ff6008f8a85c291f8cf4f6da1318a Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Sun, 23 Jan 2022 17:31:12 +0100 Subject: [PATCH 151/201] fix: pass RUSTC_WRAPPER to the cross container and enforce static builds --- .gitlab-ci.yml | 18 ++++++------------ Cross.toml | 3 --- cross/build.sh | 9 +++++---- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b5a12f3d..fac678cf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,8 +35,6 @@ variables: before_script: - 'echo "Building for target $TARGET"' - "rustup show && rustc --version && cargo --version" # Print version info for debugging - # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: - - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi # install cross-compiling prerequisites - 'apt-get update && apt-get install -y docker.io && docker version' # install docker - 'cargo install cross && cross --version' # install cross @@ -45,10 +43,14 @@ variables: - 'cp -r $CARGO_HOME/bin $SHARED_PATH/cargo' - 'cp -r $RUSTUP_HOME $SHARED_PATH' - 'export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup' + # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: + - if [ -n "${SCCACHE_BIN_URL}" ]; then export RUSTC_WRAPPER=$SHARED_PATH/cargo/bin/sccache && curl $SCCACHE_BIN_URL --output $RUSTC_WRAPPER && chmod +x $RUSTC_WRAPPER; fi script: # cross-compile conduit for target - 'time ./cross/build.sh --locked --release' - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"' + # assert that the binary is statically linked + - 'file conduit-$TARGET | grep "static\(-pie\|ally\) linked"' cache: # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci key: 'cargo-cache-$TARGET' @@ -82,16 +84,6 @@ build:release:cargo:x86_64-unknown-linux-musl: - "conduit-x86_64-unknown-linux-musl" expose_as: "Conduit for x86_64-unknown-linux-musl" -build:release:cargo:i686-unknown-linux-musl: - extends: .build-cargo-shared-settings - variables: - TARGET: "i686-unknown-linux-musl" - artifacts: - name: "conduit-i686-unknown-linux-musl" - paths: - - "conduit-i686-unknown-linux-musl" - expose_as: "Conduit for i686-unknown-linux-musl" - build:release:cargo:arm-unknown-linux-musleabihf: extends: .build-cargo-shared-settings variables: @@ -132,6 +124,8 @@ build:release:cargo:aarch64-unknown-linux-musl: # cross-compile conduit for target - 'time ./cross/build.sh --locked' - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"' + # assert that the binary is statically linked + - 'file conduit-debug-$TARGET | grep "static\(-pie\|ally\) linked"' artifacts: expire_in: 4 weeks diff --git a/Cross.toml b/Cross.toml index 22c84b97..491efcb7 100644 --- a/Cross.toml +++ b/Cross.toml @@ -7,8 +7,5 @@ image = "rust-cross:arm-unknown-linux-musleabihf" [target.armv7-unknown-linux-musleabihf] image = "rust-cross:armv7-unknown-linux-musleabihf" -[target.i686-unknown-linux-musl] -image = "rust-cross:i686-unknown-linux-musl" - [target.x86_64-unknown-linux-musl] image = "rust-cross:x86_64-unknown-linux-musl" diff --git a/cross/build.sh b/cross/build.sh index 24a2224b..34082606 100755 --- a/cross/build.sh +++ b/cross/build.sh @@ -17,13 +17,14 @@ ENV TARGET_PREFIX="/usr/local/$(echo "${TARGET:?}" | sed -e 's/armv7/arm/' -e 's # Make sure that cc-rs links libc/libstdc++ statically when cross-compiling # See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information ENV RUSTFLAGS="-L\$TARGET_PREFIX/lib" CXXSTDLIB="static=stdc++" -# Forcefully linking against libatomic and libgcc is required for arm32, otherwise symbols are missing -$([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-lgcc -Clink-arg=-latomic"') -# Forcefully linking against libc is required for 32-bit, otherwise symbols are missing -$([[ $TARGET =~ arm|i686 ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -lstatic=c"') +# Forcefully linking against libatomic, libc and libgcc is required for arm32, otherwise symbols are missing +$([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-lgcc -Clink-arg=-latomic -lstatic=c"') # Strip symbols while compiling in release mode $([[ $@ =~ -r ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-s"') +# Support a rustc wrapper like sccache when cross-compiling +ENV RUSTC_WRAPPER="$RUSTC_WRAPPER" + # Make sure that rust-bindgen uses the correct include path when cross-compiling # See https://github.com/rust-lang/rust-bindgen#environment-variables for more information ENV BINDGEN_EXTRA_CLANG_ARGS="-I\$TARGET_PREFIX/include" From c2ad2b3dd747e7a8baa5ff2f9ade8edb92204aa6 Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Sun, 23 Jan 2022 17:38:13 +0100 Subject: [PATCH 152/201] fix: pass sccache variables to cross container with build.env.passthrough --- Cross.toml | 12 ++++++++++++ cross/build.sh | 3 --- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Cross.toml b/Cross.toml index 491efcb7..a989a98f 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,3 +1,15 @@ +[build.env] +# CI uses an S3 endpoint to store sccache artifacts, so their config needs to +# be available in the cross container as well +passthrough = [ + "RUSTC_WRAPPER", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "SCCACHE_BUCKET", + "SCCACHE_ENDPOINT", + "SCCACHE_S3_USE_SSL", +] + [target.aarch64-unknown-linux-musl] image = "rust-cross:aarch64-unknown-linux-musl" diff --git a/cross/build.sh b/cross/build.sh index 34082606..4a6d4493 100755 --- a/cross/build.sh +++ b/cross/build.sh @@ -22,9 +22,6 @@ $([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-lgcc -Clin # Strip symbols while compiling in release mode $([[ $@ =~ -r ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-s"') -# Support a rustc wrapper like sccache when cross-compiling -ENV RUSTC_WRAPPER="$RUSTC_WRAPPER" - # Make sure that rust-bindgen uses the correct include path when cross-compiling # See https://github.com/rust-lang/rust-bindgen#environment-variables for more information ENV BINDGEN_EXTRA_CLANG_ARGS="-I\$TARGET_PREFIX/include" From c7560b3502d27a49f935c347458df6421459c485 Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Sun, 23 Jan 2022 18:09:14 +0100 Subject: [PATCH 153/201] fix: remove libgcc dependency in ci builds since the binary is ensured to be statically compiled --- docker/ci-binaries-packaging.Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/ci-binaries-packaging.Dockerfile b/docker/ci-binaries-packaging.Dockerfile index f4603105..a6339be3 100644 --- a/docker/ci-binaries-packaging.Dockerfile +++ b/docker/ci-binaries-packaging.Dockerfile @@ -9,6 +9,7 @@ FROM docker.io/alpine:3.15.0 AS runner + # Standard port on which Conduit launches. # You still need to map the port when using the docker command or docker-compose. EXPOSE 6167 @@ -18,10 +19,8 @@ ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" # Conduit needs: # ca-certificates: for https -# libgcc: Apparently this is needed, even if I (@jfowl) don't know exactly why. But whatever, it's not that big. RUN apk add --no-cache \ - ca-certificates \ - libgcc + ca-certificates ARG CREATED From 64c25ea4a15739f75ebe2811e84dc00280ba5fb0 Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Sun, 23 Jan 2022 18:31:40 +0100 Subject: [PATCH 154/201] fix: always print ELF information --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fac678cf..defd66ee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,7 +50,8 @@ variables: - 'time ./cross/build.sh --locked --release' - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"' # assert that the binary is statically linked - - 'file conduit-$TARGET | grep "static\(-pie\|ally\) linked"' + - 'ldd conduit-$TARGET' # print linking information + - 'file conduit-$TARGET | sed -e "/static\(-pie\|ally\) linked/!q1"' # print elf information cache: # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci key: 'cargo-cache-$TARGET' @@ -125,7 +126,8 @@ build:release:cargo:aarch64-unknown-linux-musl: - 'time ./cross/build.sh --locked' - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"' # assert that the binary is statically linked - - 'file conduit-debug-$TARGET | grep "static\(-pie\|ally\) linked"' + - 'ldd conduit-debug-$TARGET' # print linking information + - 'file conduit-debug-$TARGET | sed -e "/static\(-pie\|ally\) linked/!q1"' # print elf information artifacts: expire_in: 4 weeks From 77ad4cb8f8c69b563c890494b5d203d96195253d Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Sun, 23 Jan 2022 19:24:36 +0100 Subject: [PATCH 155/201] fix: use readelf for checking static compilation --- .gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index defd66ee..9e584a2a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -49,9 +49,9 @@ variables: # cross-compile conduit for target - 'time ./cross/build.sh --locked --release' - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"' - # assert that the binary is statically linked - - 'ldd conduit-$TARGET' # print linking information - - 'file conduit-$TARGET | sed -e "/static\(-pie\|ally\) linked/!q1"' # print elf information + # print information about linking for debugging + - 'file conduit-$TARGET' # print file information + - 'readelf --dynamic conduit-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked cache: # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci key: 'cargo-cache-$TARGET' @@ -125,9 +125,9 @@ build:release:cargo:aarch64-unknown-linux-musl: # cross-compile conduit for target - 'time ./cross/build.sh --locked' - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"' - # assert that the binary is statically linked - - 'ldd conduit-debug-$TARGET' # print linking information - - 'file conduit-debug-$TARGET | sed -e "/static\(-pie\|ally\) linked/!q1"' # print elf information + # print information about linking for debugging + - 'file conduit-debug-$TARGET' # print file information + - 'readelf --dynamic conduit-debug-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked artifacts: expire_in: 4 weeks From 067fcfc0e40ced1c2ed28f32b04940c6e74d6f5a Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Sun, 23 Jan 2022 21:19:19 +0100 Subject: [PATCH 156/201] fix: remove trailing slash from shared path --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9e584a2a..aac773c2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,7 +28,7 @@ variables: DOCKER_HOST: tcp://docker:2375/ DOCKER_TLS_CERTDIR: "" DOCKER_DRIVER: overlay2 - SHARED_PATH: $CI_PROJECT_DIR/shared/ + SHARED_PATH: $CI_PROJECT_DIR/shared CARGO_PROFILE_RELEASE_LTO: "true" CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1" CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow From acf1585fc35e7851df9c30208f654d5c085267d6 Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Mon, 24 Jan 2022 11:45:07 +0100 Subject: [PATCH 157/201] fix: make sure that libatomic is linked statically --- cross/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cross/build.sh b/cross/build.sh index 4a6d4493..8f64ff87 100755 --- a/cross/build.sh +++ b/cross/build.sh @@ -18,7 +18,7 @@ ENV TARGET_PREFIX="/usr/local/$(echo "${TARGET:?}" | sed -e 's/armv7/arm/' -e 's # See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information ENV RUSTFLAGS="-L\$TARGET_PREFIX/lib" CXXSTDLIB="static=stdc++" # Forcefully linking against libatomic, libc and libgcc is required for arm32, otherwise symbols are missing -$([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-lgcc -Clink-arg=-latomic -lstatic=c"') +$([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-static-libgcc -Clink-arg=-lgcc -lstatic=atomic -lstatic=c"') # Strip symbols while compiling in release mode $([[ $@ =~ -r ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-s"') From ff167299766d41c361398243db6bf97e9d45fa65 Mon Sep 17 00:00:00 2001 From: Maxim De Clercq Date: Tue, 25 Jan 2022 22:36:51 +0100 Subject: [PATCH 158/201] fix: correct RUSTC_WRAPPER path in cross container --- .gitlab-ci.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aac773c2..741b5327 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,6 +8,10 @@ variables: GIT_SUBMODULE_STRATEGY: recursive FF_USE_FASTZIP: 1 CACHE_COMPRESSION_LEVEL: fastest + # Docker in Docker + DOCKER_HOST: tcp://docker:2375/ + DOCKER_TLS_CERTDIR: "" + DOCKER_DRIVER: overlay2 # --------------------------------------------------------------------- # # Cargo: Compiling for different architectures # @@ -25,9 +29,6 @@ variables: tags: ["docker"] services: ["docker:dind"] variables: - DOCKER_HOST: tcp://docker:2375/ - DOCKER_TLS_CERTDIR: "" - DOCKER_DRIVER: overlay2 SHARED_PATH: $CI_PROJECT_DIR/shared CARGO_PROFILE_RELEASE_LTO: "true" CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1" @@ -43,8 +44,9 @@ variables: - 'cp -r $CARGO_HOME/bin $SHARED_PATH/cargo' - 'cp -r $RUSTUP_HOME $SHARED_PATH' - 'export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup' - # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: - - if [ -n "${SCCACHE_BIN_URL}" ]; then export RUSTC_WRAPPER=$SHARED_PATH/cargo/bin/sccache && curl $SCCACHE_BIN_URL --output $RUSTC_WRAPPER && chmod +x $RUSTC_WRAPPER; fi + # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results. + # The sccache binary is stored in the sysroot of the rustc installation since that directory is added to the path of the cross container. + - if [ -n "${SCCACHE_BIN_URL}" ]; then RUSTC_SYSROOT=$(rustc --print sysroot) && curl $SCCACHE_BIN_URL --output $RUSTC_SYSROOT/bin/sccache && chmod +x $RUSTC_SYSROOT/bin/sccache && export RUSTC_WRAPPER=sccache; fi script: # cross-compile conduit for target - 'time ./cross/build.sh --locked --release' @@ -157,9 +159,6 @@ build:debug:cargo:x86_64-unknown-linux-musl: - "build:release:cargo:armv7-unknown-linux-musleabihf" - "build:release:cargo:aarch64-unknown-linux-musl" variables: - DOCKER_HOST: tcp://docker:2375/ - DOCKER_TLS_CERTDIR: "" - DOCKER_DRIVER: overlay2 PLATFORMS: "linux/arm/v6,linux/arm/v7,linux/arm64,linux/amd64" DOCKER_FILE: "docker/ci-binaries-packaging.Dockerfile" cache: From 8472eff277faf55808dc794a8eb9023fafb75763 Mon Sep 17 00:00:00 2001 From: Andrej Kacian Date: Thu, 27 Jan 2022 00:25:20 +0100 Subject: [PATCH 159/201] Implement media download with custom filename --- src/client_server/media.rs | 64 +++++++++++++++++++++++++++++++++++++- src/main.rs | 1 + 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 0a7f4bb5..2e3cf056 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -4,7 +4,10 @@ use crate::{ }; use ruma::api::client::{ error::ErrorKind, - r0::media::{create_content, get_content, get_content_thumbnail, get_media_config}, + r0::media::{ + create_content, get_content, get_content_as_filename, get_content_thumbnail, + get_media_config, + }, }; use std::convert::TryInto; @@ -129,6 +132,65 @@ pub async fn get_content_route( } } +/// # `GET /_matrix/media/r0/download/{serverName}/{mediaId}/{fileName}` +/// +/// Load media from our server or over federation, permitting desired filename. +/// +/// - Only allows federation if `allow_remote` is true +#[cfg_attr( + feature = "conduit_bin", + get("/_matrix/media/r0/download/<_>/<_>/<_>", data = "") +)] +#[tracing::instrument(skip(db, body))] +pub async fn get_content_as_filename_route( + db: DatabaseGuard, + body: Ruma>, +) -> ConduitResult { + let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); + + if let Some(FileMeta { + content_disposition: _, + content_type, + file, + }) = db.media.get(&db.globals, &mxc).await? + { + Ok(get_content_as_filename::Response { + file, + content_type, + content_disposition: Some(format!("inline; filename={}", body.filename)), + } + .into()) + } else if &*body.server_name != db.globals.server_name() && body.allow_remote { + let get_content_response = db + .sending + .send_federation_request( + &db.globals, + &body.server_name, + get_content_as_filename::Request { + allow_remote: false, + server_name: &body.server_name, + media_id: &body.media_id, + filename: &body.filename, + }, + ) + .await?; + + db.media + .create( + mxc, + &db.globals, + &get_content_response.content_disposition.as_deref(), + &get_content_response.content_type.as_deref(), + &get_content_response.file, + ) + .await?; + + Ok(get_content_response.into()) + } else { + Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")) + } +} + /// # `POST /_matrix/media/r0/thumbnail/{serverName}/{mediaId}` /// /// Load media thumbnail from our server or over federation. diff --git a/src/main.rs b/src/main.rs index 56faa3e7..514c2448 100644 --- a/src/main.rs +++ b/src/main.rs @@ -136,6 +136,7 @@ fn setup_rocket(config: Figment, data: Arc>) -> rocket::Rocket< client_server::send_event_to_device_route, client_server::get_media_config_route, client_server::create_content_route, + client_server::get_content_as_filename_route, client_server::get_content_route, client_server::get_content_thumbnail_route, client_server::get_devices_route, From 52873c88b7e204f5b3c9295448c15780fff8084c Mon Sep 17 00:00:00 2001 From: Andrej Kacian Date: Thu, 27 Jan 2022 00:27:13 +0100 Subject: [PATCH 160/201] Fix incorrect HTTP method in doc comments of two media routes --- src/client_server/media.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 2e3cf056..5b196dff 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -74,7 +74,7 @@ pub async fn create_content_route( .into()) } -/// # `POST /_matrix/media/r0/download/{serverName}/{mediaId}` +/// # `GET /_matrix/media/r0/download/{serverName}/{mediaId}` /// /// Load media from our server or over federation. /// @@ -191,7 +191,7 @@ pub async fn get_content_as_filename_route( } } -/// # `POST /_matrix/media/r0/thumbnail/{serverName}/{mediaId}` +/// # `GET /_matrix/media/r0/thumbnail/{serverName}/{mediaId}` /// /// Load media thumbnail from our server or over federation. /// From 9c2000cb8973894512940b96996a2b5937f5cc8f Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Thu, 27 Jan 2022 16:17:55 +0100 Subject: [PATCH 161/201] Upgrade Ruma --- Cargo.lock | 36 ++++++++++++++++++------------------ Cargo.toml | 2 +- src/database/key_backups.rs | 29 ++++++++--------------------- 3 files changed, 27 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 493ac082..794a0257 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2107,7 +2107,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "assign", "js_int", @@ -2128,7 +2128,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.18.5" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "bytes", "http", @@ -2144,7 +2144,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.18.5" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2155,7 +2155,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "ruma-api", "ruma-common", @@ -2169,7 +2169,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.12.3" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "assign", "bytes", @@ -2189,7 +2189,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.6.0" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "indexmap", "js_int", @@ -2204,7 +2204,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.24.6" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "indoc", "js_int", @@ -2221,7 +2221,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.24.6" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2232,7 +2232,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.3.1" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "js_int", "ruma-api", @@ -2247,7 +2247,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.20.0" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "percent-encoding", "rand 0.8.4", @@ -2262,7 +2262,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.20.0" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "quote", "ruma-identifiers-validation", @@ -2272,7 +2272,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "thiserror", ] @@ -2280,7 +2280,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "js_int", "ruma-api", @@ -2293,7 +2293,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "js_int", "ruma-api", @@ -2308,7 +2308,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "base64 0.13.0", "bytes", @@ -2323,7 +2323,7 @@ dependencies = [ [[package]] name = "ruma-serde-macros" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2334,7 +2334,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.9.0" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "base64 0.13.0", "ed25519-dalek", @@ -2351,7 +2351,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.4.1" -source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a" +source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" dependencies = [ "itertools", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 78a4c8ff..9ba1ac05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle request # Used for matrix spec type definitions and helpers #ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } -ruma = { git = "https://github.com/ruma/ruma", rev = "08d60b3d376b63462f769d4b9bd3bbfb560d501a", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +ruma = { git = "https://github.com/ruma/ruma", rev = "82becb86c837570224964425929d1b5305784435", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { git = "https://github.com/timokoesters/ruma", rev = "50c1db7e0a3a21fc794b0cce3b64285a4c750c71", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } diff --git a/src/database/key_backups.rs b/src/database/key_backups.rs index b74bc408..2eefe481 100644 --- a/src/database/key_backups.rs +++ b/src/database/key_backups.rs @@ -7,7 +7,6 @@ use ruma::{ serde::Raw, RoomId, UserId, }; -use serde_json::json; use std::{collections::BTreeMap, sync::Arc}; use super::abstraction::Tree; @@ -212,13 +211,13 @@ impl KeyBackups { &self, user_id: &UserId, version: &str, - ) -> Result, Raw>> { + ) -> Result, RoomKeyBackup>> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xff); prefix.extend_from_slice(version.as_bytes()); prefix.push(0xff); - let mut rooms = BTreeMap::, Raw>::new(); + let mut rooms = BTreeMap::, RoomKeyBackup>::new(); for result in self .backupkeyid_backup @@ -244,7 +243,7 @@ impl KeyBackups { Error::bad_database("backupkeyid_backup room_id is invalid room id.") })?; - let key_data: serde_json::Value = serde_json::from_slice(&value).map_err(|_| { + let key_data = serde_json::from_slice(&value).map_err(|_| { Error::bad_database("KeyBackupData in backupkeyid_backup is invalid.") })?; @@ -252,25 +251,13 @@ impl KeyBackups { }) { let (room_id, session_id, key_data) = result?; - let room_key_backup = rooms.entry(room_id).or_insert_with(|| { - Raw::new(&RoomKeyBackup { + rooms + .entry(room_id) + .or_insert_with(|| RoomKeyBackup { sessions: BTreeMap::new(), }) - .expect("RoomKeyBackup serialization") - }); - - let mut object = room_key_backup - .deserialize_as::>() - .map_err(|_| Error::bad_database("RoomKeyBackup is not an object"))?; - - let sessions = object.entry("session").or_insert_with(|| json!({})); - if let serde_json::Value::Object(unsigned_object) = sessions { - unsigned_object.insert(session_id, key_data); - } - - *room_key_backup = Raw::from_json( - serde_json::value::to_raw_value(&object).expect("Value => RawValue serialization"), - ); + .sessions + .insert(session_id, key_data); } Ok(rooms) From c4317a7a962fcc4b41b1abfc273034d9827a6563 Mon Sep 17 00:00:00 2001 From: Andrej Kacian Date: Thu, 27 Jan 2022 16:12:39 +0100 Subject: [PATCH 162/201] Reduce code duplication in media download route handlers --- src/client_server/media.rs | 99 ++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 48 deletions(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 5b196dff..bd73cff5 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -74,6 +74,38 @@ pub async fn create_content_route( .into()) } +pub async fn get_remote_content( + db: &DatabaseGuard, + mxc: &str, + server_name: &ruma::ServerName, + media_id: &str +) -> ConduitResult { + let content_response = db + .sending + .send_federation_request( + &db.globals, + server_name, + get_content::Request { + allow_remote: false, + server_name, + media_id + }, + ) + .await?; + + db.media + .create( + mxc.to_string(), + &db.globals, + &content_response.content_disposition.as_deref(), + &content_response.content_type.as_deref(), + &content_response.file, + ) + .await?; + + Ok(content_response.into()) +} + /// # `GET /_matrix/media/r0/download/{serverName}/{mediaId}` /// /// Load media from our server or over federation. @@ -103,30 +135,13 @@ pub async fn get_content_route( } .into()) } else if &*body.server_name != db.globals.server_name() && body.allow_remote { - let get_content_response = db - .sending - .send_federation_request( - &db.globals, - &body.server_name, - get_content::Request { - allow_remote: false, - server_name: &body.server_name, - media_id: &body.media_id, - }, - ) - .await?; - - db.media - .create( - mxc, - &db.globals, - &get_content_response.content_disposition.as_deref(), - &get_content_response.content_type.as_deref(), - &get_content_response.file, - ) - .await?; - - Ok(get_content_response.into()) + let remote_content_response = get_remote_content( + &db, + &mxc, + &body.server_name, + &body.media_id + ).await?; + Ok(remote_content_response) } else { Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")) } @@ -161,31 +176,19 @@ pub async fn get_content_as_filename_route( } .into()) } else if &*body.server_name != db.globals.server_name() && body.allow_remote { - let get_content_response = db - .sending - .send_federation_request( - &db.globals, - &body.server_name, - get_content_as_filename::Request { - allow_remote: false, - server_name: &body.server_name, - media_id: &body.media_id, - filename: &body.filename, - }, - ) - .await?; + let remote_content_response = get_remote_content( + &db, + &mxc, + &body.server_name, + &body.media_id + ).await?; - db.media - .create( - mxc, - &db.globals, - &get_content_response.content_disposition.as_deref(), - &get_content_response.content_type.as_deref(), - &get_content_response.file, - ) - .await?; - - Ok(get_content_response.into()) + Ok(get_content_as_filename::Response { + content_disposition: Some(format!("inline: filename={}", body.filename)), + content_type: remote_content_response.0.content_type, + file: remote_content_response.0.file + } + .into()) } else { Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")) } From ccfc243c2c1fec9b859ab6accec1246fa63aef94 Mon Sep 17 00:00:00 2001 From: Andrej Kacian Date: Thu, 27 Jan 2022 17:00:08 +0100 Subject: [PATCH 163/201] Make get_remote_content() return Result instead of ConduitResult --- src/client_server/media.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index bd73cff5..dd8e7b0f 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -79,7 +79,7 @@ pub async fn get_remote_content( mxc: &str, server_name: &ruma::ServerName, media_id: &str -) -> ConduitResult { +) -> Result { let content_response = db .sending .send_federation_request( @@ -103,7 +103,7 @@ pub async fn get_remote_content( ) .await?; - Ok(content_response.into()) + Ok(content_response) } /// # `GET /_matrix/media/r0/download/{serverName}/{mediaId}` @@ -141,7 +141,7 @@ pub async fn get_content_route( &body.server_name, &body.media_id ).await?; - Ok(remote_content_response) + Ok(remote_content_response.into()) } else { Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")) } @@ -185,8 +185,8 @@ pub async fn get_content_as_filename_route( Ok(get_content_as_filename::Response { content_disposition: Some(format!("inline: filename={}", body.filename)), - content_type: remote_content_response.0.content_type, - file: remote_content_response.0.file + content_type: remote_content_response.content_type, + file: remote_content_response.file } .into()) } else { From 0f6d232cb1117d959a1984551ee2872558172767 Mon Sep 17 00:00:00 2001 From: Andrej Kacian Date: Thu, 27 Jan 2022 17:08:04 +0100 Subject: [PATCH 164/201] Style fixes from 'cargo fmt' --- src/client_server/media.rs | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index dd8e7b0f..a827d64f 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -78,7 +78,7 @@ pub async fn get_remote_content( db: &DatabaseGuard, mxc: &str, server_name: &ruma::ServerName, - media_id: &str + media_id: &str, ) -> Result { let content_response = db .sending @@ -88,7 +88,7 @@ pub async fn get_remote_content( get_content::Request { allow_remote: false, server_name, - media_id + media_id, }, ) .await?; @@ -135,12 +135,8 @@ pub async fn get_content_route( } .into()) } else if &*body.server_name != db.globals.server_name() && body.allow_remote { - let remote_content_response = get_remote_content( - &db, - &mxc, - &body.server_name, - &body.media_id - ).await?; + let remote_content_response = + get_remote_content(&db, &mxc, &body.server_name, &body.media_id).await?; Ok(remote_content_response.into()) } else { Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")) @@ -176,17 +172,13 @@ pub async fn get_content_as_filename_route( } .into()) } else if &*body.server_name != db.globals.server_name() && body.allow_remote { - let remote_content_response = get_remote_content( - &db, - &mxc, - &body.server_name, - &body.media_id - ).await?; + let remote_content_response = + get_remote_content(&db, &mxc, &body.server_name, &body.media_id).await?; Ok(get_content_as_filename::Response { content_disposition: Some(format!("inline: filename={}", body.filename)), content_type: remote_content_response.content_type, - file: remote_content_response.file + file: remote_content_response.file, } .into()) } else { From f8d1c1a8af122d8955b4b08fa564723badbb3f77 Mon Sep 17 00:00:00 2001 From: "Aode (Lion)" Date: Mon, 24 Jan 2022 18:42:15 -0600 Subject: [PATCH 165/201] Re-use a basic request in all possible cases --- src/appservice_server.rs | 6 +----- src/database/globals.rs | 40 ++++++++++++++++++++++++++++++---------- src/database/pusher.rs | 6 +----- src/server_server.rs | 23 ++++++++++++----------- 4 files changed, 44 insertions(+), 31 deletions(-) diff --git a/src/appservice_server.rs b/src/appservice_server.rs index ed886d6c..a5d795f6 100644 --- a/src/appservice_server.rs +++ b/src/appservice_server.rs @@ -46,11 +46,7 @@ where *reqwest_request.timeout_mut() = Some(Duration::from_secs(30)); let url = reqwest_request.url().clone(); - let mut response = globals - .reqwest_client()? - .build()? - .execute(reqwest_request) - .await?; + let mut response = globals.reqwest_client().execute(reqwest_request).await?; // reqwest::Response -> http::Response conversion let status = response.status(); diff --git a/src/database/globals.rs b/src/database/globals.rs index 098d8197..da91c1fb 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -39,6 +39,7 @@ pub struct Globals { keypair: Arc, dns_resolver: TokioAsyncResolver, jwt_decoding_key: Option>, + basic_client: reqwest::Client, pub(super) server_signingkeys: Arc, pub bad_event_ratelimiter: Arc, RateLimitState>>>, pub bad_signature_ratelimiter: Arc, RateLimitState>>>, @@ -132,6 +133,8 @@ impl Globals { .as_ref() .map(|secret| jsonwebtoken::DecodingKey::from_secret(secret.as_bytes()).into_static()); + let basic_client = reqwest_client_builder(&config, None)?.build()?; + let s = Self { globals, config, @@ -141,6 +144,7 @@ impl Globals { })?, actual_destination_cache: Arc::new(RwLock::new(WellKnownMap::new())), tls_name_override, + basic_client, server_signingkeys, jwt_decoding_key, bad_event_ratelimiter: Arc::new(RwLock::new(HashMap::new())), @@ -163,17 +167,15 @@ impl Globals { &self.keypair } - /// Returns a reqwest client which can be used to send requests. - pub fn reqwest_client(&self) -> Result { - let mut reqwest_client_builder = reqwest::Client::builder() - .connect_timeout(Duration::from_secs(30)) - .timeout(Duration::from_secs(60 * 3)) - .pool_max_idle_per_host(1); - if let Some(proxy) = self.config.proxy.to_proxy()? { - reqwest_client_builder = reqwest_client_builder.proxy(proxy); - } + /// Returns a reqwest client which can be used to send requests + pub fn reqwest_client(&self) -> reqwest::Client { + // can't return &Client or else we'll hold a lock around the DB across an await + self.basic_client.clone() + } - Ok(reqwest_client_builder) + /// Returns a reqwest client builder which can be customized and used to send requests. + pub fn reqwest_client_builder(&self) -> Result { + reqwest_client_builder(&self.config, Some(1)) } #[tracing::instrument(skip(self))] @@ -340,3 +342,21 @@ impl Globals { r } } + +fn reqwest_client_builder( + config: &Config, + max_idle: Option, +) -> Result { + let mut reqwest_client_builder = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(30)) + .timeout(Duration::from_secs(60 * 3)); + + if let Some(max_idle) = max_idle { + reqwest_client_builder = reqwest_client_builder.pool_max_idle_per_host(max_idle); + } + if let Some(proxy) = config.proxy.to_proxy()? { + reqwest_client_builder = reqwest_client_builder.proxy(proxy); + } + + Ok(reqwest_client_builder) +} diff --git a/src/database/pusher.rs b/src/database/pusher.rs index 97ca85d8..d63db1d7 100644 --- a/src/database/pusher.rs +++ b/src/database/pusher.rs @@ -115,11 +115,7 @@ where //*reqwest_request.timeout_mut() = Some(Duration::from_secs(5)); let url = reqwest_request.url().clone(); - let response = globals - .reqwest_client()? - .build()? - .execute(reqwest_request) - .await; + let response = globals.reqwest_client().execute(reqwest_request).await; match response { Ok(mut response) => { diff --git a/src/server_server.rs b/src/server_server.rs index 9129951b..205355f9 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -237,21 +237,25 @@ where let url = reqwest_request.url().clone(); - let mut client = globals.reqwest_client()?; - if let Some((override_name, port)) = globals + let client = if let Some((override_name, port)) = globals .tls_name_override .read() .unwrap() .get(&actual_destination.hostname()) { - client = client.resolve( - &actual_destination.hostname(), - SocketAddr::new(override_name[0], *port), - ); + globals + .reqwest_client_builder()? + .resolve( + &actual_destination.hostname(), + SocketAddr::new(override_name[0], *port), + ) + .build()? // port will be ignored - } + } else { + globals.reqwest_client() + }; - let response = client.build()?.execute(reqwest_request).await; + let response = client.execute(reqwest_request).await; match response { Ok(mut response) => { @@ -492,9 +496,6 @@ async fn request_well_known( let body: serde_json::Value = serde_json::from_str( &globals .reqwest_client() - .ok()? - .build() - .ok()? .get(&format!( "https://{}/.well-known/matrix/server", destination From 1059f35fdcf4942fd253748121c883ea38b427a7 Mon Sep 17 00:00:00 2001 From: "Aode (lion)" Date: Thu, 27 Jan 2022 10:19:28 -0600 Subject: [PATCH 166/201] use pre-constructed client for well-known requests also --- Cargo.lock | 3 +-- Cargo.toml | 2 +- src/database/globals.rs | 30 ++++++++++++++++++------------ src/server_server.rs | 20 +------------------- 4 files changed, 21 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 794a0257..21c27700 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1950,8 +1950,7 @@ dependencies = [ [[package]] name = "reqwest" version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f242f1488a539a79bac6dbe7c8609ae43b7914b7736210f239a37cccb32525" +source = "git+https://github.com/niuhuan/reqwest?branch=dns-resolver-fn#57b7cf4feb921573dfafad7d34b9ac6e44ead0bd" dependencies = [ "base64 0.13.0", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 9ba1ac05..974b4ce8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,7 +48,7 @@ rand = "0.8.4" # Used to hash passwords rust-argon2 = "0.8.3" # Used to send requests -reqwest = { version = "0.11.4", default-features = false, features = ["rustls-tls", "socks"] } +reqwest = { version = "0.11.4", default-features = false, features = ["rustls-tls", "socks"], git = "https://github.com/niuhuan/reqwest", branch = "dns-resolver-fn" } # Used for conduit::Error type thiserror = "1.0.28" # Used to generate thumbnails for images diff --git a/src/database/globals.rs b/src/database/globals.rs index da91c1fb..3278b7f6 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -10,7 +10,7 @@ use std::{ collections::{BTreeMap, HashMap}, fs, future::Future, - net::IpAddr, + net::{IpAddr, SocketAddr}, path::PathBuf, sync::{Arc, Mutex, RwLock}, time::{Duration, Instant}, @@ -39,6 +39,7 @@ pub struct Globals { keypair: Arc, dns_resolver: TokioAsyncResolver, jwt_decoding_key: Option>, + well_known_client: reqwest::Client, basic_client: reqwest::Client, pub(super) server_signingkeys: Arc, pub bad_event_ratelimiter: Arc, RateLimitState>>>, @@ -133,7 +134,16 @@ impl Globals { .as_ref() .map(|secret| jsonwebtoken::DecodingKey::from_secret(secret.as_bytes()).into_static()); - let basic_client = reqwest_client_builder(&config, None)?.build()?; + let basic_client = reqwest_client_builder(&config)?.build()?; + let name_override = Arc::clone(&tls_name_override); + let well_known_client = reqwest_client_builder(&config)? + .resolve_fn(move |domain| { + let read_guard = name_override.read().unwrap(); + let (override_name, port) = read_guard.get(&domain)?; + let first_name = override_name.get(0)?; + Some(SocketAddr::new(*first_name, *port)) + }) + .build()?; let s = Self { globals, @@ -144,6 +154,7 @@ impl Globals { })?, actual_destination_cache: Arc::new(RwLock::new(WellKnownMap::new())), tls_name_override, + well_known_client, basic_client, server_signingkeys, jwt_decoding_key, @@ -173,9 +184,10 @@ impl Globals { self.basic_client.clone() } - /// Returns a reqwest client builder which can be customized and used to send requests. - pub fn reqwest_client_builder(&self) -> Result { - reqwest_client_builder(&self.config, Some(1)) + /// Returns a client used for resolving .well-knowns + pub fn well_known_client(&self) -> reqwest::Client { + // can't return &Client or else we'll hold a lock around the DB across an await + self.well_known_client.clone() } #[tracing::instrument(skip(self))] @@ -343,17 +355,11 @@ impl Globals { } } -fn reqwest_client_builder( - config: &Config, - max_idle: Option, -) -> Result { +fn reqwest_client_builder(config: &Config) -> Result { let mut reqwest_client_builder = reqwest::Client::builder() .connect_timeout(Duration::from_secs(30)) .timeout(Duration::from_secs(60 * 3)); - if let Some(max_idle) = max_idle { - reqwest_client_builder = reqwest_client_builder.pool_max_idle_per_host(max_idle); - } if let Some(proxy) = config.proxy.to_proxy()? { reqwest_client_builder = reqwest_client_builder.proxy(proxy); } diff --git a/src/server_server.rs b/src/server_server.rs index 205355f9..978eb67f 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -237,25 +237,7 @@ where let url = reqwest_request.url().clone(); - let client = if let Some((override_name, port)) = globals - .tls_name_override - .read() - .unwrap() - .get(&actual_destination.hostname()) - { - globals - .reqwest_client_builder()? - .resolve( - &actual_destination.hostname(), - SocketAddr::new(override_name[0], *port), - ) - .build()? - // port will be ignored - } else { - globals.reqwest_client() - }; - - let response = client.execute(reqwest_request).await; + let response = globals.well_known_client().execute(reqwest_request).await; match response { Ok(mut response) => { From 529e88c7f9d8997a4fbbc84f98120d1b31d2e39e Mon Sep 17 00:00:00 2001 From: Andrej Kacian Date: Thu, 27 Jan 2022 17:47:09 +0100 Subject: [PATCH 167/201] Do not copy mxc string unnecessarily in db.get_thumbnail() --- src/client_server/media.rs | 2 +- src/database/media.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index a827d64f..8524c57e 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -207,7 +207,7 @@ pub async fn get_content_thumbnail_route( }) = db .media .get_thumbnail( - mxc.clone(), + &mxc, &db.globals, body.width .try_into() diff --git a/src/database/media.rs b/src/database/media.rs index 46630131..a4bb4025 100644 --- a/src/database/media.rs +++ b/src/database/media.rs @@ -171,7 +171,7 @@ impl Media { /// For width,height <= 96 the server uses another thumbnailing algorithm which crops the image afterwards. pub async fn get_thumbnail( &self, - mxc: String, + mxc: &str, globals: &Globals, width: u32, height: u32, From b39ddf7be9150b8baa6cecabed7730d2ab610a72 Mon Sep 17 00:00:00 2001 From: "Aode (lion)" Date: Fri, 28 Jan 2022 12:42:47 -0600 Subject: [PATCH 168/201] Rename reqwest clients, mention cheap client clones in comment --- src/appservice_server.rs | 2 +- src/database/globals.rs | 24 ++++++++++++------------ src/database/pusher.rs | 2 +- src/server_server.rs | 4 ++-- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/appservice_server.rs b/src/appservice_server.rs index a5d795f6..e78fb344 100644 --- a/src/appservice_server.rs +++ b/src/appservice_server.rs @@ -46,7 +46,7 @@ where *reqwest_request.timeout_mut() = Some(Duration::from_secs(30)); let url = reqwest_request.url().clone(); - let mut response = globals.reqwest_client().execute(reqwest_request).await?; + let mut response = globals.default_client().execute(reqwest_request).await?; // reqwest::Response -> http::Response conversion let status = response.status(); diff --git a/src/database/globals.rs b/src/database/globals.rs index 3278b7f6..decd84c3 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -39,8 +39,8 @@ pub struct Globals { keypair: Arc, dns_resolver: TokioAsyncResolver, jwt_decoding_key: Option>, - well_known_client: reqwest::Client, - basic_client: reqwest::Client, + federation_client: reqwest::Client, + default_client: reqwest::Client, pub(super) server_signingkeys: Arc, pub bad_event_ratelimiter: Arc, RateLimitState>>>, pub bad_signature_ratelimiter: Arc, RateLimitState>>>, @@ -134,9 +134,9 @@ impl Globals { .as_ref() .map(|secret| jsonwebtoken::DecodingKey::from_secret(secret.as_bytes()).into_static()); - let basic_client = reqwest_client_builder(&config)?.build()?; + let default_client = reqwest_client_builder(&config)?.build()?; let name_override = Arc::clone(&tls_name_override); - let well_known_client = reqwest_client_builder(&config)? + let federation_client = reqwest_client_builder(&config)? .resolve_fn(move |domain| { let read_guard = name_override.read().unwrap(); let (override_name, port) = read_guard.get(&domain)?; @@ -154,8 +154,8 @@ impl Globals { })?, actual_destination_cache: Arc::new(RwLock::new(WellKnownMap::new())), tls_name_override, - well_known_client, - basic_client, + federation_client, + default_client, server_signingkeys, jwt_decoding_key, bad_event_ratelimiter: Arc::new(RwLock::new(HashMap::new())), @@ -179,15 +179,15 @@ impl Globals { } /// Returns a reqwest client which can be used to send requests - pub fn reqwest_client(&self) -> reqwest::Client { - // can't return &Client or else we'll hold a lock around the DB across an await - self.basic_client.clone() + pub fn default_client(&self) -> reqwest::Client { + // Client is cheap to clone (Arc wrapper) and avoids lifetime issues + self.default_client.clone() } /// Returns a client used for resolving .well-knowns - pub fn well_known_client(&self) -> reqwest::Client { - // can't return &Client or else we'll hold a lock around the DB across an await - self.well_known_client.clone() + pub fn federation_client(&self) -> reqwest::Client { + // Client is cheap to clone (Arc wrapper) and avoids lifetime issues + self.federation_client.clone() } #[tracing::instrument(skip(self))] diff --git a/src/database/pusher.rs b/src/database/pusher.rs index d63db1d7..bbe85a8d 100644 --- a/src/database/pusher.rs +++ b/src/database/pusher.rs @@ -115,7 +115,7 @@ where //*reqwest_request.timeout_mut() = Some(Duration::from_secs(5)); let url = reqwest_request.url().clone(); - let response = globals.reqwest_client().execute(reqwest_request).await; + let response = globals.default_client().execute(reqwest_request).await; match response { Ok(mut response) => { diff --git a/src/server_server.rs b/src/server_server.rs index 978eb67f..c5e0b1a9 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -237,7 +237,7 @@ where let url = reqwest_request.url().clone(); - let response = globals.well_known_client().execute(reqwest_request).await; + let response = globals.federation_client().execute(reqwest_request).await; match response { Ok(mut response) => { @@ -477,7 +477,7 @@ async fn request_well_known( ) -> Option { let body: serde_json::Value = serde_json::from_str( &globals - .reqwest_client() + .default_client() .get(&format!( "https://{}/.well-known/matrix/server", destination From 44f7a85077e5c249ec618004b0386f3d66f01911 Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Fri, 28 Jan 2022 22:19:19 +0100 Subject: [PATCH 169/201] fix: Use default port for healthcheck as fallback Conduit can start without a specific port being configured. This adjusts the healthcheck script to tolerate that state. Closes https://gitlab.com/famedly/conduit/-/issues/222 --- docker/healthcheck.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/healthcheck.sh b/docker/healthcheck.sh index 7ca04602..efc94917 100644 --- a/docker/healthcheck.sh +++ b/docker/healthcheck.sh @@ -1,10 +1,15 @@ #!/bin/sh # If the port is not specified as env var, take it from the config file -if [ -z ${CONDUIT_PORT} ]; then +if [ -z "${CONDUIT_PORT}" ]; then CONDUIT_PORT=$(grep -m1 -o 'port\s=\s[0-9]*' conduit.toml | grep -m1 -o '[0-9]*') fi +# If the config file also does not contain a default port, just use the default one: 6167. +if [ -z "${CONDUIT_PORT}" ]; then + CONDUIT_PORT=6167 +fi + # The actual health check. # We try to first get a response on HTTP and when that fails on HTTPS and when that fails, we exit with code 1. # TODO: Change this to a single wget call. Do we have a config value that we can check for that? From 401b88d16d43f0c58e5a4ccf777815fd8d538ff8 Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Fri, 28 Jan 2022 23:23:58 +0100 Subject: [PATCH 170/201] fix: Healtcheck use netstat for port as fallback --- docker/healthcheck.sh | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/docker/healthcheck.sh b/docker/healthcheck.sh index efc94917..df7f18a5 100644 --- a/docker/healthcheck.sh +++ b/docker/healthcheck.sh @@ -1,13 +1,9 @@ #!/bin/sh -# If the port is not specified as env var, take it from the config file +# If the config file does not contain a default port and the CONDUIT_PORT env is not set, create +# try to get port from process list if [ -z "${CONDUIT_PORT}" ]; then - CONDUIT_PORT=$(grep -m1 -o 'port\s=\s[0-9]*' conduit.toml | grep -m1 -o '[0-9]*') -fi - -# If the config file also does not contain a default port, just use the default one: 6167. -if [ -z "${CONDUIT_PORT}" ]; then - CONDUIT_PORT=6167 + CONDUIT_PORT=$(netstat -tlp | grep conduit | grep -m1 -o ':[0-9]*' | grep -m1 -o '[0-9]*') fi # The actual health check. From 8ff95a5a48c055549b3652a33faddc3d89351d91 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 28 Jan 2022 22:26:56 -0800 Subject: [PATCH 171/201] fix: mention dependencies to build from source --- DEPLOY.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/DEPLOY.md b/DEPLOY.md index 38e1e286..d9f91e03 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -29,7 +29,11 @@ $ sudo wget -O /usr/local/bin/matrix-conduit $ sudo chmod +x /usr/local/bin/matrix-conduit ``` -Alternatively, you may compile the binary yourself using +Alternatively, you may compile the binary yourself + +```bash +$ sudo apt install libclang-dev build-essential +``` ```bash $ cargo build --release From 677f044d13985f794afecdb0bbf62fbab3a52dec Mon Sep 17 00:00:00 2001 From: Andrei Vasiliu Date: Sun, 30 Jan 2022 23:15:53 +0200 Subject: [PATCH 172/201] Refactor admin code to always defer command processing --- src/client_server/report.rs | 12 +- src/database/admin.rs | 304 ++++++++++++++++++------------------ src/database/rooms.rs | 8 +- 3 files changed, 155 insertions(+), 169 deletions(-) diff --git a/src/client_server/report.rs b/src/client_server/report.rs index ae069849..032e446c 100644 --- a/src/client_server/report.rs +++ b/src/client_server/report.rs @@ -1,7 +1,4 @@ -use crate::{ - database::{admin::AdminCommand, DatabaseGuard}, - ConduitResult, Error, Ruma, -}; +use crate::{database::DatabaseGuard, ConduitResult, Error, Ruma}; use ruma::{ api::client::{error::ErrorKind, r0::room::report_content}, events::room::message, @@ -50,8 +47,8 @@ pub async fn report_event_route( )); }; - db.admin.send(AdminCommand::SendMessage( - message::RoomMessageEventContent::text_html( + db.admin + .send_message(message::RoomMessageEventContent::text_html( format!( "Report received from: {}\n\n\ Event ID: {}\n\ @@ -75,8 +72,7 @@ pub async fn report_event_route( body.score, RawStr::new(&body.reason).html_escape() ), - ), - )); + )); db.flush()?; diff --git a/src/database/admin.rs b/src/database/admin.rs index dbd20e44..ea08f65a 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -19,25 +19,21 @@ use serde_json::value::to_raw_value; use tokio::sync::{MutexGuard, RwLock, RwLockReadGuard}; use tracing::warn; -pub enum AdminCommand { - RegisterAppservice(serde_yaml::Value), - UnregisterAppservice(String), - ListAppservices, - ListLocalUsers, - ShowMemoryUsage, +pub enum AdminRoomEvent { + ProcessMessage(String), SendMessage(RoomMessageEventContent), } #[derive(Clone)] pub struct Admin { - pub sender: mpsc::UnboundedSender, + pub sender: mpsc::UnboundedSender, } impl Admin { pub fn start_handler( &self, db: Arc>, - mut receiver: mpsc::UnboundedReceiver, + mut receiver: mpsc::UnboundedReceiver, ) { tokio::spawn(async move { // TODO: Use futures when we have long admin commands @@ -56,7 +52,7 @@ impl Admin { .try_into() .expect("#admins:server_name is a valid room alias"), ) - .unwrap(); + .expect("Admin room must exist"); let conduit_room = match conduit_room { None => { @@ -105,46 +101,13 @@ impl Admin { let state_lock = mutex_state.lock().await; match event { - AdminCommand::ListLocalUsers => { - match guard.users.list_local_users() { - Ok(users) => { - let mut msg: String = format!("Found {} local user account(s):\n", users.len()); - msg += &users.join("\n"); - send_message(RoomMessageEventContent::text_plain(&msg), guard, &state_lock); - } - Err(e) => { - send_message(RoomMessageEventContent::text_plain(e.to_string()), guard, &state_lock); - } - } + AdminRoomEvent::SendMessage(content) => { + send_message(content, guard, &state_lock); } - AdminCommand::RegisterAppservice(yaml) => { - guard.appservice.register_appservice(yaml).unwrap(); // TODO handle error - } - AdminCommand::UnregisterAppservice(service_name) => { - guard.appservice.unregister_appservice(&service_name).unwrap(); // TODO: see above - } - AdminCommand::ListAppservices => { - if let Ok(appservices) = guard.appservice.iter_ids().map(|ids| ids.collect::>()) { - let count = appservices.len(); - let output = format!( - "Appservices ({}): {}", - count, - appservices.into_iter().filter_map(|r| r.ok()).collect::>().join(", ") - ); - send_message(RoomMessageEventContent::text_plain(output), guard, &state_lock); - } else { - send_message(RoomMessageEventContent::text_plain("Failed to get appservices."), guard, &state_lock); - } - } - AdminCommand::ShowMemoryUsage => { - if let Ok(response) = guard._db.memory_usage() { - send_message(RoomMessageEventContent::text_plain(response), guard, &state_lock); - } else { - send_message(RoomMessageEventContent::text_plain("Failed to get database memory usage.".to_owned()), guard, &state_lock); - } - } - AdminCommand::SendMessage(message) => { - send_message(message, guard, &state_lock); + AdminRoomEvent::ProcessMessage(room_message) => { + let reply_message = process_admin_message(&*guard, room_message); + + send_message(reply_message, guard, &state_lock); } } @@ -155,67 +118,81 @@ impl Admin { }); } - pub fn send(&self, command: AdminCommand) { - self.sender.unbounded_send(command).unwrap(); + pub fn process_message(&self, room_message: String) { + self.sender + .unbounded_send(AdminRoomEvent::ProcessMessage(room_message)) + .unwrap(); + } + + pub fn send_message(&self, message_content: RoomMessageEventContent) { + self.sender + .unbounded_send(AdminRoomEvent::SendMessage(message_content)) + .unwrap(); + } +} + +// Parse and process a message from the admin room +pub fn process_admin_message(db: &Database, room_message: String) -> RoomMessageEventContent { + let mut lines = room_message.lines(); + let command_line = lines.next().expect("each string has at least one line"); + let body: Vec<_> = lines.collect(); + + let admin_command = match parse_admin_command(&command_line) { + Ok(command) => command, + Err(error) => { + let message = error + .to_string() + .replace("example.com", db.globals.server_name().as_str()); + let html_message = usage_to_html(&message); + + return RoomMessageEventContent::text_html(message, html_message); + } + }; + + match process_admin_command(db, admin_command, body) { + Ok(reply_message) => reply_message, + Err(error) => { + let markdown_message = format!( + "Encountered an error while handling the command:\n\ + ```\n{}\n```", + error, + ); + let html_message = format!( + "Encountered an error while handling the command:\n\ + 
              \n{}\n
              ", + error, + ); + + RoomMessageEventContent::text_html(markdown_message, html_message) + } } } // Parse chat messages from the admin room into an AdminCommand object -pub fn parse_admin_command(db: &Database, command_line: &str, body: Vec<&str>) -> AdminCommand { - let mut argv: Vec<_> = command_line.split_whitespace().skip(1).collect(); - - let command_name = match argv.get(0) { - Some(command) => *command, - None => { - let markdown_message = "No command given. Use `help` for a list of commands."; - let html_message = "No command given. Use help for a list of commands."; - - return AdminCommand::SendMessage(RoomMessageEventContent::text_html( - markdown_message, - html_message, - )); - } - }; +fn parse_admin_command(command_line: &str) -> std::result::Result { + // Note: argv[0] is `@conduit:servername:`, which is treated as the main command + let mut argv: Vec<_> = command_line.split_whitespace().collect(); // Replace `help command` with `command --help` // Clap has a help subcommand, but it omits the long help description. - if argv[0] == "help" { - argv.remove(0); + if argv.len() > 1 && argv[1] == "help" { + argv.remove(1); argv.push("--help"); } // Backwards compatibility with `register_appservice`-style commands let command_with_dashes; - if argv[0].contains("_") { - command_with_dashes = argv[0].replace("_", "-"); - argv[0] = &command_with_dashes; + if argv.len() > 1 && argv[1].contains("_") { + command_with_dashes = argv[1].replace("_", "-"); + argv[1] = &command_with_dashes; } - match try_parse_admin_command(db, argv, body) { - Ok(admin_command) => admin_command, - Err(error) => { - let markdown_message = format!( - "Encountered an error while handling the `{}` command:\n\ - ```\n{}\n```", - command_name, error, - ); - let html_message = format!( - "Encountered an error while handling the {} command:\n\ - 
              \n{}\n
              ", - command_name, error, - ); - - AdminCommand::SendMessage(RoomMessageEventContent::text_html( - markdown_message, - html_message, - )) - } - } + AdminCommand::try_parse_from(argv).map_err(|error| error.to_string()) } #[derive(Parser)] #[clap(name = "@conduit:example.com", version = env!("CARGO_PKG_VERSION"))] -enum AdminCommands { +enum AdminCommand { #[clap(verbatim_doc_comment)] /// Register an appservice using its registration YAML /// @@ -264,49 +241,70 @@ enum AdminCommands { DatabaseMemoryUsage, } -pub fn try_parse_admin_command( +fn process_admin_command( db: &Database, - mut argv: Vec<&str>, + command: AdminCommand, body: Vec<&str>, -) -> Result { - argv.insert(0, "@conduit:example.com:"); - let command = match AdminCommands::try_parse_from(argv) { - Ok(command) => command, - Err(error) => { - let message = error - .to_string() - .replace("example.com", db.globals.server_name().as_str()); - let html_message = usage_to_html(&message); - - return Ok(AdminCommand::SendMessage( - RoomMessageEventContent::text_html(message, html_message), - )); - } - }; - - let admin_command = match command { - AdminCommands::RegisterAppservice => { +) -> Result { + let reply_message_content = match command { + AdminCommand::RegisterAppservice => { if body.len() > 2 && body[0].trim() == "```" && body.last().unwrap().trim() == "```" { let appservice_config = body[1..body.len() - 1].join("\n"); let parsed_config = serde_yaml::from_str::(&appservice_config); match parsed_config { - Ok(yaml) => AdminCommand::RegisterAppservice(yaml), - Err(e) => AdminCommand::SendMessage(RoomMessageEventContent::text_plain( - format!("Could not parse appservice config: {}", e), + Ok(yaml) => match db.appservice.register_appservice(yaml) { + Ok(()) => RoomMessageEventContent::text_plain("Appservice registered."), + Err(e) => RoomMessageEventContent::text_plain(format!( + "Failed to register appservice: {}", + e + )), + }, + Err(e) => RoomMessageEventContent::text_plain(format!( + "Could not parse appservice config: {}", + e )), } } else { - AdminCommand::SendMessage(RoomMessageEventContent::text_plain( - "Expected code block in command body.", - )) + RoomMessageEventContent::text_plain( + "Expected code block in command body. Add --help for details.", + ) } } - AdminCommands::UnregisterAppservice { + AdminCommand::UnregisterAppservice { appservice_identifier, - } => AdminCommand::UnregisterAppservice(appservice_identifier), - AdminCommands::ListAppservices => AdminCommand::ListAppservices, - AdminCommands::ListLocalUsers => AdminCommand::ListLocalUsers, - AdminCommands::GetAuthChain { event_id } => { + } => match db.appservice.unregister_appservice(&appservice_identifier) { + Ok(()) => RoomMessageEventContent::text_plain("Appservice unregistered."), + Err(e) => RoomMessageEventContent::text_plain(format!( + "Failed to unregister appservice: {}", + e + )), + }, + AdminCommand::ListAppservices => { + if let Ok(appservices) = db.appservice.iter_ids().map(|ids| ids.collect::>()) { + let count = appservices.len(); + let output = format!( + "Appservices ({}): {}", + count, + appservices + .into_iter() + .filter_map(|r| r.ok()) + .collect::>() + .join(", ") + ); + RoomMessageEventContent::text_plain(output) + } else { + RoomMessageEventContent::text_plain("Failed to get appservices.") + } + } + AdminCommand::ListLocalUsers => match db.users.list_local_users() { + Ok(users) => { + let mut msg: String = format!("Found {} local user account(s):\n", users.len()); + msg += &users.join("\n"); + RoomMessageEventContent::text_plain(&msg) + } + Err(e) => RoomMessageEventContent::text_plain(e.to_string()), + }, + AdminCommand::GetAuthChain { event_id } => { let event_id = Arc::::from(event_id); if let Some(event) = db.rooms.get_pdu_json(&event_id)? { let room_id_str = event @@ -320,17 +318,15 @@ pub fn try_parse_admin_command( let start = Instant::now(); let count = server_server::get_auth_chain(room_id, vec![event_id], db)?.count(); let elapsed = start.elapsed(); - return Ok(AdminCommand::SendMessage( - RoomMessageEventContent::text_plain(format!( - "Loaded auth chain with length {} in {:?}", - count, elapsed - )), - )); + RoomMessageEventContent::text_plain(format!( + "Loaded auth chain with length {} in {:?}", + count, elapsed + )) } else { - AdminCommand::SendMessage(RoomMessageEventContent::text_plain("Event not found.")) + RoomMessageEventContent::text_plain("Event not found.") } } - AdminCommands::ParsePdu => { + AdminCommand::ParsePdu => { if body.len() > 2 && body[0].trim() == "```" && body.last().unwrap().trim() == "```" { let string = body[1..body.len() - 1].join("\n"); match serde_json::from_str(&string) { @@ -346,30 +342,26 @@ pub fn try_parse_admin_command( match serde_json::from_value::( serde_json::to_value(value).expect("value is json"), ) { - Ok(pdu) => { - AdminCommand::SendMessage(RoomMessageEventContent::text_plain( - format!("EventId: {:?}\n{:#?}", event_id, pdu), - )) - } - Err(e) => AdminCommand::SendMessage( - RoomMessageEventContent::text_plain(format!( - "EventId: {:?}\nCould not parse event: {}", - event_id, e - )), - ), + Ok(pdu) => RoomMessageEventContent::text_plain(format!( + "EventId: {:?}\n{:#?}", + event_id, pdu + )), + Err(e) => RoomMessageEventContent::text_plain(format!( + "EventId: {:?}\nCould not parse event: {}", + event_id, e + )), } } - Err(e) => AdminCommand::SendMessage(RoomMessageEventContent::text_plain( - format!("Invalid json in command body: {}", e), + Err(e) => RoomMessageEventContent::text_plain(format!( + "Invalid json in command body: {}", + e )), } } else { - AdminCommand::SendMessage(RoomMessageEventContent::text_plain( - "Expected code block in command body.", - )) + RoomMessageEventContent::text_plain("Expected code block in command body.") } } - AdminCommands::GetPdu { event_id } => { + AdminCommand::GetPdu { event_id } => { let mut outlier = false; let mut pdu_json = db.rooms.get_non_outlier_pdu_json(&event_id)?; if pdu_json.is_none() { @@ -380,7 +372,7 @@ pub fn try_parse_admin_command( Some(json) => { let json_text = serde_json::to_string_pretty(&json).expect("canonical json is valid json"); - AdminCommand::SendMessage(RoomMessageEventContent::text_html( + RoomMessageEventContent::text_html( format!( "{}\n```json\n{}\n```", if outlier { @@ -399,17 +391,21 @@ pub fn try_parse_admin_command( }, RawStr::new(&json_text).html_escape() ), - )) - } - None => { - AdminCommand::SendMessage(RoomMessageEventContent::text_plain("PDU not found.")) + ) } + None => RoomMessageEventContent::text_plain("PDU not found."), } } - AdminCommands::DatabaseMemoryUsage => AdminCommand::ShowMemoryUsage, + AdminCommand::DatabaseMemoryUsage => match db._db.memory_usage() { + Ok(response) => RoomMessageEventContent::text_plain(response), + Err(e) => RoomMessageEventContent::text_plain(format!( + "Failed to get database memory usage: {}", + e + )), + }, }; - Ok(admin_command) + Ok(reply_message_content) } // Utility to turn clap's `--help` text to HTML. diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 1f4566fe..2303b0dd 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -3,7 +3,6 @@ mod edus; pub use edus::RoomEdus; use crate::{ - database::admin::parse_admin_command, pdu::{EventHash, PduBuilder}, utils, Database, Error, PduEvent, Result, }; @@ -1490,12 +1489,7 @@ impl Rooms { .as_ref() == Some(&pdu.room_id) { - let mut lines = body.lines(); - let command_line = lines.next().expect("each string has at least one line"); - let body: Vec<_> = lines.collect(); - - let command = parse_admin_command(db, command_line, body); - db.admin.send(command); + db.admin.process_message(body.to_string()); } } } From cc13112592b1666e75b4e5d0d340d6124afe4071 Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Mon, 31 Jan 2022 09:27:31 +0100 Subject: [PATCH 173/201] Cleanup appservice events after removing the appservice --- src/database/admin.rs | 13 ++++++++++++- src/database/sending.rs | 31 +++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index 81e98393..9895a83b 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -112,7 +112,18 @@ impl Admin { guard.appservice.register_appservice(yaml).unwrap(); // TODO handle error } AdminCommand::UnregisterAppservice(service_name) => { - guard.appservice.unregister_appservice(&service_name).unwrap(); // TODO: see above + if let Ok(_) = guard.appservice.unregister_appservice(&service_name) { + if let Ok(_) = guard.sending.cleanup_events(&service_name) { + let msg: String = format!("OK. Appservice {} removed", service_name); + send_message(RoomMessageEventContent::text_plain(msg), guard, &state_lock); + } else { + let msg: String = format!("WARN: Appservice {} removed, but failed to cleanup events", service_name); + send_message(RoomMessageEventContent::text_plain(msg), guard, &state_lock); + } + } else { + let msg: String = format!("ERR. Appservice {} not removed", service_name); + send_message(RoomMessageEventContent::text_plain(msg), guard, &state_lock); + } } AdminCommand::ListAppservices => { if let Ok(appservices) = guard.appservice.iter_ids().map(|ids| ids.collect::>()) { diff --git a/src/database/sending.rs b/src/database/sending.rs index 69f7c444..af4ac676 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -480,6 +480,26 @@ impl Sending { hash.as_ref().to_owned() } + /// Cleanup event data + /// Used for instance after we remove an appservice registration + /// + #[tracing::instrument(skip(self))] + pub fn cleanup_events(&self, key_id: &str) -> Result<()> { + let mut prefix = b"+".to_vec(); + prefix.extend_from_slice(key_id.as_bytes()); + prefix.push(0xff); + + for (key, _) in self.servercurrentevent_data.scan_prefix(prefix.clone()) { + self.servercurrentevent_data.remove(&key).unwrap(); + } + + for (key, _) in self.servernameevent_data.scan_prefix(prefix.clone()) { + self.servernameevent_data.remove(&key).unwrap(); + } + + Ok(()) + } + #[tracing::instrument(skip(db, events, kind))] async fn handle_events( kind: OutgoingKind, @@ -520,8 +540,15 @@ impl Sending { &db.globals, db.appservice .get_registration(server.as_str()) - .unwrap() - .unwrap(), // TODO: handle error + .map_err(|e| (kind.clone(), e))? + .ok_or_else(|| { + ( + kind.clone(), + Error::bad_database( + "[Appservice] Could not load registration from db.", + ), + ) + })?, appservice::event::push_events::v1::Request { events: &pdu_jsons, txn_id: (&*base64::encode_config( From 78502aa6b10d97f3af3fe006fcdbd19b585d3b58 Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Mon, 31 Jan 2022 10:07:49 +0100 Subject: [PATCH 174/201] add error handling for register_appservice too --- src/database/admin.rs | 13 ++++++++++++- src/database/appservice.rs | 4 ++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index 9895a83b..eef6ce10 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -109,7 +109,18 @@ impl Admin { } } AdminCommand::RegisterAppservice(yaml) => { - guard.appservice.register_appservice(yaml).unwrap(); // TODO handle error + match guard.appservice.register_appservice(yaml) { + Ok(Some(id)) => { + let msg: String = format!("OK. Appservice {} created", id); + send_message(RoomMessageEventContent::text_plain(msg), guard, &state_lock); + } + Ok(None) => { + send_message(RoomMessageEventContent::text_plain("WARN. Appservice created, but its ID was not returned!"), guard, &state_lock); + } + Err(_) => { + send_message(RoomMessageEventContent::text_plain("ERR: Failed register appservice. Check server log"), guard, &state_lock); + } + } } AdminCommand::UnregisterAppservice(service_name) => { if let Ok(_) = guard.appservice.unregister_appservice(&service_name) { diff --git a/src/database/appservice.rs b/src/database/appservice.rs index 88de1f33..8b29aca9 100644 --- a/src/database/appservice.rs +++ b/src/database/appservice.rs @@ -12,7 +12,7 @@ pub struct Appservice { } impl Appservice { - pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result<()> { + pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result> { // TODO: Rumaify let id = yaml.get("id").unwrap().as_str().unwrap(); self.id_appserviceregistrations.insert( @@ -24,7 +24,7 @@ impl Appservice { .unwrap() .insert(id.to_owned(), yaml); - Ok(()) + Ok(Some(id.to_owned())) } /// Remove an appservice registration From 28d3b348d2fc23e6b2b78c468f682018ab472652 Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Mon, 31 Jan 2022 11:52:33 +0100 Subject: [PATCH 175/201] Return the ID of the appservice that was created by register_appservice --- src/database/admin.rs | 5 +---- src/database/appservice.rs | 8 +++++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index eef6ce10..a214796b 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -110,13 +110,10 @@ impl Admin { } AdminCommand::RegisterAppservice(yaml) => { match guard.appservice.register_appservice(yaml) { - Ok(Some(id)) => { + Ok(id) => { let msg: String = format!("OK. Appservice {} created", id); send_message(RoomMessageEventContent::text_plain(msg), guard, &state_lock); } - Ok(None) => { - send_message(RoomMessageEventContent::text_plain("WARN. Appservice created, but its ID was not returned!"), guard, &state_lock); - } Err(_) => { send_message(RoomMessageEventContent::text_plain("ERR: Failed register appservice. Check server log"), guard, &state_lock); } diff --git a/src/database/appservice.rs b/src/database/appservice.rs index 8b29aca9..edd5009b 100644 --- a/src/database/appservice.rs +++ b/src/database/appservice.rs @@ -12,7 +12,9 @@ pub struct Appservice { } impl Appservice { - pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result> { + /// Registers an appservice and returns the ID to the caller + /// + pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result { // TODO: Rumaify let id = yaml.get("id").unwrap().as_str().unwrap(); self.id_appserviceregistrations.insert( @@ -22,9 +24,9 @@ impl Appservice { self.cached_registrations .write() .unwrap() - .insert(id.to_owned(), yaml); + .insert(id.to_owned(), yaml.to_owned()); - Ok(Some(id.to_owned())) + Ok(id.to_owned()) } /// Remove an appservice registration From e17bbdd42d3245f2fb3730753f1feb51cd452207 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Fri, 28 Jan 2022 17:26:43 +0100 Subject: [PATCH 176/201] tests --- Cargo.toml | 2 +- src/database.rs | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 78a4c8ff..0089e7f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,7 +76,7 @@ crossbeam = { version = "0.8.1", optional = true } num_cpus = "1.13.0" threadpool = "1.8.1" heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c758867e05ad973ef800a6fe1d5d", optional = true } -rocksdb = { version = "0.17.0", default-features = false, features = ["multi-threaded-cf", "zstd"], optional = true } +rocksdb = { version = "0.17.0", default-features = true, features = ["multi-threaded-cf", "zstd"], optional = true } thread_local = "1.1.3" # used for TURN server authentication diff --git a/src/database.rs b/src/database.rs index 4f230f32..79b82088 100644 --- a/src/database.rs +++ b/src/database.rs @@ -130,7 +130,7 @@ fn default_db_cache_capacity_mb() -> f64 { } fn default_rocksdb_max_open_files() -> i32 { - 512 + 20 } fn default_pdu_cache_capacity() -> u32 { @@ -361,15 +361,15 @@ impl Database { .try_into() .expect("pdu cache capacity fits into usize"), )), - auth_chain_cache: Mutex::new(LruCache::new(1_000_000)), - shorteventid_cache: Mutex::new(LruCache::new(1_000_000)), - eventidshort_cache: Mutex::new(LruCache::new(1_000_000)), - shortstatekey_cache: Mutex::new(LruCache::new(1_000_000)), - statekeyshort_cache: Mutex::new(LruCache::new(1_000_000)), + auth_chain_cache: Mutex::new(LruCache::new(100_000)), + shorteventid_cache: Mutex::new(LruCache::new(100_000)), + eventidshort_cache: Mutex::new(LruCache::new(100_000)), + shortstatekey_cache: Mutex::new(LruCache::new(100_000)), + statekeyshort_cache: Mutex::new(LruCache::new(100_000)), our_real_users_cache: RwLock::new(HashMap::new()), appservice_in_room_cache: RwLock::new(HashMap::new()), lazy_load_waiting: Mutex::new(HashMap::new()), - stateinfo_cache: Mutex::new(LruCache::new(1000)), + stateinfo_cache: Mutex::new(LruCache::new(100)), }, account_data: account_data::AccountData { roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?, From 23aecb78c7c5ba5872a058f806cd722787eefc10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 31 Jan 2022 15:39:46 +0100 Subject: [PATCH 177/201] fix: use to_lowercase on /register/available username --- src/client_server/account.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client_server/account.rs b/src/client_server/account.rs index c4e118c9..80c6f702 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -60,7 +60,7 @@ pub async fn get_register_available_route( body: Ruma>, ) -> ConduitResult { // Validate user id - let user_id = UserId::parse_with_server_name(body.username.clone(), db.globals.server_name()) + let user_id = UserId::parse_with_server_name(body.username.to_lowercase(), db.globals.server_name()) .ok() .filter(|user_id| { !user_id.is_historical() && user_id.server_name() == db.globals.server_name() From caf9834e50b540fc48bf8cf50dc439c57c503de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 1 Feb 2022 14:42:13 +0100 Subject: [PATCH 178/201] feat: cache capacity modifier --- src/database.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/database.rs b/src/database.rs index 79b82088..449d71bd 100644 --- a/src/database.rs +++ b/src/database.rs @@ -49,6 +49,8 @@ pub struct Config { database_path: String, #[serde(default = "default_db_cache_capacity_mb")] db_cache_capacity_mb: f64, + #[serde(default = "default_conduit_cache_capacity_modifier")] + conduit_cache_capacity_modifier: f64, #[serde(default = "default_rocksdb_max_open_files")] rocksdb_max_open_files: i32, #[serde(default = "default_pdu_cache_capacity")] @@ -129,6 +131,10 @@ fn default_db_cache_capacity_mb() -> f64 { 10.0 } +fn default_conduit_cache_capacity_modifier() -> f64 { + 1.0 +} + fn default_rocksdb_max_open_files() -> i32 { 20 } @@ -361,15 +367,15 @@ impl Database { .try_into() .expect("pdu cache capacity fits into usize"), )), - auth_chain_cache: Mutex::new(LruCache::new(100_000)), - shorteventid_cache: Mutex::new(LruCache::new(100_000)), - eventidshort_cache: Mutex::new(LruCache::new(100_000)), - shortstatekey_cache: Mutex::new(LruCache::new(100_000)), - statekeyshort_cache: Mutex::new(LruCache::new(100_000)), + auth_chain_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), + shorteventid_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), + eventidshort_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), + shortstatekey_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), + statekeyshort_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), our_real_users_cache: RwLock::new(HashMap::new()), appservice_in_room_cache: RwLock::new(HashMap::new()), lazy_load_waiting: Mutex::new(HashMap::new()), - stateinfo_cache: Mutex::new(LruCache::new(100)), + stateinfo_cache: Mutex::new(LruCache::new((100.0 * config.conduit_cache_capacity_modifier) as usize)), }, account_data: account_data::AccountData { roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?, From fa4099b138b3a4cdf6727acb14c47f20ace5f38e Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Tue, 1 Feb 2022 23:51:38 +0000 Subject: [PATCH 179/201] Use prebuilt CI-containers from https://gitlab.com/jfowl/conduit-containers Also run all builds on approved MRs --- .gitlab-ci.yml | 39 ++++++++++++++++----------------------- Cross.toml | 8 ++++---- cross/build.sh | 31 ------------------------------- cross/test.sh | 8 -------- 4 files changed, 20 insertions(+), 66 deletions(-) delete mode 100755 cross/build.sh delete mode 100755 cross/test.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 741b5327..6f1a19f0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,8 +24,9 @@ variables: - if: '$CI_COMMIT_BRANCH == "master"' - if: '$CI_COMMIT_BRANCH == "next"' - if: "$CI_COMMIT_TAG" + - if: '($CI_MERGE_REQUEST_APPROVED == "true") || $BUILD_EVERYTHING' # Once MR is approved, test all builds. Or if BUILD_EVERYTHING is set. interruptible: true - image: "rust:1.58" + image: "registry.gitlab.com/jfowl/conduit-containers/rust-with-tools:latest" tags: ["docker"] services: ["docker:dind"] variables: @@ -36,27 +37,23 @@ variables: before_script: - 'echo "Building for target $TARGET"' - "rustup show && rustc --version && cargo --version" # Print version info for debugging - # install cross-compiling prerequisites - - 'apt-get update && apt-get install -y docker.io && docker version' # install docker - - 'cargo install cross && cross --version' # install cross # fix cargo and rustup mounts from this container (https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41227) - - 'mkdir -p $SHARED_PATH/cargo' - - 'cp -r $CARGO_HOME/bin $SHARED_PATH/cargo' - - 'cp -r $RUSTUP_HOME $SHARED_PATH' - - 'export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup' + - "mkdir -p $SHARED_PATH/cargo" + - "cp -r $CARGO_HOME/bin $SHARED_PATH/cargo" + - "cp -r $RUSTUP_HOME $SHARED_PATH" + - "export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup" # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results. - # The sccache binary is stored in the sysroot of the rustc installation since that directory is added to the path of the cross container. - - if [ -n "${SCCACHE_BIN_URL}" ]; then RUSTC_SYSROOT=$(rustc --print sysroot) && curl $SCCACHE_BIN_URL --output $RUSTC_SYSROOT/bin/sccache && chmod +x $RUSTC_SYSROOT/bin/sccache && export RUSTC_WRAPPER=sccache; fi + - if [ -n "${SCCACHE_ENDPOINT}" ]; then export RUSTC_WRAPPER=/sccache; fi script: # cross-compile conduit for target - - 'time ./cross/build.sh --locked --release' + - 'time cross build --target="$TARGET" --locked --release' - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"' # print information about linking for debugging - - 'file conduit-$TARGET' # print file information + - "file conduit-$TARGET" # print file information - 'readelf --dynamic conduit-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked cache: # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci - key: 'cargo-cache-$TARGET' + key: "cargo-cache-$TARGET" paths: - $SHARED_PATH/cargo/registry/index - $SHARED_PATH/cargo/registry/cache @@ -125,10 +122,10 @@ build:release:cargo:aarch64-unknown-linux-musl: key: "build_cache--$TARGET--$CI_COMMIT_BRANCH--debug" script: # cross-compile conduit for target - - 'time ./cross/build.sh --locked' + - 'time time cross build --target="$TARGET" --locked' - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"' # print information about linking for debugging - - 'file conduit-debug-$TARGET' # print file information + - "file conduit-debug-$TARGET" # print file information - 'readelf --dynamic conduit-debug-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked artifacts: expire_in: 4 weeks @@ -230,24 +227,20 @@ docker:master:dockerhub: test:cargo: stage: "test" needs: [] - image: "rust:latest" + image: "registry.gitlab.com/jfowl/conduit-containers/rust-with-tools:latest" tags: ["docker"] variables: CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow interruptible: true before_script: - # - mkdir -p $CARGO_HOME - - apt-get update -yqq - - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config libclang-dev - rustup component add clippy rustfmt - - curl "https://faulty-storage.de/gitlab-report" --output ./gitlab-report && chmod +x ./gitlab-report # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: - - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi + - if [ -n "${SCCACHE_ENDPOINT}" ]; then export RUSTC_WRAPPER=/usr/local/cargo/bin/sccache; fi script: - rustc --version && cargo --version # Print version info for debugging - cargo fmt --all -- --check - - "cargo test --color always --workspace --verbose --locked --no-fail-fast -- -Z unstable-options --format json | ./gitlab-report -p test > $CI_PROJECT_DIR/report.xml" - - "cargo clippy --color always --verbose --message-format=json | ./gitlab-report -p clippy > $CI_PROJECT_DIR/gl-code-quality-report.json" + - "cargo test --color always --workspace --verbose --locked --no-fail-fast -- -Z unstable-options --format json | gitlab-report -p test > $CI_PROJECT_DIR/report.xml" + - "cargo clippy --color always --verbose --message-format=json | gitlab-report -p clippy > $CI_PROJECT_DIR/gl-code-quality-report.json" artifacts: when: always reports: diff --git a/Cross.toml b/Cross.toml index a989a98f..a1387b43 100644 --- a/Cross.toml +++ b/Cross.toml @@ -11,13 +11,13 @@ passthrough = [ ] [target.aarch64-unknown-linux-musl] -image = "rust-cross:aarch64-unknown-linux-musl" +image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-aarch64-unknown-linux-musl:latest" [target.arm-unknown-linux-musleabihf] -image = "rust-cross:arm-unknown-linux-musleabihf" +image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-arm-unknown-linux-musleabihf:latest" [target.armv7-unknown-linux-musleabihf] -image = "rust-cross:armv7-unknown-linux-musleabihf" +image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-armv7-unknown-linux-musleabihf:latest" [target.x86_64-unknown-linux-musl] -image = "rust-cross:x86_64-unknown-linux-musl" +image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-x86_64-unknown-linux-musl:latest" diff --git a/cross/build.sh b/cross/build.sh deleted file mode 100755 index 8f64ff87..00000000 --- a/cross/build.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -set -ex - -# build custom container with libclang and static compilation -tag="rust-cross:${TARGET:?}" -docker build --tag="$tag" - << EOF -FROM rustembedded/cross:$TARGET - -# Install libclang for generating bindings with rust-bindgen -# The architecture is not relevant here since it's not used for compilation -RUN apt-get update && \ - apt-get install --assume-yes libclang-dev - -# Set the target prefix -ENV TARGET_PREFIX="/usr/local/$(echo "${TARGET:?}" | sed -e 's/armv7/arm/' -e 's/-unknown//')" - -# Make sure that cc-rs links libc/libstdc++ statically when cross-compiling -# See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information -ENV RUSTFLAGS="-L\$TARGET_PREFIX/lib" CXXSTDLIB="static=stdc++" -# Forcefully linking against libatomic, libc and libgcc is required for arm32, otherwise symbols are missing -$([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-static-libgcc -Clink-arg=-lgcc -lstatic=atomic -lstatic=c"') -# Strip symbols while compiling in release mode -$([[ $@ =~ -r ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-s"') - -# Make sure that rust-bindgen uses the correct include path when cross-compiling -# See https://github.com/rust-lang/rust-bindgen#environment-variables for more information -ENV BINDGEN_EXTRA_CLANG_ARGS="-I\$TARGET_PREFIX/include" -EOF - -# build conduit for a specific target -cross build --target="$TARGET" $@ diff --git a/cross/test.sh b/cross/test.sh deleted file mode 100755 index 0aa0909c..00000000 --- a/cross/test.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env sh -set -ex - -# Build conduit for a specific target -cross/build.sh $@ - -# Test conduit for a specific target -cross test --target="$TARGET" $@ From a5f004d7e9c783caf280884d7fd332c7bafa67ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 2 Feb 2022 12:36:55 +0100 Subject: [PATCH 180/201] fix: signature mismatch on odd send_join servers --- Cargo.toml | 2 +- src/client_server/account.rs | 19 ++++++++++--------- src/client_server/membership.rs | 17 +++++++++-------- src/database.rs | 24 ++++++++++++++++++------ 4 files changed, 38 insertions(+), 24 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0089e7f8..78a4c8ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,7 +76,7 @@ crossbeam = { version = "0.8.1", optional = true } num_cpus = "1.13.0" threadpool = "1.8.1" heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c758867e05ad973ef800a6fe1d5d", optional = true } -rocksdb = { version = "0.17.0", default-features = true, features = ["multi-threaded-cf", "zstd"], optional = true } +rocksdb = { version = "0.17.0", default-features = false, features = ["multi-threaded-cf", "zstd"], optional = true } thread_local = "1.1.3" # used for TURN server authentication diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 80c6f702..ff348545 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -60,15 +60,16 @@ pub async fn get_register_available_route( body: Ruma>, ) -> ConduitResult { // Validate user id - let user_id = UserId::parse_with_server_name(body.username.to_lowercase(), db.globals.server_name()) - .ok() - .filter(|user_id| { - !user_id.is_historical() && user_id.server_name() == db.globals.server_name() - }) - .ok_or(Error::BadRequest( - ErrorKind::InvalidUsername, - "Username is invalid.", - ))?; + let user_id = + UserId::parse_with_server_name(body.username.to_lowercase(), db.globals.server_name()) + .ok() + .filter(|user_id| { + !user_id.is_historical() && user_id.server_name() == db.globals.server_name() + }) + .ok_or(Error::BadRequest( + ErrorKind::InvalidUsername, + "Username is invalid.", + ))?; // Check if username is creative enough if db.users.exists(&user_id)? { diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 70352784..216c4c0a 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -655,7 +655,7 @@ async fn join_room_by_id_helper( db.rooms.get_or_create_shortroomid(room_id, &db.globals)?; - let pdu = PduEvent::from_id_val(event_id, join_event.clone()) + let parsed_pdu = PduEvent::from_id_val(event_id, join_event.clone()) .map_err(|_| Error::BadServerResponse("Invalid join event PDU."))?; let mut state = HashMap::new(); @@ -695,14 +695,15 @@ async fn join_room_by_id_helper( } let incoming_shortstatekey = db.rooms.get_or_create_shortstatekey( - &pdu.kind, - pdu.state_key + &parsed_pdu.kind, + parsed_pdu + .state_key .as_ref() .expect("Pdu is a membership state event"), &db.globals, )?; - state.insert(incoming_shortstatekey, pdu.event_id.clone()); + state.insert(incoming_shortstatekey, parsed_pdu.event_id.clone()); let create_shortstatekey = db .rooms @@ -738,12 +739,12 @@ async fn join_room_by_id_helper( // We append to state before appending the pdu, so we don't have a moment in time with the // pdu without it's state. This is okay because append_pdu can't fail. - let statehashid = db.rooms.append_to_state(&pdu, &db.globals)?; + let statehashid = db.rooms.append_to_state(&parsed_pdu, &db.globals)?; db.rooms.append_pdu( - &pdu, - utils::to_canonical_object(&pdu).expect("Pdu is valid canonical object"), - iter::once(&*pdu.event_id), + &parsed_pdu, + join_event, + iter::once(&*parsed_pdu.event_id), db, )?; diff --git a/src/database.rs b/src/database.rs index 449d71bd..8d245b7f 100644 --- a/src/database.rs +++ b/src/database.rs @@ -367,15 +367,27 @@ impl Database { .try_into() .expect("pdu cache capacity fits into usize"), )), - auth_chain_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), - shorteventid_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), - eventidshort_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), - shortstatekey_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), - statekeyshort_cache: Mutex::new(LruCache::new((100_000.0 * config.conduit_cache_capacity_modifier) as usize)), + auth_chain_cache: Mutex::new(LruCache::new( + (100_000.0 * config.conduit_cache_capacity_modifier) as usize, + )), + shorteventid_cache: Mutex::new(LruCache::new( + (100_000.0 * config.conduit_cache_capacity_modifier) as usize, + )), + eventidshort_cache: Mutex::new(LruCache::new( + (100_000.0 * config.conduit_cache_capacity_modifier) as usize, + )), + shortstatekey_cache: Mutex::new(LruCache::new( + (100_000.0 * config.conduit_cache_capacity_modifier) as usize, + )), + statekeyshort_cache: Mutex::new(LruCache::new( + (100_000.0 * config.conduit_cache_capacity_modifier) as usize, + )), our_real_users_cache: RwLock::new(HashMap::new()), appservice_in_room_cache: RwLock::new(HashMap::new()), lazy_load_waiting: Mutex::new(HashMap::new()), - stateinfo_cache: Mutex::new(LruCache::new((100.0 * config.conduit_cache_capacity_modifier) as usize)), + stateinfo_cache: Mutex::new(LruCache::new( + (100.0 * config.conduit_cache_capacity_modifier) as usize, + )), }, account_data: account_data::AccountData { roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?, From bfcf2db497ffab518b946922205fb9a5661d8c27 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 28 Jan 2022 22:26:56 -0800 Subject: [PATCH 181/201] fix: mention dependencies to build from source --- DEPLOY.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/DEPLOY.md b/DEPLOY.md index 38e1e286..d9f91e03 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -29,7 +29,11 @@ $ sudo wget -O /usr/local/bin/matrix-conduit $ sudo chmod +x /usr/local/bin/matrix-conduit ``` -Alternatively, you may compile the binary yourself using +Alternatively, you may compile the binary yourself + +```bash +$ sudo apt install libclang-dev build-essential +``` ```bash $ cargo build --release From da7b55b39c1ea592c0d5ec86a1988465bedaad0e Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Mon, 31 Jan 2022 09:27:31 +0100 Subject: [PATCH 182/201] Cleanup appservice events after removing the appservice --- src/database/admin.rs | 13 ++++++++++++- src/database/sending.rs | 31 +++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index 81e98393..9895a83b 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -112,7 +112,18 @@ impl Admin { guard.appservice.register_appservice(yaml).unwrap(); // TODO handle error } AdminCommand::UnregisterAppservice(service_name) => { - guard.appservice.unregister_appservice(&service_name).unwrap(); // TODO: see above + if let Ok(_) = guard.appservice.unregister_appservice(&service_name) { + if let Ok(_) = guard.sending.cleanup_events(&service_name) { + let msg: String = format!("OK. Appservice {} removed", service_name); + send_message(RoomMessageEventContent::text_plain(msg), guard, &state_lock); + } else { + let msg: String = format!("WARN: Appservice {} removed, but failed to cleanup events", service_name); + send_message(RoomMessageEventContent::text_plain(msg), guard, &state_lock); + } + } else { + let msg: String = format!("ERR. Appservice {} not removed", service_name); + send_message(RoomMessageEventContent::text_plain(msg), guard, &state_lock); + } } AdminCommand::ListAppservices => { if let Ok(appservices) = guard.appservice.iter_ids().map(|ids| ids.collect::>()) { diff --git a/src/database/sending.rs b/src/database/sending.rs index 69f7c444..af4ac676 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -480,6 +480,26 @@ impl Sending { hash.as_ref().to_owned() } + /// Cleanup event data + /// Used for instance after we remove an appservice registration + /// + #[tracing::instrument(skip(self))] + pub fn cleanup_events(&self, key_id: &str) -> Result<()> { + let mut prefix = b"+".to_vec(); + prefix.extend_from_slice(key_id.as_bytes()); + prefix.push(0xff); + + for (key, _) in self.servercurrentevent_data.scan_prefix(prefix.clone()) { + self.servercurrentevent_data.remove(&key).unwrap(); + } + + for (key, _) in self.servernameevent_data.scan_prefix(prefix.clone()) { + self.servernameevent_data.remove(&key).unwrap(); + } + + Ok(()) + } + #[tracing::instrument(skip(db, events, kind))] async fn handle_events( kind: OutgoingKind, @@ -520,8 +540,15 @@ impl Sending { &db.globals, db.appservice .get_registration(server.as_str()) - .unwrap() - .unwrap(), // TODO: handle error + .map_err(|e| (kind.clone(), e))? + .ok_or_else(|| { + ( + kind.clone(), + Error::bad_database( + "[Appservice] Could not load registration from db.", + ), + ) + })?, appservice::event::push_events::v1::Request { events: &pdu_jsons, txn_id: (&*base64::encode_config( From 8f69f02e592299dbe3713e238b94b19bfc445ec8 Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Mon, 31 Jan 2022 10:07:49 +0100 Subject: [PATCH 183/201] add error handling for register_appservice too --- src/database/admin.rs | 13 ++++++++++++- src/database/appservice.rs | 4 ++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index 9895a83b..eef6ce10 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -109,7 +109,18 @@ impl Admin { } } AdminCommand::RegisterAppservice(yaml) => { - guard.appservice.register_appservice(yaml).unwrap(); // TODO handle error + match guard.appservice.register_appservice(yaml) { + Ok(Some(id)) => { + let msg: String = format!("OK. Appservice {} created", id); + send_message(RoomMessageEventContent::text_plain(msg), guard, &state_lock); + } + Ok(None) => { + send_message(RoomMessageEventContent::text_plain("WARN. Appservice created, but its ID was not returned!"), guard, &state_lock); + } + Err(_) => { + send_message(RoomMessageEventContent::text_plain("ERR: Failed register appservice. Check server log"), guard, &state_lock); + } + } } AdminCommand::UnregisterAppservice(service_name) => { if let Ok(_) = guard.appservice.unregister_appservice(&service_name) { diff --git a/src/database/appservice.rs b/src/database/appservice.rs index 88de1f33..8b29aca9 100644 --- a/src/database/appservice.rs +++ b/src/database/appservice.rs @@ -12,7 +12,7 @@ pub struct Appservice { } impl Appservice { - pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result<()> { + pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result> { // TODO: Rumaify let id = yaml.get("id").unwrap().as_str().unwrap(); self.id_appserviceregistrations.insert( @@ -24,7 +24,7 @@ impl Appservice { .unwrap() .insert(id.to_owned(), yaml); - Ok(()) + Ok(Some(id.to_owned())) } /// Remove an appservice registration From e24d75cffc8f00d526848a93a4e2cfce54bf69a2 Mon Sep 17 00:00:00 2001 From: Torsten Flammiger Date: Mon, 31 Jan 2022 11:52:33 +0100 Subject: [PATCH 184/201] Return the ID of the appservice that was created by register_appservice --- src/database/admin.rs | 5 +---- src/database/appservice.rs | 8 +++++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index eef6ce10..a214796b 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -110,13 +110,10 @@ impl Admin { } AdminCommand::RegisterAppservice(yaml) => { match guard.appservice.register_appservice(yaml) { - Ok(Some(id)) => { + Ok(id) => { let msg: String = format!("OK. Appservice {} created", id); send_message(RoomMessageEventContent::text_plain(msg), guard, &state_lock); } - Ok(None) => { - send_message(RoomMessageEventContent::text_plain("WARN. Appservice created, but its ID was not returned!"), guard, &state_lock); - } Err(_) => { send_message(RoomMessageEventContent::text_plain("ERR: Failed register appservice. Check server log"), guard, &state_lock); } diff --git a/src/database/appservice.rs b/src/database/appservice.rs index 8b29aca9..edd5009b 100644 --- a/src/database/appservice.rs +++ b/src/database/appservice.rs @@ -12,7 +12,9 @@ pub struct Appservice { } impl Appservice { - pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result> { + /// Registers an appservice and returns the ID to the caller + /// + pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result { // TODO: Rumaify let id = yaml.get("id").unwrap().as_str().unwrap(); self.id_appserviceregistrations.insert( @@ -22,9 +24,9 @@ impl Appservice { self.cached_registrations .write() .unwrap() - .insert(id.to_owned(), yaml); + .insert(id.to_owned(), yaml.to_owned()); - Ok(Some(id.to_owned())) + Ok(id.to_owned()) } /// Remove an appservice registration From 9478c75f9dcd040cb9f03deb5ea809f117985de2 Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Tue, 1 Feb 2022 23:51:38 +0000 Subject: [PATCH 185/201] Use prebuilt CI-containers from https://gitlab.com/jfowl/conduit-containers Also run all builds on approved MRs --- .gitlab-ci.yml | 39 ++++++++++--------------- Cross.toml | 8 ++--- cross/build.sh | 31 -------------------- cross/test.sh | 8 ----- docker/ci-binaries-packaging.Dockerfile | 4 ++- docker/healthcheck.sh | 2 +- 6 files changed, 24 insertions(+), 68 deletions(-) delete mode 100755 cross/build.sh delete mode 100755 cross/test.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 741b5327..6f1a19f0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,8 +24,9 @@ variables: - if: '$CI_COMMIT_BRANCH == "master"' - if: '$CI_COMMIT_BRANCH == "next"' - if: "$CI_COMMIT_TAG" + - if: '($CI_MERGE_REQUEST_APPROVED == "true") || $BUILD_EVERYTHING' # Once MR is approved, test all builds. Or if BUILD_EVERYTHING is set. interruptible: true - image: "rust:1.58" + image: "registry.gitlab.com/jfowl/conduit-containers/rust-with-tools:latest" tags: ["docker"] services: ["docker:dind"] variables: @@ -36,27 +37,23 @@ variables: before_script: - 'echo "Building for target $TARGET"' - "rustup show && rustc --version && cargo --version" # Print version info for debugging - # install cross-compiling prerequisites - - 'apt-get update && apt-get install -y docker.io && docker version' # install docker - - 'cargo install cross && cross --version' # install cross # fix cargo and rustup mounts from this container (https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41227) - - 'mkdir -p $SHARED_PATH/cargo' - - 'cp -r $CARGO_HOME/bin $SHARED_PATH/cargo' - - 'cp -r $RUSTUP_HOME $SHARED_PATH' - - 'export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup' + - "mkdir -p $SHARED_PATH/cargo" + - "cp -r $CARGO_HOME/bin $SHARED_PATH/cargo" + - "cp -r $RUSTUP_HOME $SHARED_PATH" + - "export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup" # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results. - # The sccache binary is stored in the sysroot of the rustc installation since that directory is added to the path of the cross container. - - if [ -n "${SCCACHE_BIN_URL}" ]; then RUSTC_SYSROOT=$(rustc --print sysroot) && curl $SCCACHE_BIN_URL --output $RUSTC_SYSROOT/bin/sccache && chmod +x $RUSTC_SYSROOT/bin/sccache && export RUSTC_WRAPPER=sccache; fi + - if [ -n "${SCCACHE_ENDPOINT}" ]; then export RUSTC_WRAPPER=/sccache; fi script: # cross-compile conduit for target - - 'time ./cross/build.sh --locked --release' + - 'time cross build --target="$TARGET" --locked --release' - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"' # print information about linking for debugging - - 'file conduit-$TARGET' # print file information + - "file conduit-$TARGET" # print file information - 'readelf --dynamic conduit-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked cache: # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci - key: 'cargo-cache-$TARGET' + key: "cargo-cache-$TARGET" paths: - $SHARED_PATH/cargo/registry/index - $SHARED_PATH/cargo/registry/cache @@ -125,10 +122,10 @@ build:release:cargo:aarch64-unknown-linux-musl: key: "build_cache--$TARGET--$CI_COMMIT_BRANCH--debug" script: # cross-compile conduit for target - - 'time ./cross/build.sh --locked' + - 'time time cross build --target="$TARGET" --locked' - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"' # print information about linking for debugging - - 'file conduit-debug-$TARGET' # print file information + - "file conduit-debug-$TARGET" # print file information - 'readelf --dynamic conduit-debug-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked artifacts: expire_in: 4 weeks @@ -230,24 +227,20 @@ docker:master:dockerhub: test:cargo: stage: "test" needs: [] - image: "rust:latest" + image: "registry.gitlab.com/jfowl/conduit-containers/rust-with-tools:latest" tags: ["docker"] variables: CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow interruptible: true before_script: - # - mkdir -p $CARGO_HOME - - apt-get update -yqq - - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config libclang-dev - rustup component add clippy rustfmt - - curl "https://faulty-storage.de/gitlab-report" --output ./gitlab-report && chmod +x ./gitlab-report # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results: - - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi + - if [ -n "${SCCACHE_ENDPOINT}" ]; then export RUSTC_WRAPPER=/usr/local/cargo/bin/sccache; fi script: - rustc --version && cargo --version # Print version info for debugging - cargo fmt --all -- --check - - "cargo test --color always --workspace --verbose --locked --no-fail-fast -- -Z unstable-options --format json | ./gitlab-report -p test > $CI_PROJECT_DIR/report.xml" - - "cargo clippy --color always --verbose --message-format=json | ./gitlab-report -p clippy > $CI_PROJECT_DIR/gl-code-quality-report.json" + - "cargo test --color always --workspace --verbose --locked --no-fail-fast -- -Z unstable-options --format json | gitlab-report -p test > $CI_PROJECT_DIR/report.xml" + - "cargo clippy --color always --verbose --message-format=json | gitlab-report -p clippy > $CI_PROJECT_DIR/gl-code-quality-report.json" artifacts: when: always reports: diff --git a/Cross.toml b/Cross.toml index a989a98f..a1387b43 100644 --- a/Cross.toml +++ b/Cross.toml @@ -11,13 +11,13 @@ passthrough = [ ] [target.aarch64-unknown-linux-musl] -image = "rust-cross:aarch64-unknown-linux-musl" +image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-aarch64-unknown-linux-musl:latest" [target.arm-unknown-linux-musleabihf] -image = "rust-cross:arm-unknown-linux-musleabihf" +image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-arm-unknown-linux-musleabihf:latest" [target.armv7-unknown-linux-musleabihf] -image = "rust-cross:armv7-unknown-linux-musleabihf" +image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-armv7-unknown-linux-musleabihf:latest" [target.x86_64-unknown-linux-musl] -image = "rust-cross:x86_64-unknown-linux-musl" +image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-x86_64-unknown-linux-musl:latest" diff --git a/cross/build.sh b/cross/build.sh deleted file mode 100755 index 8f64ff87..00000000 --- a/cross/build.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -set -ex - -# build custom container with libclang and static compilation -tag="rust-cross:${TARGET:?}" -docker build --tag="$tag" - << EOF -FROM rustembedded/cross:$TARGET - -# Install libclang for generating bindings with rust-bindgen -# The architecture is not relevant here since it's not used for compilation -RUN apt-get update && \ - apt-get install --assume-yes libclang-dev - -# Set the target prefix -ENV TARGET_PREFIX="/usr/local/$(echo "${TARGET:?}" | sed -e 's/armv7/arm/' -e 's/-unknown//')" - -# Make sure that cc-rs links libc/libstdc++ statically when cross-compiling -# See https://github.com/alexcrichton/cc-rs#external-configuration-via-environment-variables for more information -ENV RUSTFLAGS="-L\$TARGET_PREFIX/lib" CXXSTDLIB="static=stdc++" -# Forcefully linking against libatomic, libc and libgcc is required for arm32, otherwise symbols are missing -$([[ $TARGET =~ arm ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-static-libgcc -Clink-arg=-lgcc -lstatic=atomic -lstatic=c"') -# Strip symbols while compiling in release mode -$([[ $@ =~ -r ]] && echo 'ENV RUSTFLAGS="$RUSTFLAGS -Clink-arg=-s"') - -# Make sure that rust-bindgen uses the correct include path when cross-compiling -# See https://github.com/rust-lang/rust-bindgen#environment-variables for more information -ENV BINDGEN_EXTRA_CLANG_ARGS="-I\$TARGET_PREFIX/include" -EOF - -# build conduit for a specific target -cross build --target="$TARGET" $@ diff --git a/cross/test.sh b/cross/test.sh deleted file mode 100755 index 0aa0909c..00000000 --- a/cross/test.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env sh -set -ex - -# Build conduit for a specific target -cross/build.sh $@ - -# Test conduit for a specific target -cross test --target="$TARGET" $@ diff --git a/docker/ci-binaries-packaging.Dockerfile b/docker/ci-binaries-packaging.Dockerfile index a6339be3..bb67bb22 100644 --- a/docker/ci-binaries-packaging.Dockerfile +++ b/docker/ci-binaries-packaging.Dockerfile @@ -19,8 +19,10 @@ ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" # Conduit needs: # ca-certificates: for https +# iproute2: for `ss` for the healthcheck script RUN apk add --no-cache \ - ca-certificates + ca-certificates \ + iproute2 ARG CREATED diff --git a/docker/healthcheck.sh b/docker/healthcheck.sh index df7f18a5..42b2e103 100644 --- a/docker/healthcheck.sh +++ b/docker/healthcheck.sh @@ -3,7 +3,7 @@ # If the config file does not contain a default port and the CONDUIT_PORT env is not set, create # try to get port from process list if [ -z "${CONDUIT_PORT}" ]; then - CONDUIT_PORT=$(netstat -tlp | grep conduit | grep -m1 -o ':[0-9]*' | grep -m1 -o '[0-9]*') + CONDUIT_PORT=$(ss -tlpn | grep conduit | grep -m1 -o ':[0-9]*' | grep -m1 -o '[0-9]*') fi # The actual health check. From e5bac5e4f53fa3e6565cca96b687dc8ff976f7f0 Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Wed, 2 Feb 2022 14:07:35 +0100 Subject: [PATCH 186/201] fix: Running in Docker --- Dockerfile | 44 ++++++++++++------------- conduit-example.toml | 1 + docker/ci-binaries-packaging.Dockerfile | 5 +-- 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/Dockerfile b/Dockerfile index b629690d..0da4aace 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ # syntax=docker/dockerfile:1 -FROM docker.io/rust:1.58-alpine AS builder +FROM docker.io/rust:1.58-bullseye AS builder WORKDIR /usr/src/conduit # Install required packages to build Conduit and it's dependencies -RUN apk add musl-dev +RUN apt update && apt -y install libclang-11-dev # == Build dependencies without our own code separately for caching == # @@ -26,28 +26,28 @@ COPY src src # Builds conduit and places the binary at /usr/src/conduit/target/release/conduit RUN touch src/main.rs && touch src/lib.rs && cargo build --release - - - # --------------------------------------------------------------------------------------------------------------- # Stuff below this line actually ends up in the resulting docker image # --------------------------------------------------------------------------------------------------------------- -FROM docker.io/alpine:3.15.0 AS runner +FROM docker.io/debian:bullseye-slim AS runner # Standard port on which Conduit launches. # You still need to map the port when using the docker command or docker-compose. EXPOSE 6167 -# Note from @jfowl: I would like to remove this in the future and just have the Docker version be configured with envs. -ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" +# Note from @jfowl: I would like to remove the config file in the future and just have the Docker version be configured with envs. +ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" \ + CONDUIT_PORT=6167 # Conduit needs: # ca-certificates: for https -# libgcc: Apparently this is needed, even if I (@jfowl) don't know exactly why. But whatever, it's not that big. -RUN apk add --no-cache \ +# iproute2 & wget: for the healthcheck script +RUN apt update && apt -y install \ ca-certificates \ - libgcc + iproute2 \ + wget +RUN rm -rf /var/lib/apt/lists/* # Created directory for the database and media files RUN mkdir -p /srv/conduit/.local/share/conduit @@ -59,20 +59,20 @@ HEALTHCHECK --start-period=5s --interval=5s CMD ./healthcheck.sh # Copy over the actual Conduit binary from the builder stage COPY --from=builder /usr/src/conduit/target/release/conduit /srv/conduit/conduit -# Improve security: Don't run stuff as root, that does not need to run as root: -# Add www-data user and group with UID 82, as used by alpine -# https://git.alpinelinux.org/aports/tree/main/nginx/nginx.pre-install +# Improve security: Don't run stuff as root, that does not need to run as root +# Add 'conduit' user and group (100:82). The UID:GID choice is to be compatible +# with previous, Alpine-based containers, where the user and group were both +# named 'www-data'. RUN set -x ; \ - addgroup -Sg 82 www-data 2>/dev/null ; \ - adduser -S -D -H -h /srv/conduit -G www-data -g www-data www-data 2>/dev/null ; \ - addgroup www-data www-data 2>/dev/null && exit 0 ; exit 1 + groupadd -r -g 82 conduit ; \ + useradd -r -M -d /srv/conduit -o -u 100 -g conduit conduit && exit 0 ; exit 1 -# Change ownership of Conduit files to www-data user and group -RUN chown -cR www-data:www-data /srv/conduit -RUN chmod +x /srv/conduit/healthcheck.sh +# Change ownership of Conduit files to conduit user and group and make the healthcheck executable: +RUN chown -cR conduit:conduit /srv/conduit && \ + chmod +x /srv/conduit/healthcheck.sh -# Change user to www-data -USER www-data +# Change user to conduit, no root permissions afterwards: +USER conduit # Set container home directory WORKDIR /srv/conduit diff --git a/conduit-example.toml b/conduit-example.toml index c0274a4d..f1578078 100644 --- a/conduit-example.toml +++ b/conduit-example.toml @@ -22,6 +22,7 @@ database_backend = "rocksdb" # The port Conduit will be running on. You need to set up a reverse proxy in # your web server (e.g. apache or nginx), so all requests to /_matrix on port # 443 and 8448 will be forwarded to the Conduit instance running on this port +# Docker users: Don't change this, you'll need to map an external port to this. port = 6167 # Max size for uploads diff --git a/docker/ci-binaries-packaging.Dockerfile b/docker/ci-binaries-packaging.Dockerfile index bb67bb22..3731bac1 100644 --- a/docker/ci-binaries-packaging.Dockerfile +++ b/docker/ci-binaries-packaging.Dockerfile @@ -14,8 +14,9 @@ FROM docker.io/alpine:3.15.0 AS runner # You still need to map the port when using the docker command or docker-compose. EXPOSE 6167 -# Note from @jfowl: I would like to remove this in the future and just have the Docker version be configured with envs. -ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" +# Note from @jfowl: I would like to remove the config file in the future and just have the Docker version be configured with envs. +ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" \ + CONDUIT_PORT=6167 # Conduit needs: # ca-certificates: for https From c4733676cf16267ffbb0b348848e87a7d103cf37 Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Wed, 2 Feb 2022 13:35:15 +0000 Subject: [PATCH 187/201] Apply feedback from Ticho --- Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0da4aace..b631f297 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM docker.io/rust:1.58-bullseye AS builder WORKDIR /usr/src/conduit # Install required packages to build Conduit and it's dependencies -RUN apt update && apt -y install libclang-11-dev +RUN apt update && apt -y install libclang-dev # == Build dependencies without our own code separately for caching == # @@ -45,9 +45,8 @@ ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" \ RUN apt update && apt -y install \ ca-certificates \ iproute2 \ - wget - -RUN rm -rf /var/lib/apt/lists/* + wget \ + && rm -rf /var/lib/apt/lists/* # Created directory for the database and media files RUN mkdir -p /srv/conduit/.local/share/conduit From 87225e70c3441c9ddd96d9fe0c4dd4e5a2c1289e Mon Sep 17 00:00:00 2001 From: Andrei Vasiliu Date: Wed, 2 Feb 2022 21:35:57 +0200 Subject: [PATCH 188/201] Parse admin command body templates from doc comments --- src/database/admin.rs | 69 ++++++++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 21 deletions(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index ea08f65a..c7150493 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -132,7 +132,7 @@ impl Admin { } // Parse and process a message from the admin room -pub fn process_admin_message(db: &Database, room_message: String) -> RoomMessageEventContent { +fn process_admin_message(db: &Database, room_message: String) -> RoomMessageEventContent { let mut lines = room_message.lines(); let command_line = lines.next().expect("each string has at least one line"); let body: Vec<_> = lines.collect(); @@ -202,7 +202,10 @@ enum AdminCommand { /// Registering a new bridge using the ID of an existing bridge will replace /// the old one. /// - /// [add-yaml-block-to-usage] + /// [commandbody] + /// # ``` + /// # yaml content here + /// # ``` RegisterAppservice, /// Unregister an appservice using its ID @@ -225,10 +228,16 @@ enum AdminCommand { event_id: Box, }, + #[clap(verbatim_doc_comment)] /// Parse and print a PDU from a JSON /// /// The PDU event is only checked for validity and is not added to the /// database. + /// + /// [commandbody] + /// # ``` + /// # PDU json content here + /// # ``` ParsePdu, /// Retrieve and print a PDU by ID from the Conduit database @@ -433,33 +442,51 @@ fn usage_to_html(text: &str) -> String { .expect("Regex compilation should not fail"); let text = re.replace_all(&text, "$1: $4"); - // // Enclose examples in code blocks - // // (?ms) enables multi-line mode and dot-matches-all - // let re = - // Regex::new("(?ms)^Example:\n(.*?)\nUSAGE:$").expect("Regex compilation should not fail"); - // let text = re.replace_all(&text, "EXAMPLE:\n
              $1
              \nUSAGE:"); + // Look for a `[commandbody]` tag. If it exists, use all lines below it that + // start with a `#` in the USAGE section. + let mut text_lines: Vec<&str> = text.lines().collect(); + let mut command_body = String::new(); - let has_yaml_block_marker = text.contains("\n[add-yaml-block-to-usage]\n"); - let text = text.replace("\n[add-yaml-block-to-usage]\n", ""); + if let Some(line_index) = text_lines.iter().position(|line| *line == "[commandbody]") { + text_lines.remove(line_index); - // Add HTML line-breaks - let text = text.replace("\n", "
              \n"); + while text_lines + .get(line_index) + .map(|line| line.starts_with("#")) + .unwrap_or(false) + { + command_body += if text_lines[line_index].starts_with("# ") { + &text_lines[line_index][2..] + } else { + &text_lines[line_index][1..] + }; + command_body += "[nobr]\n"; + text_lines.remove(line_index); + } + } - let text = if !has_yaml_block_marker { + let text = text_lines.join("\n"); + + // Improve the usage section + let text = if command_body.is_empty() { // Wrap the usage line in code tags - let re = Regex::new("(?m)^USAGE:
              \n (@conduit:.*)
              $") + let re = Regex::new("(?m)^USAGE:\n (@conduit:.*)$") .expect("Regex compilation should not fail"); - re.replace_all(&text, "USAGE:
              \n$1
              ") + re.replace_all(&text, "USAGE:\n$1").to_string() } else { // Wrap the usage line in a code block, and add a yaml block example // This makes the usage of e.g. `register-appservice` more accurate - let re = Regex::new("(?m)^USAGE:
              \n (.*?)
              \n
              \n") - .expect("Regex compilation should not fail"); - re.replace_all( - &text, - "USAGE:
              \n
              $1\n```\nyaml content here\n```
              ", - ) + let re = + Regex::new("(?m)^USAGE:\n (.*?)\n\n").expect("Regex compilation should not fail"); + re.replace_all(&text, "USAGE:\n
              $1[nobr]\n[commandbodyblock]
              ") + .replace("[commandbodyblock]", &command_body) }; - text.to_string() + // Add HTML line-breaks + let text = text + .replace("\n\n\n", "\n\n") + .replace("\n", "
              \n") + .replace("[nobr]
              ", ""); + + text } From 9ef3abacd43571300a7fbd7d35ba05d040816d8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 2 Feb 2022 18:03:50 +0100 Subject: [PATCH 189/201] fix: initial state deserialize->serialize error --- src/client_server/room.rs | 7 +++++-- src/pdu.rs | 19 ++----------------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 52d25425..a2339639 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -344,10 +344,13 @@ pub async fn create_room_route( // 6. Events listed in initial_state for event in &body.initial_state { - let pdu_builder = PduBuilder::from(event.deserialize().map_err(|e| { + let mut pdu_builder = event.deserialize_as::().map_err(|e| { warn!("Invalid initial state event: {:?}", e); Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event.") - })?); + })?; + + // Implicit state key defaults to "" + pdu_builder.state_key.get_or_insert_with(|| "".to_owned()); // Silently skip encryption events if they are not allowed if pdu_builder.event_type == EventType::RoomEncryption && !db.globals.allow_encryption() { diff --git a/src/pdu.rs b/src/pdu.rs index db9375e4..fe004609 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -1,9 +1,8 @@ use crate::Error; use ruma::{ events::{ - room::member::RoomMemberEventContent, AnyEphemeralRoomEvent, AnyInitialStateEvent, - AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, - EventType, StateEvent, + room::member::RoomMemberEventContent, AnyEphemeralRoomEvent, AnyRoomEvent, AnyStateEvent, + AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent, }, serde::{CanonicalJsonObject, CanonicalJsonValue, Raw}, state_res, EventId, MilliSecondsSinceUnixEpoch, RoomId, RoomVersionId, UInt, UserId, @@ -361,17 +360,3 @@ pub struct PduBuilder { pub state_key: Option, pub redacts: Option>, } - -/// Direct conversion prevents loss of the empty `state_key` that ruma requires. -impl From for PduBuilder { - fn from(event: AnyInitialStateEvent) -> Self { - Self { - event_type: EventType::from(event.event_type()), - content: to_raw_value(&event.content()) - .expect("AnyStateEventContent came from JSON and can thus turn back into JSON."), - unsigned: None, - state_key: Some(event.state_key().to_owned()), - redacts: None, - } - } -} From abb4b4cf0b0868fe7e5ee21298278b8c3deacb0e Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Thu, 3 Feb 2022 13:24:04 +0100 Subject: [PATCH 190/201] Remove TryFrom, TryInto imports They are no longer needed in the 2021 edition. --- src/appservice_server.rs | 7 +------ src/client_server/account.rs | 2 +- src/client_server/context.rs | 2 +- src/client_server/directory.rs | 2 -- src/client_server/media.rs | 1 - src/client_server/membership.rs | 1 - src/client_server/message.rs | 1 - src/client_server/presence.rs | 2 +- src/client_server/profile.rs | 2 +- src/client_server/room.rs | 2 +- src/client_server/sync.rs | 1 - src/database.rs | 1 - src/database/account_data.rs | 2 +- src/database/admin.rs | 2 +- src/database/pusher.rs | 2 +- src/database/rooms.rs | 1 - src/database/rooms/edus.rs | 1 - src/database/sending.rs | 1 - src/database/users.rs | 7 +------ src/pdu.rs | 2 +- src/server_server.rs | 1 - src/utils.rs | 1 - 22 files changed, 11 insertions(+), 33 deletions(-) diff --git a/src/appservice_server.rs b/src/appservice_server.rs index ed886d6c..0152c386 100644 --- a/src/appservice_server.rs +++ b/src/appservice_server.rs @@ -1,12 +1,7 @@ use crate::{utils, Error, Result}; use bytes::BytesMut; use ruma::api::{IncomingResponse, OutgoingRequest, SendAccessToken}; -use std::{ - convert::{TryFrom, TryInto}, - fmt::Debug, - mem, - time::Duration, -}; +use std::{fmt::Debug, mem, time::Duration}; use tracing::warn; pub(crate) async fn send_request( diff --git a/src/client_server/account.rs b/src/client_server/account.rs index ff348545..47e2a6a4 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, convert::TryInto, sync::Arc}; +use std::{collections::BTreeMap, sync::Arc}; use super::{DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH}; use crate::{database::DatabaseGuard, pdu::PduBuilder, utils, ConduitResult, Error, Ruma}; diff --git a/src/client_server/context.rs b/src/client_server/context.rs index e1177661..7ded48da 100644 --- a/src/client_server/context.rs +++ b/src/client_server/context.rs @@ -3,7 +3,7 @@ use ruma::{ api::client::{error::ErrorKind, r0::context::get_context}, events::EventType, }; -use std::{collections::HashSet, convert::TryFrom}; +use std::collections::HashSet; #[cfg(feature = "conduit_bin")] use rocket::get; diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 5a1bc494..719d9af4 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -1,5 +1,3 @@ -use std::convert::TryInto; - use crate::{database::DatabaseGuard, ConduitResult, Database, Error, Result, Ruma}; use ruma::{ api::{ diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 8524c57e..deea319e 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -9,7 +9,6 @@ use ruma::api::client::{ get_media_config, }, }; -use std::convert::TryInto; #[cfg(feature = "conduit_bin")] use rocket::{get, post}; diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 216c4c0a..e855dba2 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -30,7 +30,6 @@ use ruma::{ use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use std::{ collections::{hash_map::Entry, BTreeMap, HashMap, HashSet}, - convert::{TryFrom, TryInto}, iter, sync::{Arc, RwLock}, time::{Duration, Instant}, diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 7d904f90..cf4f0cb6 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -8,7 +8,6 @@ use ruma::{ }; use std::{ collections::{BTreeMap, HashSet}, - convert::TryInto, sync::Arc, }; diff --git a/src/client_server/presence.rs b/src/client_server/presence.rs index aaa78a92..cdc1e1f5 100644 --- a/src/client_server/presence.rs +++ b/src/client_server/presence.rs @@ -1,6 +1,6 @@ use crate::{database::DatabaseGuard, utils, ConduitResult, Ruma}; use ruma::api::client::r0::presence::{get_presence, set_presence}; -use std::{convert::TryInto, time::Duration}; +use std::time::Duration; #[cfg(feature = "conduit_bin")] use rocket::{get, put}; diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index 71e61da3..ef58a980 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -12,7 +12,7 @@ use ruma::{ events::{room::member::RoomMemberEventContent, EventType}, }; use serde_json::value::to_raw_value; -use std::{convert::TryInto, sync::Arc}; +use std::sync::Arc; #[cfg(feature = "conduit_bin")] use rocket::{get, put}; diff --git a/src/client_server/room.rs b/src/client_server/room.rs index a2339639..7ea31d8a 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -27,7 +27,7 @@ use ruma::{ RoomAliasId, RoomId, RoomVersionId, }; use serde_json::{json, value::to_raw_value}; -use std::{cmp::max, collections::BTreeMap, convert::TryInto, sync::Arc}; +use std::{cmp::max, collections::BTreeMap, sync::Arc}; use tracing::{info, warn}; #[cfg(feature = "conduit_bin")] diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 14aac3a1..7cfea5af 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -14,7 +14,6 @@ use ruma::{ }; use std::{ collections::{hash_map::Entry, BTreeMap, HashMap, HashSet}, - convert::TryInto, sync::Arc, time::Duration, }; diff --git a/src/database.rs b/src/database.rs index 8d245b7f..c9cbad4d 100644 --- a/src/database.rs +++ b/src/database.rs @@ -28,7 +28,6 @@ use ruma::{DeviceId, EventId, RoomId, ServerName, UserId}; use serde::{de::IgnoredAny, Deserialize}; use std::{ collections::{BTreeMap, HashMap, HashSet}, - convert::{TryFrom, TryInto}, fs::{self, remove_dir_all}, io::Write, mem::size_of, diff --git a/src/database/account_data.rs b/src/database/account_data.rs index 456283bd..ec9d09e8 100644 --- a/src/database/account_data.rs +++ b/src/database/account_data.rs @@ -6,7 +6,7 @@ use ruma::{ RoomId, UserId, }; use serde::{de::DeserializeOwned, Serialize}; -use std::{collections::HashMap, convert::TryFrom, sync::Arc}; +use std::{collections::HashMap, sync::Arc}; use super::abstraction::Tree; diff --git a/src/database/admin.rs b/src/database/admin.rs index a214796b..32972de4 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -1,4 +1,4 @@ -use std::{convert::TryInto, sync::Arc}; +use std::sync::Arc; use crate::{pdu::PduBuilder, Database}; use rocket::futures::{channel::mpsc, stream::StreamExt}; diff --git a/src/database/pusher.rs b/src/database/pusher.rs index 97ca85d8..f401834a 100644 --- a/src/database/pusher.rs +++ b/src/database/pusher.rs @@ -19,7 +19,7 @@ use ruma::{ }; use tracing::{error, info, warn}; -use std::{convert::TryFrom, fmt::Debug, mem, sync::Arc}; +use std::{fmt::Debug, mem, sync::Arc}; use super::abstraction::Tree; diff --git a/src/database/rooms.rs b/src/database/rooms.rs index c0cb1ce9..a139853b 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -35,7 +35,6 @@ use serde_json::value::to_raw_value; use std::{ borrow::Cow, collections::{BTreeMap, HashMap, HashSet}, - convert::{TryFrom, TryInto}, fmt::Debug, iter, mem::size_of, diff --git a/src/database/rooms/edus.rs b/src/database/rooms/edus.rs index eb2d3427..289a00a1 100644 --- a/src/database/rooms/edus.rs +++ b/src/database/rooms/edus.rs @@ -11,7 +11,6 @@ use ruma::{ }; use std::{ collections::{HashMap, HashSet}, - convert::TryInto, mem, sync::Arc, }; diff --git a/src/database/sending.rs b/src/database/sending.rs index af4ac676..4a032855 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -1,6 +1,5 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, - convert::TryInto, fmt::Debug, sync::Arc, time::{Duration, Instant}, diff --git a/src/database/users.rs b/src/database/users.rs index 13f9b151..681ee284 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -11,12 +11,7 @@ use ruma::{ DeviceId, DeviceKeyAlgorithm, DeviceKeyId, MilliSecondsSinceUnixEpoch, RoomAliasId, UInt, UserId, }; -use std::{ - collections::BTreeMap, - convert::{TryFrom, TryInto}, - mem, - sync::Arc, -}; +use std::{collections::BTreeMap, mem, sync::Arc}; use tracing::warn; use super::abstraction::Tree; diff --git a/src/pdu.rs b/src/pdu.rs index fe004609..ec6c961b 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -12,7 +12,7 @@ use serde_json::{ json, value::{to_raw_value, RawValue as RawJsonValue}, }; -use std::{cmp::Ordering, collections::BTreeMap, convert::TryInto, sync::Arc}; +use std::{cmp::Ordering, collections::BTreeMap, sync::Arc}; use tracing::warn; /// Content hashes of a PDU. diff --git a/src/server_server.rs b/src/server_server.rs index 9129951b..e730210a 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -60,7 +60,6 @@ use ruma::{ use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use std::{ collections::{btree_map, hash_map, BTreeMap, BTreeSet, HashMap, HashSet}, - convert::{TryFrom, TryInto}, fmt::Debug, future::Future, mem, diff --git a/src/utils.rs b/src/utils.rs index 26d71a8c..e2d71f4c 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -4,7 +4,6 @@ use rand::prelude::*; use ruma::serde::{try_from_json_map, CanonicalJsonError, CanonicalJsonObject}; use std::{ cmp, - convert::TryInto, str::FromStr, time::{SystemTime, UNIX_EPOCH}, }; From ce60fc6859ea698ed8341beea8321a949d90ad39 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Sat, 22 Jan 2022 13:27:54 +0100 Subject: [PATCH 191/201] Stop using set_env to configure tracing-subscriber --- src/main.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/main.rs b/src/main.rs index 63b22194..5fda5737 100644 --- a/src/main.rs +++ b/src/main.rs @@ -184,9 +184,6 @@ fn setup_rocket(config: Figment, data: Arc>) -> rocket::Rocket< #[rocket::main] async fn main() { - // Force log level off, so we can use our own logger - std::env::set_var("CONDUIT_LOG_LEVEL", "off"); - let raw_config = Figment::from(default_config()) .merge( @@ -197,8 +194,6 @@ async fn main() { ) .merge(Env::prefixed("CONDUIT_").global()); - std::env::set_var("RUST_LOG", "warn"); - let config = match raw_config.extract::() { Ok(s) => s, Err(e) => { @@ -244,8 +239,6 @@ async fn main() { println!("exporting"); opentelemetry::global::shutdown_tracer_provider(); } else { - std::env::set_var("RUST_LOG", &config.log); - let registry = tracing_subscriber::Registry::default(); if config.tracing_flame { let (flame_layer, _guard) = @@ -259,7 +252,7 @@ async fn main() { start.await; } else { let fmt_layer = tracing_subscriber::fmt::Layer::new(); - let filter_layer = EnvFilter::try_from_default_env() + let filter_layer = EnvFilter::try_new(&config.log) .or_else(|_| EnvFilter::try_new("info")) .unwrap(); From 974c10e739b70c5798450f5e51819e9d2beed5d3 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Thu, 3 Feb 2022 13:30:04 +0100 Subject: [PATCH 192/201] Move Config out of database module --- src/config.rs | 131 ++++++++++++++++++++++++++++++ src/{database => config}/proxy.rs | 0 src/database.rs | 130 +---------------------------- src/lib.rs | 4 +- 4 files changed, 136 insertions(+), 129 deletions(-) create mode 100644 src/config.rs rename src/{database => config}/proxy.rs (100%) diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 00000000..4c0fcc21 --- /dev/null +++ b/src/config.rs @@ -0,0 +1,131 @@ +use std::collections::BTreeMap; + +use ruma::ServerName; +use serde::{de::IgnoredAny, Deserialize}; +use tracing::warn; + +mod proxy; + +use self::proxy::ProxyConfig; + +#[derive(Clone, Debug, Deserialize)] +pub struct Config { + pub server_name: Box, + #[serde(default = "default_database_backend")] + pub database_backend: String, + pub database_path: String, + #[serde(default = "default_db_cache_capacity_mb")] + pub db_cache_capacity_mb: f64, + #[serde(default = "default_conduit_cache_capacity_modifier")] + pub conduit_cache_capacity_modifier: f64, + #[serde(default = "default_rocksdb_max_open_files")] + pub rocksdb_max_open_files: i32, + #[serde(default = "default_pdu_cache_capacity")] + pub pdu_cache_capacity: u32, + #[serde(default = "default_cleanup_second_interval")] + pub cleanup_second_interval: u32, + #[serde(default = "default_max_request_size")] + pub max_request_size: u32, + #[serde(default = "default_max_concurrent_requests")] + pub max_concurrent_requests: u16, + #[serde(default = "false_fn")] + pub allow_registration: bool, + #[serde(default = "true_fn")] + pub allow_encryption: bool, + #[serde(default = "false_fn")] + pub allow_federation: bool, + #[serde(default = "true_fn")] + pub allow_room_creation: bool, + #[serde(default = "false_fn")] + pub allow_jaeger: bool, + #[serde(default = "false_fn")] + pub tracing_flame: bool, + #[serde(default)] + pub proxy: ProxyConfig, + pub jwt_secret: Option, + #[serde(default = "Vec::new")] + pub trusted_servers: Vec>, + #[serde(default = "default_log")] + pub log: String, + #[serde(default)] + pub turn_username: String, + #[serde(default)] + pub turn_password: String, + #[serde(default = "Vec::new")] + pub turn_uris: Vec, + #[serde(default)] + pub turn_secret: String, + #[serde(default = "default_turn_ttl")] + pub turn_ttl: u64, + + #[serde(flatten)] + pub catchall: BTreeMap, +} + +const DEPRECATED_KEYS: &[&str] = &["cache_capacity"]; + +impl Config { + pub fn warn_deprecated(&self) { + let mut was_deprecated = false; + for key in self + .catchall + .keys() + .filter(|key| DEPRECATED_KEYS.iter().any(|s| s == key)) + { + warn!("Config parameter {} is deprecated", key); + was_deprecated = true; + } + + if was_deprecated { + warn!("Read conduit documentation and check your configuration if any new configuration parameters should be adjusted"); + } + } +} + +fn false_fn() -> bool { + false +} + +fn true_fn() -> bool { + true +} + +fn default_database_backend() -> String { + "sqlite".to_owned() +} + +fn default_db_cache_capacity_mb() -> f64 { + 10.0 +} + +fn default_conduit_cache_capacity_modifier() -> f64 { + 1.0 +} + +fn default_rocksdb_max_open_files() -> i32 { + 20 +} + +fn default_pdu_cache_capacity() -> u32 { + 150_000 +} + +fn default_cleanup_second_interval() -> u32 { + 1 * 60 // every minute +} + +fn default_max_request_size() -> u32 { + 20 * 1024 * 1024 // Default to 20 MB +} + +fn default_max_concurrent_requests() -> u16 { + 100 +} + +fn default_log() -> String { + "info,state_res=warn,rocket=off,_=off,sled=off".to_owned() +} + +fn default_turn_ttl() -> u64 { + 60 * 60 * 24 +} diff --git a/src/database/proxy.rs b/src/config/proxy.rs similarity index 100% rename from src/database/proxy.rs rename to src/config/proxy.rs diff --git a/src/database.rs b/src/database.rs index c9cbad4d..5deedcfe 100644 --- a/src/database.rs +++ b/src/database.rs @@ -6,7 +6,6 @@ pub mod appservice; pub mod globals; pub mod key_backups; pub mod media; -pub mod proxy; pub mod pusher; pub mod rooms; pub mod sending; @@ -14,7 +13,7 @@ pub mod transaction_ids; pub mod uiaa; pub mod users; -use crate::{utils, Error, Result}; +use crate::{utils, Config, Error, Result}; use abstraction::DatabaseEngine; use directories::ProjectDirs; use lru_cache::LruCache; @@ -24,8 +23,7 @@ use rocket::{ request::{FromRequest, Request}, Shutdown, State, }; -use ruma::{DeviceId, EventId, RoomId, ServerName, UserId}; -use serde::{de::IgnoredAny, Deserialize}; +use ruma::{DeviceId, EventId, RoomId, UserId}; use std::{ collections::{BTreeMap, HashMap, HashSet}, fs::{self, remove_dir_all}, @@ -38,130 +36,6 @@ use std::{ use tokio::sync::{OwnedRwLockReadGuard, RwLock as TokioRwLock, Semaphore}; use tracing::{debug, error, warn}; -use self::proxy::ProxyConfig; - -#[derive(Clone, Debug, Deserialize)] -pub struct Config { - server_name: Box, - #[serde(default = "default_database_backend")] - database_backend: String, - database_path: String, - #[serde(default = "default_db_cache_capacity_mb")] - db_cache_capacity_mb: f64, - #[serde(default = "default_conduit_cache_capacity_modifier")] - conduit_cache_capacity_modifier: f64, - #[serde(default = "default_rocksdb_max_open_files")] - rocksdb_max_open_files: i32, - #[serde(default = "default_pdu_cache_capacity")] - pdu_cache_capacity: u32, - #[serde(default = "default_cleanup_second_interval")] - cleanup_second_interval: u32, - #[serde(default = "default_max_request_size")] - max_request_size: u32, - #[serde(default = "default_max_concurrent_requests")] - max_concurrent_requests: u16, - #[serde(default = "false_fn")] - allow_registration: bool, - #[serde(default = "true_fn")] - allow_encryption: bool, - #[serde(default = "false_fn")] - allow_federation: bool, - #[serde(default = "true_fn")] - allow_room_creation: bool, - #[serde(default = "false_fn")] - pub allow_jaeger: bool, - #[serde(default = "false_fn")] - pub tracing_flame: bool, - #[serde(default)] - proxy: ProxyConfig, - jwt_secret: Option, - #[serde(default = "Vec::new")] - trusted_servers: Vec>, - #[serde(default = "default_log")] - pub log: String, - #[serde(default)] - turn_username: String, - #[serde(default)] - turn_password: String, - #[serde(default = "Vec::new")] - turn_uris: Vec, - #[serde(default)] - turn_secret: String, - #[serde(default = "default_turn_ttl")] - turn_ttl: u64, - - #[serde(flatten)] - catchall: BTreeMap, -} - -const DEPRECATED_KEYS: &[&str] = &["cache_capacity"]; - -impl Config { - pub fn warn_deprecated(&self) { - let mut was_deprecated = false; - for key in self - .catchall - .keys() - .filter(|key| DEPRECATED_KEYS.iter().any(|s| s == key)) - { - warn!("Config parameter {} is deprecated", key); - was_deprecated = true; - } - - if was_deprecated { - warn!("Read conduit documentation and check your configuration if any new configuration parameters should be adjusted"); - } - } -} - -fn false_fn() -> bool { - false -} - -fn true_fn() -> bool { - true -} - -fn default_database_backend() -> String { - "sqlite".to_owned() -} - -fn default_db_cache_capacity_mb() -> f64 { - 10.0 -} - -fn default_conduit_cache_capacity_modifier() -> f64 { - 1.0 -} - -fn default_rocksdb_max_open_files() -> i32 { - 20 -} - -fn default_pdu_cache_capacity() -> u32 { - 150_000 -} - -fn default_cleanup_second_interval() -> u32 { - 1 * 60 // every minute -} - -fn default_max_request_size() -> u32 { - 20 * 1024 * 1024 // Default to 20 MB -} - -fn default_max_concurrent_requests() -> u16 { - 100 -} - -fn default_log() -> String { - "info,state_res=warn,rocket=off,_=off,sled=off".to_owned() -} - -fn default_turn_ttl() -> u64 { - 60 * 60 * 24 -} - pub struct Database { _db: Arc, pub globals: globals::Globals, diff --git a/src/lib.rs b/src/lib.rs index 745eb394..030dfc3a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,6 +9,7 @@ use std::ops::Deref; +mod config; mod database; mod error; mod pdu; @@ -19,7 +20,8 @@ pub mod appservice_server; pub mod client_server; pub mod server_server; -pub use database::{Config, Database}; +pub use config::Config; +pub use database::Database; pub use error::{Error, Result}; pub use pdu::PduEvent; pub use rocket::Config as RocketConfig; From 6399a7fe4e07f9992ac8ca0412dc48c87d4d0456 Mon Sep 17 00:00:00 2001 From: Andrei Vasiliu Date: Thu, 3 Feb 2022 20:21:04 +0200 Subject: [PATCH 193/201] Remove dash from admin command help --- src/database/admin.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/database/admin.rs b/src/database/admin.rs index 8f90e4d5..34bef5f5 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -13,7 +13,7 @@ use rocket::{ }; use ruma::{ events::{room::message::RoomMessageEventContent, EventType}, - EventId, RoomId, RoomVersionId, UserId, + EventId, RoomId, RoomVersionId, ServerName, UserId, }; use serde_json::value::to_raw_value; use tokio::sync::{MutexGuard, RwLock, RwLockReadGuard}; @@ -140,10 +140,11 @@ fn process_admin_message(db: &Database, room_message: String) -> RoomMessageEven let admin_command = match parse_admin_command(&command_line) { Ok(command) => command, Err(error) => { + let server_name = db.globals.server_name(); let message = error .to_string() - .replace("example.com", db.globals.server_name().as_str()); - let html_message = usage_to_html(&message); + .replace("server.name", server_name.as_str()); + let html_message = usage_to_html(&message, server_name); return RoomMessageEventContent::text_html(message, html_message); } @@ -191,7 +192,7 @@ fn parse_admin_command(command_line: &str) -> std::result::Result String { +fn usage_to_html(text: &str, server_name: &ServerName) -> String { + // Replace `@conduit:servername:-subcmdname` with `@conduit:servername: subcmdname` + let text = text.replace( + &format!("@conduit:{}:-", server_name), + &format!("@conduit:{}: ", server_name), + ); + // For the conduit admin room, subcommands become main commands let text = text.replace("SUBCOMMAND", "COMMAND"); let text = text.replace("subcommand", "command"); From 92571d961f8ec0ce72c0c40433e2487032643060 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Thu, 3 Feb 2022 19:54:29 +0100 Subject: [PATCH 194/201] Remove mutation from default_config and set default log_level to off --- Cargo.lock | 1 + Cargo.toml | 1 + src/main.rs | 37 ++++++++++++++++++------------------- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6dbb6586..85487715 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -317,6 +317,7 @@ dependencies = [ "image", "jsonwebtoken", "lru-cache", + "maplit", "num_cpus", "opentelemetry", "opentelemetry-jaeger", diff --git a/Cargo.toml b/Cargo.toml index 05782e7c..fe60f6e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,6 +84,7 @@ hmac = "0.11.0" sha-1 = "0.9.8" # used for conduit's CLI and admin room command parsing clap = { version = "3.0.10", default-features = false, features = ["std", "derive"] } +maplit = "1.0.2" [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemalloc-ctl = { version = "0.4.2", features = ['use_std'] } diff --git a/src/main.rs b/src/main.rs index 5fda5737..b3e85c95 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,6 +9,7 @@ use std::sync::Arc; +use maplit::hashset; use opentelemetry::trace::{FutureExt, Tracer}; use rocket::{ catch, catchers, @@ -292,28 +293,26 @@ fn bad_json_catcher() -> Result<()> { } fn default_config() -> rocket::Config { - let mut config = rocket::Config::release_default(); + use rocket::config::{LogLevel, Shutdown, Sig}; - { - let mut shutdown = &mut config.shutdown; + rocket::Config { + // Disable rocket's logging to get only tracing-subscriber's log output + log_level: LogLevel::Off, + shutdown: Shutdown { + // Once shutdown is triggered, this is the amount of seconds before rocket + // will forcefully start shutting down connections, this gives enough time to /sync + // requests and the like (which havent gotten the memo, somehow) to still complete gracefully. + grace: 35, - #[cfg(unix)] - { - use rocket::config::Sig; + // After the grace period, rocket starts shutting down connections, and waits at least this + // many seconds before forcefully shutting all of them down. + mercy: 10, - shutdown.signals.insert(Sig::Term); - shutdown.signals.insert(Sig::Int); - } + #[cfg(unix)] + signals: hashset![Sig::Term, Sig::Int], - // Once shutdown is triggered, this is the amount of seconds before rocket - // will forcefully start shutting down connections, this gives enough time to /sync - // requests and the like (which havent gotten the memo, somehow) to still complete gracefully. - shutdown.grace = 35; - - // After the grace period, rocket starts shutting down connections, and waits at least this - // many seconds before forcefully shutting all of them down. - shutdown.mercy = 10; + ..Shutdown::default() + }, + ..rocket::Config::release_default() } - - config } From d23d6fbb371c4aa263d47ff430f6491283d49915 Mon Sep 17 00:00:00 2001 From: Jonas Platte Date: Thu, 3 Feb 2022 20:23:35 +0100 Subject: [PATCH 195/201] Upgrade Ruma --- Cargo.lock | 36 +++++++++++++++++----------------- Cargo.toml | 2 +- src/client_server/to_device.rs | 2 +- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 85487715..19df999a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2175,7 +2175,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" +source = "git+https://github.com/ruma/ruma?rev=f7a10a7e471b59d3096be2695c2a05d407d80df1#f7a10a7e471b59d3096be2695c2a05d407d80df1" dependencies = [ "assign", "js_int", @@ -2196,7 +2196,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.18.5" -source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" +source = "git+https://github.com/ruma/ruma?rev=f7a10a7e471b59d3096be2695c2a05d407d80df1#f7a10a7e471b59d3096be2695c2a05d407d80df1" dependencies = [ "bytes", "http", @@ -2212,7 +2212,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.18.5" -source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" +source = "git+https://github.com/ruma/ruma?rev=f7a10a7e471b59d3096be2695c2a05d407d80df1#f7a10a7e471b59d3096be2695c2a05d407d80df1" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2223,7 +2223,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" +source = "git+https://github.com/ruma/ruma?rev=f7a10a7e471b59d3096be2695c2a05d407d80df1#f7a10a7e471b59d3096be2695c2a05d407d80df1" dependencies = [ "ruma-api", "ruma-common", @@ -2237,7 +2237,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.12.3" -source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" +source = "git+https://github.com/ruma/ruma?rev=f7a10a7e471b59d3096be2695c2a05d407d80df1#f7a10a7e471b59d3096be2695c2a05d407d80df1" dependencies = [ "assign", "bytes", @@ -2257,7 +2257,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.6.0" -source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" +source = "git+https://github.com/ruma/ruma?rev=f7a10a7e471b59d3096be2695c2a05d407d80df1#f7a10a7e471b59d3096be2695c2a05d407d80df1" dependencies = [ "indexmap", "js_int", @@ -2272,7 +2272,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.24.6" -source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" +source = "git+https://github.com/ruma/ruma?rev=f7a10a7e471b59d3096be2695c2a05d407d80df1#f7a10a7e471b59d3096be2695c2a05d407d80df1" dependencies = [ "indoc", "js_int", @@ -2289,7 +2289,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.24.6" -source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" +source = "git+https://github.com/ruma/ruma?rev=f7a10a7e471b59d3096be2695c2a05d407d80df1#f7a10a7e471b59d3096be2695c2a05d407d80df1" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2300,7 +2300,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.3.1" -source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" +source = "git+https://github.com/ruma/ruma?rev=f7a10a7e471b59d3096be2695c2a05d407d80df1#f7a10a7e471b59d3096be2695c2a05d407d80df1" dependencies = [ "js_int", "ruma-api", @@ -2315,7 +2315,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.20.0" -source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" +source = "git+https://github.com/ruma/ruma?rev=f7a10a7e471b59d3096be2695c2a05d407d80df1#f7a10a7e471b59d3096be2695c2a05d407d80df1" dependencies = [ "percent-encoding", "rand 0.8.4", @@ -2330,7 +2330,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.20.0" -source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" +source = "git+https://github.com/ruma/ruma?rev=f7a10a7e471b59d3096be2695c2a05d407d80df1#f7a10a7e471b59d3096be2695c2a05d407d80df1" dependencies = [ "quote", "ruma-identifiers-validation", @@ -2340,7 +2340,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" +source = "git+https://github.com/ruma/ruma?rev=f7a10a7e471b59d3096be2695c2a05d407d80df1#f7a10a7e471b59d3096be2695c2a05d407d80df1" dependencies = [ "thiserror", ] @@ -2348,7 +2348,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" +source = "git+https://github.com/ruma/ruma?rev=f7a10a7e471b59d3096be2695c2a05d407d80df1#f7a10a7e471b59d3096be2695c2a05d407d80df1" dependencies = [ "js_int", "ruma-api", @@ -2361,7 +2361,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" +source = "git+https://github.com/ruma/ruma?rev=f7a10a7e471b59d3096be2695c2a05d407d80df1#f7a10a7e471b59d3096be2695c2a05d407d80df1" dependencies = [ "js_int", "ruma-api", @@ -2376,7 +2376,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" +source = "git+https://github.com/ruma/ruma?rev=f7a10a7e471b59d3096be2695c2a05d407d80df1#f7a10a7e471b59d3096be2695c2a05d407d80df1" dependencies = [ "base64 0.13.0", "bytes", @@ -2391,7 +2391,7 @@ dependencies = [ [[package]] name = "ruma-serde-macros" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" +source = "git+https://github.com/ruma/ruma?rev=f7a10a7e471b59d3096be2695c2a05d407d80df1#f7a10a7e471b59d3096be2695c2a05d407d80df1" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2402,7 +2402,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.9.0" -source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" +source = "git+https://github.com/ruma/ruma?rev=f7a10a7e471b59d3096be2695c2a05d407d80df1#f7a10a7e471b59d3096be2695c2a05d407d80df1" dependencies = [ "base64 0.13.0", "ed25519-dalek", @@ -2419,7 +2419,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.4.1" -source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435" +source = "git+https://github.com/ruma/ruma?rev=f7a10a7e471b59d3096be2695c2a05d407d80df1#f7a10a7e471b59d3096be2695c2a05d407d80df1" dependencies = [ "itertools", "js_int", diff --git a/Cargo.toml b/Cargo.toml index fe60f6e9..1e1b188f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle request # Used for matrix spec type definitions and helpers #ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } -ruma = { git = "https://github.com/ruma/ruma", rev = "82becb86c837570224964425929d1b5305784435", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +ruma = { git = "https://github.com/ruma/ruma", rev = "f7a10a7e471b59d3096be2695c2a05d407d80df1", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { git = "https://github.com/timokoesters/ruma", rev = "50c1db7e0a3a21fc794b0cce3b64285a4c750c71", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } diff --git a/src/client_server/to_device.rs b/src/client_server/to_device.rs index 6e764deb..e0aa9e91 100644 --- a/src/client_server/to_device.rs +++ b/src/client_server/to_device.rs @@ -54,7 +54,7 @@ pub async fn send_event_to_device_route( DirectDeviceContent { sender: sender_user.clone(), ev_type: EventType::from(&*body.event_type), - message_id: body.txn_id.to_string(), + message_id: body.txn_id.clone(), messages, }, )) From e1c0dcb6bb45432c0638f5eced7b719ea2ff1afe Mon Sep 17 00:00:00 2001 From: Andrei Vasiliu Date: Thu, 3 Feb 2022 20:52:41 +0200 Subject: [PATCH 196/201] Create admin room and hide migration messages on first run --- src/client_server/account.rs | 302 ++---------------------------- src/database.rs | 81 +++++--- src/database/admin.rs | 353 +++++++++++++++++++++++++++++++++-- 3 files changed, 419 insertions(+), 317 deletions(-) diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 47e2a6a4..a210e8ae 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -1,7 +1,11 @@ -use std::{collections::BTreeMap, sync::Arc}; +use std::sync::Arc; use super::{DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH}; -use crate::{database::DatabaseGuard, pdu::PduBuilder, utils, ConduitResult, Error, Ruma}; +use crate::{ + database::{admin::make_user_admin, DatabaseGuard}, + pdu::PduBuilder, + utils, ConduitResult, Error, Ruma, +}; use ruma::{ api::client::{ error::ErrorKind, @@ -14,25 +18,13 @@ use ruma::{ }, }, events::{ - room::{ - canonical_alias::RoomCanonicalAliasEventContent, - create::RoomCreateEventContent, - guest_access::{GuestAccess, RoomGuestAccessEventContent}, - history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, - join_rules::{JoinRule, RoomJoinRulesEventContent}, - member::{MembershipState, RoomMemberEventContent}, - message::RoomMessageEventContent, - name::RoomNameEventContent, - power_levels::RoomPowerLevelsEventContent, - topic::RoomTopicEventContent, - }, + room::member::{MembershipState, RoomMemberEventContent}, EventType, }, - identifiers::RoomName, - push, RoomAliasId, RoomId, RoomVersionId, UserId, + push, UserId, }; use serde_json::value::to_raw_value; -use tracing::info; +use tracing::{info, warn}; use register::RegistrationKind; #[cfg(feature = "conduit_bin")] @@ -253,276 +245,16 @@ pub async fn register_route( body.initial_device_display_name.clone(), )?; - // If this is the first user on this server, create the admin room - if db.users.count()? == 1 { - // Create a user for the server - let conduit_user = UserId::parse_with_server_name("conduit", db.globals.server_name()) - .expect("@conduit:server_name is valid"); - - db.users.create(&conduit_user, None)?; - - let room_id = RoomId::new(db.globals.server_name()); - - db.rooms.get_or_create_shortroomid(&room_id, &db.globals)?; - - let mutex_state = Arc::clone( - db.globals - .roomid_mutex_state - .write() - .unwrap() - .entry(room_id.clone()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; - - let mut content = RoomCreateEventContent::new(conduit_user.clone()); - content.federate = true; - content.predecessor = None; - content.room_version = RoomVersionId::V6; - - // 1. The room create event - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomCreate, - content: to_raw_value(&content).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &conduit_user, - &room_id, - &db, - &state_lock, - )?; - - // 2. Make conduit bot join - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: to_raw_value(&RoomMemberEventContent { - membership: MembershipState::Join, - displayname: None, - avatar_url: None, - is_direct: None, - third_party_invite: None, - blurhash: None, - reason: None, - join_authorized_via_users_server: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(conduit_user.to_string()), - redacts: None, - }, - &conduit_user, - &room_id, - &db, - &state_lock, - )?; - - // 3. Power levels - let mut users = BTreeMap::new(); - users.insert(conduit_user.clone(), 100.into()); - users.insert(user_id.clone(), 100.into()); - - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomPowerLevels, - content: to_raw_value(&RoomPowerLevelsEventContent { - users, - ..Default::default() - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &conduit_user, - &room_id, - &db, - &state_lock, - )?; - - // 4.1 Join Rules - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomJoinRules, - content: to_raw_value(&RoomJoinRulesEventContent::new(JoinRule::Invite)) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &conduit_user, - &room_id, - &db, - &state_lock, - )?; - - // 4.2 History Visibility - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomHistoryVisibility, - content: to_raw_value(&RoomHistoryVisibilityEventContent::new( - HistoryVisibility::Shared, - )) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &conduit_user, - &room_id, - &db, - &state_lock, - )?; - - // 4.3 Guest Access - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomGuestAccess, - content: to_raw_value(&RoomGuestAccessEventContent::new(GuestAccess::Forbidden)) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &conduit_user, - &room_id, - &db, - &state_lock, - )?; - - // 6. Events implied by name and topic - let room_name = RoomName::parse(format!("{} Admin Room", db.globals.server_name())) - .expect("Room name is valid"); - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomName, - content: to_raw_value(&RoomNameEventContent::new(Some(room_name))) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &conduit_user, - &room_id, - &db, - &state_lock, - )?; - - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomTopic, - content: to_raw_value(&RoomTopicEventContent { - topic: format!("Manage {}", db.globals.server_name()), - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &conduit_user, - &room_id, - &db, - &state_lock, - )?; - - // Room alias - let alias: Box = format!("#admins:{}", db.globals.server_name()) - .try_into() - .expect("#admins:server_name is a valid alias name"); - - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomCanonicalAlias, - content: to_raw_value(&RoomCanonicalAliasEventContent { - alias: Some(alias.clone()), - alt_aliases: Vec::new(), - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &conduit_user, - &room_id, - &db, - &state_lock, - )?; - - db.rooms.set_alias(&alias, Some(&room_id), &db.globals)?; - - // Invite and join the real user - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: to_raw_value(&RoomMemberEventContent { - membership: MembershipState::Invite, - displayname: None, - avatar_url: None, - is_direct: None, - third_party_invite: None, - blurhash: None, - reason: None, - join_authorized_via_users_server: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - }, - &conduit_user, - &room_id, - &db, - &state_lock, - )?; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: to_raw_value(&RoomMemberEventContent { - membership: MembershipState::Join, - displayname: Some(displayname), - avatar_url: None, - is_direct: None, - third_party_invite: None, - blurhash: None, - reason: None, - join_authorized_via_users_server: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - }, - &user_id, - &room_id, - &db, - &state_lock, - )?; - - // Send welcome message - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMessage, - content: to_raw_value(&RoomMessageEventContent::text_html( - "## Thank you for trying out Conduit!\n\nConduit is currently in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.\n\nHelpful links:\n> Website: https://conduit.rs\n> Git and Documentation: https://gitlab.com/famedly/conduit\n> Report issues: https://gitlab.com/famedly/conduit/-/issues\n\nHere are some rooms you can join (by typing the command):\n\nConduit room (Ask questions and get notified on updates):\n`/join #conduit:fachschaften.org`\n\nConduit lounge (Off-topic, only Conduit users are allowed to join)\n`/join #conduit-lounge:conduit.rs`".to_owned(), - "

              Thank you for trying out Conduit!

              \n

              Conduit is currently in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.

              \n

              Helpful links:

              \n
              \n

              Website: https://conduit.rs
              Git and Documentation: https://gitlab.com/famedly/conduit
              Report issues: https://gitlab.com/famedly/conduit/-/issues

              \n
              \n

              Here are some rooms you can join (by typing the command):

              \n

              Conduit room (Ask questions and get notified on updates):
              /join #conduit:fachschaften.org

              \n

              Conduit lounge (Off-topic, only Conduit users are allowed to join)
              /join #conduit-lounge:conduit.rs

              \n".to_owned(), - )) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: None, - redacts: None, - }, - &conduit_user, - &room_id, - &db, - &state_lock, - )?; - } - info!("{} registered on this server", user_id); + // If this is the first real user, grant them admin privileges + // Note: the server user, @conduit:servername, is generated first + if db.users.count()? == 2 { + make_user_admin(&db, &user_id, displayname).await?; + + warn!("Granting {} admin privileges as the first user", user_id); + } + db.flush()?; Ok(register::Response { diff --git a/src/database.rs b/src/database.rs index 5deedcfe..2b1671cd 100644 --- a/src/database.rs +++ b/src/database.rs @@ -34,7 +34,9 @@ use std::{ sync::{Arc, Mutex, RwLock}, }; use tokio::sync::{OwnedRwLockReadGuard, RwLock as TokioRwLock, Semaphore}; -use tracing::{debug, error, warn}; +use tracing::{debug, error, info, warn}; + +use self::admin::create_admin_room; pub struct Database { _db: Arc, @@ -301,10 +303,32 @@ impl Database { )?, })); - { - let db = db.read().await; + let guard = db.read().await; + + // Matrix resource ownership is based on the server name; changing it + // requires recreating the database from scratch. + if guard.users.count()? > 0 { + let conduit_user = + UserId::parse_with_server_name("conduit", guard.globals.server_name()) + .expect("@conduit:server_name is valid"); + + if !guard.users.exists(&conduit_user)? { + error!( + "The {} server user does not exist, and the database is not new.", + conduit_user + ); + return Err(Error::bad_database( + "Cannot reuse an existing database after changing the server name, please delete the old one first." + )); + } + } + + // If the database has any data, perform data migrations before starting + let latest_database_version = 11; + + if guard.users.count()? > 0 { + let db = &*guard; // MIGRATIONS - // TODO: database versions of new dbs should probably not be 0 if db.globals.database_version()? < 1 { for (roomserverid, _) in db.rooms.roomserverids.iter() { let mut parts = roomserverid.split(|&b| b == 0xff); @@ -325,7 +349,7 @@ impl Database { db.globals.bump_database_version(1)?; - println!("Migration: 0 -> 1 finished"); + warn!("Migration: 0 -> 1 finished"); } if db.globals.database_version()? < 2 { @@ -344,7 +368,7 @@ impl Database { db.globals.bump_database_version(2)?; - println!("Migration: 1 -> 2 finished"); + warn!("Migration: 1 -> 2 finished"); } if db.globals.database_version()? < 3 { @@ -362,7 +386,7 @@ impl Database { db.globals.bump_database_version(3)?; - println!("Migration: 2 -> 3 finished"); + warn!("Migration: 2 -> 3 finished"); } if db.globals.database_version()? < 4 { @@ -385,7 +409,7 @@ impl Database { db.globals.bump_database_version(4)?; - println!("Migration: 3 -> 4 finished"); + warn!("Migration: 3 -> 4 finished"); } if db.globals.database_version()? < 5 { @@ -409,7 +433,7 @@ impl Database { db.globals.bump_database_version(5)?; - println!("Migration: 4 -> 5 finished"); + warn!("Migration: 4 -> 5 finished"); } if db.globals.database_version()? < 6 { @@ -422,7 +446,7 @@ impl Database { db.globals.bump_database_version(6)?; - println!("Migration: 5 -> 6 finished"); + warn!("Migration: 5 -> 6 finished"); } if db.globals.database_version()? < 7 { @@ -549,7 +573,7 @@ impl Database { db.globals.bump_database_version(7)?; - println!("Migration: 6 -> 7 finished"); + warn!("Migration: 6 -> 7 finished"); } if db.globals.database_version()? < 8 { @@ -557,7 +581,7 @@ impl Database { for (room_id, _) in db.rooms.roomid_shortstatehash.iter() { let shortroomid = db.globals.next_count()?.to_be_bytes(); db.rooms.roomid_shortroomid.insert(&room_id, &shortroomid)?; - println!("Migration: 8"); + info!("Migration: 8"); } // Update pduids db layout let mut batch = db.rooms.pduid_pdu.iter().filter_map(|(key, v)| { @@ -608,7 +632,7 @@ impl Database { db.globals.bump_database_version(8)?; - println!("Migration: 7 -> 8 finished"); + warn!("Migration: 7 -> 8 finished"); } if db.globals.database_version()? < 9 { @@ -650,7 +674,7 @@ impl Database { println!("smaller batch done"); } - println!("Deleting starts"); + info!("Deleting starts"); let batch2: Vec<_> = db .rooms @@ -673,7 +697,7 @@ impl Database { db.globals.bump_database_version(9)?; - println!("Migration: 8 -> 9 finished"); + warn!("Migration: 8 -> 9 finished"); } if db.globals.database_version()? < 10 { @@ -692,7 +716,7 @@ impl Database { db.globals.bump_database_version(10)?; - println!("Migration: 9 -> 10 finished"); + warn!("Migration: 9 -> 10 finished"); } if db.globals.database_version()? < 11 { @@ -701,11 +725,28 @@ impl Database { .clear()?; db.globals.bump_database_version(11)?; - println!("Migration: 10 -> 11 finished"); + warn!("Migration: 10 -> 11 finished"); } - } - let guard = db.read().await; + assert_eq!(11, latest_database_version); + + info!( + "Loaded {} database with version {}", + config.database_backend, latest_database_version + ); + } else { + guard + .globals + .bump_database_version(latest_database_version)?; + + // Create the admin room and server user on first run + create_admin_room(&guard).await?; + + warn!( + "Created new {} database with version {}", + config.database_backend, latest_database_version + ); + } // This data is probably outdated guard.rooms.edus.presenceid_presence.clear()?; @@ -724,8 +765,6 @@ impl Database { #[cfg(feature = "conduit_bin")] pub async fn start_on_shutdown_tasks(db: Arc>, shutdown: Shutdown) { - use tracing::info; - tokio::spawn(async move { shutdown.await; diff --git a/src/database/admin.rs b/src/database/admin.rs index 34bef5f5..9bbfd4ea 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -1,4 +1,4 @@ -use std::{convert::TryFrom, convert::TryInto, sync::Arc, time::Instant}; +use std::{collections::BTreeMap, convert::TryFrom, convert::TryInto, sync::Arc, time::Instant}; use crate::{ error::{Error, Result}, @@ -12,12 +12,22 @@ use rocket::{ http::RawStr, }; use ruma::{ + events::room::{ + canonical_alias::RoomCanonicalAliasEventContent, + create::RoomCreateEventContent, + guest_access::{GuestAccess, RoomGuestAccessEventContent}, + history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, + join_rules::{JoinRule, RoomJoinRulesEventContent}, + member::{MembershipState, RoomMemberEventContent}, + name::RoomNameEventContent, + power_levels::RoomPowerLevelsEventContent, + topic::RoomTopicEventContent, + }, events::{room::message::RoomMessageEventContent, EventType}, - EventId, RoomId, RoomVersionId, ServerName, UserId, + identifiers::{EventId, RoomAliasId, RoomId, RoomName, RoomVersionId, ServerName, UserId}, }; use serde_json::value::to_raw_value; use tokio::sync::{MutexGuard, RwLock, RwLockReadGuard}; -use tracing::warn; pub enum AdminRoomEvent { ProcessMessage(String), @@ -52,16 +62,9 @@ impl Admin { .try_into() .expect("#admins:server_name is a valid room alias"), ) + .expect("Database data for admin room alias must be valid") .expect("Admin room must exist"); - let conduit_room = match conduit_room { - None => { - warn!("Conduit instance does not have an #admins room. Logging to that room will not work. Restart Conduit after creating a user to fix this."); - return; - } - Some(r) => r, - }; - drop(guard); let send_message = |message: RoomMessageEventContent, @@ -500,3 +503,331 @@ fn usage_to_html(text: &str, server_name: &ServerName) -> String { text } + +/// Create the admin room. +/// +/// Users in this room are considered admins by conduit, and the room can be +/// used to issue admin commands by talking to the server user inside it. +pub(crate) async fn create_admin_room(db: &Database) -> Result<()> { + let room_id = RoomId::new(db.globals.server_name()); + + db.rooms.get_or_create_shortroomid(&room_id, &db.globals)?; + + let mutex_state = Arc::clone( + db.globals + .roomid_mutex_state + .write() + .unwrap() + .entry(room_id.clone()) + .or_default(), + ); + let state_lock = mutex_state.lock().await; + + // Create a user for the server + let conduit_user = UserId::parse_with_server_name("conduit", db.globals.server_name()) + .expect("@conduit:server_name is valid"); + + db.users.create(&conduit_user, None)?; + + let mut content = RoomCreateEventContent::new(conduit_user.clone()); + content.federate = true; + content.predecessor = None; + content.room_version = RoomVersionId::V6; + + // 1. The room create event + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomCreate, + content: to_raw_value(&content).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db, + &state_lock, + )?; + + // 2. Make conduit bot join + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: to_raw_value(&RoomMemberEventContent { + membership: MembershipState::Join, + displayname: None, + avatar_url: None, + is_direct: None, + third_party_invite: None, + blurhash: None, + reason: None, + join_authorized_via_users_server: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(conduit_user.to_string()), + redacts: None, + }, + &conduit_user, + &room_id, + &db, + &state_lock, + )?; + + // 3. Power levels + let mut users = BTreeMap::new(); + users.insert(conduit_user.clone(), 100.into()); + + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomPowerLevels, + content: to_raw_value(&RoomPowerLevelsEventContent { + users, + ..Default::default() + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db, + &state_lock, + )?; + + // 4.1 Join Rules + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomJoinRules, + content: to_raw_value(&RoomJoinRulesEventContent::new(JoinRule::Invite)) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db, + &state_lock, + )?; + + // 4.2 History Visibility + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomHistoryVisibility, + content: to_raw_value(&RoomHistoryVisibilityEventContent::new( + HistoryVisibility::Shared, + )) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db, + &state_lock, + )?; + + // 4.3 Guest Access + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomGuestAccess, + content: to_raw_value(&RoomGuestAccessEventContent::new(GuestAccess::Forbidden)) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db, + &state_lock, + )?; + + // 5. Events implied by name and topic + let room_name = RoomName::parse(format!("{} Admin Room", db.globals.server_name())) + .expect("Room name is valid"); + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomName, + content: to_raw_value(&RoomNameEventContent::new(Some(room_name))) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db, + &state_lock, + )?; + + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomTopic, + content: to_raw_value(&RoomTopicEventContent { + topic: format!("Manage {}", db.globals.server_name()), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db, + &state_lock, + )?; + + // 6. Room alias + let alias: Box = format!("#admins:{}", db.globals.server_name()) + .try_into() + .expect("#admins:server_name is a valid alias name"); + + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomCanonicalAlias, + content: to_raw_value(&RoomCanonicalAliasEventContent { + alias: Some(alias.clone()), + alt_aliases: Vec::new(), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db, + &state_lock, + )?; + + db.rooms.set_alias(&alias, Some(&room_id), &db.globals)?; + + Ok(()) +} + +/// Invite the user to the conduit admin room. +/// +/// In conduit, this is equivalent to granting admin privileges. +pub(crate) async fn make_user_admin( + db: &Database, + user_id: &UserId, + displayname: String, +) -> Result<()> { + let admin_room_alias: Box = format!("#admins:{}", db.globals.server_name()) + .try_into() + .expect("#admins:server_name is a valid alias name"); + let room_id = db + .rooms + .id_from_alias(&admin_room_alias)? + .expect("Admin room must exist"); + + let mutex_state = Arc::clone( + db.globals + .roomid_mutex_state + .write() + .unwrap() + .entry(room_id.clone()) + .or_default(), + ); + let state_lock = mutex_state.lock().await; + + // Use the server user to grant the new admin's power level + let conduit_user = UserId::parse_with_server_name("conduit", db.globals.server_name()) + .expect("@conduit:server_name is valid"); + + // Invite and join the real user + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: to_raw_value(&RoomMemberEventContent { + membership: MembershipState::Invite, + displayname: None, + avatar_url: None, + is_direct: None, + third_party_invite: None, + blurhash: None, + reason: None, + join_authorized_via_users_server: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + }, + &conduit_user, + &room_id, + &db, + &state_lock, + )?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: to_raw_value(&RoomMemberEventContent { + membership: MembershipState::Join, + displayname: Some(displayname), + avatar_url: None, + is_direct: None, + third_party_invite: None, + blurhash: None, + reason: None, + join_authorized_via_users_server: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + }, + &user_id, + &room_id, + &db, + &state_lock, + )?; + + // Set power level + let mut users = BTreeMap::new(); + users.insert(conduit_user.to_owned(), 100.into()); + users.insert(user_id.to_owned(), 100.into()); + + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomPowerLevels, + content: to_raw_value(&RoomPowerLevelsEventContent { + users, + ..Default::default() + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db, + &state_lock, + )?; + + // Send welcome message + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMessage, + content: to_raw_value(&RoomMessageEventContent::text_html( + "## Thank you for trying out Conduit!\n\nConduit is currently in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.\n\nHelpful links:\n> Website: https://conduit.rs\n> Git and Documentation: https://gitlab.com/famedly/conduit\n> Report issues: https://gitlab.com/famedly/conduit/-/issues\n\nHere are some rooms you can join (by typing the command):\n\nConduit room (Ask questions and get notified on updates):\n`/join #conduit:fachschaften.org`\n\nConduit lounge (Off-topic, only Conduit users are allowed to join)\n`/join #conduit-lounge:conduit.rs`".to_owned(), + "

              Thank you for trying out Conduit!

              \n

              Conduit is currently in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.

              \n

              Helpful links:

              \n
              \n

              Website: https://conduit.rs
              Git and Documentation: https://gitlab.com/famedly/conduit
              Report issues: https://gitlab.com/famedly/conduit/-/issues

              \n
              \n

              Here are some rooms you can join (by typing the command):

              \n

              Conduit room (Ask questions and get notified on updates):
              /join #conduit:fachschaften.org

              \n

              Conduit lounge (Off-topic, only Conduit users are allowed to join)
              /join #conduit-lounge:conduit.rs

              \n".to_owned(), + )) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: None, + redacts: None, + }, + &conduit_user, + &room_id, + &db, + &state_lock, + )?; + + Ok(()) +} From 72cd52e57c7afa1f051b488e6385b59617fffa4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Fri, 4 Feb 2022 13:30:42 +0100 Subject: [PATCH 197/201] fix: lazy loading for /context --- src/client_server/context.rs | 99 +++++++++++++++++++++++++++--------- 1 file changed, 74 insertions(+), 25 deletions(-) diff --git a/src/client_server/context.rs b/src/client_server/context.rs index 7ded48da..02148f41 100644 --- a/src/client_server/context.rs +++ b/src/client_server/context.rs @@ -1,9 +1,13 @@ use crate::{database::DatabaseGuard, ConduitResult, Error, Ruma}; use ruma::{ - api::client::{error::ErrorKind, r0::context::get_context}, + api::client::{ + error::ErrorKind, + r0::{context::get_context, filter::LazyLoadOptions}, + }, events::EventType, }; -use std::collections::HashSet; +use std::{collections::HashSet, convert::TryFrom}; +use tracing::error; #[cfg(feature = "conduit_bin")] use rocket::get; @@ -26,12 +30,15 @@ pub async fn get_context_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); - if !db.rooms.is_joined(sender_user, &body.room_id)? { - return Err(Error::BadRequest( - ErrorKind::Forbidden, - "You don't have permission to view this room.", - )); - } + // Load filter + let filter = body.filter.clone().unwrap_or_default(); + + let (lazy_load_enabled, lazy_load_send_redundant) = match filter.lazy_load_options { + LazyLoadOptions::Enabled { + include_redundant_members: redundant, + } => (true, redundant), + _ => (false, false), + }; let mut lazy_loaded = HashSet::new(); @@ -53,20 +60,30 @@ pub async fn get_context_route( "Base event not found.", ))?; + let room_id = base_event.room_id.clone(); + + if !db.rooms.is_joined(sender_user, &room_id)? { + return Err(Error::BadRequest( + ErrorKind::Forbidden, + "You don't have permission to view this room.", + )); + } + if !db.rooms.lazy_load_was_sent_before( sender_user, sender_device, - &body.room_id, + &room_id, &base_event.sender, - )? { - lazy_loaded.insert(base_event.sender.clone()); + )? || lazy_load_send_redundant + { + lazy_loaded.insert(base_event.sender.as_str().to_owned()); } let base_event = base_event.to_room_event(); let events_before: Vec<_> = db .rooms - .pdus_until(sender_user, &body.room_id, base_token)? + .pdus_until(sender_user, &room_id, base_token)? .take( u32::try_from(body.limit).map_err(|_| { Error::BadRequest(ErrorKind::InvalidParam, "Limit value is invalid.") @@ -80,10 +97,11 @@ pub async fn get_context_route( if !db.rooms.lazy_load_was_sent_before( sender_user, sender_device, - &body.room_id, + &room_id, &event.sender, - )? { - lazy_loaded.insert(event.sender.clone()); + )? || lazy_load_send_redundant + { + lazy_loaded.insert(event.sender.as_str().to_owned()); } } @@ -99,7 +117,7 @@ pub async fn get_context_route( let events_after: Vec<_> = db .rooms - .pdus_after(sender_user, &body.room_id, base_token)? + .pdus_after(sender_user, &room_id, base_token)? .take( u32::try_from(body.limit).map_err(|_| { Error::BadRequest(ErrorKind::InvalidParam, "Limit value is invalid.") @@ -113,13 +131,28 @@ pub async fn get_context_route( if !db.rooms.lazy_load_was_sent_before( sender_user, sender_device, - &body.room_id, + &room_id, &event.sender, - )? { - lazy_loaded.insert(event.sender.clone()); + )? || lazy_load_send_redundant + { + lazy_loaded.insert(event.sender.as_str().to_owned()); } } + let shortstatehash = match db.rooms.pdu_shortstatehash( + events_after + .last() + .map_or(&*body.event_id, |(_, e)| &*e.event_id), + )? { + Some(s) => s, + None => db + .rooms + .current_shortstatehash(&room_id)? + .expect("All rooms have state"), + }; + + let state_ids = db.rooms.state_full_ids(shortstatehash)?; + let end_token = events_after .last() .and_then(|(pdu_id, _)| db.rooms.pdu_count(pdu_id).ok()) @@ -131,12 +164,28 @@ pub async fn get_context_route( .collect(); let mut state = Vec::new(); - for ll_id in &lazy_loaded { - if let Some(member_event) = - db.rooms - .room_state_get(&body.room_id, &EventType::RoomMember, ll_id.as_str())? - { - state.push(member_event.to_state_event()); + + for (shortstatekey, id) in state_ids { + let (event_type, state_key) = db.rooms.get_statekey_from_short(shortstatekey)?; + + if event_type != EventType::RoomMember { + let pdu = match db.rooms.get_pdu(&id)? { + Some(pdu) => pdu, + None => { + error!("Pdu in state not found: {}", id); + continue; + } + }; + state.push(pdu.to_state_event()); + } else if !lazy_load_enabled || lazy_loaded.contains(&state_key) { + let pdu = match db.rooms.get_pdu(&id)? { + Some(pdu) => pdu, + None => { + error!("Pdu in state not found: {}", id); + continue; + } + }; + state.push(pdu.to_state_event()); } } From 8d8edddb2e1074bf36a14b244c7ed9a0eec054dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Fri, 4 Feb 2022 16:59:30 +0100 Subject: [PATCH 198/201] feat: allow disabling jemalloc via feature --- Cargo.toml | 8 ++++---- src/main.rs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index aac840b5..dd31c849 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -86,17 +86,17 @@ sha-1 = "0.9.8" clap = { version = "3.0.10", default-features = false, features = ["std", "derive"] } maplit = "1.0.2" -[target.'cfg(not(target_env = "msvc"))'.dependencies] -tikv-jemalloc-ctl = { version = "0.4.2", features = ['use_std'] } -tikv-jemallocator = { version = "0.4.1", features = ['unprefixed_malloc_on_supported_platforms'] } +tikv-jemalloc-ctl = { version = "0.4.2", features = ["use_std"], optional = true } +tikv-jemallocator = { version = "0.4.1", features = ["unprefixed_malloc_on_supported_platforms"], optional = true } [features] -default = ["conduit_bin", "backend_sqlite", "backend_rocksdb"] +default = ["conduit_bin", "backend_sqlite", "backend_rocksdb", "jemalloc"] backend_sled = ["sled"] backend_persy = ["persy", "parking_lot"] backend_sqlite = ["sqlite"] backend_heed = ["heed", "crossbeam"] backend_rocksdb = ["rocksdb"] +jemalloc = ["tikv-jemalloc-ctl", "tikv-jemallocator"] sqlite = ["rusqlite", "parking_lot", "tokio/signal"] conduit_bin = [] # TODO: add rocket to this when it is optional diff --git a/src/main.rs b/src/main.rs index b3e85c95..ea09dd5b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,10 +26,10 @@ use tracing_subscriber::{prelude::*, EnvFilter}; pub use conduit::*; // Re-export everything from the library crate pub use rocket::State; -#[cfg(not(target_env = "msvc"))] +#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] use tikv_jemallocator::Jemalloc; -#[cfg(not(target_env = "msvc"))] +#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; From eb0b2c429faf9bfa3c472c198deca8f7f07f46f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Fri, 4 Feb 2022 17:15:21 +0100 Subject: [PATCH 199/201] fix: crash on empty search --- src/client_server/search.rs | 9 +++++---- src/database/rooms.rs | 29 +++++++++++++++-------------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/client_server/search.rs b/src/client_server/search.rs index 59c9480a..f492292c 100644 --- a/src/client_server/search.rs +++ b/src/client_server/search.rs @@ -44,11 +44,12 @@ pub async fn search_events_route( )); } - let search = db + if let Some(search) = db .rooms - .search_pdus(&room_id, &search_criteria.search_term)?; - - searches.push(search.0.peekable()); + .search_pdus(&room_id, &search_criteria.search_term)? + { + searches.push(search.0.peekable()); + } } let skip = match body.next_batch.as_ref().map(|s| s.parse()) { diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 2c271d16..0abd2e79 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -2771,7 +2771,7 @@ impl Rooms { &'a self, room_id: &RoomId, search_string: &str, - ) -> Result<(impl Iterator> + 'a, Vec)> { + ) -> Result> + 'a, Vec)>> { let prefix = self .get_shortroomid(room_id)? .expect("room exists") @@ -2799,19 +2799,20 @@ impl Rooms { .map(|(key, _)| key[key.len() - size_of::()..].to_vec()) }); - Ok(( - utils::common_elements(iterators, |a, b| { - // We compare b with a because we reversed the iterator earlier - b.cmp(a) - }) - .unwrap() - .map(move |id| { - let mut pduid = prefix_clone.clone(); - pduid.extend_from_slice(&id); - pduid - }), - words, - )) + Ok(utils::common_elements(iterators, |a, b| { + // We compare b with a because we reversed the iterator earlier + b.cmp(a) + }) + .map(|iter| { + ( + iter.map(move |id| { + let mut pduid = prefix_clone.clone(); + pduid.extend_from_slice(&id); + pduid + }), + words, + ) + })) } #[tracing::instrument(skip(self))] From dd03608f173b1cc450183eae2b2be46684e868fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Fri, 4 Feb 2022 17:24:45 +0100 Subject: [PATCH 200/201] use our own reqwest fork --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 209adea4..859564bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2018,7 +2018,7 @@ dependencies = [ [[package]] name = "reqwest" version = "0.11.9" -source = "git+https://github.com/niuhuan/reqwest?branch=dns-resolver-fn#57b7cf4feb921573dfafad7d34b9ac6e44ead0bd" +source = "git+https://github.com/timokoesters/reqwest?rev=57b7cf4feb921573dfafad7d34b9ac6e44ead0bd#57b7cf4feb921573dfafad7d34b9ac6e44ead0bd" dependencies = [ "base64 0.13.0", "bytes", diff --git a/Cargo.toml b/Cargo.toml index aac840b5..fb861380 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,7 +48,7 @@ rand = "0.8.4" # Used to hash passwords rust-argon2 = "0.8.3" # Used to send requests -reqwest = { version = "0.11.4", default-features = false, features = ["rustls-tls", "socks"], git = "https://github.com/niuhuan/reqwest", branch = "dns-resolver-fn" } +reqwest = { default-features = false, features = ["rustls-tls", "socks"], git = "https://github.com/timokoesters/reqwest", rev = "57b7cf4feb921573dfafad7d34b9ac6e44ead0bd" } # Used for conduit::Error type thiserror = "1.0.28" # Used to generate thumbnails for images From 103dc7e09b4cdcefca6817b448f6b45677988e84 Mon Sep 17 00:00:00 2001 From: Jonas Zohren Date: Fri, 4 Feb 2022 17:57:59 +0100 Subject: [PATCH 201/201] Pre-0.3 doc adjustments --- APPSERVICES.md | 4 ++-- Cargo.lock | 2 +- Cargo.toml | 2 +- DEPLOY.md | 27 ++++++++++++++++----------- README.md | 18 ++++++------------ docker/README.md | 2 +- 6 files changed, 27 insertions(+), 28 deletions(-) diff --git a/APPSERVICES.md b/APPSERVICES.md index 545772a6..8ca015a0 100644 --- a/APPSERVICES.md +++ b/APPSERVICES.md @@ -2,7 +2,7 @@ ## Getting help -If you run into any problems while setting up an Appservice, write an email to `timo@koesters.xyz`, ask us in `#conduit:matrix.org` or [open an issue on GitLab](https://gitlab.com/famedly/conduit/-/issues/new). +If you run into any problems while setting up an Appservice, write an email to `timo@koesters.xyz`, ask us in [#conduit:fachschaften.org](https://matrix.to/#/#conduit:fachschaften.org) or [open an issue on GitLab](https://gitlab.com/famedly/conduit/-/issues/new). ## Set up the appservice - general instructions @@ -46,7 +46,7 @@ could help. To remove an appservice go to your admin room and execute -```@conduit:your.server.name: unregister-appservice ``` +`@conduit:your.server.name: unregister-appservice ` where `` one of the output of `list-appservices`. diff --git a/Cargo.lock b/Cargo.lock index 859564bd..632b4cea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -304,7 +304,7 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" [[package]] name = "conduit" -version = "0.2.0" +version = "0.3.0" dependencies = [ "base64 0.13.0", "bytes", diff --git a/Cargo.toml b/Cargo.toml index fb861380..587e26bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ authors = ["timokoesters "] homepage = "https://conduit.rs" repository = "https://gitlab.com/famedly/conduit" readme = "README.md" -version = "0.2.0" +version = "0.3.0" rust-version = "1.56" edition = "2021" diff --git a/DEPLOY.md b/DEPLOY.md index d9f91e03..c3da6975 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -1,9 +1,9 @@ # Deploying Conduit -## Getting help - -If you run into any problems while setting up Conduit, write an email to `timo@koesters.xyz`, ask us -in `#conduit:matrix.org` or [open an issue on GitLab](https://gitlab.com/famedly/conduit/-/issues/new). +> ## Getting help +> +> If you run into any problems while setting up Conduit, write an email to `timo@koesters.xyz`, ask us +> in `#conduit:fachschaften.org` or [open an issue on GitLab](https://gitlab.com/famedly/conduit/-/issues/new). ## Installing Conduit @@ -12,17 +12,21 @@ only offer Linux binaries. You may simply download the binary that fits your machine. Run `uname -m` to see what you need. Now copy the right url: -| CPU Architecture | Download stable version | -| ------------------------------------------- | ------------------------------ | -| x84_64 / amd64 (Most servers and computers) | [Download][x84_64-musl-master] | -| armv6 | [Download][armv6-musl-master] | -| armv7 (e.g. Raspberry Pi by default) | [Download][armv7-musl-master] | -| armv8 / aarch64 | [Download][armv8-musl-master] | +| CPU Architecture | Download stable version | Download development version | +| ------------------------------------------- | ------------------------------ | ---------------------------- | +| x84_64 / amd64 (Most servers and computers) | [Download][x84_64-musl-master] | [Download][x84_64-musl-next] | +| armv6 | [Download][armv6-musl-master] | [Download][armv6-musl-next] | +| armv7 (e.g. Raspberry Pi by default) | [Download][armv7-musl-master] | [Download][armv7-musl-next] | +| armv8 / aarch64 | [Download][armv8-musl-master] | [Download][armv8-musl-next] | [x84_64-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-musl?job=build:release:cargo:x86_64-unknown-linux-musl [armv6-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-arm-unknown-linux-musleabihf?job=build:release:cargo:arm-unknown-linux-musleabihf [armv7-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-armv7-unknown-linux-musleabihf?job=build:release:cargo:armv7-unknown-linux-musleabihf [armv8-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-aarch64-unknown-linux-musl?job=build:release:cargo:aarch64-unknown-linux-musl +[x84_64-musl-next]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/next/raw/conduit-x86_64-unknown-linux-musl?job=build:release:cargo:x86_64-unknown-linux-musl +[armv6-musl-next]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/next/raw/conduit-arm-unknown-linux-musleabihf?job=build:release:cargo:arm-unknown-linux-musleabihf +[armv7-musl-next]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/next/raw/conduit-armv7-unknown-linux-musleabihf?job=build:release:cargo:armv7-unknown-linux-musleabihf +[armv8-musl-next]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/next/raw/conduit-aarch64-unknown-linux-musl?job=build:release:cargo:aarch64-unknown-linux-musl ```bash $ sudo wget -O /usr/local/bin/matrix-conduit @@ -240,4 +244,5 @@ $ curl https://your.server.name/_matrix/client/versions $ curl https://your.server.name:8448/_matrix/client/versions ``` -If you want to set up an appservice, take a look at the [Appservice Guide](APPSERVICES.md). +- To check if your server can talk with other homeservers, you can use the [Matrix Federation Tester](https://federationtester.matrix.org/) +- If you want to set up an appservice, take a look at the [Appservice Guide](APPSERVICES.md). diff --git a/README.md b/README.md index e667d18d..a4f09298 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # Conduit + ### A Matrix homeserver written in Rust #### What is the goal? @@ -7,7 +8,6 @@ An efficient Matrix homeserver that's easy to set up and just works. You can ins it on a mini-computer like the Raspberry Pi to host Matrix for your family, friends or company. - #### Can I try it out? Yes! You can test our Conduit instance by opening a Matrix client ( or Element Android for @@ -17,7 +17,6 @@ It is hosted on a ODROID HC 2 with 2GB RAM and a SAMSUNG Exynos 5422 CPU, which was used in the Samsung Galaxy S5. It joined many big rooms including Matrix HQ. - #### What is the current status? As of 2021-09-01, Conduit is Beta, meaning you can join and participate in most @@ -31,26 +30,23 @@ There are still a few important features missing: Check out the [Conduit 1.0 Release Milestone](https://gitlab.com/famedly/conduit/-/milestones/3). - #### How can I deploy my own? -Simple install (this was tested the most): [DEPLOY.md](DEPLOY.md)\ -Debian package: [debian/README.Debian](debian/README.Debian)\ -Docker: [docker/README.md](docker/README.md) +- Simple install (this was tested the most): [DEPLOY.md](DEPLOY.md) +- Debian package: [debian/README.Debian](debian/README.Debian) +- Docker: [docker/README.md](docker/README.md) If you want to connect an Appservice to Conduit, take a look at [APPSERVICES.md](APPSERVICES.md). - #### How can I contribute? 1. Look for an issue you would like to work on and make sure it's not assigned to other users 2. Ask someone to assign the issue to you (comment on the issue or chat in - #conduit:nordgedanken.dev) -3. Fork the repo and work on the issue. #conduit:nordgedanken.dev is happy to help :) + [#conduit:fachschaften.org](https://matrix.to/#/#conduit:fachschaften.org)) +3. Fork the repo and work on the issue.[#conduit:fachschaften.org](https://matrix.to/#/#conduit:fachschaften.org) is happy to help :) 4. Submit a MR - #### Thanks to Thanks to Famedly, Prototype Fund (DLR and German BMBF) and all other individuals for financially supporting this project. @@ -60,13 +56,11 @@ Thanks to the contributors to Conduit and all libraries we use, for example: - Ruma: A clean library for the Matrix Spec in Rust - Rocket: A flexible web framework - #### Donate Liberapay: \ Bitcoin: `bc1qnnykf986tw49ur7wx9rpw2tevpsztvar5x8w4n` - #### Logo Lightning Bolt Logo: https://github.com/mozilla/fxemoji/blob/gh-pages/svgs/nature/u26A1-bolt.svg \ diff --git a/docker/README.md b/docker/README.md index 1f38d66a..d8867385 100644 --- a/docker/README.md +++ b/docker/README.md @@ -35,7 +35,7 @@ or you can skip the build step and pull the image from one of the following regi | GitLab Registry | [registry.gitlab.com/famedly/conduit/matrix-conduit:latest][gl] | ![Image Size][shield] | [dh]: https://hub.docker.com/r/matrixconduit/matrix-conduit -[gl]: https://gitlab.com/famedly/conduit/container_registry/ +[gl]: https://gitlab.com/famedly/conduit/container_registry/2497937 [shield]: https://img.shields.io/docker/image-size/matrixconduit/matrix-conduit/latest The `-d` flag lets the container run in detached mode. You now need to supply a `conduit.toml` config file, an example can be found [here](../conduit-example.toml).