magmaus3/continuwuity
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: sending slowness

Browse source
This commit is contained in:
Timo Kösters 2020-09-15 16:13:54 +02:00
parent 0b263208e3
commit b7ab57897b
No known key found for this signature in database
GPG key ID: 24DA7517711A2BA4
15 changed files with 574 additions and 417 deletions
Download patch file Download diff file Expand all files Collapse all files

21
src/database/globals.rs
View file

@ -1,12 +1,13 @@
use crate::{utils, Error, Result};
use ruma::ServerName;
use std::convert::TryInto;
use std::{convert::TryInto, sync::Arc};
pub const COUNTER: &str = "c";
#[derive(Clone)]
pub struct Globals {
pub(super) globals: sled::Tree,
keypair: ruma::signatures::Ed25519KeyPair,
keypair: Arc<ruma::signatures::Ed25519KeyPair>,
reqwest_client: reqwest::Client,
server_name: Box<ServerName>,
max_request_size: u32,
@ -16,13 +17,15 @@ pub struct Globals {
impl Globals {
pub fn load(globals: sled::Tree, config: &rocket::Config) -> Result<Self> {
let keypair = ruma::signatures::Ed25519KeyPair::new(
&*globals
.update_and_fetch("keypair", utils::generate_keypair)?
.expect("utils::generate_keypair always returns Some"),
"key1".to_owned(),
)
.map_err(|_| Error::bad_database("Private or public keys are invalid."))?;
let keypair = Arc::new(
ruma::signatures::Ed25519KeyPair::new(
&*globals
.update_and_fetch("keypair", utils::generate_keypair)?
.expect("utils::generate_keypair always returns Some"),
"key1".to_owned(),
)
.map_err(|_| Error::bad_database("Private or public keys are invalid."))?,
);
Ok(Self {
globals,

58
src/database/rooms.rs
View file

@ -1,14 +1,12 @@
mod edus;
pub use edus::RoomEdus;
use rocket::futures;
use crate::{pdu::PduBuilder, server_server, utils, Error, PduEvent, Result};
use log::{error, warn};
use crate::{pdu::PduBuilder, utils, Error, PduEvent, Result};
use log::error;
use ring::digest;
use ruma::{
api::client::error::ErrorKind,
api::federation,
events::{
ignored_user_list,
room::{
@ -27,7 +25,6 @@ use std::{
convert::{TryFrom, TryInto},
mem,
sync::Arc,
time::SystemTime,
};
/// The unique identifier of each state group.
@ -36,6 +33,7 @@ use std::{
/// hashing the entire state.
pub type StateHashId = Vec<u8>;
#[derive(Clone)]
pub struct Rooms {
pub edus: edus::RoomEdus,
pub(super) pduid_pdu: sled::Tree, // PduId = RoomId + Count
@ -415,6 +413,16 @@ impl Rooms {
})
}
/// Returns the pdu.
pub fn get_pdu_json_from_id(&self, pdu_id: &IVec) -> Result<Option<serde_json::Value>> {
self.pduid_pdu.get(pdu_id)?.map_or(Ok(None), |pdu| {
Ok(Some(
serde_json::from_slice(&pdu)
.map_err(|_| Error::bad_database("Invalid PDU in db."))?,
))
})
}
/// Removes a pdu and creates a new one with the same id.
fn replace_pdu(&self, pdu_id: &IVec, pdu: &PduEvent) -> Result<()> {
if self.pduid_pdu.get(&pdu_id)?.is_some() {
@ -613,6 +621,7 @@ impl Rooms {
sender: &UserId,
room_id: &RoomId,
globals: &super::globals::Globals,
sending: &super::sending::Sending,
account_data: &super::account_data::AccountData,
) -> Result<EventId> {
let PduBuilder {
@ -829,39 +838,12 @@ impl Rooms {
self.append_to_state(&pdu_id, &pdu)?;
}
pdu_json
.as_object_mut()
.expect("json is object")
.remove("event_id");
let raw_json =
serde_json::from_value::<Raw<_>>(pdu_json).expect("Raw::from_value always works");
let pdus = &[raw_json];
let transaction_id = utils::random_string(16);
for result in futures::future::join_all(
self.room_servers(room_id)
.filter_map(|r| r.ok())
.filter(|server| &**server != globals.server_name())
.map(|server| {
server_server::send_request(
&globals,
server,
federation::transactions::send_transaction_message::v1::Request {
origin: globals.server_name(),
pdus,
edus: &[],
origin_server_ts: SystemTime::now(),
transaction_id: &transaction_id,
},
)
}),
)
.await {
if let Err(e) = result {
warn!("{}", e);
}
for server in self
.room_servers(room_id)
.filter_map(|r| r.ok())
.filter(|server| &**server != globals.server_name())
{
sending.send_pdu(server, &pdu_id)?;
}
Ok(pdu.event_id)

1
src/database/rooms/edus.rs
View file

@ -13,6 +13,7 @@ use std::{
convert::{TryFrom, TryInto},
};
#[derive(Clone)]
pub struct RoomEdus {
pub(in super::super) readreceiptid_readreceipt: sled::Tree, // ReadReceiptId = RoomId + Count + UserId
pub(in super::super) roomuserid_privateread: sled::Tree, // RoomUserId = Room + User, PrivateRead = Count

83
src/database/sending.rs Normal file
View file

@ -0,0 +1,83 @@
use std::{convert::TryFrom, time::SystemTime};
use crate::{server_server, utils, Error, Result};
use rocket::futures::stream::{FuturesUnordered, StreamExt};
use ruma::{api::federation, Raw, ServerName};
use tokio::select;
pub struct Sending {
/// The state for a given state hash.
pub(super) serverpduids: sled::Tree, // ServerPduId = ServerName + PduId
}
impl Sending {
pub fn start_handler(&self, globals: &super::globals::Globals, rooms: &super::rooms::Rooms) {
let serverpduids = self.serverpduids.clone();
let rooms = rooms.clone();
let globals = globals.clone();
tokio::spawn(async move {
let mut futures = FuturesUnordered::new();
let mut subscriber = serverpduids.watch_prefix(b"");
loop {
select! {
Some(_) = futures.next() => {},
Some(event) = &mut subscriber => {
let serverpduid = if let sled::Event::Insert {key, ..} = event {
key
} else
{ return Err::<(), Error>(Error::bad_database("")); };
let mut parts = serverpduid.splitn(2, |&b| b == 0xff);
let server = Box::<ServerName>::try_from(
utils::string_from_bytes(parts.next().expect("splitn will always return 1 or more elements"))
.map_err(|_| Error::bad_database("ServerName in serverpduid bytes are invalid."))?
).map_err(|_| Error::bad_database("ServerName in serverpduid is invalid."))?;
let pdu_id = parts.next().ok_or_else(|| Error::bad_database("Invalid serverpduid in db."))?;
let mut pdu_json = rooms.get_pdu_json_from_id(&pdu_id.into())?.ok_or_else(|| Error::bad_database("Event in serverpduids not found in db."))?;
pdu_json
.as_object_mut()
.expect("json is object")
.remove("event_id");
let raw_json =
serde_json::from_value::<Raw<_>>(pdu_json).expect("Raw::from_value always works");
let globals = &globals;
futures.push(
async move {
let pdus = vec![raw_json];
let transaction_id = utils::random_string(16);
server_server::send_request(
&globals,
server,
federation::transactions::send_transaction_message::v1::Request {
origin: globals.server_name(),
pdus: &pdus,
edus: &[],
origin_server_ts: SystemTime::now(),
transaction_id: &transaction_id,
},
).await
}
);
},
}
}
});
}
/*
*/
pub fn send_pdu(&self, server: Box<ServerName>, pdu_id: &[u8]) -> Result<()> {
let mut key = server.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(pdu_id);
self.serverpduids.insert(key, b"")?;
Ok(())
}
}