magmaus3/continuwuity
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'correct-sendtxn' into pushers

Browse source
This commit is contained in:
Timo Kösters 2021-03-15 09:48:19 +01:00
commit 21f785d530
No known key found for this signature in database
GPG key ID: 24DA7517711A2BA4
61 changed files with 3003 additions and 1919 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/database/account_data.rs
View file

@ -74,6 +74,7 @@ impl AccountData {
}
/// Returns all changes to the account data that happened after `since`.
#[tracing::instrument(skip(self))]
pub fn changes_since(
&self,
room_id: Option<&RoomId>,

3
src/database/admin.rs
View file

@ -7,7 +7,6 @@ use ruma::{
events::{room::message, EventType},
UserId,
};
use tokio::select;
pub enum AdminCommand {
RegisterAppservice(serde_yaml::Value),
@ -67,7 +66,7 @@ impl Admin {
};
loop {
select! {
tokio::select! {
Some(event) = receiver.next() => {
match event {
AdminCommand::RegisterAppservice(yaml) => {

82
src/database/globals.rs
View file

@ -14,20 +14,26 @@ use trust_dns_resolver::TokioAsyncResolver;
pub const COUNTER: &str = "c";
pub type DestinationCache = Arc<RwLock<HashMap<Box<ServerName>, (String, Option<String>)>>>;
type WellKnownMap = HashMap<Box<ServerName>, (String, Option<String>)>;
#[derive(Clone)]
pub struct Globals {
pub actual_destination_cache: Arc<RwLock<WellKnownMap>>, // actual_destination, host
pub(super) globals: sled::Tree,
config: Config,
keypair: Arc<ruma::signatures::Ed25519KeyPair>,
reqwest_client: reqwest::Client,
pub actual_destination_cache: DestinationCache, // actual_destination, host
dns_resolver: TokioAsyncResolver,
pub(super) servertimeout_signingkey: sled::Tree, // ServerName -> algorithm:key + pubkey
jwt_decoding_key: Option<jsonwebtoken::DecodingKey<'static>>,
pub(super) servertimeout_signingkey: sled::Tree, // ServerName + Timeout Timestamp -> algorithm:key + pubkey
}
impl Globals {
pub fn load(globals: sled::Tree, server_keys: sled::Tree, config: Config) -> Result<Self> {
pub fn load(
globals: sled::Tree,
servertimeout_signingkey: sled::Tree,
config: Config,
) -> Result<Self> {
let bytes = &*globals
.update_and_fetch("keypair", utils::generate_keypair)?
.expect("utils::generate_keypair always returns Some");
@ -69,6 +75,11 @@ impl Globals {
.build()
.unwrap();
let jwt_decoding_key = config
.jwt_secret
.as_ref()
.map(|secret| jsonwebtoken::DecodingKey::from_secret(secret.as_bytes()).into_static());
Ok(Self {
globals,
config,
@ -78,7 +89,8 @@ impl Globals {
Error::bad_config("Failed to set up trust dns resolver with system config.")
})?,
actual_destination_cache: Arc::new(RwLock::new(HashMap::new())),
servertimeout_signingkey: server_keys,
servertimeout_signingkey,
jwt_decoding_key,
})
}
@ -129,46 +141,48 @@ impl Globals {
self.config.allow_federation
}
pub fn trusted_servers(&self) -> &[Box<ServerName>] {
&self.config.trusted_servers
}
pub fn dns_resolver(&self) -> &TokioAsyncResolver {
&self.dns_resolver
}
pub fn jwt_decoding_key(&self) -> Option<&jsonwebtoken::DecodingKey<'_>> {
self.jwt_decoding_key.as_ref()
}
/// TODO: the key valid until timestamp is only honored in room version > 4
/// Remove the outdated keys and insert the new ones.
///
/// This doesn't actually check that the keys provided are newer than the old set.
pub fn add_signing_key(&self, origin: &ServerName, keys: &ServerSigningKeys) -> Result<()> {
// Remove outdated keys
let now = crate::utils::millis_since_unix_epoch();
for item in self.servertimeout_signingkey.scan_prefix(origin.as_bytes()) {
let (k, _) = item?;
let valid_until = k
.splitn(2, |&b| b == 0xff)
.nth(1)
.map(crate::utils::u64_from_bytes)
.ok_or_else(|| Error::bad_database("Invalid signing keys."))?
.map_err(|_| Error::bad_database("Invalid signing key valid until bytes"))?;
let mut key1 = origin.as_bytes().to_vec();
key1.push(0xff);
if now > valid_until {
self.servertimeout_signingkey.remove(k)?;
}
}
let mut key2 = key1.clone();
let mut key = origin.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(
&(keys
.valid_until_ts
.duration_since(std::time::UNIX_EPOCH)
.expect("time is valid")
.as_millis() as u64)
.to_be_bytes(),
);
let ts = keys
.valid_until_ts
.duration_since(std::time::UNIX_EPOCH)
.expect("time is valid")
.as_millis() as u64;
key1.extend_from_slice(&ts.to_be_bytes());
key2.extend_from_slice(&(ts + 1).to_be_bytes());
self.servertimeout_signingkey.insert(
key,
key1,
serde_json::to_vec(&keys.verify_keys).expect("ServerSigningKeys are a valid string"),
)?;
self.servertimeout_signingkey.insert(
key2,
serde_json::to_vec(&keys.old_verify_keys)
.expect("ServerSigningKeys are a valid string"),
)?;
Ok(())
}
@ -177,7 +191,10 @@ impl Globals {
&self,
origin: &ServerName,
) -> Result<BTreeMap<ServerSigningKeyId, VerifyKey>> {
let mut response = BTreeMap::new();
let now = crate::utils::millis_since_unix_epoch();
for item in self.servertimeout_signingkey.scan_prefix(origin.as_bytes()) {
let (k, bytes) = item?;
let valid_until = k
@ -188,10 +205,11 @@ impl Globals {
.map_err(|_| Error::bad_database("Invalid signing key valid until bytes"))?;
// If these keys are still valid use em!
if valid_until > now {
return serde_json::from_slice(&bytes)
.map_err(|_| Error::bad_database("Invalid BTreeMap<> of signing keys"));
let btree: BTreeMap<_, _> = serde_json::from_slice(&bytes)
.map_err(|_| Error::bad_database("Invalid BTreeMap<> of signing keys"))?;
response.extend(btree);
}
}
Ok(BTreeMap::default())
Ok(response)
}
}

32
src/database/key_backups.rs
View file

@ -2,7 +2,7 @@ use crate::{utils, Error, Result};
use ruma::{
api::client::{
error::ErrorKind,
r0::backup::{BackupAlgorithm, KeyData, Sessions},
r0::backup::{BackupAlgorithm, KeyBackupData, RoomKeyBackup},
},
RoomId, UserId,
};
@ -129,7 +129,7 @@ impl KeyBackups {
version: &str,
room_id: &RoomId,
session_id: &str,
key_data: &KeyData,
key_data: &KeyBackupData,
globals: &super::globals::Globals,
) -> Result<()> {
let mut key = user_id.to_string().as_bytes().to_vec();
@ -153,7 +153,7 @@ impl KeyBackups {
self.backupkeyid_backup.insert(
&key,
&*serde_json::to_string(&key_data).expect("KeyData::to_string always works"),
&*serde_json::to_string(&key_data).expect("KeyBackupData::to_string always works"),
)?;
Ok(())
@ -182,13 +182,17 @@ impl KeyBackups {
.to_string())
}
pub fn get_all(&self, user_id: &UserId, version: &str) -> Result<BTreeMap<RoomId, Sessions>> {
pub fn get_all(
&self,
user_id: &UserId,
version: &str,
) -> Result<BTreeMap<RoomId, RoomKeyBackup>> {
let mut prefix = user_id.to_string().as_bytes().to_vec();
prefix.push(0xff);
prefix.extend_from_slice(version.as_bytes());
prefix.push(0xff);
let mut rooms = BTreeMap::<RoomId, Sessions>::new();
let mut rooms = BTreeMap::<RoomId, RoomKeyBackup>::new();
for result in self.backupkeyid_backup.scan_prefix(&prefix).map(|r| {
let (key, value) = r?;
@ -211,15 +215,16 @@ impl KeyBackups {
)
.map_err(|_| Error::bad_database("backupkeyid_backup room_id is invalid room id."))?;
let key_data = serde_json::from_slice(&value)
.map_err(|_| Error::bad_database("KeyData in backupkeyid_backup is invalid."))?;
let key_data = serde_json::from_slice(&value).map_err(|_| {
Error::bad_database("KeyBackupData in backupkeyid_backup is invalid.")
})?;
Ok::<_, Error>((room_id, session_id, key_data))
}) {
let (room_id, session_id, key_data) = result?;
rooms
.entry(room_id)
.or_insert_with(|| Sessions {
.or_insert_with(|| RoomKeyBackup {
sessions: BTreeMap::new(),
})
.sessions
@ -234,7 +239,7 @@ impl KeyBackups {
user_id: &UserId,
version: &str,
room_id: &RoomId,
) -> BTreeMap<String, KeyData> {
) -> BTreeMap<String, KeyBackupData> {
let mut prefix = user_id.to_string().as_bytes().to_vec();
prefix.push(0xff);
prefix.extend_from_slice(version.as_bytes());
@ -257,7 +262,7 @@ impl KeyBackups {
})?;
let key_data = serde_json::from_slice(&value).map_err(|_| {
Error::bad_database("KeyData in backupkeyid_backup is invalid.")
Error::bad_database("KeyBackupData in backupkeyid_backup is invalid.")
})?;
Ok::<_, Error>((session_id, key_data))
@ -272,7 +277,7 @@ impl KeyBackups {
version: &str,
room_id: &RoomId,
session_id: &str,
) -> Result<Option<KeyData>> {
) -> Result<Option<KeyBackupData>> {
let mut key = user_id.to_string().as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(version.as_bytes());
@ -284,8 +289,9 @@ impl KeyBackups {
self.backupkeyid_backup
.get(&key)?
.map(|value| {
serde_json::from_slice(&value)
.map_err(|_| Error::bad_database("KeyData in backupkeyid_backup is invalid."))
serde_json::from_slice(&value).map_err(|_| {
Error::bad_database("KeyBackupData in backupkeyid_backup is invalid.")
})
})
.transpose()
}

14
src/database/pusher.rs
View file

@ -11,7 +11,7 @@ use ruma::{
},
events::room::{
member::{MemberEventContent, MembershipState},
message::{MessageEventContent, TextMessageEventContent},
message::{MessageEventContent, MessageType, TextMessageEventContent},
power_levels::PowerLevelsEventContent,
},
events::EventType,
@ -265,8 +265,8 @@ pub async fn send_push_notice(
.map_err(|_| {
Error::bad_database("PDU contained bad message content")
})?;
if let MessageEventContent::Text(TextMessageEventContent { body, .. }) =
&msg_content
if let MessageType::Text(TextMessageEventContent { body, .. }) =
&msg_content.msgtype
{
if body.contains(user.localpart()) {
let tweaks = rule
@ -305,8 +305,8 @@ pub async fn send_push_notice(
.map_err(|_| {
Error::bad_database("PDU contained bad message content")
})?;
if let MessageEventContent::Text(TextMessageEventContent { body, .. }) =
&msg_content
if let MessageType::Text(TextMessageEventContent { body, .. }) =
&msg_content.msgtype
{
let power_level_cmp = |pl: PowerLevelsEventContent| {
&pl.notifications.room
@ -346,8 +346,8 @@ pub async fn send_push_notice(
.map_err(|_| {
Error::bad_database("PDU contained bad message content")
})?;
if let MessageEventContent::Text(TextMessageEventContent { body, .. }) =
&msg_content
if let MessageType::Text(TextMessageEventContent { body, .. }) =
&msg_content.msgtype
{
if body.contains(user.localpart()) {
let tweaks = rule

255
src/database/rooms.rs
View file

@ -3,7 +3,7 @@ mod edus;
pub use edus::RoomEdus;
use crate::{pdu::PduBuilder, utils, Database, Error, PduEvent, Result};
use log::error;
use log::{error, info, warn};
use regex::Regex;
use ring::digest;
use ruma::{
@ -63,16 +63,24 @@ pub struct Rooms {
/// Remember the state hash at events in the past.
pub(super) pduid_statehash: sled::Tree,
/// The state for a given state hash.
pub(super) statekey_short: sled::Tree, // StateKey = EventType + StateKey, Short = Count
pub(super) stateid_pduid: sled::Tree, // StateId = StateHash + Short, PduId = Count (without roomid)
///
/// StateKey = EventType + StateKey, Short = Count
pub(super) statekey_short: sled::Tree,
/// StateId = StateHash + Short, PduId = Count (without roomid)
pub(super) stateid_pduid: sled::Tree,
/// Any pdu that has passed the steps up to auth with auth_events.
pub(super) pduid_outlierpdu: sled::Tree,
/// RoomId + EventId -> outlier PDU.
/// Any pdu that has passed the steps 1-8 in the incoming event /federation/send/txn.
pub(super) eventid_outlierpdu: sled::Tree,
/// RoomId + EventId -> Parent PDU EventId.
pub(super) prevevent_parent: sled::Tree,
}
impl Rooms {
/// Builds a StateMap by iterating over all keys that start
/// with state_hash, this gives the full state for the given state_hash.
#[tracing::instrument(skip(self))]
pub fn state_full(
&self,
room_id: &RoomId,
@ -81,16 +89,17 @@ impl Rooms {
self.stateid_pduid
.scan_prefix(&state_hash)
.values()
.map(|pduid_short| {
let mut pduid = room_id.as_bytes().to_vec();
pduid.push(0xff);
pduid.extend_from_slice(&pduid_short?);
match self.pduid_pdu.get(&pduid)? {
.map(|short_id| {
let short_id = short_id?;
let mut long_id = room_id.as_bytes().to_vec();
long_id.push(0xff);
long_id.extend_from_slice(&short_id);
match self.pduid_pdu.get(&long_id)? {
Some(b) => serde_json::from_slice::<PduEvent>(&b)
.map_err(|_| Error::bad_database("Invalid PDU in db.")),
None => self
.pduid_outlierpdu
.get(pduid)?
.eventid_outlierpdu
.get(short_id)?
.map(|b| {
serde_json::from_slice::<PduEvent>(&b)
.map_err(|_| Error::bad_database("Invalid PDU in db."))
@ -117,6 +126,7 @@ impl Rooms {
}
/// Returns a single PDU from `room_id` with key (`event_type`, `state_key`).
#[tracing::instrument(skip(self))]
pub fn state_get(
&self,
room_id: &RoomId,
@ -128,6 +138,8 @@ impl Rooms {
key.push(0xff);
key.extend_from_slice(&state_key.as_bytes());
info!("Looking for {} {:?}", event_type, state_key);
let short = self.statekey_short.get(&key)?;
if let Some(short) = short {
@ -135,42 +147,52 @@ impl Rooms {
stateid.push(0xff);
stateid.extend_from_slice(&short);
info!("trying to find pduid/eventid. short: {:?}", stateid);
self.stateid_pduid
.get(&stateid)?
.map_or(Ok(None), |pdu_id_short| {
let mut pdu_id = room_id.as_bytes().to_vec();
pdu_id.push(0xff);
pdu_id.extend_from_slice(&pdu_id_short);
.map_or(Ok(None), |short_id| {
info!("found in stateid_pduid");
let mut long_id = room_id.as_bytes().to_vec();
long_id.push(0xff);
long_id.extend_from_slice(&short_id);
Ok::<_, Error>(Some((
pdu_id.clone().into(),
match self.pduid_pdu.get(&pdu_id)? {
Some(b) => serde_json::from_slice::<PduEvent>(&b)
Ok::<_, Error>(Some(match self.pduid_pdu.get(&long_id)? {
Some(b) => (
long_id.clone().into(),
serde_json::from_slice::<PduEvent>(&b)
.map_err(|_| Error::bad_database("Invalid PDU in db."))?,
None => self
.pduid_outlierpdu
.get(pdu_id)?
.map(|b| {
serde_json::from_slice::<PduEvent>(&b)
.map_err(|_| Error::bad_database("Invalid PDU in db."))
})
.ok_or_else(|| {
Error::bad_database("Event is not in pdu tree or outliers.")
})??,
},
)))
),
None => {
info!("looking in outliers");
(
short_id.clone().into(),
self.eventid_outlierpdu
.get(&short_id)?
.map(|b| {
serde_json::from_slice::<PduEvent>(&b)
.map_err(|_| Error::bad_database("Invalid PDU in db."))
})
.ok_or_else(|| {
Error::bad_database("Event is not in pdu tree or outliers.")
})??,
)
}
}))
})
} else {
info!("short id not found");
Ok(None)
}
}
/// Returns the state hash for this pdu.
#[tracing::instrument(skip(self))]
pub fn pdu_state_hash(&self, pdu_id: &[u8]) -> Result<Option<StateHashId>> {
Ok(self.pduid_statehash.get(pdu_id)?)
}
/// Returns the last state hash key added to the db for the given room.
#[tracing::instrument(skip(self))]
pub fn current_state_hash(&self, room_id: &RoomId) -> Result<Option<StateHashId>> {
Ok(self.roomid_statehash.get(room_id.as_bytes())?)
}
@ -198,9 +220,11 @@ impl Rooms {
&event_type,
&state_key
.as_deref()
.expect("found a non state event in auth events"),
.ok_or_else(|| Error::bad_database("Saved auth event with no state key."))?,
)? {
events.insert((event_type, state_key), pdu);
} else {
warn!("Could not find {} {:?} in state", event_type, state_key);
}
}
Ok(events)
@ -239,11 +263,11 @@ impl Rooms {
globals: &super::globals::Globals,
) -> Result<()> {
let state_hash =
self.calculate_hash(&state.values().map(|pdu_id| &**pdu_id).collect::<Vec<_>>())?;
self.calculate_hash(&state.values().map(|long_id| &**long_id).collect::<Vec<_>>())?;
let mut prefix = state_hash.to_vec();
prefix.push(0xff);
for ((event_type, state_key), pdu_id) in state {
for ((event_type, state_key), long_id) in state {
let mut statekey = event_type.as_ref().as_bytes().to_vec();
statekey.push(0xff);
statekey.extend_from_slice(&state_key.as_bytes());
@ -259,14 +283,13 @@ impl Rooms {
}
};
let pdu_id_short = pdu_id
.splitn(2, |&b| b == 0xff)
.nth(1)
.ok_or_else(|| Error::bad_database("Invalid pduid in state."))?;
// If it's a pdu id we remove the room id, if it's an event id we leave it the same
let short_id = long_id.splitn(2, |&b| b == 0xff).nth(1).unwrap_or(&long_id);
let mut state_id = prefix.clone();
state_id.extend_from_slice(&short.to_be_bytes());
self.stateid_pduid.insert(state_id, pdu_id_short)?;
info!("inserting {:?} into {:?}", short_id, state_id);
self.stateid_pduid.insert(state_id, short_id)?;
}
self.roomid_statehash
@ -276,6 +299,7 @@ impl Rooms {
}
/// Returns the full room state.
#[tracing::instrument(skip(self))]
pub fn room_state_full(
&self,
room_id: &RoomId,
@ -288,6 +312,7 @@ impl Rooms {
}
/// Returns a single PDU from `room_id` with key (`event_type`, `state_key`).
#[tracing::instrument(skip(self))]
pub fn room_state_get(
&self,
room_id: &RoomId,
@ -302,6 +327,7 @@ impl Rooms {
}
/// Returns the `count` of this pdu's id.
#[tracing::instrument(skip(self))]
pub fn pdu_count(&self, pdu_id: &[u8]) -> Result<u64> {
Ok(
utils::u64_from_bytes(&pdu_id[pdu_id.len() - mem::size_of::<u64>()..pdu_id.len()])
@ -316,21 +342,32 @@ impl Rooms {
.map_or(Ok(None), |pdu_id| self.pdu_count(&pdu_id).map(Some))
}
pub fn latest_pdu_count(&self, room_id: &RoomId) -> Result<u64> {
self.pduid_pdu
.scan_prefix(room_id.as_bytes())
.last()
.map(|b| self.pdu_count(&b?.0))
.transpose()
.map(|op| op.unwrap_or_default())
}
/// Returns the json of a pdu.
pub fn get_pdu_json(&self, event_id: &EventId) -> Result<Option<serde_json::Value>> {
self.eventid_pduid
.get(event_id.as_bytes())?
.map_or(Ok(None), |pdu_id| {
Ok(Some(
serde_json::from_slice(&match self.pduid_pdu.get(&pdu_id)? {
Some(b) => b,
None => self.pduid_outlierpdu.get(pdu_id)?.ok_or_else(|| {
Error::bad_database("Event is not in pdu tree or outliers.")
})?,
})
.map_err(|_| Error::bad_database("Invalid PDU in db."))?,
))
.map_or_else::<Result<_>, _, _>(
|| Ok(self.eventid_outlierpdu.get(event_id.as_bytes())?),
|pduid| {
Ok(Some(self.pduid_pdu.get(&pduid)?.ok_or_else(|| {
Error::bad_database("Invalid pduid in eventid_pduid.")
})?))
},
)?
.map(|pdu| {
Ok(serde_json::from_slice(&pdu)
.map_err(|_| Error::bad_database("Invalid PDU in db."))?)
})
.transpose()
}
/// Returns the pdu's id.
@ -340,24 +377,36 @@ impl Rooms {
.map_or(Ok(None), |pdu_id| Ok(Some(pdu_id)))
}
/// Returns the pdu.
pub fn get_pdu(&self, event_id: &EventId) -> Result<Option<PduEvent>> {
self.eventid_pduid
.get(event_id.as_bytes())?
.map_or(Ok(None), |pdu_id| {
Ok(Some(
serde_json::from_slice(&match self.pduid_pdu.get(&pdu_id)? {
Some(b) => b,
None => self.pduid_outlierpdu.get(pdu_id)?.ok_or_else(|| {
Error::bad_database("Event is not in pdu tree or outliers.")
})?,
})
.map_err(|_| Error::bad_database("Invalid PDU in db."))?,
))
})
pub fn get_long_id(&self, event_id: &EventId) -> Result<Vec<u8>> {
Ok(self
.get_pdu_id(event_id)?
.map_or_else(|| event_id.as_bytes().to_vec(), |pduid| pduid.to_vec()))
}
/// Returns the pdu.
///
/// Checks the `eventid_outlierpdu` Tree if not found in the timeline.
pub fn get_pdu(&self, event_id: &EventId) -> Result<Option<PduEvent>> {
self.eventid_pduid
.get(event_id.as_bytes())?
.map_or_else::<Result<_>, _, _>(
|| Ok(self.eventid_outlierpdu.get(event_id.as_bytes())?),
|pduid| {
Ok(Some(self.pduid_pdu.get(&pduid)?.ok_or_else(|| {
Error::bad_database("Invalid pduid in eventid_pduid.")
})?))
},
)?
.map(|pdu| {
Ok(serde_json::from_slice(&pdu)
.map_err(|_| Error::bad_database("Invalid PDU in db."))?)
})
.transpose()
}
/// Returns the pdu.
///
/// This does __NOT__ check the outliers `Tree`.
pub fn get_pdu_from_id(&self, pdu_id: &IVec) -> Result<Option<PduEvent>> {
self.pduid_pdu.get(pdu_id)?.map_or(Ok(None), |pdu| {
Ok(Some(
@ -421,7 +470,7 @@ impl Rooms {
/// Replace the leaves of a room.
///
/// The provided `event_ids` become the new leaves, this enables an event having multiple
/// The provided `event_ids` become the new leaves, this allows a room to have multiple
/// `prev_events`.
pub fn replace_pdu_leaves(&self, room_id: &RoomId, event_ids: &[EventId]) -> Result<()> {
let mut prefix = room_id.as_bytes().to_vec();
@ -440,39 +489,38 @@ impl Rooms {
Ok(())
}
/// Returns the pdu from the outlier tree.
pub fn get_pdu_outlier(&self, event_id: &EventId) -> Result<Option<PduEvent>> {
if let Some(id) = self.eventid_pduid.get(event_id.as_bytes())? {
self.pduid_outlierpdu.get(id)?.map_or(Ok(None), |pdu| {
serde_json::from_slice(&pdu).map_err(|_| Error::bad_database("Invalid PDU in db."))
})
} else {
Ok(None)
}
pub fn is_pdu_referenced(&self, pdu: &PduEvent) -> Result<bool> {
let mut key = pdu.room_id().as_bytes().to_vec();
key.extend_from_slice(pdu.event_id().as_bytes());
self.prevevent_parent.contains_key(key).map_err(Into::into)
}
/// Returns true if the event_id was previously inserted.
pub fn append_pdu_outlier(&self, pdu_id: &[u8], pdu: &PduEvent) -> Result<bool> {
log::info!("Number of outlier pdu's {}", self.pduid_outlierpdu.len());
/// Returns the pdu from the outlier tree.
pub fn get_pdu_outlier(&self, event_id: &EventId) -> Result<Option<PduEvent>> {
self.eventid_outlierpdu
.get(event_id.as_bytes())?
.map_or(Ok(None), |pdu| {
serde_json::from_slice(&pdu).map_err(|_| Error::bad_database("Invalid PDU in db."))
})
}
// we need to be able to find it by event_id
self.eventid_pduid
.insert(pdu.event_id.as_bytes(), &*pdu_id)?;
/// Append the PDU as an outlier.
///
/// Any event given to this will be processed (state-res) on another thread.
pub fn add_pdu_outlier(&self, pdu: &PduEvent) -> Result<()> {
self.eventid_outlierpdu.insert(
&pdu.event_id.as_bytes(),
&*serde_json::to_string(&pdu).expect("PduEvent is always a valid String"),
)?;
let res = self
.pduid_outlierpdu
.insert(
pdu_id,
&*serde_json::to_string(&pdu).expect("PduEvent is always a valid String"),
)
.map(|op| op.is_some())?;
Ok(res)
Ok(())
}
/// Creates a new persisted data unit and adds it to a room.
///
/// By this point the incoming event should be fully authenticated, no auth happens
/// in `append_pdu`.
#[allow(clippy::too_many_arguments)]
pub fn append_pdu(
&self,
pdu: &PduEvent,
@ -509,9 +557,12 @@ impl Rooms {
}
}
// We no longer keep this pdu as an outlier
if let Some(id) = self.eventid_pduid.remove(pdu.event_id().as_bytes())? {
self.pduid_outlierpdu.remove(id)?;
// We must keep track of all events that have been referenced.
for leaf in leaves {
let mut key = pdu.room_id().as_bytes().to_vec();
key.extend_from_slice(leaf.as_bytes());
self.prevevent_parent
.insert(key, pdu.event_id().as_bytes())?;
}
self.replace_pdu_leaves(&pdu.room_id, leaves)?;
@ -527,6 +578,8 @@ impl Rooms {
.expect("CanonicalJsonObject is always a valid String"),
)?;
// This also replaces the eventid of any outliers with the correct
// pduid, removing the place holder.
self.eventid_pduid
.insert(pdu.event_id.as_bytes(), &*pdu_id)?;
@ -836,12 +889,12 @@ impl Rooms {
content.clone(),
prev_event,
None, // TODO: third party invite
&auth_events
dbg!(&auth_events
.iter()
.map(|((ty, key), pdu)| {
Ok(((ty.clone(), key.clone()), Arc::new(pdu.clone())))
})
.collect::<Result<StateMap<_>>>()?,
.collect::<Result<StateMap<_>>>()?),
)
.map_err(|e| {
log::error!("{}", e);
@ -1025,6 +1078,7 @@ impl Rooms {
let user_is_joined =
|bridge_user_id| self.is_joined(&bridge_user_id, room_id).unwrap_or(false);
let matching_users = |users: &Regex| {
users.is_match(pdu.sender.as_str())
|| pdu.kind == EventType::RoomMember
@ -1053,6 +1107,7 @@ impl Rooms {
}
/// Returns an iterator over all PDUs in a room.
#[tracing::instrument(skip(self))]
pub fn all_pdus(
&self,
user_id: &UserId,
@ -1063,6 +1118,7 @@ impl Rooms {
/// Returns a double-ended iterator over all events in a room that happened after the event with id `since`
/// in chronological order.
#[tracing::instrument(skip(self))]
pub fn pdus_since(
&self,
user_id: &UserId,
@ -1129,6 +1185,7 @@ impl Rooms {
/// Returns an iterator over all events and their token in a room that happened after the event
/// with id `from` in chronological order.
#[tracing::instrument(skip(self))]
pub fn pdus_after(
&self,
user_id: &UserId,
@ -1177,7 +1234,7 @@ impl Rooms {
}
/// Update current membership data.
fn update_membership(
pub fn update_membership(
&self,
room_id: &RoomId,
user_id: &UserId,
@ -1482,6 +1539,7 @@ impl Rooms {
))
}
#[tracing::instrument(skip(self))]
pub fn get_shared_rooms<'a>(
&'a self,
users: Vec<UserId>,
@ -1521,7 +1579,7 @@ impl Rooms {
})
}
/// Returns an iterator over all joined members of a room.
/// Returns an iterator of all servers participating in this room.
pub fn room_servers(&self, room_id: &RoomId) -> impl Iterator<Item = Result<Box<ServerName>>> {
let mut prefix = room_id.as_bytes().to_vec();
prefix.push(0xff);
@ -1543,6 +1601,7 @@ impl Rooms {
}
/// Returns an iterator over all joined members of a room.
#[tracing::instrument(skip(self))]
pub fn room_members(&self, room_id: &RoomId) -> impl Iterator<Item = Result<UserId>> {
let mut prefix = room_id.as_bytes().to_vec();
prefix.push(0xff);
@ -1591,6 +1650,7 @@ impl Rooms {
}
/// Returns an iterator over all invited members of a room.
#[tracing::instrument(skip(self))]
pub fn room_members_invited(&self, room_id: &RoomId) -> impl Iterator<Item = Result<UserId>> {
let mut prefix = room_id.as_bytes().to_vec();
prefix.push(0xff);
@ -1615,6 +1675,7 @@ impl Rooms {
}
/// Returns an iterator over all rooms this user joined.
#[tracing::instrument(skip(self))]
pub fn rooms_joined(&self, user_id: &UserId) -> impl Iterator<Item = Result<RoomId>> {
self.userroomid_joined
.scan_prefix(user_id.as_bytes())
@ -1636,6 +1697,7 @@ impl Rooms {
}
/// Returns an iterator over all rooms a user was invited to.
#[tracing::instrument(skip(self))]
pub fn rooms_invited(&self, user_id: &UserId) -> impl Iterator<Item = Result<RoomId>> {
let mut prefix = user_id.as_bytes().to_vec();
prefix.push(0xff);
@ -1660,6 +1722,7 @@ impl Rooms {
}
/// Returns an iterator over all rooms a user left.
#[tracing::instrument(skip(self))]
pub fn rooms_left(&self, user_id: &UserId) -> impl Iterator<Item = Result<RoomId>> {
let mut prefix = user_id.as_bytes().to_vec();
prefix.push(0xff);

5
src/database/rooms/edus.rs
View file

@ -70,6 +70,7 @@ impl RoomEdus {
}
/// Returns an iterator over the most recent read_receipts in a room that happened after the event with id `since`.
#[tracing::instrument(skip(self))]
pub fn readreceipts_since(
&self,
room_id: &RoomId,
@ -115,6 +116,7 @@ impl RoomEdus {
}
/// Returns the private read marker.
#[tracing::instrument(skip(self))]
pub fn private_read_get(&self, room_id: &RoomId, user_id: &UserId) -> Result<Option<u64>> {
let mut key = room_id.to_string().as_bytes().to_vec();
key.push(0xff);
@ -256,6 +258,7 @@ impl RoomEdus {
}
/// Returns the count of the last typing update in this room.
#[tracing::instrument(skip(self, globals))]
pub fn last_typing_update(
&self,
room_id: &RoomId,
@ -339,6 +342,7 @@ impl RoomEdus {
}
/// Resets the presence timeout, so the user will stay in their current presence state.
#[tracing::instrument(skip(self))]
pub fn ping_presence(&self, user_id: &UserId) -> Result<()> {
self.userid_lastpresenceupdate.insert(
&user_id.to_string().as_bytes(),
@ -429,6 +433,7 @@ impl RoomEdus {
}
/// Returns an iterator over the most recent presence updates that happened after the event with id `since`.
#[tracing::instrument(skip(self, globals, rooms))]
pub fn presence_since(
&self,
room_id: &RoomId,

148
src/database/sending.rs
View file

@ -6,9 +6,12 @@ use std::{
time::{Duration, Instant, SystemTime},
};
use crate::{appservice_server, server_server, utils, Database, Error, PduEvent, Result};
use crate::{
appservice_server, database::pusher, server_server, utils, Database, Error, PduEvent, Result,
};
use federation::transactions::send_transaction_message;
use log::info;
use log::{info, warn};
use ring::digest;
use rocket::futures::stream::{FuturesUnordered, StreamExt};
use ruma::{
api::{appservice, federation, OutgoingRequest},
@ -37,30 +40,66 @@ impl Sending {
pub fn start_handler(&self, db: &Database) {
let servernamepduids = self.servernamepduids.clone();
let servercurrentpdus = self.servercurrentpdus.clone();
let db = db.clone();
tokio::spawn(async move {
let mut futures = FuturesUnordered::new();
// Retry requests we could not finish yet
let mut current_transactions = HashMap::new();
let mut current_transactions = HashMap::<OutgoingKind, Vec<IVec>>::new();
for (outgoing_kind, pdu) in servercurrentpdus
for (key, outgoing_kind, pdu) in servercurrentpdus
.iter()
.filter_map(|r| r.ok())
.filter_map(|(key, _)| Self::parse_servercurrentpdus(key).ok())
.filter(|(_, pdu)| !pdu.is_empty()) // Skip reservation key
.take(50)
// This should not contain more than 50 anyway
.filter_map(|(key, _)| {
Self::parse_servercurrentpdus(&key)
.ok()
.map(|(k, p)| (key, k, p))
})
{
current_transactions
if pdu.is_empty() {
// Remove old reservation key
servercurrentpdus.remove(key).unwrap();
continue;
}
let entry = current_transactions
.entry(outgoing_kind)
.or_insert_with(Vec::new)
.push(pdu);
.or_insert_with(Vec::new);
if entry.len() > 30 {
warn!("Dropping some current pdus because too many were queued. This should not happen.");
servercurrentpdus.remove(key).unwrap();
continue;
}
entry.push(pdu);
}
for (outgoing_kind, pdus) in current_transactions {
futures.push(Self::handle_event(outgoing_kind, pdus, &db));
// Create new reservation
let mut prefix = match &outgoing_kind {
OutgoingKind::Appservice(server) => {
let mut p = b"+".to_vec();
p.extend_from_slice(server.as_bytes());
p
}
OutgoingKind::Push(id) => {
let mut p = b"$".to_vec();
p.extend_from_slice(&id);
p
}
OutgoingKind::Normal(server) => {
let mut p = Vec::new();
p.extend_from_slice(server.as_bytes());
p
}
};
prefix.push(0xff);
servercurrentpdus.insert(prefix, &[]).unwrap();
futures.push(Self::handle_event(outgoing_kind.clone(), pdus, &db));
}
let mut last_failed_try: HashMap<OutgoingKind, (u32, Instant)> = HashMap::new();
@ -109,7 +148,7 @@ impl Sending {
.map(|k| {
k.subslice(prefix.len(), k.len() - prefix.len())
})
.take(50)
.take(30)
.collect::<Vec<_>>();
if !new_pdus.is_empty() {
@ -255,6 +294,7 @@ impl Sending {
});
}
#[tracing::instrument(skip(self))]
pub fn send_push_pdu(&self, pdu_id: &[u8]) -> Result<()> {
// Make sure we don't cause utf8 errors when parsing to a String...
let pduid = String::from_utf8_lossy(pdu_id).as_bytes().to_vec();
@ -273,6 +313,7 @@ impl Sending {
Ok(())
}
#[tracing::instrument(skip(self))]
pub fn send_pdu(&self, server: &ServerName, pdu_id: &[u8]) -> Result<()> {
let mut key = server.as_bytes().to_vec();
key.push(0xff);
@ -282,6 +323,7 @@ impl Sending {
Ok(())
}
#[tracing::instrument(skip(self))]
pub fn send_pdu_appservice(&self, appservice_id: &str, pdu_id: &[u8]) -> Result<()> {
let mut key = b"+".to_vec();
key.extend_from_slice(appservice_id.as_bytes());
@ -292,13 +334,21 @@ impl Sending {
Ok(())
}
// TODO this is the whole DB but is it better to clone smaller parts than the whole thing??
#[tracing::instrument]
fn calculate_hash(keys: &[IVec]) -> Vec<u8> {
// We only hash the pdu's event ids, not the whole pdu
let bytes = keys.join(&0xff);
let hash = digest::digest(&digest::SHA256, &bytes);
hash.as_ref().to_owned()
}
#[tracing::instrument(skip(db))]
async fn handle_event(
kind: OutgoingKind,
pdu_ids: Vec<IVec>,
db: &Database,
) -> std::result::Result<OutgoingKind, (OutgoingKind, Error)> {
match dbg!(kind) {
match dbg!(&kind) {
OutgoingKind::Appservice(server) => {
let pdu_jsons = pdu_ids
.iter()
@ -320,7 +370,8 @@ impl Sending {
})
.filter_map(|r| r.ok())
.collect::<Vec<_>>();
appservice_server::send_request(
let permit = db.sending.maximum_requests.acquire().await;
let response = appservice_server::send_request(
&db.globals,
db.appservice
.get_registration(server.as_str())
@ -328,12 +379,19 @@ impl Sending {
.unwrap(), // TODO: handle error
appservice::event::push_events::v1::Request {
events: &pdu_jsons,
txn_id: &utils::random_string(16),
txn_id: &base64::encode_config(
Self::calculate_hash(&pdu_ids),
base64::URL_SAFE_NO_PAD,
),
},
)
.await
.map(|_response| OutgoingKind::Appservice(server.clone()))
.map_err(|e| (OutgoingKind::Appservice(server.clone()), e))
.map(|_response| kind.clone())
.map_err(|e| (kind, e));
drop(permit);
response
}
OutgoingKind::Push(id) => {
let pdus = pdu_ids
@ -403,22 +461,23 @@ impl Sending {
uint!(0)
};
dbg!(
crate::database::pusher::send_push_notice(
&user,
unread,
&pushers,
rules_for_user,
pdu,
db,
)
.await
let permit = db.sending.maximum_requests.acquire().await;
let _response = pusher::send_push_notice(
&user,
unread,
&pushers,
rules_for_user,
pdu,
db,
)
.map_err(|e| (OutgoingKind::Push(id.clone()), e))?;
.await
.map(|_response| kind.clone())
.map_err(|e| (kind.clone(), e));
drop(permit);
}
}
Ok(OutgoingKind::Push(id))
Ok(OutgoingKind::Push(id.clone()))
}
OutgoingKind::Normal(server) => {
let pdu_jsons = pdu_ids
@ -449,7 +508,10 @@ impl Sending {
.filter_map(|r| r.ok())
.collect::<Vec<_>>();
server_server::send_request(
let permit = db.sending.maximum_requests.acquire().await;
info!("sending pdus to {}: {:#?}", server, pdu_jsons);
let response = server_server::send_request(
&db.globals,
&*server,
send_transaction_message::v1::Request {
@ -457,17 +519,27 @@ impl Sending {
pdus: &pdu_jsons,
edus: &[],
origin_server_ts: SystemTime::now(),
transaction_id: &utils::random_string(16),
transaction_id: &base64::encode_config(
Self::calculate_hash(&pdu_ids),
base64::URL_SAFE_NO_PAD,
),
},
)
.await
.map(|_response| OutgoingKind::Normal(server.clone()))
.map_err(|e| (OutgoingKind::Normal(server.clone()), e))
.map(|response| {
info!("server response: {:?}", response);
kind.clone()
})
.map_err(|e| (kind, e));
drop(permit);
response
}
}
}
fn parse_servercurrentpdus(key: IVec) -> Result<(OutgoingKind, IVec)> {
fn parse_servercurrentpdus(key: &IVec) -> Result<(OutgoingKind, IVec)> {
let mut parts = key.splitn(2, |&b| b == 0xff);
let server = parts.next().expect("splitn always returns one element");
let pdu = parts
@ -501,6 +573,7 @@ impl Sending {
})
}
#[tracing::instrument(skip(self, globals))]
pub async fn send_federation_request<T: OutgoingRequest>(
&self,
globals: &crate::database::globals::Globals,
@ -517,6 +590,7 @@ impl Sending {
response
}
#[tracing::instrument(skip(self, globals))]
pub async fn send_appservice_request<T: OutgoingRequest>(
&self,
globals: &crate::database::globals::Globals,

7
src/database/users.rs
View file

@ -251,7 +251,7 @@ impl Users {
}
/// Replaces the access token of one device.
fn set_token(&self, user_id: &UserId, device_id: &DeviceId, token: &str) -> Result<()> {
pub fn set_token(&self, user_id: &UserId, device_id: &DeviceId, token: &str) -> Result<()> {
let mut userdeviceid = user_id.to_string().as_bytes().to_vec();
userdeviceid.push(0xff);
userdeviceid.extend_from_slice(device_id.as_bytes());
@ -311,6 +311,7 @@ impl Users {
Ok(())
}
#[tracing::instrument(skip(self))]
pub fn last_one_time_keys_update(&self, user_id: &UserId) -> Result<u64> {
self.userid_lastonetimekeyupdate
.get(&user_id.to_string().as_bytes())?
@ -364,6 +365,7 @@ impl Users {
.transpose()
}
#[tracing::instrument(skip(self))]
pub fn count_one_time_keys(
&self,
user_id: &UserId,
@ -563,6 +565,7 @@ impl Users {
Ok(())
}
#[tracing::instrument(skip(self))]
pub fn keys_changed(
&self,
user_or_room_id: &str,
@ -738,6 +741,7 @@ impl Users {
Ok(())
}
#[tracing::instrument(skip(self))]
pub fn get_to_device_events(
&self,
user_id: &UserId,
@ -760,6 +764,7 @@ impl Users {
Ok(events)
}
#[tracing::instrument(skip(self))]
pub fn remove_to_device_events(
&self,
user_id: &UserId,