Database Refactor

combine service/users data w/ mod unit

split sliding sync related out of service/users

instrument database entry points

remove increment crap from database interface

de-wrap all database get() calls

de-wrap all database insert() calls

de-wrap all database remove() calls

refactor database interface for async streaming

add query key serializer for database

implement Debug for result handle

add query deserializer for database

add deserialization trait for option handle

start a stream utils suite

de-wrap/asyncify/type-query count_one_time_keys()

de-wrap/asyncify users count

add admin query users command suite

de-wrap/asyncify users exists

de-wrap/partially asyncify user filter related

asyncify/de-wrap users device/keys related

asyncify/de-wrap user auth/misc related

asyncify/de-wrap users blurhash

asyncify/de-wrap account_data get; merge Data into Service

partial asyncify/de-wrap uiaa; merge Data into Service

partially asyncify/de-wrap transaction_ids get; merge Data into Service

partially asyncify/de-wrap key_backups; merge Data into Service

asyncify/de-wrap pusher service getters; merge Data into Service

asyncify/de-wrap rooms alias getters/some iterators

asyncify/de-wrap rooms directory getters/iterator

partially asyncify/de-wrap rooms lazy-loading

partially asyncify/de-wrap rooms metadata

asyncify/dewrap rooms outlier

asyncify/dewrap rooms pdu_metadata

dewrap/partially asyncify rooms read receipt

de-wrap rooms search service

de-wrap/partially asyncify rooms user service

partial de-wrap rooms state_compressor

de-wrap rooms state_cache

de-wrap room state et al

de-wrap rooms timeline service

additional users device/keys related

de-wrap/asyncify sender

asyncify services

refactor database to TryFuture/TryStream

refactor services for TryFuture/TryStream

asyncify api handlers

additional asyncification for admin module

abstract stream related; support reverse streams

additional stream conversions

asyncify state-res related

Signed-off-by: Jason Volk <jason@zemos.net>

src/core/utils/algorithm.rs

@@ -1,25 +0,0 @@
-use std::cmp::Ordering;
-
-#[allow(clippy::impl_trait_in_params)]
-pub fn common_elements(
-	mut iterators: impl Iterator<Item = impl Iterator<Item = Vec<u8>>>, check_order: impl Fn(&[u8], &[u8]) -> Ordering,
-) -> Option<impl Iterator<Item = Vec<u8>>> {
-	let first_iterator = iterators.next()?;
-	let mut other_iterators = iterators.map(Iterator::peekable).collect::<Vec<_>>();
-
-	Some(first_iterator.filter(move |target| {
-		other_iterators.iter_mut().all(|it| {
-			while let Some(element) = it.peek() {
-				match check_order(element, target) {
-					Ordering::Greater => return false, // We went too far
-					Ordering::Equal => return true,    // Element is in both iters
-					Ordering::Less => {
-						// Keep searching
-						it.next();
-					},
-				}
-			}
-			false
-		})
-	}))
-}

src/core/utils/mod.rs

@@ -1,4 +1,3 @@
-pub mod algorithm;
 pub mod bytes;
 pub mod content_disposition;
 pub mod debug;
@@ -9,25 +8,30 @@ pub mod json;
 pub mod math;
 pub mod mutex_map;
 pub mod rand;
+pub mod set;
+pub mod stream;
 pub mod string;
 pub mod sys;
 mod tests;
 pub mod time;
 
+pub use ::conduit_macros::implement;
 pub use ::ctor::{ctor, dtor};
-pub use algorithm::common_elements;
-pub use bytes::{increment, u64_from_bytes, u64_from_u8, u64_from_u8x8};
-pub use conduit_macros::implement;
-pub use debug::slice_truncated as debug_slice_truncated;
-pub use hash::calculate_hash;
-pub use html::Escape as HtmlEscape;
-pub use json::{deserialize_from_str, to_canonical_object};
-pub use math::clamp;
-pub use mutex_map::{Guard as MutexMapGuard, MutexMap};
-pub use rand::string as random_string;
-pub use string::{str_from_bytes, string_from_bytes};
-pub use sys::available_parallelism;
-pub use time::now_millis as millis_since_unix_epoch;
+
+pub use self::{
+	bytes::{increment, u64_from_bytes, u64_from_u8, u64_from_u8x8},
+	debug::slice_truncated as debug_slice_truncated,
+	hash::calculate_hash,
+	html::Escape as HtmlEscape,
+	json::{deserialize_from_str, to_canonical_object},
+	math::clamp,
+	mutex_map::{Guard as MutexMapGuard, MutexMap},
+	rand::string as random_string,
+	stream::{IterStream, ReadyExt, TryReadyExt},
+	string::{str_from_bytes, string_from_bytes},
+	sys::available_parallelism,
+	time::now_millis as millis_since_unix_epoch,
+};
 
 #[inline]
 pub fn exchange<T>(state: &mut T, source: T) -> T { std::mem::replace(state, source) }

src/core/utils/set.rs

@@ -0,0 +1,47 @@
+use std::cmp::{Eq, Ord};
+
+use crate::{is_equal_to, is_less_than};
+
+/// Intersection of sets
+///
+/// Outputs the set of elements common to all input sets. Inputs do not have to
+/// be sorted. If inputs are sorted a more optimized function is available in
+/// this suite and should be used.
+pub fn intersection<Item, Iter, Iters>(mut input: Iters) -> impl Iterator<Item = Item> + Send
+where
+	Iters: Iterator<Item = Iter> + Clone + Send,
+	Iter: Iterator<Item = Item> + Send,
+	Item: Eq + Send,
+{
+	input.next().into_iter().flat_map(move |first| {
+		let input = input.clone();
+		first.filter(move |targ| {
+			input
+				.clone()
+				.all(|mut other| other.any(is_equal_to!(*targ)))
+		})
+	})
+}
+
+/// Intersection of sets
+///
+/// Outputs the set of elements common to all input sets. Inputs must be sorted.
+pub fn intersection_sorted<Item, Iter, Iters>(mut input: Iters) -> impl Iterator<Item = Item> + Send
+where
+	Iters: Iterator<Item = Iter> + Clone + Send,
+	Iter: Iterator<Item = Item> + Send,
+	Item: Eq + Ord + Send,
+{
+	input.next().into_iter().flat_map(move |first| {
+		let mut input = input.clone().collect::<Vec<_>>();
+		first.filter(move |targ| {
+			input.iter_mut().all(|it| {
+				it.by_ref()
+					.skip_while(is_less_than!(targ))
+					.peekable()
+					.peek()
+					.is_some_and(is_equal_to!(targ))
+			})
+		})
+	})
+}

src/core/utils/stream/cloned.rs

@@ -0,0 +1,20 @@
+use std::clone::Clone;
+
+use futures::{stream::Map, Stream, StreamExt};
+
+pub trait Cloned<'a, T, S>
+where
+	S: Stream<Item = &'a T>,
+	T: Clone + 'a,
+{
+	fn cloned(self) -> Map<S, fn(&T) -> T>;
+}
+
+impl<'a, T, S> Cloned<'a, T, S> for S
+where
+	S: Stream<Item = &'a T>,
+	T: Clone + 'a,
+{
+	#[inline]
+	fn cloned(self) -> Map<S, fn(&T) -> T> { self.map(Clone::clone) }
+}

src/core/utils/stream/expect.rs

@@ -0,0 +1,17 @@
+use futures::{Stream, StreamExt, TryStream};
+
+use crate::Result;
+
+pub trait TryExpect<'a, Item> {
+	fn expect_ok(self) -> impl Stream<Item = Item> + Send + 'a;
+}
+
+impl<'a, T, Item> TryExpect<'a, Item> for T
+where
+	T: Stream<Item = Result<Item>> + TryStream + Send + 'a,
+{
+	#[inline]
+	fn expect_ok(self: T) -> impl Stream<Item = Item> + Send + 'a {
+		self.map(|res| res.expect("stream expectation failure"))
+	}
+}

src/core/utils/stream/ignore.rs

@@ -0,0 +1,21 @@
+use futures::{future::ready, Stream, StreamExt, TryStream};
+
+use crate::{Error, Result};
+
+pub trait TryIgnore<'a, Item> {
+	fn ignore_err(self) -> impl Stream<Item = Item> + Send + 'a;
+
+	fn ignore_ok(self) -> impl Stream<Item = Error> + Send + 'a;
+}
+
+impl<'a, T, Item> TryIgnore<'a, Item> for T
+where
+	T: Stream<Item = Result<Item>> + TryStream + Send + 'a,
+	Item: Send + 'a,
+{
+	#[inline]
+	fn ignore_err(self: T) -> impl Stream<Item = Item> + Send + 'a { self.filter_map(|res| ready(res.ok())) }
+
+	#[inline]
+	fn ignore_ok(self: T) -> impl Stream<Item = Error> + Send + 'a { self.filter_map(|res| ready(res.err())) }
+}

src/core/utils/stream/iter_stream.rs

@@ -0,0 +1,27 @@
+use futures::{
+	stream,
+	stream::{Stream, TryStream},
+	StreamExt,
+};
+
+pub trait IterStream<I: IntoIterator + Send> {
+	/// Convert an Iterator into a Stream
+	fn stream(self) -> impl Stream<Item = <I as IntoIterator>::Item> + Send;
+
+	/// Convert an Iterator into a TryStream
+	fn try_stream(self) -> impl TryStream<Ok = <I as IntoIterator>::Item, Error = crate::Error> + Send;
+}
+
+impl<I> IterStream<I> for I
+where
+	I: IntoIterator + Send,
+	<I as IntoIterator>::IntoIter: Send,
+{
+	#[inline]
+	fn stream(self) -> impl Stream<Item = <I as IntoIterator>::Item> + Send { stream::iter(self) }
+
+	#[inline]
+	fn try_stream(self) -> impl TryStream<Ok = <I as IntoIterator>::Item, Error = crate::Error> + Send {
+		self.stream().map(Ok)
+	}
+}

src/core/utils/stream/mod.rs

@@ -0,0 +1,13 @@
+mod cloned;
+mod expect;
+mod ignore;
+mod iter_stream;
+mod ready;
+mod try_ready;
+
+pub use cloned::Cloned;
+pub use expect::TryExpect;
+pub use ignore::TryIgnore;
+pub use iter_stream::IterStream;
+pub use ready::ReadyExt;
+pub use try_ready::TryReadyExt;

src/core/utils/stream/ready.rs

@@ -0,0 +1,109 @@
+//! Synchronous combinator extensions to futures::Stream
+
+use futures::{
+	future::{ready, Ready},
+	stream::{Any, Filter, FilterMap, Fold, ForEach, SkipWhile, Stream, StreamExt, TakeWhile},
+};
+
+/// Synchronous combinators to augment futures::StreamExt. Most Stream
+/// combinators take asynchronous arguments, but often only simple predicates
+/// are required to steer a Stream like an Iterator. This suite provides a
+/// convenience to reduce boilerplate by de-cluttering non-async predicates.
+///
+/// This interface is not necessarily complete; feel free to add as-needed.
+pub trait ReadyExt<Item, S>
+where
+	S: Stream<Item = Item> + Send + ?Sized,
+	Self: Stream + Send + Sized,
+{
+	fn ready_any<F>(self, f: F) -> Any<Self, Ready<bool>, impl FnMut(S::Item) -> Ready<bool>>
+	where
+		F: Fn(S::Item) -> bool;
+
+	fn ready_filter<'a, F>(self, f: F) -> Filter<Self, Ready<bool>, impl FnMut(&S::Item) -> Ready<bool> + 'a>
+	where
+		F: Fn(&S::Item) -> bool + 'a;
+
+	fn ready_filter_map<F, U>(self, f: F) -> FilterMap<Self, Ready<Option<U>>, impl FnMut(S::Item) -> Ready<Option<U>>>
+	where
+		F: Fn(S::Item) -> Option<U>;
+
+	fn ready_fold<T, F>(self, init: T, f: F) -> Fold<Self, Ready<T>, T, impl FnMut(T, S::Item) -> Ready<T>>
+	where
+		F: Fn(T, S::Item) -> T;
+
+	fn ready_for_each<F>(self, f: F) -> ForEach<Self, Ready<()>, impl FnMut(S::Item) -> Ready<()>>
+	where
+		F: FnMut(S::Item);
+
+	fn ready_take_while<'a, F>(self, f: F) -> TakeWhile<Self, Ready<bool>, impl FnMut(&S::Item) -> Ready<bool> + 'a>
+	where
+		F: Fn(&S::Item) -> bool + 'a;
+
+	fn ready_skip_while<'a, F>(self, f: F) -> SkipWhile<Self, Ready<bool>, impl FnMut(&S::Item) -> Ready<bool> + 'a>
+	where
+		F: Fn(&S::Item) -> bool + 'a;
+}
+
+impl<Item, S> ReadyExt<Item, S> for S
+where
+	S: Stream<Item = Item> + Send + ?Sized,
+	Self: Stream + Send + Sized,
+{
+	#[inline]
+	fn ready_any<F>(self, f: F) -> Any<Self, Ready<bool>, impl FnMut(S::Item) -> Ready<bool>>
+	where
+		F: Fn(S::Item) -> bool,
+	{
+		self.any(move |t| ready(f(t)))
+	}
+
+	#[inline]
+	fn ready_filter<'a, F>(self, f: F) -> Filter<Self, Ready<bool>, impl FnMut(&S::Item) -> Ready<bool> + 'a>
+	where
+		F: Fn(&S::Item) -> bool + 'a,
+	{
+		self.filter(move |t| ready(f(t)))
+	}
+
+	#[inline]
+	fn ready_filter_map<F, U>(self, f: F) -> FilterMap<Self, Ready<Option<U>>, impl FnMut(S::Item) -> Ready<Option<U>>>
+	where
+		F: Fn(S::Item) -> Option<U>,
+	{
+		self.filter_map(move |t| ready(f(t)))
+	}
+
+	#[inline]
+	fn ready_fold<T, F>(self, init: T, f: F) -> Fold<Self, Ready<T>, T, impl FnMut(T, S::Item) -> Ready<T>>
+	where
+		F: Fn(T, S::Item) -> T,
+	{
+		self.fold(init, move |a, t| ready(f(a, t)))
+	}
+
+	#[inline]
+	#[allow(clippy::unit_arg)]
+	fn ready_for_each<F>(self, mut f: F) -> ForEach<Self, Ready<()>, impl FnMut(S::Item) -> Ready<()>>
+	where
+		F: FnMut(S::Item),
+	{
+		self.for_each(move |t| ready(f(t)))
+	}
+
+	#[inline]
+	fn ready_take_while<'a, F>(self, f: F) -> TakeWhile<Self, Ready<bool>, impl FnMut(&S::Item) -> Ready<bool> + 'a>
+	where
+		F: Fn(&S::Item) -> bool + 'a,
+	{
+		self.take_while(move |t| ready(f(t)))
+	}
+
+	#[inline]
+	fn ready_skip_while<'a, F>(self, f: F) -> SkipWhile<Self, Ready<bool>, impl FnMut(&S::Item) -> Ready<bool> + 'a>
+	where
+		F: Fn(&S::Item) -> bool + 'a,
+	{
+		self.skip_while(move |t| ready(f(t)))
+	}
+}

src/core/utils/stream/try_ready.rs

@@ -0,0 +1,35 @@
+//! Synchronous combinator extensions to futures::TryStream
+
+use futures::{
+	future::{ready, Ready},
+	stream::{AndThen, TryStream, TryStreamExt},
+};
+
+use crate::Result;
+
+/// Synchronous combinators to augment futures::TryStreamExt.
+///
+/// This interface is not necessarily complete; feel free to add as-needed.
+pub trait TryReadyExt<T, E, S>
+where
+	S: TryStream<Ok = T, Error = E, Item = Result<T, E>> + Send + ?Sized,
+	Self: TryStream + Send + Sized,
+{
+	fn ready_and_then<U, F>(self, f: F) -> AndThen<Self, Ready<Result<U, E>>, impl FnMut(S::Ok) -> Ready<Result<U, E>>>
+	where
+		F: Fn(S::Ok) -> Result<U, E>;
+}
+
+impl<T, E, S> TryReadyExt<T, E, S> for S
+where
+	S: TryStream<Ok = T, Error = E, Item = Result<T, E>> + Send + ?Sized,
+	Self: TryStream + Send + Sized,
+{
+	#[inline]
+	fn ready_and_then<U, F>(self, f: F) -> AndThen<Self, Ready<Result<U, E>>, impl FnMut(S::Ok) -> Ready<Result<U, E>>>
+	where
+		F: Fn(S::Ok) -> Result<U, E>,
+	{
+		self.and_then(move |t| ready(f(t)))
+	}
+}

src/core/utils/tests.rs

@@ -107,3 +107,133 @@ async fn mutex_map_contend() {
 	tokio::try_join!(join_b, join_a).expect("joined");
 	assert!(map.is_empty(), "Must be empty");
 }
+
+#[test]
+#[allow(clippy::iter_on_single_items, clippy::many_single_char_names)]
+fn set_intersection_none() {
+	use utils::set::intersection;
+
+	let a: [&str; 0] = [];
+	let b: [&str; 0] = [];
+	let i = [a.iter(), b.iter()];
+	let r = intersection(i.into_iter());
+	assert_eq!(r.count(), 0);
+
+	let a: [&str; 0] = [];
+	let b = ["abc", "def"];
+	let i = [a.iter(), b.iter()];
+	let r = intersection(i.into_iter());
+	assert_eq!(r.count(), 0);
+	let i = [b.iter(), a.iter()];
+	let r = intersection(i.into_iter());
+	assert_eq!(r.count(), 0);
+	let i = [a.iter()];
+	let r = intersection(i.into_iter());
+	assert_eq!(r.count(), 0);
+
+	let a = ["foo", "bar", "baz"];
+	let b = ["def", "hij", "klm", "nop"];
+	let i = [a.iter(), b.iter()];
+	let r = intersection(i.into_iter());
+	assert_eq!(r.count(), 0);
+}
+
+#[test]
+#[allow(clippy::iter_on_single_items, clippy::many_single_char_names)]
+fn set_intersection_all() {
+	use utils::set::intersection;
+
+	let a = ["foo"];
+	let b = ["foo"];
+	let i = [a.iter(), b.iter()];
+	let r = intersection(i.into_iter());
+	assert!(r.eq(["foo"].iter()));
+
+	let a = ["foo", "bar"];
+	let b = ["bar", "foo"];
+	let i = [a.iter(), b.iter()];
+	let r = intersection(i.into_iter());
+	assert!(r.eq(["foo", "bar"].iter()));
+	let i = [b.iter()];
+	let r = intersection(i.into_iter());
+	assert!(r.eq(["bar", "foo"].iter()));
+
+	let a = ["foo", "bar", "baz"];
+	let b = ["baz", "foo", "bar"];
+	let c = ["bar", "baz", "foo"];
+	let i = [a.iter(), b.iter(), c.iter()];
+	let r = intersection(i.into_iter());
+	assert!(r.eq(["foo", "bar", "baz"].iter()));
+}
+
+#[test]
+#[allow(clippy::iter_on_single_items, clippy::many_single_char_names)]
+fn set_intersection_some() {
+	use utils::set::intersection;
+
+	let a = ["foo"];
+	let b = ["bar", "foo"];
+	let i = [a.iter(), b.iter()];
+	let r = intersection(i.into_iter());
+	assert!(r.eq(["foo"].iter()));
+	let i = [b.iter(), a.iter()];
+	let r = intersection(i.into_iter());
+	assert!(r.eq(["foo"].iter()));
+
+	let a = ["abcdef", "foo", "hijkl", "abc"];
+	let b = ["hij", "bar", "baz", "abc", "foo"];
+	let c = ["abc", "xyz", "foo", "ghi"];
+	let i = [a.iter(), b.iter(), c.iter()];
+	let r = intersection(i.into_iter());
+	assert!(r.eq(["foo", "abc"].iter()));
+}
+
+#[test]
+#[allow(clippy::iter_on_single_items, clippy::many_single_char_names)]
+fn set_intersection_sorted_some() {
+	use utils::set::intersection_sorted;
+
+	let a = ["bar"];
+	let b = ["bar", "foo"];
+	let i = [a.iter(), b.iter()];
+	let r = intersection_sorted(i.into_iter());
+	assert!(r.eq(["bar"].iter()));
+	let i = [b.iter(), a.iter()];
+	let r = intersection_sorted(i.into_iter());
+	assert!(r.eq(["bar"].iter()));
+
+	let a = ["aaa", "ccc", "eee", "ggg"];
+	let b = ["aaa", "bbb", "ccc", "ddd", "eee"];
+	let c = ["bbb", "ccc", "eee", "fff"];
+	let i = [a.iter(), b.iter(), c.iter()];
+	let r = intersection_sorted(i.into_iter());
+	assert!(r.eq(["ccc", "eee"].iter()));
+}
+
+#[test]
+#[allow(clippy::iter_on_single_items, clippy::many_single_char_names)]
+fn set_intersection_sorted_all() {
+	use utils::set::intersection_sorted;
+
+	let a = ["foo"];
+	let b = ["foo"];
+	let i = [a.iter(), b.iter()];
+	let r = intersection_sorted(i.into_iter());
+	assert!(r.eq(["foo"].iter()));
+
+	let a = ["bar", "foo"];
+	let b = ["bar", "foo"];
+	let i = [a.iter(), b.iter()];
+	let r = intersection_sorted(i.into_iter());
+	assert!(r.eq(["bar", "foo"].iter()));
+	let i = [b.iter()];
+	let r = intersection_sorted(i.into_iter());
+	assert!(r.eq(["bar", "foo"].iter()));
+
+	let a = ["bar", "baz", "foo"];
+	let b = ["bar", "baz", "foo"];
+	let c = ["bar", "baz", "foo"];
+	let i = [a.iter(), b.iter(), c.iter()];
+	let r = intersection_sorted(i.into_iter());
+	assert!(r.eq(["bar", "baz", "foo"].iter()));
+}