Database Refactor

combine service/users data w/ mod unit

split sliding sync related out of service/users

instrument database entry points

remove increment crap from database interface

de-wrap all database get() calls

de-wrap all database insert() calls

de-wrap all database remove() calls

refactor database interface for async streaming

add query key serializer for database

implement Debug for result handle

add query deserializer for database

add deserialization trait for option handle

start a stream utils suite

de-wrap/asyncify/type-query count_one_time_keys()

de-wrap/asyncify users count

add admin query users command suite

de-wrap/asyncify users exists

de-wrap/partially asyncify user filter related

asyncify/de-wrap users device/keys related

asyncify/de-wrap user auth/misc related

asyncify/de-wrap users blurhash

asyncify/de-wrap account_data get; merge Data into Service

partial asyncify/de-wrap uiaa; merge Data into Service

partially asyncify/de-wrap transaction_ids get; merge Data into Service

partially asyncify/de-wrap key_backups; merge Data into Service

asyncify/de-wrap pusher service getters; merge Data into Service

asyncify/de-wrap rooms alias getters/some iterators

asyncify/de-wrap rooms directory getters/iterator

partially asyncify/de-wrap rooms lazy-loading

partially asyncify/de-wrap rooms metadata

asyncify/dewrap rooms outlier

asyncify/dewrap rooms pdu_metadata

dewrap/partially asyncify rooms read receipt

de-wrap rooms search service

de-wrap/partially asyncify rooms user service

partial de-wrap rooms state_compressor

de-wrap rooms state_cache

de-wrap room state et al

de-wrap rooms timeline service

additional users device/keys related

de-wrap/asyncify sender

asyncify services

refactor database to TryFuture/TryStream

refactor services for TryFuture/TryStream

asyncify api handlers

additional asyncification for admin module

abstract stream related; support reverse streams

additional stream conversions

asyncify state-res related

Signed-off-by: Jason Volk <jason@zemos.net>

src/service/media/data.rs

@@ -2,10 +2,11 @@ use std::sync::Arc;
 
 use conduit::{
 	debug, debug_info, trace,
-	utils::{str_from_bytes, string_from_bytes},
+	utils::{str_from_bytes, stream::TryIgnore, string_from_bytes, ReadyExt},
 	Err, Error, Result,
 };
 use database::{Database, Map};
+use futures::StreamExt;
 use ruma::{api::client::error::ErrorKind, http_headers::ContentDisposition, Mxc, OwnedMxcUri, UserId};
 
 use super::{preview::UrlPreviewData, thumbnail::Dim};
@@ -59,7 +60,7 @@ impl Data {
 				.unwrap_or_default(),
 		);
 
-		self.mediaid_file.insert(&key, &[])?;
+		self.mediaid_file.insert(&key, &[]);
 
 		if let Some(user) = user {
 			let mut key: Vec<u8> = Vec::new();
@@ -68,13 +69,13 @@ impl Data {
 			key.extend_from_slice(b"/");
 			key.extend_from_slice(mxc.media_id.as_bytes());
 			let user = user.as_bytes().to_vec();
-			self.mediaid_user.insert(&key, &user)?;
+			self.mediaid_user.insert(&key, &user);
 		}
 
 		Ok(key)
 	}
 
-	pub(super) fn delete_file_mxc(&self, mxc: &Mxc<'_>) -> Result<()> {
+	pub(super) async fn delete_file_mxc(&self, mxc: &Mxc<'_>) {
 		debug!("MXC URI: {mxc}");
 
 		let mut prefix: Vec<u8> = Vec::new();
@@ -85,25 +86,31 @@ impl Data {
 		prefix.push(0xFF);
 
 		trace!("MXC db prefix: {prefix:?}");
-		for (key, _) in self.mediaid_file.scan_prefix(prefix.clone()) {
-			debug!("Deleting key: {:?}", key);
-			self.mediaid_file.remove(&key)?;
-		}
+		self.mediaid_file
+			.raw_keys_prefix(&prefix)
+			.ignore_err()
+			.ready_for_each(|key| {
+				debug!("Deleting key: {:?}", key);
+				self.mediaid_file.remove(key);
+			})
+			.await;
 
-		for (key, value) in self.mediaid_user.scan_prefix(prefix.clone()) {
-			if key.starts_with(&prefix) {
-				let user = str_from_bytes(&value).unwrap_or_default();
+		self.mediaid_user
+			.raw_stream_prefix(&prefix)
+			.ignore_err()
+			.ready_for_each(|(key, val)| {
+				if key.starts_with(&prefix) {
+					let user = str_from_bytes(val).unwrap_or_default();
+					debug_info!("Deleting key {key:?} which was uploaded by user {user}");
 
-				debug_info!("Deleting key \"{key:?}\" which was uploaded by user {user}");
-				self.mediaid_user.remove(&key)?;
-			}
-		}
-
-		Ok(())
+					self.mediaid_user.remove(key);
+				}
+			})
+			.await;
 	}
 
 	/// Searches for all files with the given MXC
-	pub(super) fn search_mxc_metadata_prefix(&self, mxc: &Mxc<'_>) -> Result<Vec<Vec<u8>>> {
+	pub(super) async fn search_mxc_metadata_prefix(&self, mxc: &Mxc<'_>) -> Result<Vec<Vec<u8>>> {
 		debug!("MXC URI: {mxc}");
 
 		let mut prefix: Vec<u8> = Vec::new();
@@ -115,9 +122,10 @@ impl Data {
 
 		let keys: Vec<Vec<u8>> = self
 			.mediaid_file
-			.scan_prefix(prefix)
-			.map(|(key, _)| key)
-			.collect();
+			.keys_prefix_raw(&prefix)
+			.ignore_err()
+			.collect()
+			.await;
 
 		if keys.is_empty() {
 			return Err!(Database("Failed to find any keys in database for `{mxc}`",));
@@ -128,7 +136,7 @@ impl Data {
 		Ok(keys)
 	}
 
-	pub(super) fn search_file_metadata(&self, mxc: &Mxc<'_>, dim: &Dim) -> Result<Metadata> {
+	pub(super) async fn search_file_metadata(&self, mxc: &Mxc<'_>, dim: &Dim) -> Result<Metadata> {
 		let mut prefix: Vec<u8> = Vec::new();
 		prefix.extend_from_slice(b"mxc://");
 		prefix.extend_from_slice(mxc.server_name.as_bytes());
@@ -139,10 +147,13 @@ impl Data {
 		prefix.extend_from_slice(&dim.height.to_be_bytes());
 		prefix.push(0xFF);
 
-		let (key, _) = self
+		let key = self
 			.mediaid_file
-			.scan_prefix(prefix)
+			.raw_keys_prefix(&prefix)
+			.ignore_err()
+			.map(ToOwned::to_owned)
 			.next()
+			.await
 			.ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Media not found"))?;
 
 		let mut parts = key.rsplit(|&b| b == 0xFF);
@@ -177,28 +188,31 @@ impl Data {
 	}
 
 	/// Gets all the MXCs associated with a user
-	pub(super) fn get_all_user_mxcs(&self, user_id: &UserId) -> Vec<OwnedMxcUri> {
-		let user_id = user_id.as_bytes().to_vec();
-
+	pub(super) async fn get_all_user_mxcs(&self, user_id: &UserId) -> Vec<OwnedMxcUri> {
 		self.mediaid_user
-			.iter()
-			.filter_map(|(key, user)| {
-				if *user == user_id {
-					let mxc_s = string_from_bytes(&key).ok()?;
-					Some(OwnedMxcUri::from(mxc_s))
-				} else {
-					None
-				}
-			})
+			.stream()
+			.ignore_err()
+			.ready_filter_map(|(key, user): (&str, &UserId)| (user == user_id).then(|| key.into()))
 			.collect()
+			.await
 	}
 
 	/// Gets all the media keys in our database (this includes all the metadata
 	/// associated with it such as width, height, content-type, etc)
-	pub(crate) fn get_all_media_keys(&self) -> Vec<Vec<u8>> { self.mediaid_file.iter().map(|(key, _)| key).collect() }
+	pub(crate) async fn get_all_media_keys(&self) -> Vec<Vec<u8>> {
+		self.mediaid_file
+			.raw_keys()
+			.ignore_err()
+			.map(<[u8]>::to_vec)
+			.collect()
+			.await
+	}
 
 	#[inline]
-	pub(super) fn remove_url_preview(&self, url: &str) -> Result<()> { self.url_previews.remove(url.as_bytes()) }
+	pub(super) fn remove_url_preview(&self, url: &str) -> Result<()> {
+		self.url_previews.remove(url.as_bytes());
+		Ok(())
+	}
 
 	pub(super) fn set_url_preview(
 		&self, url: &str, data: &UrlPreviewData, timestamp: std::time::Duration,
@@ -233,11 +247,13 @@ impl Data {
 		value.push(0xFF);
 		value.extend_from_slice(&data.image_height.unwrap_or(0).to_be_bytes());
 
-		self.url_previews.insert(url.as_bytes(), &value)
+		self.url_previews.insert(url.as_bytes(), &value);
+
+		Ok(())
 	}
 
-	pub(super) fn get_url_preview(&self, url: &str) -> Option<UrlPreviewData> {
-		let values = self.url_previews.get(url.as_bytes()).ok()??;
+	pub(super) async fn get_url_preview(&self, url: &str) -> Result<UrlPreviewData> {
+		let values = self.url_previews.qry(url).await?;
 
 		let mut values = values.split(|&b| b == 0xFF);
 
@@ -291,7 +307,7 @@ impl Data {
 			x => x,
 		};
 
-		Some(UrlPreviewData {
+		Ok(UrlPreviewData {
 			title,
 			description,
 			image,

src/service/media/migrations.rs

@@ -7,7 +7,11 @@ use std::{
 	time::Instant,
 };
 
-use conduit::{debug, debug_info, debug_warn, error, info, warn, Config, Result};
+use conduit::{
+	debug, debug_info, debug_warn, error, info,
+	utils::{stream::TryIgnore, ReadyExt},
+	warn, Config, Result,
+};
 
 use crate::{globals, Services};
 
@@ -23,12 +27,17 @@ pub(crate) async fn migrate_sha256_media(services: &Services) -> Result<()> {
 
 	// Move old media files to new names
 	let mut changes = Vec::<(PathBuf, PathBuf)>::new();
-	for (key, _) in mediaid_file.iter() {
-		let old = services.media.get_media_file_b64(&key);
-		let new = services.media.get_media_file_sha256(&key);
-		debug!(?key, ?old, ?new, num = changes.len(), "change");
-		changes.push((old, new));
-	}
+	mediaid_file
+		.raw_keys()
+		.ignore_err()
+		.ready_for_each(|key| {
+			let old = services.media.get_media_file_b64(key);
+			let new = services.media.get_media_file_sha256(key);
+			debug!(?key, ?old, ?new, num = changes.len(), "change");
+			changes.push((old, new));
+		})
+		.await;
+
 	// move the file to the new location
 	for (old_path, path) in changes {
 		if old_path.exists() {
@@ -41,11 +50,11 @@ pub(crate) async fn migrate_sha256_media(services: &Services) -> Result<()> {
 
 	// Apply fix from when sha256_media was backward-incompat and bumped the schema
 	// version from 13 to 14. For users satisfying these conditions we can go back.
-	if services.globals.db.database_version()? == 14 && globals::migrations::DATABASE_VERSION == 13 {
+	if services.globals.db.database_version().await == 14 && globals::migrations::DATABASE_VERSION == 13 {
 		services.globals.db.bump_database_version(13)?;
 	}
 
-	db["global"].insert(b"feat_sha256_media", &[])?;
+	db["global"].insert(b"feat_sha256_media", &[]);
 	info!("Finished applying sha256_media");
 	Ok(())
 }
@@ -71,7 +80,7 @@ pub(crate) async fn checkup_sha256_media(services: &Services) -> Result<()> {
 		.filter_map(|ent| ent.map_or(None, |ent| Some(ent.path().into_os_string())))
 		.collect();
 
-	for key in media.db.get_all_media_keys() {
+	for key in media.db.get_all_media_keys().await {
 		let new_path = media.get_media_file_sha256(&key).into_os_string();
 		let old_path = media.get_media_file_b64(&key).into_os_string();
 		if let Err(e) = handle_media_check(&dbs, config, &files, &key, &new_path, &old_path).await {
@@ -112,8 +121,8 @@ async fn handle_media_check(
 			"Media is missing at all paths. Removing from database..."
 		);
 
-		mediaid_file.remove(key)?;
-		mediaid_user.remove(key)?;
+		mediaid_file.remove(key);
+		mediaid_user.remove(key);
 	}
 
 	if config.media_compat_file_link && !old_exists && new_exists {

src/service/media/mod.rs

@@ -97,7 +97,7 @@ impl Service {
 
 	/// Deletes a file in the database and from the media directory via an MXC
 	pub async fn delete(&self, mxc: &Mxc<'_>) -> Result<()> {
-		if let Ok(keys) = self.db.search_mxc_metadata_prefix(mxc) {
+		if let Ok(keys) = self.db.search_mxc_metadata_prefix(mxc).await {
 			for key in keys {
 				trace!(?mxc, "MXC Key: {key:?}");
 				debug_info!(?mxc, "Deleting from filesystem");
@@ -107,7 +107,7 @@ impl Service {
 				}
 
 				debug_info!(?mxc, "Deleting from database");
-				_ = self.db.delete_file_mxc(mxc);
+				self.db.delete_file_mxc(mxc).await;
 			}
 
 			Ok(())
@@ -120,7 +120,7 @@ impl Service {
 	///
 	/// currently, this is only practical for local users
 	pub async fn delete_from_user(&self, user: &UserId) -> Result<usize> {
-		let mxcs = self.db.get_all_user_mxcs(user);
+		let mxcs = self.db.get_all_user_mxcs(user).await;
 		let mut deletion_count: usize = 0;
 
 		for mxc in mxcs {
@@ -150,7 +150,7 @@ impl Service {
 			content_disposition,
 			content_type,
 			key,
-		}) = self.db.search_file_metadata(mxc, &Dim::default())
+		}) = self.db.search_file_metadata(mxc, &Dim::default()).await
 		{
 			let mut content = Vec::new();
 			let path = self.get_media_file(&key);
@@ -170,7 +170,7 @@ impl Service {
 
 	/// Gets all the MXC URIs in our media database
 	pub async fn get_all_mxcs(&self) -> Result<Vec<OwnedMxcUri>> {
-		let all_keys = self.db.get_all_media_keys();
+		let all_keys = self.db.get_all_media_keys().await;
 
 		let mut mxcs = Vec::with_capacity(all_keys.len());
 
@@ -209,7 +209,7 @@ impl Service {
 	pub async fn delete_all_remote_media_at_after_time(
 		&self, time: SystemTime, before: bool, after: bool, yes_i_want_to_delete_local_media: bool,
 	) -> Result<usize> {
-		let all_keys = self.db.get_all_media_keys();
+		let all_keys = self.db.get_all_media_keys().await;
 		let mut remote_mxcs = Vec::with_capacity(all_keys.len());
 
 		for key in all_keys {
@@ -343,9 +343,10 @@ impl Service {
 	}
 
 	#[inline]
-	pub fn get_metadata(&self, mxc: &Mxc<'_>) -> Option<FileMeta> {
+	pub async fn get_metadata(&self, mxc: &Mxc<'_>) -> Option<FileMeta> {
 		self.db
 			.search_file_metadata(mxc, &Dim::default())
+			.await
 			.map(|metadata| FileMeta {
 				content_disposition: metadata.content_disposition,
 				content_type: metadata.content_type,

src/service/media/preview.rs

@@ -71,16 +71,16 @@ pub async fn download_image(&self, url: &str) -> Result<UrlPreviewData> {
 
 #[implement(Service)]
 pub async fn get_url_preview(&self, url: &str) -> Result<UrlPreviewData> {
-	if let Some(preview) = self.db.get_url_preview(url) {
+	if let Ok(preview) = self.db.get_url_preview(url).await {
 		return Ok(preview);
 	}
 
 	// ensure that only one request is made per URL
 	let _request_lock = self.url_preview_mutex.lock(url).await;
 
-	match self.db.get_url_preview(url) {
-		Some(preview) => Ok(preview),
-		None => self.request_url_preview(url).await,
+	match self.db.get_url_preview(url).await {
+		Ok(preview) => Ok(preview),
+		Err(_) => self.request_url_preview(url).await,
 	}
 }
 

src/service/media/thumbnail.rs

@@ -54,9 +54,9 @@ impl super::Service {
 		// 0, 0 because that's the original file
 		let dim = dim.normalized();
 
-		if let Ok(metadata) = self.db.search_file_metadata(mxc, &dim) {
+		if let Ok(metadata) = self.db.search_file_metadata(mxc, &dim).await {
 			self.get_thumbnail_saved(metadata).await
-		} else if let Ok(metadata) = self.db.search_file_metadata(mxc, &Dim::default()) {
+		} else if let Ok(metadata) = self.db.search_file_metadata(mxc, &Dim::default()).await {
 			self.get_thumbnail_generate(mxc, &dim, metadata).await
 		} else {
 			Ok(None)