magmaus3/continuwuity
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor for structured Mxc type

Browse source 
Signed-off-by: Jason Volk <jason@zemos.net>
This commit is contained in:
Jason Volk 2024-08-16 01:13:20 +00:00
parent 54e6a41404
commit 152ae705a0
10 changed files with 133 additions and 120 deletions
Download patch file Download diff file Expand all files Collapse all files

59
src/service/media/data.rs
View file

@ -1,8 +1,12 @@
use std::sync::Arc;
use conduit::{debug, debug_info, utils::string_from_bytes, Error, Result};
use conduit::{
debug, debug_info, trace,
utils::{str_from_bytes, string_from_bytes},
Err, Error, Result,
};
use database::{Database, Map};
use ruma::{api::client::error::ErrorKind, http_headers::ContentDisposition};
use ruma::{api::client::error::ErrorKind, http_headers::ContentDisposition, Mxc, UserId};
use super::preview::UrlPreviewData;
@ -29,10 +33,12 @@ impl Data {
}
pub(super) fn create_file_metadata(
&self, sender_user: Option<&str>, mxc: &str, width: u32, height: u32,
&self, mxc: &Mxc<'_>, user: Option<&UserId>, width: u32, height: u32,
content_disposition: Option<&ContentDisposition>, content_type: Option<&str>,
) -> Result<Vec<u8>> {
let mut key = mxc.as_bytes().to_vec();
let mut key: Vec<u8> = Vec::new();
key.extend_from_slice(mxc.server_name.as_bytes());
key.extend_from_slice(mxc.media_id.as_bytes());
key.push(0xFF);
key.extend_from_slice(&width.to_be_bytes());
key.extend_from_slice(&height.to_be_bytes());
@ -53,8 +59,10 @@ impl Data {
self.mediaid_file.insert(&key, &[])?;
if let Some(user) = sender_user {
let key = mxc.as_bytes().to_vec();
if let Some(user) = user {
let mut key: Vec<u8> = Vec::new();
key.extend_from_slice(mxc.server_name.as_bytes());
key.extend_from_slice(mxc.media_id.as_bytes());
let user = user.as_bytes().to_vec();
self.mediaid_user.insert(&key, &user)?;
}
@ -62,22 +70,23 @@ impl Data {
Ok(key)
}
pub(super) fn delete_file_mxc(&self, mxc: &str) -> Result<()> {
debug!("MXC URI: {:?}", mxc);
pub(super) fn delete_file_mxc(&self, mxc: &Mxc<'_>) -> Result<()> {
debug!("MXC URI: {mxc}");
let mut prefix = mxc.as_bytes().to_vec();
let mut prefix: Vec<u8> = Vec::new();
prefix.extend_from_slice(mxc.server_name.as_bytes());
prefix.extend_from_slice(mxc.media_id.as_bytes());
prefix.push(0xFF);
debug!("MXC db prefix: {prefix:?}");
for (key, _) in self.mediaid_file.scan_prefix(prefix) {
trace!("MXC db prefix: {prefix:?}");
for (key, _) in self.mediaid_file.scan_prefix(prefix.clone()) {
debug!("Deleting key: {:?}", key);
self.mediaid_file.remove(&key)?;
}
for (key, value) in self.mediaid_user.scan_prefix(mxc.as_bytes().to_vec()) {
if key == mxc.as_bytes().to_vec() {
let user = string_from_bytes(&value).unwrap_or_default();
for (key, value) in self.mediaid_user.scan_prefix(prefix.clone()) {
if key.starts_with(&prefix) {
let user = str_from_bytes(&value).unwrap_or_default();
debug_info!("Deleting key \"{key:?}\" which was uploaded by user {user}");
self.mediaid_user.remove(&key)?;
@ -88,10 +97,12 @@ impl Data {
}
/// Searches for all files with the given MXC
pub(super) fn search_mxc_metadata_prefix(&self, mxc: &str) -> Result<Vec<Vec<u8>>> {
debug!("MXC URI: {:?}", mxc);
pub(super) fn search_mxc_metadata_prefix(&self, mxc: &Mxc<'_>) -> Result<Vec<Vec<u8>>> {
debug!("MXC URI: {mxc}");
let mut prefix = mxc.as_bytes().to_vec();
let mut prefix: Vec<u8> = Vec::new();
prefix.extend_from_slice(mxc.server_name.as_bytes());
prefix.extend_from_slice(mxc.media_id.as_bytes());
prefix.push(0xFF);
let keys: Vec<Vec<u8>> = self
@ -101,18 +112,18 @@ impl Data {
.collect();
if keys.is_empty() {
return Err(Error::bad_database(
"Failed to find any keys in database with the provided MXC.",
));
return Err!(Database("Failed to find any keys in database for `{mxc}`",));
}
debug!("Got the following keys: {:?}", keys);
debug!("Got the following keys: {keys:?}");
Ok(keys)
}
pub(super) fn search_file_metadata(&self, mxc: &str, width: u32, height: u32) -> Result<Metadata> {
let mut prefix = mxc.as_bytes().to_vec();
pub(super) fn search_file_metadata(&self, mxc: &Mxc<'_>, width: u32, height: u32) -> Result<Metadata> {
let mut prefix: Vec<u8> = Vec::new();
prefix.extend_from_slice(mxc.server_name.as_bytes());
prefix.extend_from_slice(mxc.media_id.as_bytes());
prefix.push(0xFF);
prefix.extend_from_slice(&width.to_be_bytes());
prefix.extend_from_slice(&height.to_be_bytes());

19
src/service/media/mod.rs
View file

@ -10,7 +10,7 @@ use async_trait::async_trait;
use base64::{engine::general_purpose, Engine as _};
use conduit::{debug, debug_error, err, error, trace, utils, utils::MutexMap, Err, Result, Server};
use data::{Data, Metadata};
use ruma::{http_headers::ContentDisposition, OwnedMxcUri, OwnedUserId};
use ruma::{http_headers::ContentDisposition, Mxc, OwnedMxcUri, UserId};
use tokio::{
fs,
io::{AsyncReadExt, AsyncWriteExt, BufReader},
@ -68,17 +68,13 @@ impl crate::Service for Service {
impl Service {
/// Uploads a file.
pub async fn create(
&self, sender_user: Option<OwnedUserId>, mxc: &str, content_disposition: Option<&ContentDisposition>,
&self, mxc: &Mxc<'_>, user: Option<&UserId>, content_disposition: Option<&ContentDisposition>,
content_type: Option<&str>, file: &[u8],
) -> Result<()> {
// Width, Height = 0 if it's not a thumbnail
let key = if let Some(user) = sender_user {
self.db
.create_file_metadata(Some(user.as_str()), mxc, 0, 0, content_disposition, content_type)?
} else {
self.db
.create_file_metadata(None, mxc, 0, 0, content_disposition, content_type)?
};
let key = self
.db
.create_file_metadata(mxc, user, 0, 0, content_disposition, content_type)?;
//TODO: Dangling metadata in database if creation fails
let mut f = self.create_media_file(&key).await?;
@ -88,7 +84,7 @@ impl Service {
}
/// Deletes a file in the database and from the media directory via an MXC
pub async fn delete(&self, mxc: &str) -> Result<()> {
pub async fn delete(&self, mxc: &Mxc<'_>) -> Result<()> {
if let Ok(keys) = self.db.search_mxc_metadata_prefix(mxc) {
for key in keys {
trace!(?mxc, ?key, "Deleting from filesystem");
@ -111,7 +107,7 @@ impl Service {
}
/// Downloads a file.
pub async fn get(&self, mxc: &str) -> Result<Option<FileMeta>> {
pub async fn get(&self, mxc: &Mxc<'_>) -> Result<Option<FileMeta>> {
if let Ok(Metadata {
content_disposition,
content_type,
@ -213,6 +209,7 @@ impl Service {
debug!("Deleting media now in the past {user_duration:?}.");
let mut deletion_count: usize = 0;
for mxc in remote_mxcs {
let mxc: Mxc<'_> = mxc.as_str().try_into()?;
debug!("Deleting MXC {mxc} from database and filesystem");
self.delete(&mxc).await?;
deletion_count = deletion_count.saturating_add(1);

14
src/service/media/preview.rs
View file

@ -4,6 +4,7 @@ use conduit::{debug, utils, warn, Err, Result};
use conduit_core::implement;
use image::ImageReader as ImgReader;
use ipaddress::IPAddress;
use ruma::Mxc;
use serde::Serialize;
use url::Url;
use webpage::HTML;
@ -44,13 +45,12 @@ pub async fn set_url_preview(&self, url: &str, data: &UrlPreviewData) -> Result<
pub async fn download_image(&self, url: &str) -> Result<UrlPreviewData> {
let client = &self.services.client.url_preview;
let image = client.get(url).send().await?.bytes().await?;
let mxc = format!(
"mxc://{}/{}",
self.services.globals.server_name(),
utils::random_string(MXC_LENGTH)
);
let mxc = Mxc {
server_name: self.services.globals.server_name(),
media_id: &utils::random_string(MXC_LENGTH),
};
self.create(None, &mxc, None, None, &image).await?;
self.create(&mxc, None, None, None, &image).await?;
let (width, height) = match ImgReader::new(Cursor::new(&image)).with_guessed_format() {
Err(_) => (None, None),
@ -61,7 +61,7 @@ pub async fn download_image(&self, url: &str) -> Result<UrlPreviewData> {
};
Ok(UrlPreviewData {
image: Some(mxc),
image: Some(mxc.to_string()),
image_size: Some(image.len()),
image_width: width,
image_height: height,

47
src/service/media/remote.rs
View file

@ -1,25 +1,25 @@
use std::time::Duration;
use conduit::{debug_warn, err, implement, utils::content_disposition::make_content_disposition, Err, Error, Result};
use ruma::{
api::client::media::{get_content, get_content_thumbnail},
ServerName,
};
use ruma::{api::client::media, Mxc};
#[implement(super::Service)]
#[allow(deprecated)]
pub async fn fetch_remote_thumbnail(
&self, mxc: &str, body: &get_content_thumbnail::v3::Request,
) -> Result<get_content_thumbnail::v3::Response> {
let server_name = &body.server_name;
self.check_fetch_authorized(mxc, server_name)?;
pub async fn fetch_remote_thumbnail_legacy(
&self, body: &media::get_content_thumbnail::v3::Request,
) -> Result<media::get_content_thumbnail::v3::Response> {
let mxc = Mxc {
server_name: &body.server_name,
media_id: &body.media_id,
};
self.check_fetch_authorized(&mxc)?;
let reponse = self
.services
.sending
.send_federation_request(
server_name,
get_content_thumbnail::v3::Request {
mxc.server_name,
media::get_content_thumbnail::v3::Request {
allow_remote: body.allow_remote,
height: body.height,
width: body.width,
@ -34,8 +34,8 @@ pub async fn fetch_remote_thumbnail(
.await?;
self.upload_thumbnail(
&mxc,
None,
mxc,
None,
reponse.content_type.as_deref(),
body.width
@ -53,20 +53,19 @@ pub async fn fetch_remote_thumbnail(
#[implement(super::Service)]
#[allow(deprecated)]
pub async fn fetch_remote_content(
&self, mxc: &str, server_name: &ServerName, media_id: String, allow_redirect: bool, timeout_ms: Duration,
) -> Result<get_content::v3::Response, Error> {
self.check_fetch_authorized(mxc, server_name)?;
pub async fn fetch_remote_content_legacy(
&self, mxc: &Mxc<'_>, allow_redirect: bool, timeout_ms: Duration,
) -> Result<media::get_content::v3::Response, Error> {
self.check_fetch_authorized(mxc)?;
let response = self
.services
.sending
.send_federation_request(
server_name,
get_content::v3::Request {
mxc.server_name,
media::get_content::v3::Request {
allow_remote: true,
server_name: server_name.to_owned(),
media_id,
server_name: mxc.server_name.into(),
media_id: mxc.media_id.into(),
timeout_ms,
allow_redirect,
},
@ -77,8 +76,8 @@ pub async fn fetch_remote_content(
make_content_disposition(response.content_disposition.as_ref(), response.content_type.as_deref(), None);
self.create(
None,
mxc,
None,
Some(&content_disposition),
response.content_type.as_deref(),
&response.file,
@ -89,14 +88,14 @@ pub async fn fetch_remote_content(
}
#[implement(super::Service)]
fn check_fetch_authorized(&self, mxc: &str, server_name: &ServerName) -> Result<()> {
fn check_fetch_authorized(&self, mxc: &Mxc<'_>) -> Result<()> {
if self
.services
.server
.config
.prevent_media_downloads_from
.iter()
.any(|entry| entry == server_name)
.any(|entry| entry == mxc.server_name)
{
// we'll lie to the client and say the blocked server's media was not found and
// log. the client has no way of telling anyways so this is a security bonus.

20
src/service/media/thumbnail.rs
View file

@ -2,7 +2,7 @@ use std::{cmp, io::Cursor, num::Saturating as Sat};
use conduit::{checked, Result};
use image::{imageops::FilterType, DynamicImage};
use ruma::{http_headers::ContentDisposition, OwnedUserId};
use ruma::{http_headers::ContentDisposition, Mxc, UserId};
use tokio::{
fs,
io::{AsyncReadExt, AsyncWriteExt},
@ -14,16 +14,12 @@ impl super::Service {
/// Uploads or replaces a file thumbnail.
#[allow(clippy::too_many_arguments)]
pub async fn upload_thumbnail(
&self, sender_user: Option<OwnedUserId>, mxc: &str, content_disposition: Option<&ContentDisposition>,
&self, mxc: &Mxc<'_>, user: Option<&UserId>, content_disposition: Option<&ContentDisposition>,
content_type: Option<&str>, width: u32, height: u32, file: &[u8],
) -> Result<()> {
let key = if let Some(user) = sender_user {
self.db
.create_file_metadata(Some(user.as_str()), mxc, width, height, content_disposition, content_type)?
} else {
self.db
.create_file_metadata(None, mxc, width, height, content_disposition, content_type)?
};
let key = self
.db
.create_file_metadata(mxc, user, width, height, content_disposition, content_type)?;
//TODO: Dangling metadata in database if creation fails
let mut f = self.create_media_file(&key).await?;
@ -46,7 +42,7 @@ impl super::Service {
/// For width,height <= 96 the server uses another thumbnailing algorithm
/// which crops the image afterwards.
#[tracing::instrument(skip(self), name = "thumbnail", level = "debug")]
pub async fn get_thumbnail(&self, mxc: &str, width: u32, height: u32) -> Result<Option<FileMeta>> {
pub async fn get_thumbnail(&self, mxc: &Mxc<'_>, width: u32, height: u32) -> Result<Option<FileMeta>> {
// 0, 0 because that's the original file
let (width, height, crop) = thumbnail_properties(width, height).unwrap_or((0, 0, false));
@ -76,7 +72,7 @@ impl super::Service {
/// Generate a thumbnail
#[tracing::instrument(skip(self), name = "generate", level = "debug")]
async fn get_thumbnail_generate(
&self, mxc: &str, width: u32, height: u32, crop: bool, data: Metadata,
&self, mxc: &Mxc<'_>, width: u32, height: u32, crop: bool, data: Metadata,
) -> Result<Option<FileMeta>> {
let mut content = Vec::new();
let path = self.get_media_file(&data.key);
@ -100,8 +96,8 @@ impl super::Service {
// Save thumbnail in database so we don't have to generate it again next time
let thumbnail_key = self.db.create_file_metadata(
None,
mxc,
None,
width,
height,
data.content_disposition.as_ref(),