admin command to delete all remote media within the past x time

Signed-off-by: strawberry <strawberry@puppygock.gay>

src/database/key_value/media.rs

@@ -52,12 +52,11 @@ impl service::media::Data for KeyValueDatabase {
             debug!("Deleting key: {:?}", key);
             self.mediaid_file.remove(&key)?;
         }
-        //return Err(Error::bad_database("Media not found."));
 
         Ok(())
     }
 
-    /// Searches for all files with the given MXC (e.g. thumbnail and original image)
+    /// Searches for all files with the given MXC
     fn search_mxc_metadata_prefix(&self, mxc: String) -> Result<Vec<Vec<u8>>> {
         debug!("MXC URI: {:?}", mxc);
 
@@ -126,6 +125,17 @@ impl service::media::Data for KeyValueDatabase {
         Ok((content_disposition, content_type, key))
     }
 
+    /// Gets all the media keys in our database (this includes all the metadata associated with it such as width, height, content-type, etc)
+    fn get_all_media_keys(&self) -> Result<Vec<Vec<u8>>> {
+        let mut keys: Vec<Vec<u8>> = vec![];
+
+        for (key, _) in self.mediaid_file.iter() {
+            keys.push(key);
+        }
+
+        Ok(keys)
+    }
+
     fn remove_url_preview(&self, url: &str) -> Result<()> {
         self.url_previews.remove(url.as_bytes())
     }

src/service/admin/mod.rs

@@ -96,6 +96,12 @@ enum MediaCommand {
 
     /// - Deletes a codeblock list of MXC URLs from our database and on the filesystem
     DeleteList,
+
+    // - Deletes all remote media in the last <x> amount of time using filesystem metadata first created at date.
+    DeletePastRemoteMedia {
+        /// - The duration (at or after), e.g. "5m" to delete all media in the past 5 minutes
+        duration: String,
+    },
 }
 
 #[cfg_attr(test, derive(Debug))]
@@ -785,6 +791,17 @@ impl Service {
                         ));
                     }
                 }
+                MediaCommand::DeletePastRemoteMedia { duration } => {
+                    let deleted_count = services()
+                        .media
+                        .delete_all_remote_media_at_after_time(duration)
+                        .await?;
+
+                    return Ok(RoomMessageEventContent::text_plain(format!(
+                        "Deleted {} total files.",
+                        deleted_count
+                    )));
+                }
             },
             AdminCommand::Users(command) => match command {
                 UserCommand::List => match services().users.list_local_users() {

src/service/media/data.rs

@@ -22,6 +22,8 @@ pub trait Data: Send + Sync {
 
     fn search_mxc_metadata_prefix(&self, mxc: String) -> Result<Vec<Vec<u8>>>;
 
+    fn get_all_media_keys(&self) -> Result<Vec<Vec<u8>>>;
+
     fn remove_url_preview(&self, url: &str) -> Result<()>;
 
     fn set_url_preview(

src/service/media/mod.rs

@@ -7,14 +7,15 @@ use std::{
 };
 
 pub(crate) use data::Data;
+use ruma::OwnedMxcUri;
 use serde::Serialize;
 use tracing::{debug, error};
 
-use crate::{services, Error, Result};
+use crate::{services, utils, Error, Result};
 use image::imageops::FilterType;
 
 use tokio::{
-    fs::File,
+    fs::{self, File},
     io::{AsyncReadExt, AsyncWriteExt, BufReader},
     sync::Mutex,
 };
@@ -174,6 +175,112 @@ impl Service {
         }
     }
 
+    /// Deletes all remote only media files in the given at or after time/duration. Returns a u32
+    /// with the amount of media files deleted.
+    pub async fn delete_all_remote_media_at_after_time(&self, time: String) -> Result<u32> {
+        if let Ok(all_keys) = self.db.get_all_media_keys() {
+            let user_duration: SystemTime = match cyborgtime::parse_duration(&time) {
+                Ok(duration) => {
+                    debug!("Parsed duration: {:?}", duration);
+                    debug!("System time now: {:?}", SystemTime::now());
+                    SystemTime::now() - duration
+                }
+                Err(e) => {
+                    error!("Failed to parse user-specified time duration: {}", e);
+                    return Err(Error::bad_database(
+                        "Failed to parse user-specified time duration.",
+                    ));
+                }
+            };
+
+            let mut remote_mxcs: Vec<String> = vec![];
+
+            for key in all_keys {
+                debug!("Full MXC key from database: {:?}", key);
+
+                // we need to get the MXC URL from the first part of the key (the first 0xff / 255 push)
+                // this code does look kinda crazy but blame conduit for using magic keys
+                let mut parts = key.split(|&b| b == 0xff);
+                let mxc = parts
+                    .next()
+                    .map(|bytes| {
+                        utils::string_from_bytes(bytes).map_err(|e| {
+                            error!("Failed to parse MXC unicode bytes from our database: {}", e);
+                            Error::bad_database(
+                                "Failed to parse MXC unicode bytes from our database",
+                            )
+                        })
+                    })
+                    .transpose()?;
+
+                let mxc_s = match mxc {
+                    Some(mxc) => mxc,
+                    None => {
+                        return Err(Error::bad_database(
+                            "Parsed MXC URL unicode bytes from database but still is None",
+                        ));
+                    }
+                };
+
+                debug!("Parsed MXC key to URL: {}", mxc_s);
+
+                let mxc = OwnedMxcUri::from(mxc_s);
+                if mxc.server_name() == Ok(services().globals.server_name()) {
+                    debug!("Ignoring local media MXC: {}", mxc);
+                    // ignore our own MXC URLs as this would be local media.
+                    continue;
+                }
+
+                let path = if cfg!(feature = "sha256_media") {
+                    services().globals.get_media_file_new(&key)
+                } else {
+                    #[allow(deprecated)]
+                    services().globals.get_media_file(&key)
+                };
+
+                debug!("MXC path: {:?}", path);
+
+                let file_metadata = fs::metadata(path.clone()).await?;
+                debug!("File metadata: {:?}", file_metadata);
+
+                let file_created_at = file_metadata.created()?;
+                debug!("File created at: {:?}", file_created_at);
+
+                if file_created_at >= user_duration {
+                    debug!("File is within user duration, pushing to list of file paths and keys to delete.");
+                    remote_mxcs.push(mxc.to_string());
+                } else {
+                    // don't need to log this even in debug as it would be noisy
+                    continue;
+                }
+            }
+
+            debug!("Finished going through all our media in database for eligible keys to delete, checking if these are empty");
+
+            if remote_mxcs.is_empty() {
+                return Err(Error::bad_database(
+                    "Did not found any eligible MXCs to delete.",
+                ));
+            }
+
+            debug!("Deleting media now in the past \"{:?}\".", user_duration);
+
+            let mut deletion_count = 0;
+
+            for mxc in remote_mxcs {
+                debug!("Deleting MXC {mxc} from database and filesystem");
+                self.delete(mxc).await?;
+                deletion_count += 1;
+            }
+
+            Ok(deletion_count)
+        } else {
+            Err(Error::bad_database(
+                "Failed to get all our media keys (filesystem or database issue?).",
+            ))
+        }
+    }
+
     /// Returns width, height of the thumbnail and whether it should be cropped. Returns None when
     /// the server should send the original file.
     pub fn thumbnail_properties(&self, width: u32, height: u32) -> Option<(u32, u32, bool)> {