admin command to delete all remote media within the past x time

Signed-off-by: strawberry <strawberry@puppygock.gay>

Cargo.lock

@@ -227,6 +227,12 @@ version = "0.21.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
 
+[[package]]
+name = "base64"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51"
+
 [[package]]
 name = "base64ct"
 version = "1.6.0"
@@ -412,9 +418,10 @@ dependencies = [
  "axum",
  "axum-server",
  "axum-server-dual-protocol",
- "base64",
+ "base64 0.22.0",
  "bytes",
  "clap",
+ "cyborgtime",
  "either",
  "figment",
  "futures-util",
@@ -562,6 +569,12 @@ dependencies = [
  "syn 2.0.50",
 ]
 
+[[package]]
+name = "cyborgtime"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "817fa642fb0ee7fe42e95783e00e0969927b96091bdd4b9b1af082acd943913b"
+
 [[package]]
 name = "data-encoding"
 version = "2.5.0"
@@ -894,7 +907,7 @@ version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270"
 dependencies = [
- "base64",
+ "base64 0.21.7",
  "bytes",
  "headers-core",
  "http",
@@ -1206,7 +1219,7 @@ version = "9.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5c7ea04a7c5c055c175f189b6dc6ba036fd62306b58c66c9f6389036c503a3f4"
 dependencies = [
- "base64",
+ "base64 0.21.7",
  "js-sys",
  "pem",
  "ring",
@@ -1746,7 +1759,7 @@ version = "3.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1b8fcc794035347fb64beda2d3b462595dd2753e3f268d89c5aae77e8cf2c310"
 dependencies = [
- "base64",
+ "base64 0.21.7",
  "serde",
 ]
 
@@ -2015,7 +2028,7 @@ version = "0.11.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251"
 dependencies = [
- "base64",
+ "base64 0.21.7",
  "bytes",
  "encoding_rs",
  "futures-core",
@@ -2089,7 +2102,7 @@ dependencies = [
 [[package]]
 name = "ruma"
 version = "0.9.4"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
 dependencies = [
  "assign",
  "js_int",
@@ -2108,7 +2121,7 @@ dependencies = [
 [[package]]
 name = "ruma-appservice-api"
 version = "0.9.0"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
 dependencies = [
  "js_int",
  "ruma-common",
@@ -2120,7 +2133,7 @@ dependencies = [
 [[package]]
 name = "ruma-client-api"
 version = "0.17.4"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
 dependencies = [
  "as_variant",
  "assign",
@@ -2139,10 +2152,10 @@ dependencies = [
 [[package]]
 name = "ruma-common"
 version = "0.12.1"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
 dependencies = [
  "as_variant",
- "base64",
+ "base64 0.21.7",
  "bytes",
  "form_urlencoded",
  "http",
@@ -2167,7 +2180,7 @@ dependencies = [
 [[package]]
 name = "ruma-events"
 version = "0.27.11"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
 dependencies = [
  "as_variant",
  "indexmap",
@@ -2189,7 +2202,7 @@ dependencies = [
 [[package]]
 name = "ruma-federation-api"
 version = "0.8.0"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
 dependencies = [
  "js_int",
  "ruma-common",
@@ -2201,7 +2214,7 @@ dependencies = [
 [[package]]
 name = "ruma-identifiers-validation"
 version = "0.9.3"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
 dependencies = [
  "js_int",
  "thiserror",
@@ -2210,7 +2223,7 @@ dependencies = [
 [[package]]
 name = "ruma-identity-service-api"
 version = "0.8.0"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
 dependencies = [
  "js_int",
  "ruma-common",
@@ -2220,7 +2233,7 @@ dependencies = [
 [[package]]
 name = "ruma-macros"
 version = "0.12.0"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
 dependencies = [
  "once_cell",
  "proc-macro-crate",
@@ -2235,7 +2248,7 @@ dependencies = [
 [[package]]
 name = "ruma-push-gateway-api"
 version = "0.8.0"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
 dependencies = [
  "js_int",
  "ruma-common",
@@ -2247,9 +2260,9 @@ dependencies = [
 [[package]]
 name = "ruma-signatures"
 version = "0.14.0"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
 dependencies = [
- "base64",
+ "base64 0.21.7",
  "ed25519-dalek",
  "pkcs8",
  "rand",
@@ -2263,7 +2276,7 @@ dependencies = [
 [[package]]
 name = "ruma-state-res"
 version = "0.10.0"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
 dependencies = [
  "itertools 0.11.0",
  "js_int",
@@ -2339,7 +2352,7 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c"
 dependencies = [
- "base64",
+ "base64 0.21.7",
 ]
 
 [[package]]

Cargo.toml

@@ -115,6 +115,8 @@ axum-server-dual-protocol = { version = "0.5.2", optional = true }
 # to get the client IP address of requests
 #axum-client-ip = "0.4.2"
 
+# to parse user-friendly time durations in admin commands
+cyborgtime = "2.1.1"
 
 [target.'cfg(unix)'.dependencies]
 nix = { version = "0.28.0", features = ["resource"] }

src/database/key_value/media.rs

@@ -52,12 +52,11 @@ impl service::media::Data for KeyValueDatabase {
             debug!("Deleting key: {:?}", key);
             self.mediaid_file.remove(&key)?;
         }
-        //return Err(Error::bad_database("Media not found."));
 
         Ok(())
     }
 
-    /// Searches for all files with the given MXC (e.g. thumbnail and original image)
+    /// Searches for all files with the given MXC
     fn search_mxc_metadata_prefix(&self, mxc: String) -> Result<Vec<Vec<u8>>> {
         debug!("MXC URI: {:?}", mxc);
 
@@ -126,6 +125,17 @@ impl service::media::Data for KeyValueDatabase {
         Ok((content_disposition, content_type, key))
     }
 
+    /// Gets all the media keys in our database (this includes all the metadata associated with it such as width, height, content-type, etc)
+    fn get_all_media_keys(&self) -> Result<Vec<Vec<u8>>> {
+        let mut keys: Vec<Vec<u8>> = vec![];
+
+        for (key, _) in self.mediaid_file.iter() {
+            keys.push(key);
+        }
+
+        Ok(keys)
+    }
+
     fn remove_url_preview(&self, url: &str) -> Result<()> {
         self.url_previews.remove(url.as_bytes())
     }

src/service/admin/mod.rs

@@ -96,6 +96,12 @@ enum MediaCommand {
 
     /// - Deletes a codeblock list of MXC URLs from our database and on the filesystem
     DeleteList,
+
+    // - Deletes all remote media in the last <x> amount of time using filesystem metadata first created at date.
+    DeletePastRemoteMedia {
+        /// - The duration (at or after), e.g. "5m" to delete all media in the past 5 minutes
+        duration: String,
+    },
 }
 
 #[cfg_attr(test, derive(Debug))]
@@ -785,6 +791,17 @@ impl Service {
                         ));
                     }
                 }
+                MediaCommand::DeletePastRemoteMedia { duration } => {
+                    let deleted_count = services()
+                        .media
+                        .delete_all_remote_media_at_after_time(duration)
+                        .await?;
+
+                    return Ok(RoomMessageEventContent::text_plain(format!(
+                        "Deleted {} total files.",
+                        deleted_count
+                    )));
+                }
             },
             AdminCommand::Users(command) => match command {
                 UserCommand::List => match services().users.list_local_users() {

src/service/media/data.rs

@@ -22,6 +22,8 @@ pub trait Data: Send + Sync {
 
     fn search_mxc_metadata_prefix(&self, mxc: String) -> Result<Vec<Vec<u8>>>;
 
+    fn get_all_media_keys(&self) -> Result<Vec<Vec<u8>>>;
+
     fn remove_url_preview(&self, url: &str) -> Result<()>;
 
     fn set_url_preview(

src/service/media/mod.rs

@@ -7,14 +7,15 @@ use std::{
 };
 
 pub(crate) use data::Data;
+use ruma::OwnedMxcUri;
 use serde::Serialize;
 use tracing::{debug, error};
 
-use crate::{services, Error, Result};
+use crate::{services, utils, Error, Result};
 use image::imageops::FilterType;
 
 use tokio::{
-    fs::File,
+    fs::{self, File},
     io::{AsyncReadExt, AsyncWriteExt, BufReader},
     sync::Mutex,
 };
@@ -174,6 +175,112 @@ impl Service {
         }
     }
 
+    /// Deletes all remote only media files in the given at or after time/duration. Returns a u32
+    /// with the amount of media files deleted.
+    pub async fn delete_all_remote_media_at_after_time(&self, time: String) -> Result<u32> {
+        if let Ok(all_keys) = self.db.get_all_media_keys() {
+            let user_duration: SystemTime = match cyborgtime::parse_duration(&time) {
+                Ok(duration) => {
+                    debug!("Parsed duration: {:?}", duration);
+                    debug!("System time now: {:?}", SystemTime::now());
+                    SystemTime::now() - duration
+                }
+                Err(e) => {
+                    error!("Failed to parse user-specified time duration: {}", e);
+                    return Err(Error::bad_database(
+                        "Failed to parse user-specified time duration.",
+                    ));
+                }
+            };
+
+            let mut remote_mxcs: Vec<String> = vec![];
+
+            for key in all_keys {
+                debug!("Full MXC key from database: {:?}", key);
+
+                // we need to get the MXC URL from the first part of the key (the first 0xff / 255 push)
+                // this code does look kinda crazy but blame conduit for using magic keys
+                let mut parts = key.split(|&b| b == 0xff);
+                let mxc = parts
+                    .next()
+                    .map(|bytes| {
+                        utils::string_from_bytes(bytes).map_err(|e| {
+                            error!("Failed to parse MXC unicode bytes from our database: {}", e);
+                            Error::bad_database(
+                                "Failed to parse MXC unicode bytes from our database",
+                            )
+                        })
+                    })
+                    .transpose()?;
+
+                let mxc_s = match mxc {
+                    Some(mxc) => mxc,
+                    None => {
+                        return Err(Error::bad_database(
+                            "Parsed MXC URL unicode bytes from database but still is None",
+                        ));
+                    }
+                };
+
+                debug!("Parsed MXC key to URL: {}", mxc_s);
+
+                let mxc = OwnedMxcUri::from(mxc_s);
+                if mxc.server_name() == Ok(services().globals.server_name()) {
+                    debug!("Ignoring local media MXC: {}", mxc);
+                    // ignore our own MXC URLs as this would be local media.
+                    continue;
+                }
+
+                let path = if cfg!(feature = "sha256_media") {
+                    services().globals.get_media_file_new(&key)
+                } else {
+                    #[allow(deprecated)]
+                    services().globals.get_media_file(&key)
+                };
+
+                debug!("MXC path: {:?}", path);
+
+                let file_metadata = fs::metadata(path.clone()).await?;
+                debug!("File metadata: {:?}", file_metadata);
+
+                let file_created_at = file_metadata.created()?;
+                debug!("File created at: {:?}", file_created_at);
+
+                if file_created_at >= user_duration {
+                    debug!("File is within user duration, pushing to list of file paths and keys to delete.");
+                    remote_mxcs.push(mxc.to_string());
+                } else {
+                    // don't need to log this even in debug as it would be noisy
+                    continue;
+                }
+            }
+
+            debug!("Finished going through all our media in database for eligible keys to delete, checking if these are empty");
+
+            if remote_mxcs.is_empty() {
+                return Err(Error::bad_database(
+                    "Did not found any eligible MXCs to delete.",
+                ));
+            }
+
+            debug!("Deleting media now in the past \"{:?}\".", user_duration);
+
+            let mut deletion_count = 0;
+
+            for mxc in remote_mxcs {
+                debug!("Deleting MXC {mxc} from database and filesystem");
+                self.delete(mxc).await?;
+                deletion_count += 1;
+            }
+
+            Ok(deletion_count)
+        } else {
+            Err(Error::bad_database(
+                "Failed to get all our media keys (filesystem or database issue?).",
+            ))
+        }
+    }
+
     /// Returns width, height of the thumbnail and whether it should be cropped. Returns None when
     /// the server should send the original file.
     pub fn thumbnail_properties(&self, width: u32, height: u32) -> Option<(u32, u32, bool)> {