From ee548bd2e700fd904610e344cb0bc1a51421ed74 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 2 Mar 2024 11:00:53 -0500 Subject: [PATCH] admin command to delete all remote media within the past x time Signed-off-by: strawberry --- Cargo.lock | 53 +++++++++------ Cargo.toml | 2 + src/database/key_value/media.rs | 14 +++- src/service/admin/mod.rs | 17 +++++ src/service/media/data.rs | 2 + src/service/media/mod.rs | 111 +++++++++++++++++++++++++++++++- 6 files changed, 175 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2e219cbb..c4ffd95b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -227,6 +227,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" + [[package]] name = "base64ct" version = "1.6.0" @@ -412,9 +418,10 @@ dependencies = [ "axum", "axum-server", "axum-server-dual-protocol", - "base64", + "base64 0.22.0", "bytes", "clap", + "cyborgtime", "either", "figment", "futures-util", @@ -562,6 +569,12 @@ dependencies = [ "syn 2.0.50", ] +[[package]] +name = "cyborgtime" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "817fa642fb0ee7fe42e95783e00e0969927b96091bdd4b9b1af082acd943913b" + [[package]] name = "data-encoding" version = "2.5.0" @@ -894,7 +907,7 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270" dependencies = [ - "base64", + "base64 0.21.7", "bytes", "headers-core", "http", @@ -1206,7 +1219,7 @@ version = "9.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c7ea04a7c5c055c175f189b6dc6ba036fd62306b58c66c9f6389036c503a3f4" dependencies = [ - "base64", + "base64 0.21.7", "js-sys", "pem", "ring", @@ -1746,7 +1759,7 @@ version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b8fcc794035347fb64beda2d3b462595dd2753e3f268d89c5aae77e8cf2c310" dependencies = [ - "base64", + "base64 0.21.7", "serde", ] @@ -2015,7 +2028,7 @@ version = "0.11.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" dependencies = [ - "base64", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", @@ -2089,7 +2102,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.9.4" -source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a" +source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e" dependencies = [ "assign", "js_int", @@ -2108,7 +2121,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a" +source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e" dependencies = [ "js_int", "ruma-common", @@ -2120,7 +2133,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.17.4" -source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a" +source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e" dependencies = [ "as_variant", "assign", @@ -2139,10 +2152,10 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.12.1" -source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a" +source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e" dependencies = [ "as_variant", - "base64", + "base64 0.21.7", "bytes", "form_urlencoded", "http", @@ -2167,7 +2180,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.27.11" -source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a" +source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e" dependencies = [ "as_variant", "indexmap", @@ -2189,7 +2202,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.8.0" -source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a" +source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e" dependencies = [ "js_int", "ruma-common", @@ -2201,7 +2214,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.3" -source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a" +source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e" dependencies = [ "js_int", "thiserror", @@ -2210,7 +2223,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.8.0" -source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a" +source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e" dependencies = [ "js_int", "ruma-common", @@ -2220,7 +2233,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.12.0" -source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a" +source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e" dependencies = [ "once_cell", "proc-macro-crate", @@ -2235,7 +2248,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.8.0" -source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a" +source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e" dependencies = [ "js_int", "ruma-common", @@ -2247,9 +2260,9 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.14.0" -source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a" +source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e" dependencies = [ - "base64", + "base64 0.21.7", "ed25519-dalek", "pkcs8", "rand", @@ -2263,7 +2276,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a" +source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e" dependencies = [ "itertools 0.11.0", "js_int", @@ -2339,7 +2352,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64", + "base64 0.21.7", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d03dfa94..d5a20770 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -115,6 +115,8 @@ axum-server-dual-protocol = { version = "0.5.2", optional = true } # to get the client IP address of requests #axum-client-ip = "0.4.2" +# to parse user-friendly time durations in admin commands +cyborgtime = "2.1.1" [target.'cfg(unix)'.dependencies] nix = { version = "0.28.0", features = ["resource"] } diff --git a/src/database/key_value/media.rs b/src/database/key_value/media.rs index dcafc74e..f901bd55 100644 --- a/src/database/key_value/media.rs +++ b/src/database/key_value/media.rs @@ -52,12 +52,11 @@ impl service::media::Data for KeyValueDatabase { debug!("Deleting key: {:?}", key); self.mediaid_file.remove(&key)?; } - //return Err(Error::bad_database("Media not found.")); Ok(()) } - /// Searches for all files with the given MXC (e.g. thumbnail and original image) + /// Searches for all files with the given MXC fn search_mxc_metadata_prefix(&self, mxc: String) -> Result>> { debug!("MXC URI: {:?}", mxc); @@ -126,6 +125,17 @@ impl service::media::Data for KeyValueDatabase { Ok((content_disposition, content_type, key)) } + /// Gets all the media keys in our database (this includes all the metadata associated with it such as width, height, content-type, etc) + fn get_all_media_keys(&self) -> Result>> { + let mut keys: Vec> = vec![]; + + for (key, _) in self.mediaid_file.iter() { + keys.push(key); + } + + Ok(keys) + } + fn remove_url_preview(&self, url: &str) -> Result<()> { self.url_previews.remove(url.as_bytes()) } diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 0ea67abf..eb61de3f 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -96,6 +96,12 @@ enum MediaCommand { /// - Deletes a codeblock list of MXC URLs from our database and on the filesystem DeleteList, + + // - Deletes all remote media in the last amount of time using filesystem metadata first created at date. + DeletePastRemoteMedia { + /// - The duration (at or after), e.g. "5m" to delete all media in the past 5 minutes + duration: String, + }, } #[cfg_attr(test, derive(Debug))] @@ -785,6 +791,17 @@ impl Service { )); } } + MediaCommand::DeletePastRemoteMedia { duration } => { + let deleted_count = services() + .media + .delete_all_remote_media_at_after_time(duration) + .await?; + + return Ok(RoomMessageEventContent::text_plain(format!( + "Deleted {} total files.", + deleted_count + ))); + } }, AdminCommand::Users(command) => match command { UserCommand::List => match services().users.list_local_users() { diff --git a/src/service/media/data.rs b/src/service/media/data.rs index 0404b548..bb44de80 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -22,6 +22,8 @@ pub trait Data: Send + Sync { fn search_mxc_metadata_prefix(&self, mxc: String) -> Result>>; + fn get_all_media_keys(&self) -> Result>>; + fn remove_url_preview(&self, url: &str) -> Result<()>; fn set_url_preview( diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 7f8311c5..7d021e52 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -7,14 +7,15 @@ use std::{ }; pub(crate) use data::Data; +use ruma::OwnedMxcUri; use serde::Serialize; use tracing::{debug, error}; -use crate::{services, Error, Result}; +use crate::{services, utils, Error, Result}; use image::imageops::FilterType; use tokio::{ - fs::File, + fs::{self, File}, io::{AsyncReadExt, AsyncWriteExt, BufReader}, sync::Mutex, }; @@ -174,6 +175,112 @@ impl Service { } } + /// Deletes all remote only media files in the given at or after time/duration. Returns a u32 + /// with the amount of media files deleted. + pub async fn delete_all_remote_media_at_after_time(&self, time: String) -> Result { + if let Ok(all_keys) = self.db.get_all_media_keys() { + let user_duration: SystemTime = match cyborgtime::parse_duration(&time) { + Ok(duration) => { + debug!("Parsed duration: {:?}", duration); + debug!("System time now: {:?}", SystemTime::now()); + SystemTime::now() - duration + } + Err(e) => { + error!("Failed to parse user-specified time duration: {}", e); + return Err(Error::bad_database( + "Failed to parse user-specified time duration.", + )); + } + }; + + let mut remote_mxcs: Vec = vec![]; + + for key in all_keys { + debug!("Full MXC key from database: {:?}", key); + + // we need to get the MXC URL from the first part of the key (the first 0xff / 255 push) + // this code does look kinda crazy but blame conduit for using magic keys + let mut parts = key.split(|&b| b == 0xff); + let mxc = parts + .next() + .map(|bytes| { + utils::string_from_bytes(bytes).map_err(|e| { + error!("Failed to parse MXC unicode bytes from our database: {}", e); + Error::bad_database( + "Failed to parse MXC unicode bytes from our database", + ) + }) + }) + .transpose()?; + + let mxc_s = match mxc { + Some(mxc) => mxc, + None => { + return Err(Error::bad_database( + "Parsed MXC URL unicode bytes from database but still is None", + )); + } + }; + + debug!("Parsed MXC key to URL: {}", mxc_s); + + let mxc = OwnedMxcUri::from(mxc_s); + if mxc.server_name() == Ok(services().globals.server_name()) { + debug!("Ignoring local media MXC: {}", mxc); + // ignore our own MXC URLs as this would be local media. + continue; + } + + let path = if cfg!(feature = "sha256_media") { + services().globals.get_media_file_new(&key) + } else { + #[allow(deprecated)] + services().globals.get_media_file(&key) + }; + + debug!("MXC path: {:?}", path); + + let file_metadata = fs::metadata(path.clone()).await?; + debug!("File metadata: {:?}", file_metadata); + + let file_created_at = file_metadata.created()?; + debug!("File created at: {:?}", file_created_at); + + if file_created_at >= user_duration { + debug!("File is within user duration, pushing to list of file paths and keys to delete."); + remote_mxcs.push(mxc.to_string()); + } else { + // don't need to log this even in debug as it would be noisy + continue; + } + } + + debug!("Finished going through all our media in database for eligible keys to delete, checking if these are empty"); + + if remote_mxcs.is_empty() { + return Err(Error::bad_database( + "Did not found any eligible MXCs to delete.", + )); + } + + debug!("Deleting media now in the past \"{:?}\".", user_duration); + + let mut deletion_count = 0; + + for mxc in remote_mxcs { + debug!("Deleting MXC {mxc} from database and filesystem"); + self.delete(mxc).await?; + deletion_count += 1; + } + + Ok(deletion_count) + } else { + Err(Error::bad_database( + "Failed to get all our media keys (filesystem or database issue?).", + )) + } + } + /// Returns width, height of the thumbnail and whether it should be cropped. Returns None when /// the server should send the original file. pub fn thumbnail_properties(&self, width: u32, height: u32) -> Option<(u32, u32, bool)> {