TudbuT/conduit
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

admin command to delete all remote media within the past x time

Browse source 
Signed-off-by: strawberry <strawberry@puppygock.gay>
This commit is contained in:
strawberry 2024-03-02 11:00:53 -05:00 • committed by June
parent 5c94c9c0d4
commit ee548bd2e7
6 changed files with 175 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
Cargo.lock generated
View file

@ -227,6 +227,12 @@ version = "0.21.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
[[package]]
name = "base64"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51"
[[package]]
name = "base64ct"
version = "1.6.0"
@ -412,9 +418,10 @@ dependencies = [
"axum",
"axum-server",
"axum-server-dual-protocol",
"base64",
"base64 0.22.0",
"bytes",
"clap",
"cyborgtime",
"either",
"figment",
"futures-util",
@ -562,6 +569,12 @@ dependencies = [
"syn 2.0.50",
]
[[package]]
name = "cyborgtime"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "817fa642fb0ee7fe42e95783e00e0969927b96091bdd4b9b1af082acd943913b"
[[package]]
name = "data-encoding"
version = "2.5.0"
@ -894,7 +907,7 @@ version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270"
dependencies = [
"base64",
"base64 0.21.7",
"bytes",
"headers-core",
"http",
@ -1206,7 +1219,7 @@ version = "9.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c7ea04a7c5c055c175f189b6dc6ba036fd62306b58c66c9f6389036c503a3f4"
dependencies = [
"base64",
"base64 0.21.7",
"js-sys",
"pem",
"ring",
@ -1746,7 +1759,7 @@ version = "3.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b8fcc794035347fb64beda2d3b462595dd2753e3f268d89c5aae77e8cf2c310"
dependencies = [
"base64",
"base64 0.21.7",
"serde",
]
@ -2015,7 +2028,7 @@ version = "0.11.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251"
dependencies = [
"base64",
"base64 0.21.7",
"bytes",
"encoding_rs",
"futures-core",
@ -2089,7 +2102,7 @@ dependencies = [
[[package]]
name = "ruma"
version = "0.9.4"
source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"assign",
"js_int",
@ -2108,7 +2121,7 @@ dependencies = [
[[package]]
name = "ruma-appservice-api"
version = "0.9.0"
source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"js_int",
"ruma-common",
@ -2120,7 +2133,7 @@ dependencies = [
[[package]]
name = "ruma-client-api"
version = "0.17.4"
source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"as_variant",
"assign",
@ -2139,10 +2152,10 @@ dependencies = [
[[package]]
name = "ruma-common"
version = "0.12.1"
source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"as_variant",
"base64",
"base64 0.21.7",
"bytes",
"form_urlencoded",
"http",
@ -2167,7 +2180,7 @@ dependencies = [
[[package]]
name = "ruma-events"
version = "0.27.11"
source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"as_variant",
"indexmap",
@ -2189,7 +2202,7 @@ dependencies = [
[[package]]
name = "ruma-federation-api"
version = "0.8.0"
source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"js_int",
"ruma-common",
@ -2201,7 +2214,7 @@ dependencies = [
[[package]]
name = "ruma-identifiers-validation"
version = "0.9.3"
source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"js_int",
"thiserror",
@ -2210,7 +2223,7 @@ dependencies = [
[[package]]
name = "ruma-identity-service-api"
version = "0.8.0"
source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"js_int",
"ruma-common",
@ -2220,7 +2233,7 @@ dependencies = [
[[package]]
name = "ruma-macros"
version = "0.12.0"
source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"once_cell",
"proc-macro-crate",
@ -2235,7 +2248,7 @@ dependencies = [
[[package]]
name = "ruma-push-gateway-api"
version = "0.8.0"
source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"js_int",
"ruma-common",
@ -2247,9 +2260,9 @@ dependencies = [
[[package]]
name = "ruma-signatures"
version = "0.14.0"
source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"base64",
"base64 0.21.7",
"ed25519-dalek",
"pkcs8",
"rand",
@ -2263,7 +2276,7 @@ dependencies = [
[[package]]
name = "ruma-state-res"
version = "0.10.0"
source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"itertools 0.11.0",
"js_int",
@ -2339,7 +2352,7 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c"
dependencies = [
"base64",
"base64 0.21.7",
]
[[package]]

2
Cargo.toml
View file

@ -115,6 +115,8 @@ axum-server-dual-protocol = { version = "0.5.2", optional = true }
# to get the client IP address of requests
#axum-client-ip = "0.4.2"
# to parse user-friendly time durations in admin commands
cyborgtime = "2.1.1"
[target.'cfg(unix)'.dependencies]
nix = { version = "0.28.0", features = ["resource"] }

14
src/database/key_value/media.rs
View file

@ -52,12 +52,11 @@ impl service::media::Data for KeyValueDatabase {
debug!("Deleting key: {:?}", key);
self.mediaid_file.remove(&key)?;
}
//return Err(Error::bad_database("Media not found."));
Ok(())
}
/// Searches for all files with the given MXC (e.g. thumbnail and original image)
/// Searches for all files with the given MXC
fn search_mxc_metadata_prefix(&self, mxc: String) -> Result<Vec<Vec<u8>>> {
debug!("MXC URI: {:?}", mxc);
@ -126,6 +125,17 @@ impl service::media::Data for KeyValueDatabase {
Ok((content_disposition, content_type, key))
}
/// Gets all the media keys in our database (this includes all the metadata associated with it such as width, height, content-type, etc)
fn get_all_media_keys(&self) -> Result<Vec<Vec<u8>>> {
let mut keys: Vec<Vec<u8>> = vec![];
for (key, _) in self.mediaid_file.iter() {
keys.push(key);
}
Ok(keys)
}
fn remove_url_preview(&self, url: &str) -> Result<()> {
self.url_previews.remove(url.as_bytes())
}

17
src/service/admin/mod.rs
View file

@ -96,6 +96,12 @@ enum MediaCommand {
/// - Deletes a codeblock list of MXC URLs from our database and on the filesystem
DeleteList,
// - Deletes all remote media in the last <x> amount of time using filesystem metadata first created at date.
DeletePastRemoteMedia {
/// - The duration (at or after), e.g. "5m" to delete all media in the past 5 minutes
duration: String,
},
}
#[cfg_attr(test, derive(Debug))]
@ -785,6 +791,17 @@ impl Service {
));
}
}
MediaCommand::DeletePastRemoteMedia { duration } => {
let deleted_count = services()
.media
.delete_all_remote_media_at_after_time(duration)
.await?;
return Ok(RoomMessageEventContent::text_plain(format!(
"Deleted {} total files.",
deleted_count
)));
}
},
AdminCommand::Users(command) => match command {
UserCommand::List => match services().users.list_local_users() {

2
src/service/media/data.rs
View file

@ -22,6 +22,8 @@ pub trait Data: Send + Sync {
fn search_mxc_metadata_prefix(&self, mxc: String) -> Result<Vec<Vec<u8>>>;
fn get_all_media_keys(&self) -> Result<Vec<Vec<u8>>>;
fn remove_url_preview(&self, url: &str) -> Result<()>;
fn set_url_preview(

111
src/service/media/mod.rs
View file

@ -7,14 +7,15 @@ use std::{
};
pub(crate) use data::Data;
use ruma::OwnedMxcUri;
use serde::Serialize;
use tracing::{debug, error};
use crate::{services, Error, Result};
use crate::{services, utils, Error, Result};
use image::imageops::FilterType;
use tokio::{
fs::File,
fs::{self, File},
io::{AsyncReadExt, AsyncWriteExt, BufReader},
sync::Mutex,
};
@ -174,6 +175,112 @@ impl Service {
}
}
/// Deletes all remote only media files in the given at or after time/duration. Returns a u32
/// with the amount of media files deleted.
pub async fn delete_all_remote_media_at_after_time(&self, time: String) -> Result<u32> {
if let Ok(all_keys) = self.db.get_all_media_keys() {
let user_duration: SystemTime = match cyborgtime::parse_duration(&time) {
Ok(duration) => {
debug!("Parsed duration: {:?}", duration);
debug!("System time now: {:?}", SystemTime::now());
SystemTime::now() - duration
}
Err(e) => {
error!("Failed to parse user-specified time duration: {}", e);
return Err(Error::bad_database(
"Failed to parse user-specified time duration.",
));
}
};
let mut remote_mxcs: Vec<String> = vec![];
for key in all_keys {
debug!("Full MXC key from database: {:?}", key);
// we need to get the MXC URL from the first part of the key (the first 0xff / 255 push)
// this code does look kinda crazy but blame conduit for using magic keys
let mut parts = key.split(|&b| b == 0xff);
let mxc = parts
.next()
.map(|bytes| {
utils::string_from_bytes(bytes).map_err(|e| {
error!("Failed to parse MXC unicode bytes from our database: {}", e);
Error::bad_database(
"Failed to parse MXC unicode bytes from our database",
)
})
})
.transpose()?;
let mxc_s = match mxc {
Some(mxc) => mxc,
None => {
return Err(Error::bad_database(
"Parsed MXC URL unicode bytes from database but still is None",
));
}
};
debug!("Parsed MXC key to URL: {}", mxc_s);
let mxc = OwnedMxcUri::from(mxc_s);
if mxc.server_name() == Ok(services().globals.server_name()) {
debug!("Ignoring local media MXC: {}", mxc);
// ignore our own MXC URLs as this would be local media.
continue;
}
let path = if cfg!(feature = "sha256_media") {
services().globals.get_media_file_new(&key)
} else {
#[allow(deprecated)]
services().globals.get_media_file(&key)
};
debug!("MXC path: {:?}", path);
let file_metadata = fs::metadata(path.clone()).await?;
debug!("File metadata: {:?}", file_metadata);
let file_created_at = file_metadata.created()?;
debug!("File created at: {:?}", file_created_at);
if file_created_at >= user_duration {
debug!("File is within user duration, pushing to list of file paths and keys to delete.");
remote_mxcs.push(mxc.to_string());
} else {
// don't need to log this even in debug as it would be noisy
continue;
}
}
debug!("Finished going through all our media in database for eligible keys to delete, checking if these are empty");
if remote_mxcs.is_empty() {
return Err(Error::bad_database(
"Did not found any eligible MXCs to delete.",
));
}
debug!("Deleting media now in the past \"{:?}\".", user_duration);
let mut deletion_count = 0;
for mxc in remote_mxcs {
debug!("Deleting MXC {mxc} from database and filesystem");
self.delete(mxc).await?;
deletion_count += 1;
}
Ok(deletion_count)
} else {
Err(Error::bad_database(
"Failed to get all our media keys (filesystem or database issue?).",
))
}
}
/// Returns width, height of the thumbnail and whether it should be cropped. Returns None when
/// the server should send the original file.
pub fn thumbnail_properties(&self, width: u32, height: u32) -> Option<(u32, u32, bool)> {