TudbuT/conduit
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

media: decomplexify get_all_media_keys for deleting all MXC URIs

Browse source 
wow this was terrible, early strawberry code

Signed-off-by: strawberry <strawberry@puppygock.gay>
This commit is contained in:
strawberry 2024-05-22 21:19:36 -04:00 • committed by June 🍓🦴
parent cb73ae3732
commit d49507bc21
3 changed files with 107 additions and 119 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
src/service/key_value/media.rs
View file

@ -1,3 +1,4 @@
use conduit::debug_info;
use ruma::api::client::error::ErrorKind;
use tracing::debug;
@ -55,7 +56,7 @@ impl crate::media::Data for KeyValueDatabase {
if key == mxc.as_bytes().to_vec() {
let user = string_from_bytes(&value).unwrap_or_default();
debug!("Deleting key \"{key:?}\" which was uploaded by user {user}");
debug_info!("Deleting key \"{key:?}\" which was uploaded by user {user}");
self.mediaid_user.remove(&key)?;
}
}
@ -70,11 +71,11 @@ impl crate::media::Data for KeyValueDatabase {
let mut prefix = mxc.as_bytes().to_vec();
prefix.push(0xFF);
let mut keys: Vec<Vec<u8>> = vec![];
for (key, _) in self.mediaid_file.scan_prefix(prefix) {
keys.push(key);
}
let keys: Vec<Vec<u8>> = self
.mediaid_file
.scan_prefix(prefix)
.map(|(key, _)| key)
.collect();
if keys.is_empty() {
return Err(Error::bad_database(
@ -100,7 +101,7 @@ impl crate::media::Data for KeyValueDatabase {
.mediaid_file
.scan_prefix(prefix)
.next()
.ok_or(Error::BadRequest(ErrorKind::NotFound, "Media not found"))?;
.ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Media not found"))?;
let mut parts = key.rsplit(|&b| b == 0xFF);
@ -129,15 +130,7 @@ impl crate::media::Data for KeyValueDatabase {
/// Gets all the media keys in our database (this includes all the metadata
/// associated with it such as width, height, content-type, etc)
fn get_all_media_keys(&self) -> Result<Vec<Vec<u8>>> {
let mut keys: Vec<Vec<u8>> = vec![];
for (key, _) in self.mediaid_file.iter() {
keys.push(key);
}
Ok(keys)
}
fn get_all_media_keys(&self) -> Vec<Vec<u8>> { self.mediaid_file.iter().map(|(key, _)| key).collect() }
fn remove_url_preview(&self, url: &str) -> Result<()> { self.url_previews.remove(url.as_bytes()) }

2
src/service/media/data.rs
View file

@ -15,7 +15,7 @@ pub trait Data: Send + Sync {
fn search_mxc_metadata_prefix(&self, mxc: String) -> Result<Vec<Vec<u8>>>;
fn get_all_media_keys(&self) -> Result<Vec<Vec<u8>>>;
fn get_all_media_keys(&self) -> Vec<Vec<u8>>;
// TODO: use this
#[allow(dead_code)]

199
src/service/media/mod.rs
View file

@ -183,115 +183,110 @@ impl Service {
/// Deletes all remote only media files in the given at or after
/// time/duration. Returns a u32 with the amount of media files deleted.
pub async fn delete_all_remote_media_at_after_time(&self, time: String) -> Result<u32> {
if let Ok(all_keys) = self.db.get_all_media_keys() {
let user_duration: SystemTime = match cyborgtime::parse_duration(&time) {
Ok(duration) => {
debug!("Parsed duration: {:?}", duration);
debug!("System time now: {:?}", SystemTime::now());
SystemTime::now().checked_sub(duration).ok_or_else(|| {
Error::bad_database("Duration specified is not valid against the current system time")
})?
},
Err(e) => {
error!("Failed to parse user-specified time duration: {}", e);
return Err(Error::bad_database("Failed to parse user-specified time duration."));
},
let all_keys = self.db.get_all_media_keys();
let user_duration: SystemTime = match cyborgtime::parse_duration(&time) {
Ok(duration) => {
debug!("Parsed duration: {:?}", duration);
debug!("System time now: {:?}", SystemTime::now());
SystemTime::now().checked_sub(duration).ok_or_else(|| {
Error::bad_database("Duration specified is not valid against the current system time")
})?
},
Err(e) => {
error!("Failed to parse user-specified time duration: {}", e);
return Err(Error::bad_database("Failed to parse user-specified time duration."));
},
};
let mut remote_mxcs: Vec<String> = vec![];
for key in all_keys {
debug!("Full MXC key from database: {:?}", key);
// we need to get the MXC URL from the first part of the key (the first 0xff /
// 255 push). this is all necessary because of conduit using magic keys for
// media
let mut parts = key.split(|&b| b == 0xFF);
let mxc = parts
.next()
.map(|bytes| {
utils::string_from_bytes(bytes).map_err(|e| {
error!("Failed to parse MXC unicode bytes from our database: {}", e);
Error::bad_database("Failed to parse MXC unicode bytes from our database")
})
})
.transpose()?;
let Some(mxc_s) = mxc else {
return Err(Error::bad_database(
"Parsed MXC URL unicode bytes from database but still is None",
));
};
let mut remote_mxcs: Vec<String> = vec![];
debug!("Parsed MXC key to URL: {}", mxc_s);
for key in all_keys {
debug!("Full MXC key from database: {:?}", key);
// we need to get the MXC URL from the first part of the key (the first 0xff /
// 255 push). this is all necessary because of conduit using magic keys for
// media
let mut parts = key.split(|&b| b == 0xFF);
let mxc = parts
.next()
.map(|bytes| {
utils::string_from_bytes(bytes).map_err(|e| {
error!("Failed to parse MXC unicode bytes from our database: {}", e);
Error::bad_database("Failed to parse MXC unicode bytes from our database")
})
})
.transpose()?;
let Some(mxc_s) = mxc else {
return Err(Error::bad_database(
"Parsed MXC URL unicode bytes from database but still is None",
));
};
debug!("Parsed MXC key to URL: {}", mxc_s);
let mxc = OwnedMxcUri::from(mxc_s);
if mxc.server_name() == Ok(services().globals.server_name()) {
debug!("Ignoring local media MXC: {}", mxc);
// ignore our own MXC URLs as this would be local media.
continue;
}
let path;
#[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental
#[cfg(feature = "sha256_media")]
{
path = services().globals.get_media_file_new(&key);
};
#[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental
#[cfg(not(feature = "sha256_media"))]
{
path = services().globals.get_media_file(&key);
};
debug!("MXC path: {:?}", path);
let file_metadata = fs::metadata(path.clone()).await?;
debug!("File metadata: {:?}", file_metadata);
let file_created_at = match file_metadata.created() {
Ok(value) => value,
Err(err) if err.kind() == std::io::ErrorKind::Unsupported => {
debug!("btime is unsupported, using mtime instead");
file_metadata.modified()?
},
Err(err) => return Err(err.into()),
};
debug!("File created at: {:?}", file_created_at);
if file_created_at >= user_duration {
debug!("File is within user duration, pushing to list of file paths and keys to delete.");
remote_mxcs.push(mxc.to_string());
}
let mxc = OwnedMxcUri::from(mxc_s);
if mxc.server_name() == Ok(services().globals.server_name()) {
debug!("Ignoring local media MXC: {}", mxc);
// ignore our own MXC URLs as this would be local media.
continue;
}
debug!(
"Finished going through all our media in database for eligible keys to delete, checking if these are \
empty"
);
let path;
if remote_mxcs.is_empty() {
return Err(Error::bad_database("Did not found any eligible MXCs to delete."));
#[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental
#[cfg(feature = "sha256_media")]
{
path = services().globals.get_media_file_new(&key);
};
#[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental
#[cfg(not(feature = "sha256_media"))]
{
path = services().globals.get_media_file(&key);
};
debug!("MXC path: {:?}", path);
let file_metadata = fs::metadata(path.clone()).await?;
debug!("File metadata: {:?}", file_metadata);
let file_created_at = match file_metadata.created() {
Ok(value) => value,
Err(err) if err.kind() == std::io::ErrorKind::Unsupported => {
debug!("btime is unsupported, using mtime instead");
file_metadata.modified()?
},
Err(err) => return Err(err.into()),
};
debug!("File created at: {:?}", file_created_at);
if file_created_at >= user_duration {
debug!("File is within user duration, pushing to list of file paths and keys to delete.");
remote_mxcs.push(mxc.to_string());
}
debug!("Deleting media now in the past \"{:?}\".", user_duration);
let mut deletion_count = 0;
for mxc in remote_mxcs {
debug!("Deleting MXC {mxc} from database and filesystem");
self.delete(mxc).await?;
deletion_count += 1;
}
Ok(deletion_count)
} else {
Err(Error::bad_database(
"Failed to get all our media keys (filesystem or database issue?).",
))
}
debug!(
"Finished going through all our media in database for eligible keys to delete, checking if these are empty"
);
if remote_mxcs.is_empty() {
return Err(Error::bad_database("Did not found any eligible MXCs to delete."));
}
debug!("Deleting media now in the past \"{:?}\".", user_duration);
let mut deletion_count = 0;
for mxc in remote_mxcs {
debug!("Deleting MXC {mxc} from database and filesystem");
self.delete(mxc).await?;
deletion_count += 1;
}
Ok(deletion_count)
}
/// Returns width, height of the thumbnail and whether it should be cropped.
@ -529,7 +524,7 @@ mod tests {
fn search_mxc_metadata_prefix(&self, _mxc: String) -> Result<Vec<Vec<u8>>> { todo!() }
fn get_all_media_keys(&self) -> Result<Vec<Vec<u8>>> { todo!() }
fn get_all_media_keys(&self) -> Vec<Vec<u8>> { todo!() }
fn search_file_metadata(
&self, _mxc: String, _width: u32, _height: u32,