documentation, fix in flush-loop

Cargo.toml

@@ -2,6 +2,9 @@
 name = "microdb"
 version = "0.1.0"
 edition = "2021"
+description = "A very small in-program database with cache, disk storage, etc."
+license = "MIT"
+repository = "https://github.com/tudbut/microdb"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

README.md

@@ -1,9 +1,42 @@
 # MicroDB
 
-A microsized database.
+A microsized database for use in programs with too much data for the RAM, but not necessarily for your
+next incredibly successful Discord clone (tho I suppose you could make that work too).
 
-## Very unfinished as of now
+## Completed features
 
-Only basic byte-oriented operations, caching, and base storage are figured out.
+[x] Disk storage
+[x] Getting, setting, allocation, deallocation
+[x] Caching
+[x] Automatic recovery on error
+[x] Serialization for basic types (numbers, strings, vecs, options, results)
+[x] Easy-to-implement serialization
+[ ] Derivable serialization
+[ ] (maybe) Multi-client support over TCP
+[ ] (maybe) Mirroring operations to backup server (needs TCP)
 
-Development is, however, active.
+## How to use it
+
+MicroDB runs where your application does: Saving, cache synchronization, etc all happen simply in another thread of your application.
+
+To get started, create a DB:
+```rs
+let db = MicroDB::create(
+    "example_db.data.mdb",
+    "example_db.meta.mdb",
+    MicroDB::sensible_cache_period(
+        /*requests of unique objects per second*/10.0, 
+        /*max ram usage*/0.1, 
+        /*average object size in mb*/0.01, 
+        /*safety (how important staying within ram spec is)*/1.0),
+    MicroDB::sensible_block_size(
+        /*object amount*/500.0, 
+        /*average object size in bytes*/10_0000.0, 
+        /*object size fluctuation in bytes*/0.0, 
+        /*storage tightness*/1.0
+    ),
+)
+```
+Or load one using ::new and leave out the block_size arg.
+
+And now you're good to go!

examples/main.rs

@@ -42,10 +42,8 @@ fn main() {
     let db = MicroDB::create(
         "example_db.data.mdb",
         "example_db.meta.mdb",
-        // 0,
-        MicroDB::sensible_cache_period(10.0, 0.1, 0.1, 1.0),
-        // 1,
-        MicroDB::sensible_block_size(500.0, 2.0, 0.0, 1.0),
+        dbg!(MicroDB::sensible_cache_period(10.0, 0.01, 0.1, 1.0)),
+        dbg!(MicroDB::sensible_block_size(500.0, 10_000.0, 0.0, 1.0)),
     )
     .unwrap();
     println!("\nSetting test --raw--> vec![true; 500]");

src/data/traits.rs

@@ -2,6 +2,7 @@ use std::io;
 
 use crate::MicroDB;
 
+/// A path which escapes its inner path
 #[derive(Clone)]
 pub struct Escape<P: Path>(pub P);
 
@@ -11,6 +12,7 @@ impl<T: Path> Path for Escape<T> {
     }
 }
 
+/// A path which unescapes its inner path
 #[derive(Clone)]
 pub struct Unescape<P: Path>(pub P);
 
@@ -20,8 +22,11 @@ impl<T: Path> Path for Unescape<T> {
     }
 }
 
+/// Anything that can be used as a path in a MicroDB
 pub trait Path: Sized + Clone {
+    /// Turns the path into a string for storage in the [`crate::FAlloc`]
     fn to_db_path(self) -> String;
+    /// Returns a sub-path of this path. Separator is added automatically, but escaping is NOT done.
     fn sub_path<P: Path>(&self, other: P) -> String {
         self.clone().to_db_path() + "/" + &other.to_db_path()
     }
@@ -29,14 +34,23 @@ pub trait Path: Sized + Clone {
 
 /// An object which the DB can represent in bytes
 pub trait RawObj: Sized {
+    /// Turns the object into a byte-array for storage.
     fn to_db(self) -> Vec<u8>;
+    /// Turns a byte-array back into this object. None means failure to decode.
     fn from_db(x: Vec<u8>) -> Option<Self>;
 }
 
 /// A composite object, made of other ComObjects and RawObjects.
 pub trait ComObj: Sized {
+    /// Turns the object into multiple DB-objects, which can be ComObjects or RawObjects.
+    /// To write, use the set_raw and set_com methods on the db. Don't forget to always use
+    /// [`Path::sub_path`].
     fn to_db<P: Path>(self, path: P, db: &MicroDB) -> Result<(), io::Error>;
+    /// Removes the object and its sub-objects from the DB. Don't forget to always use
+    /// [`Path::sub_path`].
     fn remove<P: Path>(path: P, db: &MicroDB) -> Result<(), io::Error>;
+    /// Turns the DB object back into the original object. None means failure to decode.
+    /// Don't forget to always use [`Path::sub_path`].
     fn from_db<P: Path>(path: P, db: &MicroDB) -> Result<Option<Self>, io::Error>;
 }
 

src/db.rs

@@ -9,12 +9,14 @@ pub struct MicroDB {
 }
 
 impl MicroDB {
+    /// Loads a database. Can NOT be used to create one.
     pub fn new<S: ToString>(data: S, alloc: S, cache_period: u128) -> Result<Self, io::Error> {
         Ok(Self {
             storage: FAlloc::new(data, alloc, cache_period)?,
         })
     }
 
+    /// Creates a database. Can NOT be used to load one.
     pub fn create<S: ToString>(
         data: S,
         alloc: S,
@@ -67,7 +69,7 @@ impl MicroDB {
     }
 
     /// Gracefully shuts down the DB, saving in the process.
-    /// Please use [`shutdown`] instead if possible. This variant
+    /// Please use [`Self::shutdown`] instead if possible. This variant
     /// will force a shutdown across all threads without the guarantee that
     /// this is the only thread with access to it.
     pub fn shutdown_here(&self) -> Result<(), io::Error> {
@@ -105,7 +107,7 @@ impl MicroDB {
     /// This function will not clean up the old substructure. It may create database junk until
     /// the next time that substructure is cleaned by some other function. Use this only if you
     /// know that the types of the previous inhabitant and the new one are the same and that the
-    /// types aren't dynamic (like Vec<T> is).
+    /// types aren't dynamic (like [`Vec<T>`] is).
     pub fn set_com_hard<T: ComObj, P: Path>(&self, path: P, object: T) -> Result<(), io::Error> {
         T::to_db(object, path, self)
     }
@@ -138,6 +140,18 @@ impl MicroDB {
     }
 }
 
+/// Convenience macro to extract a value from the database and return Ok(None) if not found.
+///
+/// Example usage:
+/// ```ignore
+/// fn from_db<P: Path>(path: P, db: &MicroDB) -> Result<Option<Self>, std::io::Error> {
+///     Ok(Some(Self {
+///         username: extract!(db.get_raw(path.sub_path("username"))),
+///         email_address: extract!(db.get_raw(path.sub_path("email"))),
+///         password_hash: extract!(db.get_raw(path.sub_path("pass"))),
+///     }))
+/// }
+/// ```
 #[macro_export]
 macro_rules! extract {
     ($val:expr) => {

src/lib.rs

@@ -1,3 +1,5 @@
+//! See [`MicroDB`], [`FAlloc`], and [`crate::data::traits`].
+
 pub mod data;
 pub mod db;
 pub mod storage;

src/storage.rs

@@ -51,6 +51,11 @@ struct InnerFAlloc {
     shutdown: bool,
 }
 
+/// The storage used in a MicroDB. Effectively, this is a primitive file system.
+/// Tasks:
+/// - space allocation
+/// - caching
+/// - disk operations
 #[derive(Debug)]
 pub struct FAlloc {
     inner: Arc<Mutex<InnerFAlloc>>,
@@ -155,7 +160,9 @@ impl AllocationTable {
                 while i < (index + 1).min(self.free.len() - 1) {
                     if self.free[i].0 + self.free[i].1 == self.free[i + 1].0 {
                         self.free[i].1 += self.free.remove(i + 1).1;
-                        index -= 1;
+                        if index > 0 {
+                            index -= 1;
+                        }
                     }
                     i += 1;
                 }
@@ -314,7 +321,11 @@ impl FAlloc {
                         continue;
                     }
                 }
-                if let Err(e) = inner.flush_cache(true).and(inner.data.sync_all()) {
+                if let Err(e) = inner
+                    .flush_cache(true)
+                    .and(inner.alloc.save())
+                    .and(inner.data.sync_all())
+                {
                     inner.shutdown = true;
                     recovery = true;
                     println!("The database was unable to write to disk. Depending on where this error happened, your data might be mostly fine. Error: {e:?}. Recovery will be attempted every 30 seconds.");
@@ -349,6 +360,7 @@ impl FAlloc {
         Ok(Self { inner })
     }
 
+    /// Loads a database. Can NOT be used to create one.
     pub fn new<S: ToString>(data: S, alloc: S, cache_period: u128) -> Result<Self, io::Error> {
         Self::internal_new(
             File::options()
@@ -362,6 +374,7 @@ impl FAlloc {
         )
     }
 
+    /// Creates a database. Can NOT be used to load one.
     pub fn create<S: ToString>(
         data: S,
         alloc: S,
@@ -389,6 +402,9 @@ impl FAlloc {
         })
     }
 
+    /// Tries to find an item in cache, returning Ok(None) if it wasn't found in the cache.
+    /// This is the only function where a recently-deleted element will be an empty vector
+    /// instead of being a None.
     pub fn cache_lookup(&self, path: Option<&str>) -> Result<Option<Vec<u8>>, io::Error> {
         let mut this = self.inner.lock().unwrap();
         if this.shutdown {
@@ -462,6 +478,7 @@ impl FAlloc {
         Ok(())
     }
 
+    /// Deletes all data that is BELOW the path in the tree. The path itself is NOT deleted.
     pub fn delete_substructure(&self, path: &str) -> Result<(), io::Error> {
         let mut this = self.inner.lock().unwrap();
         if this.shutdown {
@@ -500,7 +517,7 @@ impl FAlloc {
     }
 
     /// Gracefully shuts down the allocator, saving in the process.
-    /// Please use [`shutdown`] instead if possible. This variant
+    /// Please use [`Self::shutdown`] instead if possible. This variant
     /// will force a shutdown across all threads without the guarantee that
     /// this is the only thread with access to it.
     pub fn shutdown_here(&self) -> Result<(), io::Error> {