Sqlite

src/database.rs

@@ -19,16 +19,23 @@ use abstraction::DatabaseEngine;
 use directories::ProjectDirs;
 use log::error;
 use lru_cache::LruCache;
-use rocket::futures::{channel::mpsc, stream::FuturesUnordered, StreamExt};
+use rocket::{
+    futures::{channel::mpsc, stream::FuturesUnordered, StreamExt},
+    outcome::IntoOutcome,
+    request::{FromRequest, Request},
+    try_outcome, State,
+};
 use ruma::{DeviceId, ServerName, UserId};
-use serde::Deserialize;
+use serde::{de::IgnoredAny, Deserialize};
 use std::{
-    collections::HashMap,
+    collections::{BTreeMap, HashMap},
     fs::{self, remove_dir_all},
     io::Write,
+    ops::Deref,
+    path::Path,
     sync::{Arc, RwLock},
 };
-use tokio::sync::Semaphore;
+use tokio::sync::{OwnedRwLockReadGuard, RwLock as TokioRwLock, Semaphore};
 
 use self::proxy::ProxyConfig;
 
@@ -36,8 +43,16 @@ use self::proxy::ProxyConfig;
 pub struct Config {
     server_name: Box<ServerName>,
     database_path: String,
-    #[serde(default = "default_cache_capacity")]
-    cache_capacity: u32,
+    #[serde(default = "default_db_cache_capacity_mb")]
+    db_cache_capacity_mb: f64,
+    #[serde(default = "default_sqlite_read_pool_size")]
+    sqlite_read_pool_size: usize,
+    #[serde(default = "true_fn")]
+    sqlite_wal_clean_timer: bool,
+    #[serde(default = "default_sqlite_wal_clean_second_interval")]
+    sqlite_wal_clean_second_interval: u32,
+    #[serde(default = "default_sqlite_wal_clean_second_timeout")]
+    sqlite_wal_clean_second_timeout: u32,
     #[serde(default = "default_max_request_size")]
     max_request_size: u32,
     #[serde(default = "default_max_concurrent_requests")]
@@ -57,6 +72,29 @@ pub struct Config {
     trusted_servers: Vec<Box<ServerName>>,
     #[serde(default = "default_log")]
     pub log: String,
+
+    #[serde(flatten)]
+    catchall: BTreeMap<String, IgnoredAny>,
+}
+
+const DEPRECATED_KEYS: &[&str] = &["cache_capacity"];
+
+impl Config {
+    pub fn warn_deprecated(&self) {
+        let mut was_deprecated = false;
+        for key in self
+            .catchall
+            .keys()
+            .filter(|key| DEPRECATED_KEYS.iter().any(|s| s == key))
+        {
+            log::warn!("Config parameter {} is deprecated", key);
+            was_deprecated = true;
+        }
+
+        if was_deprecated {
+            log::warn!("Read conduit documentation and check your configuration if any new configuration parameters should be adjusted");
+        }
+    }
 }
 
 fn false_fn() -> bool {
@@ -67,8 +105,20 @@ fn true_fn() -> bool {
     true
 }
 
-fn default_cache_capacity() -> u32 {
-    1024 * 1024 * 1024
+fn default_db_cache_capacity_mb() -> f64 {
+    200.0
+}
+
+fn default_sqlite_read_pool_size() -> usize {
+    num_cpus::get().max(1)
+}
+
+fn default_sqlite_wal_clean_second_interval() -> u32 {
+    60 * 60
+}
+
+fn default_sqlite_wal_clean_second_timeout() -> u32 {
+    2
 }
 
 fn default_max_request_size() -> u32 {
@@ -84,12 +134,16 @@ fn default_log() -> String {
 }
 
 #[cfg(feature = "sled")]
-pub type Engine = abstraction::SledEngine;
+pub type Engine = abstraction::sled::Engine;
 
 #[cfg(feature = "rocksdb")]
-pub type Engine = abstraction::RocksDbEngine;
+pub type Engine = abstraction::rocksdb::Engine;
+
+#[cfg(feature = "sqlite")]
+pub type Engine = abstraction::sqlite::Engine;
 
 pub struct Database {
+    _db: Arc<Engine>,
     pub globals: globals::Globals,
     pub users: users::Users,
     pub uiaa: uiaa::Uiaa,
@@ -117,8 +171,37 @@ impl Database {
         Ok(())
     }
 
+    fn check_sled_or_sqlite_db(config: &Config) -> Result<()> {
+        let path = Path::new(&config.database_path);
+
+        #[cfg(feature = "backend_sqlite")]
+        {
+            let sled_exists = path.join("db").exists();
+            let sqlite_exists = path.join("conduit.db").exists();
+            if sled_exists {
+                if sqlite_exists {
+                    // most likely an in-place directory, only warn
+                    log::warn!("Both sled and sqlite databases are detected in database directory");
+                    log::warn!("Currently running from the sqlite database, but consider removing sled database files to free up space")
+                } else {
+                    log::error!(
+                        "Sled database detected, conduit now uses sqlite for database operations"
+                    );
+                    log::error!("This database must be converted to sqlite, go to https://github.com/ShadowJonathan/conduit_toolbox#conduit_sled_to_sqlite");
+                    return Err(Error::bad_config(
+                        "sled database detected, migrate to sqlite",
+                    ));
+                }
+            }
+        }
+
+        Ok(())
+    }
+
     /// Load an existing database or create a new one.
-    pub async fn load_or_create(config: Config) -> Result<Arc<Self>> {
+    pub async fn load_or_create(config: Config) -> Result<Arc<TokioRwLock<Self>>> {
+        Self::check_sled_or_sqlite_db(&config)?;
+
         let builder = Engine::open(&config)?;
 
         if config.max_request_size < 1024 {
@@ -128,7 +211,8 @@ impl Database {
         let (admin_sender, admin_receiver) = mpsc::unbounded();
         let (sending_sender, sending_receiver) = mpsc::unbounded();
 
-        let db = Arc::new(Self {
+        let db = Arc::new(TokioRwLock::from(Self {
+            _db: builder.clone(),
             users: users::Users {
                 userid_password: builder.open_tree("userid_password")?,
                 userid_displayname: builder.open_tree("userid_displayname")?,
@@ -231,100 +315,112 @@ impl Database {
             globals: globals::Globals::load(
                 builder.open_tree("global")?,
                 builder.open_tree("server_signingkeys")?,
-                config,
+                config.clone(),
             )?,
-        });
+        }));
 
-        // MIGRATIONS
-        // TODO: database versions of new dbs should probably not be 0
-        if db.globals.database_version()? < 1 {
-            for (roomserverid, _) in db.rooms.roomserverids.iter() {
-                let mut parts = roomserverid.split(|&b| b == 0xff);
-                let room_id = parts.next().expect("split always returns one element");
-                let servername = match parts.next() {
-                    Some(s) => s,
-                    None => {
-                        error!("Migration: Invalid roomserverid in db.");
+        {
+            let db = db.read().await;
+            // MIGRATIONS
+            // TODO: database versions of new dbs should probably not be 0
+            if db.globals.database_version()? < 1 {
+                for (roomserverid, _) in db.rooms.roomserverids.iter() {
+                    let mut parts = roomserverid.split(|&b| b == 0xff);
+                    let room_id = parts.next().expect("split always returns one element");
+                    let servername = match parts.next() {
+                        Some(s) => s,
+                        None => {
+                            error!("Migration: Invalid roomserverid in db.");
+                            continue;
+                        }
+                    };
+                    let mut serverroomid = servername.to_vec();
+                    serverroomid.push(0xff);
+                    serverroomid.extend_from_slice(room_id);
+
+                    db.rooms.serverroomids.insert(&serverroomid, &[])?;
+                }
+
+                db.globals.bump_database_version(1)?;
+
+                println!("Migration: 0 -> 1 finished");
+            }
+
+            if db.globals.database_version()? < 2 {
+                // We accidentally inserted hashed versions of "" into the db instead of just ""
+                for (userid, password) in db.users.userid_password.iter() {
+                    let password = utils::string_from_bytes(&password);
+
+                    let empty_hashed_password = password.map_or(false, |password| {
+                        argon2::verify_encoded(&password, b"").unwrap_or(false)
+                    });
+
+                    if empty_hashed_password {
+                        db.users.userid_password.insert(&userid, b"")?;
+                    }
+                }
+
+                db.globals.bump_database_version(2)?;
+
+                println!("Migration: 1 -> 2 finished");
+            }
+
+            if db.globals.database_version()? < 3 {
+                // Move media to filesystem
+                for (key, content) in db.media.mediaid_file.iter() {
+                    if content.len() == 0 {
                         continue;
                     }
-                };
-                let mut serverroomid = servername.to_vec();
-                serverroomid.push(0xff);
-                serverroomid.extend_from_slice(room_id);
 
-                db.rooms.serverroomids.insert(&serverroomid, &[])?;
-            }
-
-            db.globals.bump_database_version(1)?;
-
-            println!("Migration: 0 -> 1 finished");
-        }
-
-        if db.globals.database_version()? < 2 {
-            // We accidentally inserted hashed versions of "" into the db instead of just ""
-            for (userid, password) in db.users.userid_password.iter() {
-                let password = utils::string_from_bytes(&password);
-
-                let empty_hashed_password = password.map_or(false, |password| {
-                    argon2::verify_encoded(&password, b"").unwrap_or(false)
-                });
-
-                if empty_hashed_password {
-                    db.users.userid_password.insert(&userid, b"")?;
-                }
-            }
-
-            db.globals.bump_database_version(2)?;
-
-            println!("Migration: 1 -> 2 finished");
-        }
-
-        if db.globals.database_version()? < 3 {
-            // Move media to filesystem
-            for (key, content) in db.media.mediaid_file.iter() {
-                if content.len() == 0 {
-                    continue;
+                    let path = db.globals.get_media_file(&key);
+                    let mut file = fs::File::create(path)?;
+                    file.write_all(&content)?;
+                    db.media.mediaid_file.insert(&key, &[])?;
                 }
 
-                let path = db.globals.get_media_file(&key);
-                let mut file = fs::File::create(path)?;
-                file.write_all(&content)?;
-                db.media.mediaid_file.insert(&key, &[])?;
+                db.globals.bump_database_version(3)?;
+
+                println!("Migration: 2 -> 3 finished");
             }
 
-            db.globals.bump_database_version(3)?;
-
-            println!("Migration: 2 -> 3 finished");
-        }
-
-        if db.globals.database_version()? < 4 {
-            // Add federated users to db as deactivated
-            for our_user in db.users.iter() {
-                let our_user = our_user?;
-                if db.users.is_deactivated(&our_user)? {
-                    continue;
-                }
-                for room in db.rooms.rooms_joined(&our_user) {
-                    for user in db.rooms.room_members(&room?) {
-                        let user = user?;
-                        if user.server_name() != db.globals.server_name() {
-                            println!("Migration: Creating user {}", user);
-                            db.users.create(&user, None)?;
+            if db.globals.database_version()? < 4 {
+                // Add federated users to db as deactivated
+                for our_user in db.users.iter() {
+                    let our_user = our_user?;
+                    if db.users.is_deactivated(&our_user)? {
+                        continue;
+                    }
+                    for room in db.rooms.rooms_joined(&our_user) {
+                        for user in db.rooms.room_members(&room?) {
+                            let user = user?;
+                            if user.server_name() != db.globals.server_name() {
+                                println!("Migration: Creating user {}", user);
+                                db.users.create(&user, None)?;
+                            }
                         }
                     }
                 }
+
+                db.globals.bump_database_version(4)?;
+
+                println!("Migration: 3 -> 4 finished");
             }
-
-            db.globals.bump_database_version(4)?;
-
-            println!("Migration: 3 -> 4 finished");
         }
 
-        // This data is probably outdated
-        db.rooms.edus.presenceid_presence.clear()?;
+        let guard = db.read().await;
 
-        db.admin.start_handler(Arc::clone(&db), admin_receiver);
-        db.sending.start_handler(Arc::clone(&db), sending_receiver);
+        // This data is probably outdated
+        guard.rooms.edus.presenceid_presence.clear()?;
+
+        guard.admin.start_handler(Arc::clone(&db), admin_receiver);
+        guard
+            .sending
+            .start_handler(Arc::clone(&db), sending_receiver);
+
+        drop(guard);
+
+        #[cfg(feature = "sqlite")]
+        Self::start_wal_clean_task(&db, &config).await;
 
         Ok(db)
     }
@@ -413,13 +509,113 @@ impl Database {
                 .watch_prefix(&userid_bytes),
         );
 
+        futures.push(Box::pin(self.globals.rotate.watch()));
+
         // Wait until one of them finds something
         futures.next().await;
     }
 
     pub async fn flush(&self) -> Result<()> {
-        // noop while we don't use sled 1.0
-        //self._db.flush_async().await?;
-        Ok(())
+        let start = std::time::Instant::now();
+
+        let res = self._db.flush();
+
+        log::debug!("flush: took {:?}", start.elapsed());
+
+        res
+    }
+
+    #[cfg(feature = "sqlite")]
+    pub fn flush_wal(&self) -> Result<()> {
+        self._db.flush_wal()
+    }
+
+    #[cfg(feature = "sqlite")]
+    pub async fn start_wal_clean_task(lock: &Arc<TokioRwLock<Self>>, config: &Config) {
+        use tokio::{
+            select,
+            signal::unix::{signal, SignalKind},
+            time::{interval, timeout},
+        };
+
+        use std::{
+            sync::Weak,
+            time::{Duration, Instant},
+        };
+
+        let weak: Weak<TokioRwLock<Database>> = Arc::downgrade(&lock);
+
+        let lock_timeout = Duration::from_secs(config.sqlite_wal_clean_second_timeout as u64);
+        let timer_interval = Duration::from_secs(config.sqlite_wal_clean_second_interval as u64);
+        let do_timer = config.sqlite_wal_clean_timer;
+
+        tokio::spawn(async move {
+            let mut i = interval(timer_interval);
+            let mut s = signal(SignalKind::hangup()).unwrap();
+
+            loop {
+                select! {
+                    _ = i.tick(), if do_timer => {
+                        log::info!(target: "wal-trunc", "Timer ticked")
+                    }
+                    _ = s.recv() => {
+                        log::info!(target: "wal-trunc", "Received SIGHUP")
+                    }
+                };
+
+                if let Some(arc) = Weak::upgrade(&weak) {
+                    log::info!(target: "wal-trunc", "Rotating sync helpers...");
+                    // This actually creates a very small race condition between firing this and trying to acquire the subsequent write lock.
+                    // Though it is not a huge deal if the write lock doesn't "catch", as it'll harmlessly time out.
+                    arc.read().await.globals.rotate.fire();
+
+                    log::info!(target: "wal-trunc", "Locking...");
+                    let guard = {
+                        if let Ok(guard) = timeout(lock_timeout, arc.write()).await {
+                            guard
+                        } else {
+                            log::info!(target: "wal-trunc", "Lock failed in timeout, canceled.");
+                            continue;
+                        }
+                    };
+                    log::info!(target: "wal-trunc", "Locked, flushing...");
+                    let start = Instant::now();
+                    if let Err(e) = guard.flush_wal() {
+                        log::error!(target: "wal-trunc", "Errored: {}", e);
+                    } else {
+                        log::info!(target: "wal-trunc", "Flushed in {:?}", start.elapsed());
+                    }
+                } else {
+                    break;
+                }
+            }
+        });
+    }
+}
+
+pub struct DatabaseGuard(OwnedRwLockReadGuard<Database>);
+
+impl Deref for DatabaseGuard {
+    type Target = OwnedRwLockReadGuard<Database>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+#[rocket::async_trait]
+impl<'r> FromRequest<'r> for DatabaseGuard {
+    type Error = ();
+
+    async fn from_request(req: &'r Request<'_>) -> rocket::request::Outcome<Self, ()> {
+        let db = try_outcome!(req.guard::<State<'_, Arc<TokioRwLock<Database>>>>().await);
+
+        Ok(DatabaseGuard(Arc::clone(&db).read_owned().await)).or_forward(())
+    }
+}
+
+impl Into<DatabaseGuard> for OwnedRwLockReadGuard<Database> {
+    fn into(self) -> DatabaseGuard {
+        DatabaseGuard(self)
     }
 }