Database Refactor
combine service/users data w/ mod unit split sliding sync related out of service/users instrument database entry points remove increment crap from database interface de-wrap all database get() calls de-wrap all database insert() calls de-wrap all database remove() calls refactor database interface for async streaming add query key serializer for database implement Debug for result handle add query deserializer for database add deserialization trait for option handle start a stream utils suite de-wrap/asyncify/type-query count_one_time_keys() de-wrap/asyncify users count add admin query users command suite de-wrap/asyncify users exists de-wrap/partially asyncify user filter related asyncify/de-wrap users device/keys related asyncify/de-wrap user auth/misc related asyncify/de-wrap users blurhash asyncify/de-wrap account_data get; merge Data into Service partial asyncify/de-wrap uiaa; merge Data into Service partially asyncify/de-wrap transaction_ids get; merge Data into Service partially asyncify/de-wrap key_backups; merge Data into Service asyncify/de-wrap pusher service getters; merge Data into Service asyncify/de-wrap rooms alias getters/some iterators asyncify/de-wrap rooms directory getters/iterator partially asyncify/de-wrap rooms lazy-loading partially asyncify/de-wrap rooms metadata asyncify/dewrap rooms outlier asyncify/dewrap rooms pdu_metadata dewrap/partially asyncify rooms read receipt de-wrap rooms search service de-wrap/partially asyncify rooms user service partial de-wrap rooms state_compressor de-wrap rooms state_cache de-wrap room state et al de-wrap rooms timeline service additional users device/keys related de-wrap/asyncify sender asyncify services refactor database to TryFuture/TryStream refactor services for TryFuture/TryStream asyncify api handlers additional asyncification for admin module abstract stream related; support reverse streams additional stream conversions asyncify state-res related Signed-off-by: Jason Volk <jason@zemos.net>
This commit is contained in:
Jason Volk
strawberry
parent
6001014078
commit
946ca364e0
203 changed files with 12202 additions and 10709 deletions
|
|
@ -1,15 +1,39 @@
|
|||
use std::{ffi::CStr, future::Future, mem::size_of, pin::Pin, sync::Arc};
|
||||
mod count;
|
||||
mod keys;
|
||||
mod keys_from;
|
||||
mod keys_prefix;
|
||||
mod rev_keys;
|
||||
mod rev_keys_from;
|
||||
mod rev_keys_prefix;
|
||||
mod rev_stream;
|
||||
mod rev_stream_from;
|
||||
mod rev_stream_prefix;
|
||||
mod stream;
|
||||
mod stream_from;
|
||||
mod stream_prefix;
|
||||
|
||||
use conduit::{utils, Result};
|
||||
use rocksdb::{
|
||||
AsColumnFamilyRef, ColumnFamily, Direction, IteratorMode, ReadOptions, WriteBatchWithTransaction, WriteOptions,
|
||||
use std::{
|
||||
convert::AsRef,
|
||||
ffi::CStr,
|
||||
fmt,
|
||||
fmt::{Debug, Display},
|
||||
future::Future,
|
||||
io::Write,
|
||||
pin::Pin,
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use conduit::{err, Result};
|
||||
use futures::future;
|
||||
use rocksdb::{AsColumnFamilyRef, ColumnFamily, ReadOptions, WriteBatchWithTransaction, WriteOptions};
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::{
|
||||
or_else, result,
|
||||
slice::{Byte, Key, KeyVal, OwnedKey, OwnedKeyValPair, OwnedVal, Val},
|
||||
keyval::{OwnedKey, OwnedVal},
|
||||
ser,
|
||||
util::{map_err, or_else},
|
||||
watchers::Watchers,
|
||||
Engine, Handle, Iter,
|
||||
Engine, Handle,
|
||||
};
|
||||
|
||||
pub struct Map {
|
||||
|
|
@ -21,8 +45,6 @@ pub struct Map {
|
|||
read_options: ReadOptions,
|
||||
}
|
||||
|
||||
type OwnedKeyValPairIter<'a> = Box<dyn Iterator<Item = OwnedKeyValPair> + Send + 'a>;
|
||||
|
||||
impl Map {
|
||||
pub(crate) fn open(db: &Arc<Engine>, name: &str) -> Result<Arc<Self>> {
|
||||
Ok(Arc::new(Self {
|
||||
|
|
@ -35,14 +57,125 @@ impl Map {
|
|||
}))
|
||||
}
|
||||
|
||||
pub fn get(&self, key: &Key) -> Result<Option<Handle<'_>>> {
|
||||
let read_options = &self.read_options;
|
||||
let res = self.db.db.get_pinned_cf_opt(&self.cf(), key, read_options);
|
||||
|
||||
Ok(result(res)?.map(Handle::from))
|
||||
#[tracing::instrument(skip(self), fields(%self), level = "trace")]
|
||||
pub fn del<K>(&self, key: &K)
|
||||
where
|
||||
K: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let mut buf = Vec::<u8>::with_capacity(64);
|
||||
self.bdel(key, &mut buf);
|
||||
}
|
||||
|
||||
pub fn multi_get(&self, keys: &[&Key]) -> Result<Vec<Option<OwnedVal>>> {
|
||||
#[tracing::instrument(skip(self, buf), fields(%self), level = "trace")]
|
||||
pub fn bdel<K, B>(&self, key: &K, buf: &mut B)
|
||||
where
|
||||
K: Serialize + ?Sized + Debug,
|
||||
B: Write + AsRef<[u8]>,
|
||||
{
|
||||
let key = ser::serialize(buf, key).expect("failed to serialize deletion key");
|
||||
self.remove(&key);
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace")]
|
||||
pub fn remove<K>(&self, key: &K)
|
||||
where
|
||||
K: AsRef<[u8]> + ?Sized + Debug,
|
||||
{
|
||||
let write_options = &self.write_options;
|
||||
self.db
|
||||
.db
|
||||
.delete_cf_opt(&self.cf(), key, write_options)
|
||||
.or_else(or_else)
|
||||
.expect("database remove error");
|
||||
|
||||
if !self.db.corked() {
|
||||
self.db.flush().expect("database flush error");
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, value), fields(%self), level = "trace")]
|
||||
pub fn insert<K, V>(&self, key: &K, value: &V)
|
||||
where
|
||||
K: AsRef<[u8]> + ?Sized + Debug,
|
||||
V: AsRef<[u8]> + ?Sized,
|
||||
{
|
||||
let write_options = &self.write_options;
|
||||
self.db
|
||||
.db
|
||||
.put_cf_opt(&self.cf(), key, value, write_options)
|
||||
.or_else(or_else)
|
||||
.expect("database insert error");
|
||||
|
||||
if !self.db.corked() {
|
||||
self.db.flush().expect("database flush error");
|
||||
}
|
||||
|
||||
self.watchers.wake(key.as_ref());
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self), fields(%self), level = "trace")]
|
||||
pub fn insert_batch<'a, I, K, V>(&'a self, iter: I)
|
||||
where
|
||||
I: Iterator<Item = &'a (K, V)> + Send + Debug,
|
||||
K: AsRef<[u8]> + Sized + Debug + 'a,
|
||||
V: AsRef<[u8]> + Sized + 'a,
|
||||
{
|
||||
let mut batch = WriteBatchWithTransaction::<false>::default();
|
||||
for (key, val) in iter {
|
||||
batch.put_cf(&self.cf(), key.as_ref(), val.as_ref());
|
||||
}
|
||||
|
||||
let write_options = &self.write_options;
|
||||
self.db
|
||||
.db
|
||||
.write_opt(batch, write_options)
|
||||
.or_else(or_else)
|
||||
.expect("database insert batch error");
|
||||
|
||||
if !self.db.corked() {
|
||||
self.db.flush().expect("database flush error");
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self), fields(%self), level = "trace")]
|
||||
pub fn qry<K>(&self, key: &K) -> impl Future<Output = Result<Handle<'_>>> + Send
|
||||
where
|
||||
K: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let mut buf = Vec::<u8>::with_capacity(64);
|
||||
self.bqry(key, &mut buf)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, buf), fields(%self), level = "trace")]
|
||||
pub fn bqry<K, B>(&self, key: &K, buf: &mut B) -> impl Future<Output = Result<Handle<'_>>> + Send
|
||||
where
|
||||
K: Serialize + ?Sized + Debug,
|
||||
B: Write + AsRef<[u8]>,
|
||||
{
|
||||
let key = ser::serialize(buf, key).expect("failed to serialize query key");
|
||||
let val = self.get(key);
|
||||
future::ready(val)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self), fields(%self), level = "trace")]
|
||||
pub fn get<K>(&self, key: &K) -> Result<Handle<'_>>
|
||||
where
|
||||
K: AsRef<[u8]> + ?Sized + Debug,
|
||||
{
|
||||
self.db
|
||||
.db
|
||||
.get_pinned_cf_opt(&self.cf(), key, &self.read_options)
|
||||
.map_err(map_err)?
|
||||
.map(Handle::from)
|
||||
.ok_or(err!(Request(NotFound("Not found in database"))))
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self), fields(%self), level = "trace")]
|
||||
pub fn multi_get<'a, I, K>(&self, keys: I) -> Vec<Option<OwnedVal>>
|
||||
where
|
||||
I: Iterator<Item = &'a K> + ExactSizeIterator + Send + Debug,
|
||||
K: AsRef<[u8]> + Sized + Debug + 'a,
|
||||
{
|
||||
// Optimization can be `true` if key vector is pre-sorted **by the column
|
||||
// comparator**.
|
||||
const SORTED: bool = false;
|
||||
|
|
@ -57,140 +190,25 @@ impl Map {
|
|||
match res {
|
||||
Ok(Some(res)) => ret.push(Some((*res).to_vec())),
|
||||
Ok(None) => ret.push(None),
|
||||
Err(e) => return or_else(e),
|
||||
Err(e) => or_else(e).expect("database multiget error"),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(ret)
|
||||
ret
|
||||
}
|
||||
|
||||
pub fn insert(&self, key: &Key, value: &Val) -> Result<()> {
|
||||
let write_options = &self.write_options;
|
||||
self.db
|
||||
.db
|
||||
.put_cf_opt(&self.cf(), key, value, write_options)
|
||||
.or_else(or_else)?;
|
||||
|
||||
if !self.db.corked() {
|
||||
self.db.flush()?;
|
||||
}
|
||||
|
||||
self.watchers.wake(key);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn insert_batch<'a, I>(&'a self, iter: I) -> Result<()>
|
||||
#[inline]
|
||||
pub fn watch_prefix<'a, K>(&'a self, prefix: &K) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>
|
||||
where
|
||||
I: Iterator<Item = KeyVal<'a>>,
|
||||
K: AsRef<[u8]> + ?Sized + Debug,
|
||||
{
|
||||
let mut batch = WriteBatchWithTransaction::<false>::default();
|
||||
for KeyVal(key, value) in iter {
|
||||
batch.put_cf(&self.cf(), key, value);
|
||||
}
|
||||
|
||||
let write_options = &self.write_options;
|
||||
let res = self.db.db.write_opt(batch, write_options);
|
||||
|
||||
if !self.db.corked() {
|
||||
self.db.flush()?;
|
||||
}
|
||||
|
||||
result(res)
|
||||
}
|
||||
|
||||
pub fn remove(&self, key: &Key) -> Result<()> {
|
||||
let write_options = &self.write_options;
|
||||
let res = self.db.db.delete_cf_opt(&self.cf(), key, write_options);
|
||||
|
||||
if !self.db.corked() {
|
||||
self.db.flush()?;
|
||||
}
|
||||
|
||||
result(res)
|
||||
}
|
||||
|
||||
pub fn remove_batch<'a, I>(&'a self, iter: I) -> Result<()>
|
||||
where
|
||||
I: Iterator<Item = &'a Key>,
|
||||
{
|
||||
let mut batch = WriteBatchWithTransaction::<false>::default();
|
||||
for key in iter {
|
||||
batch.delete_cf(&self.cf(), key);
|
||||
}
|
||||
|
||||
let write_options = &self.write_options;
|
||||
let res = self.db.db.write_opt(batch, write_options);
|
||||
|
||||
if !self.db.corked() {
|
||||
self.db.flush()?;
|
||||
}
|
||||
|
||||
result(res)
|
||||
}
|
||||
|
||||
pub fn iter(&self) -> OwnedKeyValPairIter<'_> {
|
||||
let mode = IteratorMode::Start;
|
||||
let read_options = read_options_default();
|
||||
Box::new(Iter::new(&self.db, &self.cf, read_options, &mode))
|
||||
}
|
||||
|
||||
pub fn iter_from(&self, from: &Key, reverse: bool) -> OwnedKeyValPairIter<'_> {
|
||||
let direction = if reverse {
|
||||
Direction::Reverse
|
||||
} else {
|
||||
Direction::Forward
|
||||
};
|
||||
let mode = IteratorMode::From(from, direction);
|
||||
let read_options = read_options_default();
|
||||
Box::new(Iter::new(&self.db, &self.cf, read_options, &mode))
|
||||
}
|
||||
|
||||
pub fn scan_prefix(&self, prefix: OwnedKey) -> OwnedKeyValPairIter<'_> {
|
||||
let mode = IteratorMode::From(&prefix, Direction::Forward);
|
||||
let read_options = read_options_default();
|
||||
Box::new(Iter::new(&self.db, &self.cf, read_options, &mode).take_while(move |(k, _)| k.starts_with(&prefix)))
|
||||
}
|
||||
|
||||
pub fn increment(&self, key: &Key) -> Result<[Byte; size_of::<u64>()]> {
|
||||
let old = self.get(key)?;
|
||||
let new = utils::increment(old.as_deref());
|
||||
self.insert(key, &new)?;
|
||||
|
||||
if !self.db.corked() {
|
||||
self.db.flush()?;
|
||||
}
|
||||
|
||||
Ok(new)
|
||||
}
|
||||
|
||||
pub fn increment_batch<'a, I>(&'a self, iter: I) -> Result<()>
|
||||
where
|
||||
I: Iterator<Item = &'a Key>,
|
||||
{
|
||||
let mut batch = WriteBatchWithTransaction::<false>::default();
|
||||
for key in iter {
|
||||
let old = self.get(key)?;
|
||||
let new = utils::increment(old.as_deref());
|
||||
batch.put_cf(&self.cf(), key, new);
|
||||
}
|
||||
|
||||
let write_options = &self.write_options;
|
||||
let res = self.db.db.write_opt(batch, write_options);
|
||||
|
||||
if !self.db.corked() {
|
||||
self.db.flush()?;
|
||||
}
|
||||
|
||||
result(res)
|
||||
}
|
||||
|
||||
pub fn watch_prefix<'a>(&'a self, prefix: &Key) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
|
||||
self.watchers.watch(prefix)
|
||||
self.watchers.watch(prefix.as_ref())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn property_integer(&self, name: &CStr) -> Result<u64> { self.db.property_integer(&self.cf(), name) }
|
||||
|
||||
#[inline]
|
||||
pub fn property(&self, name: &str) -> Result<String> { self.db.property(&self.cf(), name) }
|
||||
|
||||
#[inline]
|
||||
|
|
@ -199,12 +217,12 @@ impl Map {
|
|||
fn cf(&self) -> impl AsColumnFamilyRef + '_ { &*self.cf }
|
||||
}
|
||||
|
||||
impl<'a> IntoIterator for &'a Map {
|
||||
type IntoIter = Box<dyn Iterator<Item = Self::Item> + Send + 'a>;
|
||||
type Item = OwnedKeyValPair;
|
||||
impl Debug for Map {
|
||||
fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { write!(out, "Map {{name: {0}}}", self.name) }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn into_iter(self) -> Self::IntoIter { self.iter() }
|
||||
impl Display for Map {
|
||||
fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { write!(out, "{0}", self.name) }
|
||||
}
|
||||
|
||||
fn open(db: &Arc<Engine>, name: &str) -> Result<Arc<ColumnFamily>> {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue