magmaus3/continuwuity
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

batch queries to maximize throughput

Browse source 
query-side streams for first level of callsites

Signed-off-by: Jason Volk <jason@zemos.net>
This commit is contained in:
Jason Volk 2025-01-01 06:08:20 +00:00 • committed by strawberry
parent 1792711d09
commit 2259e2c82f
13 changed files with 191 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/service/rooms/auth_chain/mod.rs
View file

@ -71,7 +71,7 @@ impl Service {
let event_ids = self
.services
.short
.multi_get_eventid_from_short(chain.iter())
.multi_get_eventid_from_short(chain.into_iter().stream())
.ready_filter_map(Result::ok)
.collect()
.await;

12
src/service/rooms/short/mod.rs
View file

@ -69,7 +69,7 @@ where
{
self.db
.eventid_shorteventid
.get_batch(event_ids.clone())
.get_batch(event_ids.clone().stream())
.zip(event_ids.into_iter().stream())
.map(|(result, event_id)| match result {
| Ok(ref short) => utils::u64_from_u8(short),
@ -162,20 +162,18 @@ where
}
#[implement(Service)]
pub fn multi_get_eventid_from_short<'a, Id, I>(
pub fn multi_get_eventid_from_short<'a, Id, S>(
&'a self,
shorteventid: I,
shorteventid: S,
) -> impl Stream<Item = Result<Id>> + Send + 'a
where
I: Iterator<Item = &'a ShortEventId> + Send + 'a,
S: Stream<Item = ShortEventId> + Send + 'a,
Id: for<'de> Deserialize<'de> + Sized + ToOwned + 'a,
<Id as ToOwned>::Owned: Borrow<EventId>,
{
const BUFSIZE: usize = size_of::<ShortEventId>();
self.db
.shorteventid_eventid
.aqry_batch::<BUFSIZE, _, _>(shorteventid)
.qry_batch(shorteventid)
.map(Deserialized::deserialized)
}

2
src/service/rooms/state/mod.rs
View file

@ -467,7 +467,7 @@ impl Service {
let auth_pdus = self
.services
.short
.multi_get_eventid_from_short(event_ids.iter())
.multi_get_eventid_from_short(event_ids.into_iter().stream())
.zip(state_keys.into_iter().stream())
.ready_filter_map(|(event_id, tsk)| Some((tsk, event_id.ok()?)))
.broad_filter_map(|(tsk, event_id): (_, OwnedEventId)| async move {

6
src/service/rooms/state_accessor/data.rs
View file

@ -1,7 +1,7 @@
use std::{borrow::Borrow, collections::HashMap, sync::Arc};
use conduwuit::{
at, err, ref_at,
at, err,
utils::stream::{BroadbandExt, IterStream, ReadyExt},
PduEvent, Result,
};
@ -69,7 +69,7 @@ impl Data {
let full_pdus = self
.services
.short
.multi_get_eventid_from_short(short_ids.iter().map(ref_at!(1)))
.multi_get_eventid_from_short(short_ids.into_iter().map(at!(1)).stream())
.ready_filter_map(Result::ok)
.broad_filter_map(|event_id: OwnedEventId| async move {
self.services.timeline.get_pdu(&event_id).await.ok()
@ -93,7 +93,7 @@ impl Data {
let full_ids = self
.services
.short
.multi_get_eventid_from_short(short_ids.iter().map(ref_at!(1)))
.multi_get_eventid_from_short(short_ids.iter().map(at!(1)).stream())
.zip(short_ids.iter().stream().map(at!(0)))
.ready_filter_map(|(event_id, shortstatekey)| Some((shortstatekey, event_id.ok()?)))
.collect()