batch queries to maximize throughput

query-side streams for first level of callsites

Signed-off-by: Jason Volk <jason@zemos.net>

src/core/config/mod.rs

@@ -1653,6 +1653,21 @@ pub struct Config {
 	#[serde(default = "default_stream_width_scale")]
 	pub stream_width_scale: f32,
 
+	/// Sets the initial amplification factor. This controls batch sizes of
+	/// requests made by each pool worker, multiplying the throughput of each
+	/// stream. This value is somewhat abstract from specific hardware
+	/// characteristics and can be significantly larger than any thread count or
+	/// queue size. This is because each database query may require several
+	/// index lookups, thus many database queries in a batch may make progress
+	/// independently while also sharing index and data blocks which may or may
+	/// not be cached. It is worthwhile to submit huge batches to reduce
+	/// complexity. The maximum value is 32768, though sufficient hardware is
+	/// still advised for that.
+	///
+	/// default: 1024
+	#[serde(default = "default_stream_amplification")]
+	pub stream_amplification: usize,
+
 	/// Number of sender task workers; determines sender parallelism. Default is
 	/// '0' which means the value is determined internally, likely matching the
 	/// number of tokio worker-threads or number of cores, etc. Override by
@@ -2467,3 +2482,5 @@ fn default_db_pool_queue_mult() -> usize { 4 }
 fn default_stream_width_default() -> usize { 32 }
 
 fn default_stream_width_scale() -> f32 { 1.0 }
+
+fn default_stream_amplification() -> usize { 1024 }

src/core/utils/stream/band.rs

@@ -3,9 +3,15 @@ use std::sync::atomic::{AtomicUsize, Ordering};
 /// Stream concurrency factor; this is a live value.
 static WIDTH: AtomicUsize = AtomicUsize::new(32);
 
-/// Practicable limits on the stream width
+/// Stream throughput amplifier; this is a live value.
+static AMPLIFICATION: AtomicUsize = AtomicUsize::new(1024);
+
+/// Practicable limits on the stream width.
 pub const WIDTH_LIMIT: (usize, usize) = (1, 1024);
 
+/// Practicable limits on the stream amplifier.
+pub const AMPLIFICATION_LIMIT: (usize, usize) = (32, 32768);
+
 /// Sets the live concurrency factor. The first return value is the previous
 /// width which was replaced. The second return value is the value which was set
 /// after any applied limits.
@@ -14,6 +20,14 @@ pub fn set_width(width: usize) -> (usize, usize) {
 	(WIDTH.swap(width, Ordering::Relaxed), width)
 }
 
+/// Sets the live concurrency amplification. The first return value is the
+/// previous width which was replaced. The second return value is the value
+/// which was set after any applied limits.
+pub fn set_amplification(width: usize) -> (usize, usize) {
+	let width = width.clamp(AMPLIFICATION_LIMIT.0, AMPLIFICATION_LIMIT.1);
+	(AMPLIFICATION.swap(width, Ordering::Relaxed), width)
+}
+
 /// Used by stream operations where the concurrency factor hasn't been manually
 /// supplied by the caller (most uses). Instead we provide a default value which
 /// is adjusted at startup for the specific system and also dynamically.
@@ -24,3 +38,13 @@ pub fn automatic_width() -> usize {
 	debug_assert!(width <= WIDTH_LIMIT.1, "WIDTH is probably too large");
 	width
 }
+
+/// Used by stream operations where the amplification hasn't been manually
+/// supplied by the caller. Instead we provide a computed value.
+#[inline]
+pub fn automatic_amplification() -> usize {
+	let amplification = AMPLIFICATION.load(Ordering::Relaxed);
+	debug_assert!(amplification >= AMPLIFICATION_LIMIT.0, "amplification is too low");
+	debug_assert!(amplification <= AMPLIFICATION_LIMIT.1, "amplification is too high");
+	amplification
+}

src/core/utils/stream/mod.rs

@@ -10,7 +10,10 @@ mod try_broadband;
 mod try_ready;
 mod wideband;
 
-pub use band::{automatic_width, set_width, WIDTH_LIMIT};
+pub use band::{
+	automatic_amplification, automatic_width, set_amplification, set_width, AMPLIFICATION_LIMIT,
+	WIDTH_LIMIT,
+};
 pub use broadband::BroadbandExt;
 pub use cloned::Cloned;
 pub use expect::TryExpect;