handle serde_json for deserialized()

Signed-off-by: Jason Volk <jason@zemos.net>

src/database/de.rs

@@ -58,10 +58,15 @@ impl<'de> Deserializer<'de> {
 	}
 
 	#[inline]
-	fn record_trail(&mut self) -> &'de [u8] {
+	fn record_next_peek_byte(&self) -> Option<u8> {
-		let record = &self.buf[self.pos..];
+		let started = self.pos != 0;
-		self.inc_pos(record.len());
+		let buf = &self.buf[self.pos..];
-		record
+		debug_assert!(
+			!started || buf[0] == Self::SEP,
+			"Missing expected record separator at current position"
+		);
+
+		buf.get::<usize>(started.into()).copied()
 	}
 
 	#[inline]
@@ -75,6 +80,13 @@ impl<'de> Deserializer<'de> {
 		self.inc_pos(started.into());
 	}
 
+	#[inline]
+	fn record_trail(&mut self) -> &'de [u8] {
+		let record = &self.buf[self.pos..];
+		self.inc_pos(record.len());
+		record
+	}
+
 	#[inline]
 	fn inc_pos(&mut self, n: usize) {
 		self.pos = self.pos.saturating_add(n);
@@ -85,13 +97,6 @@ impl<'de> Deserializer<'de> {
 impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> {
 	type Error = Error;
 
-	fn deserialize_map<V>(self, _visitor: V) -> Result<V::Value>
-	where
-		V: Visitor<'de>,
-	{
-		unimplemented!("deserialize Map not implemented")
-	}
-
 	fn deserialize_seq<V>(self, visitor: V) -> Result<V::Value>
 	where
 		V: Visitor<'de>,
@@ -113,13 +118,23 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> {
 		visitor.visit_seq(self)
 	}
 
-	fn deserialize_struct<V>(
+	fn deserialize_map<V>(self, visitor: V) -> Result<V::Value>
-		self, _name: &'static str, _fields: &'static [&'static str], _visitor: V,
-	) -> Result<V::Value>
 	where
 		V: Visitor<'de>,
 	{
-		unimplemented!("deserialize Struct not implemented")
+		let input = self.record_next();
+		let mut d = serde_json::Deserializer::from_slice(input);
+		d.deserialize_map(visitor).map_err(Into::into)
+	}
+
+	fn deserialize_struct<V>(self, name: &'static str, fields: &'static [&'static str], visitor: V) -> Result<V::Value>
+	where
+		V: Visitor<'de>,
+	{
+		let input = self.record_next();
+		let mut d = serde_json::Deserializer::from_slice(input);
+		d.deserialize_struct(name, fields, visitor)
+			.map_err(Into::into)
 	}
 
 	fn deserialize_unit_struct<V>(self, name: &'static str, visitor: V) -> Result<V::Value>
@@ -134,11 +149,14 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> {
 		visitor.visit_unit()
 	}
 
-	fn deserialize_newtype_struct<V>(self, _name: &'static str, _visitor: V) -> Result<V::Value>
+	fn deserialize_newtype_struct<V>(self, name: &'static str, visitor: V) -> Result<V::Value>
 	where
 		V: Visitor<'de>,
 	{
-		unimplemented!("deserialize Newtype Struct not implemented")
+		match name {
+			"$serde_json::private::RawValue" => visitor.visit_map(self),
+			_ => visitor.visit_newtype_struct(self),
+		}
 	}
 
 	fn deserialize_enum<V>(
@@ -228,19 +246,31 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> {
 	}
 
 	fn deserialize_unit<V: Visitor<'de>>(self, _visitor: V) -> Result<V::Value> {
-		unimplemented!("deserialize Unit Struct not implemented")
+		unimplemented!("deserialize Unit not implemented")
 	}
 
-	fn deserialize_identifier<V: Visitor<'de>>(self, _visitor: V) -> Result<V::Value> {
+	// this only used for $serde_json::private::RawValue at this time; see MapAccess
-		unimplemented!("deserialize Identifier not implemented")
+	fn deserialize_identifier<V: Visitor<'de>>(self, visitor: V) -> Result<V::Value> {
+		let input = "$serde_json::private::RawValue";
+		visitor.visit_borrowed_str(input)
 	}
 
 	fn deserialize_ignored_any<V: Visitor<'de>>(self, _visitor: V) -> Result<V::Value> {
 		unimplemented!("deserialize Ignored Any not implemented")
 	}
 
-	fn deserialize_any<V: Visitor<'de>>(self, _visitor: V) -> Result<V::Value> {
+	fn deserialize_any<V: Visitor<'de>>(self, visitor: V) -> Result<V::Value> {
-		unimplemented!("deserialize any not implemented")
+		debug_assert_eq!(
+			conduit::debug::type_name::<V>(),
+			"serde_json::value::de::<impl serde::de::Deserialize for \
+			 serde_json::value::Value>::deserialize::ValueVisitor",
+			"deserialize_any: type not expected"
+		);
+
+		match self.record_next_peek_byte() {
+			Some(b'{') => self.deserialize_map(visitor),
+			_ => self.deserialize_str(visitor),
+		}
 	}
 }
 
@@ -259,3 +289,23 @@ impl<'a, 'de: 'a> de::SeqAccess<'de> for &'a mut Deserializer<'de> {
 		seed.deserialize(&mut **self).map(Some)
 	}
 }
+
+// this only used for $serde_json::private::RawValue at this time. our db
+// schema doesn't have its own map format; we use json for that anyway
+impl<'a, 'de: 'a> de::MapAccess<'de> for &'a mut Deserializer<'de> {
+	type Error = Error;
+
+	fn next_key_seed<K>(&mut self, seed: K) -> Result<Option<K::Value>>
+	where
+		K: DeserializeSeed<'de>,
+	{
+		seed.deserialize(&mut **self).map(Some)
+	}
+
+	fn next_value_seed<V>(&mut self, seed: V) -> Result<V::Value>
+	where
+		V: DeserializeSeed<'de>,
+	{
+		seed.deserialize(&mut **self)
+	}
+}

src/database/deserialized.rs

@@ -9,11 +9,6 @@ pub trait Deserialized {
 		F: FnOnce(T) -> U,
 		T: for<'de> Deserialize<'de>;
 
-	fn map_json<T, U, F>(self, f: F) -> Result<U>
-	where
-		F: FnOnce(T) -> U,
-		T: for<'de> Deserialize<'de>;
-
 	#[inline]
 	fn deserialized<T>(self) -> Result<T>
 	where
@@ -22,13 +17,4 @@ pub trait Deserialized {
 	{
 		self.map_de(identity::<T>)
 	}
-
-	#[inline]
-	fn deserialized_json<T>(self) -> Result<T>
-	where
-		T: for<'de> Deserialize<'de>,
-		Self: Sized,
-	{
-		self.map_json(identity::<T>)
-	}
 }

src/database/handle.rs

@@ -48,15 +48,6 @@ impl AsRef<Slice> for Handle<'_> {
 }
 
 impl Deserialized for Result<Handle<'_>> {
-	#[inline]
-	fn map_json<T, U, F>(self, f: F) -> Result<U>
-	where
-		F: FnOnce(T) -> U,
-		T: for<'de> Deserialize<'de>,
-	{
-		self?.map_json(f)
-	}
-
 	#[inline]
 	fn map_de<T, U, F>(self, f: F) -> Result<U>
 	where
@@ -68,15 +59,6 @@ impl Deserialized for Result<Handle<'_>> {
 }
 
 impl<'a> Deserialized for Result<&'a Handle<'a>> {
-	#[inline]
-	fn map_json<T, U, F>(self, f: F) -> Result<U>
-	where
-		F: FnOnce(T) -> U,
-		T: for<'de> Deserialize<'de>,
-	{
-		self.and_then(|handle| handle.map_json(f))
-	}
-
 	#[inline]
 	fn map_de<T, U, F>(self, f: F) -> Result<U>
 	where
@@ -88,16 +70,6 @@ impl<'a> Deserialized for Result<&'a Handle<'a>> {
 }
 
 impl<'a> Deserialized for &'a Handle<'a> {
-	fn map_json<T, U, F>(self, f: F) -> Result<U>
-	where
-		F: FnOnce(T) -> U,
-		T: for<'de> Deserialize<'de>,
-	{
-		serde_json::from_slice::<T>(self.as_ref())
-			.map_err(Into::into)
-			.map(f)
-	}
-
 	fn map_de<T, U, F>(self, f: F) -> Result<U>
 	where
 		F: FnOnce(T) -> U,

src/service/account_data/mod.rs

@@ -108,7 +108,7 @@ pub async fn get(
 		.qry(&key)
 		.and_then(|roomuserdataid| self.db.roomuserdataid_accountdata.qry(&roomuserdataid))
 		.await
-		.deserialized_json()
+		.deserialized()
 }
 
 /// Returns all changes to the account data that happened after `since`.

src/service/appservice/data.rs

@@ -40,7 +40,7 @@ impl Data {
 		self.id_appserviceregistrations
 			.qry(id)
 			.await
-			.deserialized_json()
+			.deserialized()
 			.map_err(|e| err!(Database("Invalid appservice {id:?} registration: {e:?}")))
 	}
 

src/service/globals/data.rs

@@ -305,10 +305,7 @@ impl Data {
 	}
 
 	pub async fn signing_keys_for(&self, origin: &ServerName) -> Result<ServerSigningKeys> {
-		self.server_signingkeys
+		self.server_signingkeys.qry(origin).await.deserialized()
-			.qry(origin)
-			.await
-			.deserialized_json()
 	}
 
 	pub async fn database_version(&self) -> u64 { self.global.qry("version").await.deserialized().unwrap_or(0) }

src/service/key_backups/mod.rs

@@ -166,11 +166,7 @@ pub async fn get_latest_backup(&self, user_id: &UserId) -> Result<(String, Raw<B
 #[implement(Service)]
 pub async fn get_backup(&self, user_id: &UserId, version: &str) -> Result<Raw<BackupAlgorithm>> {
 	let key = (user_id, version);
-	self.db
+	self.db.backupid_algorithm.qry(&key).await.deserialized()
-		.backupid_algorithm
-		.qry(&key)
-		.await
-		.deserialized_json()
 }
 
 #[implement(Service)]
@@ -278,11 +274,7 @@ pub async fn get_session(
 ) -> Result<Raw<KeyBackupData>> {
 	let key = (user_id, version, room_id, session_id);
 
-	self.db
+	self.db.backupkeyid_backup.qry(&key).await.deserialized()
-		.backupkeyid_backup
-		.qry(&key)
-		.await
-		.deserialized_json()
 }
 
 #[implement(Service)]

src/service/pusher/mod.rs

@@ -90,7 +90,7 @@ impl Service {
 			.senderkey_pusher
 			.qry(&senderkey)
 			.await
-			.deserialized_json()
+			.deserialized()
 	}
 
 	pub async fn get_pushers(&self, sender: &UserId) -> Vec<Pusher> {

src/service/rooms/outlier/mod.rs

@@ -33,7 +33,7 @@ pub async fn get_outlier_pdu_json(&self, event_id: &EventId) -> Result<Canonical
 		.eventid_outlierpdu
 		.qry(event_id)
 		.await
-		.deserialized_json()
+		.deserialized()
 }
 
 /// Returns the pdu from the outlier tree.
@@ -43,7 +43,7 @@ pub async fn get_pdu_outlier(&self, event_id: &EventId) -> Result<PduEvent> {
 		.eventid_outlierpdu
 		.qry(event_id)
 		.await
-		.deserialized_json()
+		.deserialized()
 }
 
 /// Append the PDU as an outlier.

src/service/rooms/state_cache/data.rs

@@ -156,10 +156,7 @@ impl Data {
 		&self, user_id: &UserId, room_id: &RoomId,
 	) -> Result<Vec<Raw<AnyStrippedStateEvent>>> {
 		let key = (user_id, room_id);
-		self.userroomid_invitestate
+		self.userroomid_invitestate.qry(&key).await.deserialized()
-			.qry(&key)
-			.await
-			.deserialized_json()
 	}
 
 	#[tracing::instrument(skip(self), level = "debug")]
@@ -167,10 +164,7 @@ impl Data {
 		&self, user_id: &UserId, room_id: &RoomId,
 	) -> Result<Vec<Raw<AnyStrippedStateEvent>>> {
 		let key = (user_id, room_id);
-		self.userroomid_leftstate
+		self.userroomid_leftstate.qry(&key).await.deserialized()
-			.qry(&key)
-			.await
-			.deserialized_json()
 	}
 
 	/// Returns an iterator over all rooms a user left.

src/service/rooms/timeline/data.rs

@@ -90,17 +90,14 @@ impl Data {
 			return Ok(pdu);
 		}
 
-		self.eventid_outlierpdu
+		self.eventid_outlierpdu.qry(event_id).await.deserialized()
-			.qry(event_id)
-			.await
-			.deserialized_json()
 	}
 
 	/// Returns the json of a pdu.
 	pub(super) async fn get_non_outlier_pdu_json(&self, event_id: &EventId) -> Result<CanonicalJsonObject> {
 		let pduid = self.get_pdu_id(event_id).await?;
 
-		self.pduid_pdu.qry(&pduid).await.deserialized_json()
+		self.pduid_pdu.qry(&pduid).await.deserialized()
 	}
 
 	/// Returns the pdu's id.
@@ -113,7 +110,7 @@ impl Data {
 	pub(super) async fn get_non_outlier_pdu(&self, event_id: &EventId) -> Result<PduEvent> {
 		let pduid = self.get_pdu_id(event_id).await?;
 
-		self.pduid_pdu.qry(&pduid).await.deserialized_json()
+		self.pduid_pdu.qry(&pduid).await.deserialized()
 	}
 
 	/// Like get_non_outlier_pdu(), but without the expense of fetching and
@@ -137,7 +134,7 @@ impl Data {
 		self.eventid_outlierpdu
 			.qry(event_id)
 			.await
-			.deserialized_json()
+			.deserialized()
 			.map(Arc::new)
 	}
 
@@ -162,12 +159,12 @@ impl Data {
 	///
 	/// This does __NOT__ check the outliers `Tree`.
 	pub(super) async fn get_pdu_from_id(&self, pdu_id: &[u8]) -> Result<PduEvent> {
-		self.pduid_pdu.qry(pdu_id).await.deserialized_json()
+		self.pduid_pdu.qry(pdu_id).await.deserialized()
 	}
 
 	/// Returns the pdu as a `BTreeMap<String, CanonicalJsonValue>`.
 	pub(super) async fn get_pdu_json_from_id(&self, pdu_id: &[u8]) -> Result<CanonicalJsonObject> {
-		self.pduid_pdu.qry(pdu_id).await.deserialized_json()
+		self.pduid_pdu.qry(pdu_id).await.deserialized()
 	}
 
 	pub(super) async fn append_pdu(&self, pdu_id: &[u8], pdu: &PduEvent, json: &CanonicalJsonObject, count: u64) {

src/service/uiaa/mod.rs

@@ -238,6 +238,6 @@ async fn get_uiaa_session(&self, user_id: &UserId, device_id: &DeviceId, session
 		.userdevicesessionid_uiaainfo
 		.qry(&key)
 		.await
-		.deserialized_json()
+		.deserialized()
 		.map_err(|_| err!(Request(Forbidden("UIAA session does not exist."))))
 }

src/service/users/mod.rs

@@ -577,7 +577,7 @@ impl Service {
 			.qry(&key)
 			.await
 			.map_err(|_| err!(Request(InvalidParam("Tried to sign nonexistent key."))))?
-			.deserialized_json()
+			.deserialized()
 			.map_err(|e| err!(Database("key in keyid_key is invalid. {e:?}")))?;
 
 		let signatures = cross_signing_key
@@ -652,7 +652,7 @@ impl Service {
 
 	pub async fn get_device_keys<'a>(&'a self, user_id: &'a UserId, device_id: &DeviceId) -> Result<Raw<DeviceKeys>> {
 		let key_id = (user_id, device_id);
-		self.db.keyid_key.qry(&key_id).await.deserialized_json()
+		self.db.keyid_key.qry(&key_id).await.deserialized()
 	}
 
 	pub async fn get_key<F>(
@@ -666,7 +666,7 @@ impl Service {
 			.keyid_key
 			.qry(key_id)
 			.await
-			.deserialized_json::<serde_json::Value>()?;
+			.deserialized::<serde_json::Value>()?;
 
 		let cleaned = clean_signatures(key, sender_user, user_id, allowed_signatures)?;
 		let raw_value = serde_json::value::to_raw_value(&cleaned)?;
@@ -700,7 +700,7 @@ impl Service {
 	pub async fn get_user_signing_key(&self, user_id: &UserId) -> Result<Raw<CrossSigningKey>> {
 		let key_id = self.db.userid_usersigningkeyid.qry(user_id).await?;
 
-		self.db.keyid_key.qry(&*key_id).await.deserialized_json()
+		self.db.keyid_key.qry(&*key_id).await.deserialized()
 	}
 
 	pub async fn add_to_device_event(
@@ -791,7 +791,7 @@ impl Service {
 			.userdeviceid_metadata
 			.qry(&(user_id, device_id))
 			.await
-			.deserialized_json()
+			.deserialized()
 	}
 
 	pub async fn get_devicelist_version(&self, user_id: &UserId) -> Result<u64> {
@@ -830,7 +830,7 @@ impl Service {
 			.userfilterid_filter
 			.qry(&(user_id, filter_id))
 			.await
-			.deserialized_json()
+			.deserialized()
 	}
 
 	/// Creates an OpenID token, which can be used to prove that a user has