fix: de-index pdus when redacted

bit of code dedupe as well

Co-authored-by: strawberry <strawberry@puppygock.gay>
Signed-off-by: strawberry <strawberry@puppygock.gay>

src/service/rooms/search/data.rs

@@ -7,6 +7,8 @@ type SearchPdusResult<'a> = Result<Option<(Box<dyn Iterator<Item = Vec<u8>> + 'a
 pub trait Data: Send + Sync {
 	fn index_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()>;
 
+	fn deindex_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()>;
+
 	fn search_pdus<'a>(&'a self, room_id: &RoomId, search_string: &str) -> SearchPdusResult<'a>;
 }
 
@@ -34,6 +36,22 @@ impl Data for KeyValueDatabase {
 		self.tokenids.insert_batch(&mut batch)
 	}
 
+	fn deindex_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> {
+		let batch = tokenize(message_body).map(|word| {
+			let mut key = shortroomid.to_be_bytes().to_vec();
+			key.extend_from_slice(word.as_bytes());
+			key.push(0xFF);
+			key.extend_from_slice(pdu_id); // TODO: currently we save the room id a second time here
+			key
+		});
+
+		for token in batch {
+			self.tokenids.remove(&token)?;
+		}
+
+		Ok(())
+	}
+
 	fn search_pdus<'a>(&'a self, room_id: &RoomId, search_string: &str) -> SearchPdusResult<'a> {
 		let prefix = services()
 			.rooms

src/service/rooms/search/mod.rs

@@ -17,6 +17,11 @@ impl Service {
 		self.db.index_pdu(shortroomid, pdu_id, message_body)
 	}
 
+	#[tracing::instrument(skip(self))]
+	pub fn deindex_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> {
+		self.db.deindex_pdu(shortroomid, pdu_id, message_body)
+	}
+
 	#[tracing::instrument(skip(self))]
 	pub fn search_pdus<'a>(
 		&'a self, room_id: &RoomId, search_string: &str,