From 7da44afe0e101d6096bd7a10a80c95a029bfc33e Mon Sep 17 00:00:00 2001 From: cameronvoell Date: Tue, 3 Dec 2024 12:23:11 -0800 Subject: [PATCH] added specification and rationale sections --- ...ip-48-content-type-local-db-integration.md | 204 +++++++++++++++++- 1 file changed, 196 insertions(+), 8 deletions(-) diff --git a/XIPs/xip-48-content-type-local-db-integration.md b/XIPs/xip-48-content-type-local-db-integration.md index f8526a8..607d055 100644 --- a/XIPs/xip-48-content-type-local-db-integration.md +++ b/XIPs/xip-48-content-type-local-db-integration.md @@ -12,19 +12,176 @@ created: 2024-11-27 ## Abstract -This XIP proposes a new way of managing content types in XMTP in order to make complex content types like reactions, replies, and message filtering easier for XMTP integrators. +This XIP proposes a way of managing XMTP content types in protobufs and rust in order to make complex content types like reactions, replies, and message filtering easier for XMTP integrators. ## Motivation -During the upgrade from XMTP V2 (Direct Messaging Only) to XMTP V3 (Groups via the MLS protocol), one developer experience improvement was that the new "V3" SDKs manage a local SQLITE database containing all of a user's groups and messages. At the same time, our core "libxmtp" library that manages all the database logic remained completely agnostic to the message "content types" that were being stored in the database. While this separation of content types from the database logic was great for keeping complexity down during initial developement, it has made some quieries that would be common in a production consumer messaging app impossible without forcing developers to implement their own extra local persistent data management. An example of this is when the SDK returnsall message types in order received for a group, how can you efficiently render the emoji reactions, replies and read reciepts for each message in your UI? +During the upgrade from XMTP V2 (Direct Messaging Only) to XMTP V3 (Groups via the MLS protocol), one developer experience improvement was that the new "V3" SDKs manage a local SQLITE database containing all of a user's groups and messages. At the same time, our core "libxmtp" library that manages all the database logic remained completely agnostic to the message "content types" that were being stored in the database. While this separation of content types from the database logic was great for keeping complexity down during initial developement, it has made some quieries that would be common in a production consumer messaging app impossible without forcing developers to implement their own extra local persistent data management. An example of this is when the SDK returnsall message types in order received for a group, how can you efficiently render the emoji reactions, replies and read reciepts for each message in your UI? In order to address these types of queries, this XIP proposes a way to integrate XMTP content types with our core library and local SQLITE storage. -In order to address these types of queries, this XIP proposes a way to integrate XMTP content types with our core library and local SQLITE storage. +In addition to enabling queries that will make integrators lives easier, storing content types in protobufs and rust will allow us to re-use encoding and decoding logic across platforms. ## Specification -Previously, each XMTP SDK implmented their own ContentType implementations that would aim to serialize and deserialize message contents to the same JSON format. - -This XIP proposes new ContentTypes be defined using protobuf definitions that are integrated into our core rust library. Then binding objects and encoding/decoding functions can be generated for each language all in our libxmtp rust repo. +Previously, each XMTP SDK included their own ContentType implementations that would aim to serialize and deserialize message contents to the same JSON format. + +This XIP proposes new ContentTypes be defined using protobuf definitions that are integrated into our core rust library. Then binding objects and encoding/decoding functions can be generated for each language all in our libxmtp rust repo. + +### SDK Code Before Content Types in Rust + +#### xmtp-ios: +```swift +public struct Reaction: Codable { + public var reference: String + public var action: ReactionAction + public var content: String + public var schema: ReactionSchema + ... +} + +public struct ReactionCodec: ContentCodec { + + public func encode(content: Reaction, client _: Client) throws -> EncodedContent { + var encodedContent = EncodedContent() + + encodedContent.type = ContentTypeReaction + encodedContent.content = try JSONEncoder().encode(content) + + return encodedContent + } + ... +``` + +#### xmtp-android: +```kotlin +data class Reaction( + val reference: String, + val action: ReactionAction, + val content: String, + val schema: ReactionSchema, +) + +data class ReactionCodec(override var contentType: ContentTypeId = ContentTypeReaction) : + ContentCodec { + + override fun encode(content: Reaction): EncodedContent { + val gson = GsonBuilder() + .registerTypeAdapter(Reaction::class.java, ReactionSerializer()) + .create() + + return EncodedContent.newBuilder().also { + it.type = ContentTypeReaction + it.content = gson.toJson(content).toByteStringUtf8() + }.build() + } + ... +``` +### SDK Code After Content Types in Rust + +#### libxmtp: +By moving content types to rust, we can define Foreign Function Interface (FFI) objects, as well as encoding and decoding functions that can be re-used in both xmtp-ios and xmtp-android. + +```rust +#[derive(uniffi::Record, Clone, Default)] +pub struct FfiReaction { + pub reference: String, + pub reference_inbox_id: String, + pub action: FfiReactionAction, + pub content: String, + pub schema: FfiReactionSchema, +} + + +#[uniffi::export] +pub fn encode_reaction(reaction: FfiReaction) -> Result, GenericError> { + // Use ReactionCodec to encode the reaction + let encoded = ReactionCodec::encode(reaction.into()) + .map_err(|e| GenericError::Generic { err: e.to_string() })?; + + // Encode the EncodedContent to bytes + let mut buf = Vec::new(); + encoded + .encode(&mut buf) + .map_err(|e| GenericError::Generic { err: e.to_string() })?; + + Ok(buf) +} +``` + +#### Updated xmtp-ios content codec (using FfiReaction and encodeReaction defined in Rust): +```swift +public struct ReactionCodec: ContentCodec { + + public func encode(content: FfiReaction, client _: Client) throws -> EncodedContent { + return try EncodedContent(serializedBytes: encodeReaction(content)) + } + ... +``` + +#### Updated xmtp-android content codec (using FfiReaction and encodeReaction defined in Rust): +```kotlin +data class ReactionCodec(override var contentType: ContentTypeId = ContentTypeReaction) : + ContentCodec { + + override fun encode(content: FfiReaction): EncodedContent { + return EncodedContent.parseFrom(encodeReaction(content)) + } + ... +``` + +In addition to consolidating encode/decode logic to our libxmtp repo, we can use libxmtp defined protobuf definitions for deserializing message contents in order to store content type specific data in our local database. + +An example of how this could work with the "Reaction" content type is below: + +```rust + + /// Helper function to extract queryable content fields from a message + fn extract_queryable_content_fields(message: &[u8]) -> QueryableContentFields { + // Attempt to decode the message as EncodedContent + let encoded_content = match EncodedContent::decode(message) { + Ok(content) => content, + Err(e) => { + tracing::debug!("Failed to decode message as EncodedContent: {}", e); + return QueryableContentFields { + parent_id: None, + is_readable: None, + }; + } + }; + let encoded_content_clone = encoded_content.clone(); + + // Check if it's a reaction message + let parent_id = match encoded_content.r#type { + Some(content_type) if content_type.type_id == ReactionCodec::TYPE_ID => { + // Attempt to decode as reaction + match ReactionCodec::decode(encoded_content_clone) { + Ok(reaction) => { + // Decode hex string into bytes + match hex::decode(&reaction.reference) { + Ok(bytes) => Some(bytes), + Err(e) => { + tracing::debug!( + "Failed to decode reaction reference as hex: {}", + e + ); + None + } + } + } + Err(e) => { + tracing::debug!("Failed to decode reaction: {}", e); + None + } + } + } + _ => None, + }; + + QueryableContentFields { + parent_id, + is_readable: None, + } + } +``` @@ -42,7 +228,9 @@ Talk about alternative of decoding JSON in rust. -Content types that are implemented in the core library will be treated as new content type versions that will not be automatically supported by older SDK versions. +As alluded to above, one downside of moving content types from JSON to protobufs is that clients on older versions will be unable to decode the new content types using existing JSON deserialization logic. We have a few options for mitigating this: + +1. ## Test cases*