Merge pull request #269 from ReagentX/develop

Beach Strawberry
ReagentX · Jun 10, 2024 · c694773 · c694773
2 parents 4deeac9 + 1e19691
commit c694773
Show file tree

Hide file tree

Showing 26 changed files with 1,045 additions and 308 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/README.md b/README.md
@@ -18,7 +18,7 @@ Documentation for the library is located [here](imessage-database/README.md).
 
 ### Supported Features
 
-This crate supports every iMessage feature as of macOS 14.1.1 (23B81) and iOS 17.1.1 (21B91):
+This crate supports every iMessage feature as of macOS 14.5 (23F79) and iOS 17.5.1 (21F90):
 
 - Multi-part messages
 - Replies/Threads

diff --git a/build.sh b/build.sh
@@ -17,7 +17,7 @@ if [ -n "$VERSION" ]; then
     # Otherwise it returns `invalid command code C`
     sed -i '' "s/version = \"0.0.0\"/version = \"$VERSION\"/g" imessage-database/Cargo.toml
     sed -i '' "s/version = \"0.0.0\"/version = \"$VERSION\"/g" imessage-exporter/Cargo.toml
-    sed -i '' s/'{path = "..\/imessage-database"}'/\"$VERSION\"/g imessage-exporter/Cargo.toml
+    sed -i '' s/'{ path = "..\/imessage-database" }'/\"$VERSION\"/g imessage-exporter/Cargo.toml
 
     if [ -n "$PUBLISH" ]; then
         echo 'Publishing database library...'

diff --git a/docs/diagnostics.md b/docs/diagnostics.md
@@ -0,0 +1,93 @@
+# Diagnostics
+
+Diagnostic output from `imessage-exporter` looks like:
+
+```txt
+iMessage Database Diagnostics
+
+Handle diagnostic data:
+    Contacts with more than one ID: 2
+Message diagnostic data:
+    Total messages: 183453
+    Messages not associated with a chat: 43210
+    Messages belonging to more than one chat: 36
+Attachment diagnostic data:
+    Total attachments: 49422
+        Data referenced in table: 44.13 GB
+        Data present on disk: 31.31 GB
+    Missing files: 15037 (30%)
+        No path provided: 14929
+        No file located: 108
+Thread diagnostic data:
+    Chats with no handles: 2
+Global diagnostic data:
+    Total database size: 339.88 MB
+    Duplicated contacts: 78
+    Duplicated chats: 16
+```
+
+## Handle diagnostic data
+
+### Contacts with more than one ID
+
+The number of contacts that have multiple entries in the `handle` table, deduplicated by matching their `person_centric_id` across rows. The `person_centric_id` is a field used by Apple to disambiguate contacts. Further deduplication also happens, as noted below.
+
+## Message diagnostic data
+
+### Total messages
+
+The total number of rows in the `messages` table.
+
+### Messages not associated with a chat
+
+If a message exists in the `messages` table but does not have an entry in the `chat_message_join` table, it is considered orphaned and will be listed in either the `Orphaned.html` or `Orphaned.txt` file in the export directory. Likely, these come from messages that were deleted and the chat removed from the `chat_message_join` table, but the corresponding messages were not removed from the `messages` table.
+
+### Messages belonging to more than one chat
+
+If a message exists in the `messages` table and maps to multiple chats in `chat_message_join`, the message will exist in all of those chats when exported.
+
+## Attachment diagnostic data
+
+### Total attachments
+
+The total number of rows in the `attachments` table
+
+#### Data referenced in table
+
+The sum of the `total_bytes` column in the `attachments` table. I don't know why they are different, but the former is the actual storage taken up by iMessage attachments.
+
+#### Data present on disk
+
+Represents the total size of the attachments listed in the `attachments` when following the listed path to the respective file. Missing files may have been removed by the user or not properly downloaded from iCloud.
+
+### Missing files
+
+The first line shows the count and the percentage of files missing. In the example above, `15037 (30%)` means that `15,037` files (`30%` of the total number of attachments) are referenced in the table but do not exist.
+
+There are two different types of missing files:
+
+#### No path provided
+
+This means there was a row in the `attachments` table that did not contain a path to a file.
+
+#### No file located
+
+This means there was a path provided, but there was no file at the specified location.
+
+## Thread diagnostic data
+
+Emits the count of chats that contain no chat participants.
+
+## Global diagnostic data
+
+### Total database size
+
+The total size of the database file on the disk.
+
+### Duplicated contacts
+
+Duplicated contacts occur when a single contact has multiple valid phone numbers or iMessage email addresses. The iMessage database stores handles as rows, and multiple rows can match to the same contact.
+
+### Duplicated chats
+
+The number of separate chats that contain the same participants. See the [duplicates](/docs/tables/duplicates.md) for a detailed explanation of the logic used to determine this number.
diff --git a/imessage-database/Cargo.toml b/imessage-database/Cargo.toml
@@ -10,7 +10,7 @@ repository = "https://github.com/ReagentX/imessage-exporter"
 version = "0.0.0"
 
 [dependencies]
-chrono = "0.4.31"
-plist = "1.6.0"
-rusqlite = { version = "0.30.0", features = ["blob", "bundled"] }
+chrono = "0.4.38"
+plist = "1.6.1"
+rusqlite = { version = "0.31.0", features = ["blob", "bundled"] }
 sha1 = "0.10.6"
diff --git a/imessage-database/src/message_types/edited.rs b/imessage-database/src/message_types/edited.rs
@@ -14,6 +14,22 @@ use crate::{
     },
 };
 
+#[derive(Debug, PartialEq, Eq)]
+pub struct EditedEvent<'a> {
+    /// The date the messages were edited
+    pub date: i64,
+    /// The content of the edited messages in [`streamtyped`](crate::util::streamtyped) format
+    pub text: String,
+    /// A GUID reference to another message
+    pub guid: Option<&'a str>,
+}
+
+impl<'a> EditedEvent<'a> {
+    fn new(date: i64, text: String, guid: Option<&'a str>) -> Self {
+        Self { date, text, guid }
+    }
+}
+
 /// iMessage permits editing sent messages up to five times
 /// within 15 minutes of sending the first message and unsending
 /// sent messages within 2 minutes.
@@ -34,12 +50,7 @@ use crate::{
 /// Apple describes editing and unsending messages [here](https://support.apple.com/guide/iphone/unsend-and-edit-messages-iphe67195653/ios).
 #[derive(Debug, PartialEq, Eq)]
 pub struct EditedMessage<'a> {
-    /// The dates the messages were edited
-    pub dates: Vec<i64>,
-    /// The content of the edited messages in [`streamtyped`](crate::util::streamtyped) format
-    pub texts: Vec<String>,
-    /// A GUID reference to another message
-    pub guids: Vec<Option<&'a str>>,
+    pub events: Vec<EditedEvent<'a>>,
 }
 
 impl<'a> BalloonProvider<'a> for EditedMessage<'a> {
@@ -75,9 +86,7 @@ impl<'a> BalloonProvider<'a> for EditedMessage<'a> {
 
             let guid = message_data.get("bcg").and_then(|item| item.as_string());
 
-            edited.dates.push(timestamp);
-            edited.texts.push(text);
-            edited.guids.push(guid);
+            edited.events.push(EditedEvent::new(timestamp, text, guid));
         }
 
         Ok(edited)
@@ -87,44 +96,35 @@ impl<'a> BalloonProvider<'a> for EditedMessage<'a> {
 impl<'a> EditedMessage<'a> {
     /// A new empty edited message
     fn empty() -> Self {
-        EditedMessage {
-            dates: Vec::new(),
-            texts: Vec::new(),
-            guids: Vec::new(),
-        }
+        EditedMessage { events: Vec::new() }
     }
 
     /// A new message with a preallocated capacity
     fn with_capacity(capacity: usize) -> Self {
         EditedMessage {
-            dates: Vec::with_capacity(capacity),
-            texts: Vec::with_capacity(capacity),
-            guids: Vec::with_capacity(capacity),
+            events: Vec::with_capacity(capacity),
         }
     }
 
     /// `true` if the message was deleted, `false` if it was edited
     pub fn is_deleted(&self) -> bool {
-        self.texts.is_empty()
+        self.events.is_empty()
     }
 
     /// Gets a tuple for the message at the provided position
-    pub fn item_at(&self, position: usize) -> Option<(&i64, &str, &Option<&str>)> {
-        Some((
-            self.dates.get(position)?,
-            self.texts.get(position)?,
-            self.guids.get(position)?,
-        ))
+    pub fn item_at(&self, position: usize) -> Option<&EditedEvent> {
+        self.events.get(position)
     }
 
     /// Gets the number of items in the edit history
     pub fn items(&self) -> usize {
-        self.texts.len()
+        self.events.len()
     }
 }
 
 #[cfg(test)]
 mod tests {
+    use crate::message_types::edited::EditedEvent;
     use crate::message_types::{edited::EditedMessage, variants::BalloonProvider};
     use plist::Value;
     use std::env::current_dir;
@@ -141,29 +141,18 @@ mod tests {
         let parsed = EditedMessage::from_map(&plist).unwrap();
 
         let expected = EditedMessage {
-            dates: vec![
-                690513474000000000,
-                690513480000000000,
-                690513485000000000,
-                690513494000000000,
+            events: vec![
+                EditedEvent::new(690513474000000000, "First message  ".to_string(), None),
+                EditedEvent::new(690513480000000000, "Edit 1".to_string(), None),
+                EditedEvent::new(690513485000000000, "Edit 2".to_string(), None),
+                EditedEvent::new(690513494000000000, "Edited message".to_string(), None),
             ],
-            texts: vec![
-                "First message  ".to_string(),
-                "Edit 1".to_string(),
-                "Edit 2".to_string(),
-                "Edited message".to_string(),
-            ],
-            guids: vec![None, None, None, None],
         };
 
         assert_eq!(parsed, expected);
         assert_eq!(parsed.items(), 4);
 
-        let expected_item = Some((
-            expected.dates.first().unwrap(),
-            expected.texts.first().unwrap().as_str(),
-            expected.guids.first().unwrap(),
-        ));
+        let expected_item = Some(expected.events.first().unwrap());
         assert_eq!(parsed.item_at(0), expected_item);
     }
 
@@ -178,22 +167,20 @@ mod tests {
         let parsed = EditedMessage::from_map(&plist).unwrap();
 
         let expected = EditedMessage {
-            dates: vec![690514004000000000, 690514772000000000],
-            texts: vec![
-                "here we go!".to_string(),
-                "https://github.com/ReagentX/imessage-exporter/issues/10".to_string(),
+            events: vec![
+                EditedEvent::new(690514004000000000, "here we go!".to_string(), None),
+                EditedEvent::new(
+                    690514772000000000,
+                    "https://github.com/ReagentX/imessage-exporter/issues/10".to_string(),
+                    Some("292BF9C6-C9B8-4827-BE65-6EA1C9B5B384"),
+                ),
             ],
-            guids: vec![None, Some("292BF9C6-C9B8-4827-BE65-6EA1C9B5B384")],
         };
 
         assert_eq!(parsed, expected);
         assert_eq!(parsed.items(), 2);
 
-        let expected_item = Some((
-            expected.dates.first().unwrap(),
-            expected.texts.first().unwrap().as_str(),
-            expected.guids.first().unwrap(),
-        ));
+        let expected_item = Some(expected.events.first().unwrap());
         assert_eq!(parsed.item_at(0), expected_item);
     }
 
@@ -208,27 +195,30 @@ mod tests {
         let parsed = EditedMessage::from_map(&plist).unwrap();
 
         let expected = EditedMessage {
-            dates: vec![690514809000000000, 690514819000000000, 690514834000000000],
-            texts: vec![
-                "This is a normal message".to_string(),
-                "Edit to a url https://github.com/ReagentX/imessage-exporter/issues/10".to_string(),
-                "And edit it back to a normal message...".to_string(),
-            ],
-            guids: vec![
-                None,
-                Some("0B9103FE-280C-4BD0-A66F-4EDEE3443247"),
-                Some("0D93DF88-05BA-4418-9B20-79918ADD9923"),
+            events: vec![
+                EditedEvent::new(
+                    690514809000000000,
+                    "This is a normal message".to_string(),
+                    None,
+                ),
+                EditedEvent::new(
+                    690514819000000000,
+                    "Edit to a url https://github.com/ReagentX/imessage-exporter/issues/10"
+                        .to_string(),
+                    Some("0B9103FE-280C-4BD0-A66F-4EDEE3443247"),
+                ),
+                EditedEvent::new(
+                    690514834000000000,
+                    "And edit it back to a normal message...".to_string(),
+                    Some("0D93DF88-05BA-4418-9B20-79918ADD9923"),
+                ),
             ],
         };
 
         assert_eq!(parsed, expected);
         assert_eq!(parsed.items(), 3);
 
-        let expected_item = Some((
-            expected.dates.first().unwrap(),
-            expected.texts.first().unwrap().as_str(),
-            expected.guids.first().unwrap(),
-        ));
+        let expected_item = Some(expected.events.first().unwrap());
         assert_eq!(parsed.item_at(0), expected_item);
     }
 
@@ -242,11 +232,7 @@ mod tests {
         let plist = Value::from_reader(plist_data).unwrap();
         let parsed = EditedMessage::from_map(&plist).unwrap();
 
-        let expected = EditedMessage {
-            dates: vec![],
-            texts: vec![],
-            guids: vec![],
-        };
+        let expected = EditedMessage { events: vec![] };
 
         assert_eq!(parsed, expected);
         assert!(parsed.is_deleted());

diff --git a/imessage-database/src/message_types/placemark.rs b/imessage-database/src/message_types/placemark.rs
@@ -13,15 +13,25 @@ use crate::{
 /// Representation of Apple's [`CLPlacemark`](https://developer.apple.com/documentation/corelocation/clplacemark) object
 #[derive(Debug, PartialEq, Eq, Default)]
 pub struct Placemark<'a> {
+    /// The name of the placemark
     pub name: Option<&'a str>,
+    /// The full address formatted associated with the placemark
     pub address: Option<&'a str>,
+    /// The state or province associated with the placemark
     pub state: Option<&'a str>,
+    /// The city associated with the placemark
     pub city: Option<&'a str>,
+    /// The abbreviated country or region name
     pub iso_country_code: Option<&'a str>,
+    /// The postal code associated with the placemark
     pub postal_code: Option<&'a str>,
+    /// The name of the country or region associated with the placemark
     pub country: Option<&'a str>,
+    /// The street associated with the placemark
     pub street: Option<&'a str>,
+    /// Additional administrative area information for the placemark
     pub sub_administrative_area: Option<&'a str>,
+    /// Additional city-level information for the placemark
     pub sub_locality: Option<&'a str>,
 }
 
@@ -67,7 +77,7 @@ pub struct PlacemarkMessage<'a> {
     pub original_url: Option<&'a str>,
     /// The full street address of the location
     pub place_name: Option<&'a str>,
-    /// The short description of the app in the App Store
+    /// [Placemark] data for the specified location
     pub placemark: Placemark<'a>,
 }