More frame validation improvements (consider remaining filesize). Fix…

… blocked/muted/official icons in conversation-title (html). XMLdatabase: dont add skip recipients if already present. Nicer error output from sqlitedb.

attachmentframe/attachmentframe.h

@@ -58,7 +58,7 @@ class AttachmentFrame : public FrameWithAttachment
   inline uint64_t attachmentId() const;
   inline void setAttachmentId(uint64_t rid);
   inline std::pair<unsigned char *, uint64_t> getData() const override;
-  inline virtual bool validate() const override;
+  inline virtual bool validate(uint64_t available) const override;
   inline std::string getHumanData() const override;
   inline unsigned int getField(std::string_view const &str) const;
   inline void setLengthField(uint32_t newlength);
@@ -230,7 +230,7 @@ inline std::pair<unsigned char *, uint64_t> AttachmentFrame::getData() const
 
 }
 
-inline bool AttachmentFrame::validate() const
+inline bool AttachmentFrame::validate(uint64_t available) const
 {
   if (d_framedata.empty())
     return false;
@@ -239,7 +239,7 @@ inline bool AttachmentFrame::validate() const
   int rowid_fieldsize = 0;
   int foundlength = 0;
   int length_fieldsize = 0;
-  int length = 0;
+  unsigned int length = 0;
   int foundattachmentid = 0;
   for (auto const &p : d_framedata)
   {
@@ -265,6 +265,7 @@ inline bool AttachmentFrame::validate() const
 
   return foundlength == 1 && foundattachmentid <= 1 && foundrowid == 1 &&
     length_fieldsize <= 8 && rowid_fieldsize <= 8 &&
+    length <= available &&
     length < 1 * 1024 * 1024 * 1024; // lets cap a valid attachment size at 1 gigabyte.
   // From what I've found, the current (theoretical) maximum is 500Mb for video on
   // Android.

autoversion.h

@@ -20,6 +20,6 @@
 #ifndef VERSION_H_
 #define VERSION_H_
 
-#define VERSIONDATE "20250128.084955"
+#define VERSIONDATE "20250128.134652"
 
 #endif

avatarframe/avatarframe.h

@@ -57,7 +57,7 @@ class AvatarFrame : public FrameWithAttachment
   inline std::string recipient() const;
   inline void setRecipient(std::string  const &r);
   inline std::pair<unsigned char *, uint64_t> getData() const override;
-  inline virtual bool validate() const override;
+  inline virtual bool validate(uint64_t available) const override;
   inline std::string getHumanData() const override;
   inline unsigned int getField(std::string_view const &str) const;
   inline std::optional<std::string> mimetype() const;
@@ -231,14 +231,14 @@ inline std::pair<unsigned char *, uint64_t> AvatarFrame::getData() const
   return {data, size};
 }
 
-inline bool AvatarFrame::validate() const
+inline bool AvatarFrame::validate(uint64_t available) const
 {
   if (d_framedata.empty())
     return false;
 
   int foundlength = 0;
   int length_fieldsize = 0;
-  int length = 0;
+  unsigned int length = 0;
   int foundname_or_recipient = 0;
   for (auto const &p : d_framedata)
   {
@@ -259,6 +259,7 @@ inline bool AvatarFrame::validate() const
   }
 
   return foundlength == 1 && foundname_or_recipient == 1 &&
+    length <= available &&
     length_fieldsize <= 8; // && length < some_max_avatar_size;
 }
 

backupframe/backupframe.h

@@ -1,5 +1,5 @@
 /*
-  Copyright (C) 2019-2024  Selwin van Dijk
+  Copyright (C) 2019-2025  Selwin van Dijk
 
   This file is part of signalbackup-tools.
 
@@ -97,7 +97,7 @@ class BackupFrame
   inline virtual std::pair<unsigned char *, uint64_t> getData() const;
   inline virtual std::string getHumanData() const;
   inline bool setNewData(unsigned int field, unsigned char *data, uint64_t size);
-  inline virtual bool validate() const;
+  inline virtual bool validate(uint64_t available) const;
   inline virtual uint64_t dataSize() const;
  protected:
   inline uint32_t bytesToUint32(unsigned char const *data, size_t len) const;
@@ -543,7 +543,7 @@ inline bool BackupFrame::setNewData(unsigned int field, unsigned char *data, uin
   return true;
 }
 
-inline bool BackupFrame::validate() const
+inline bool BackupFrame::validate(uint64_t) const
 {
   return true;
 }

databaseversionframe/databaseversionframe.h

@@ -1,5 +1,5 @@
 /*
-  Copyright (C) 2019-2024  Selwin van Dijk
+  Copyright (C) 2019-2025  Selwin van Dijk
 
   This file is part of signalbackup-tools.
 
@@ -45,7 +45,7 @@ class DatabaseVersionFrame : public BackupFrame
   inline uint32_t version() const;
   inline virtual void printInfo() const override;
   inline virtual std::pair<unsigned char *, uint64_t> getData() const override;
-  inline virtual bool validate() const override;
+  inline virtual bool validate(uint64_t) const override;
   inline std::string getHumanData() const override;
   //inline virtual bool setNewData(std::string const &field, std::string const &data) override;
   inline unsigned int getField(std::string_view const &str) const;
@@ -153,7 +153,7 @@ inline std::pair<unsigned char *, uint64_t> DatabaseVersionFrame::getData() cons
   return {data, size};
 }
 
-inline bool DatabaseVersionFrame::validate() const
+inline bool DatabaseVersionFrame::validate(uint64_t) const
 {
   if (d_framedata.empty())
     return false;

endframe/endframe.h

@@ -1,5 +1,5 @@
 /*
-  Copyright (C) 2019-2024  Selwin van Dijk
+  Copyright (C) 2019-2025  Selwin van Dijk
 
   This file is part of signalbackup-tools.
 
@@ -34,7 +34,7 @@ class EndFrame: public BackupFrame
   inline virtual void printInfo() const override;
   inline virtual FRAMETYPE frameType() const override;
   inline std::pair<unsigned char *, uint64_t> getData() const override;
-  inline virtual bool validate() const override;
+  inline virtual bool validate(uint64_t available) const override;
  private:
   inline uint64_t dataSize() const override;
 };
@@ -98,9 +98,10 @@ inline std::pair<unsigned char *, uint64_t> EndFrame::getData() const
   return {data, size};
 }
 
-inline bool EndFrame::validate() const
+inline bool EndFrame::validate(uint64_t available) const
 {
   return d_framedata.size() == 1 && std::get<2>(d_framedata.front()) == 8 &&
+    available == 0 &&
     (bytesToUint64(std::get<1>(d_framedata.front()), std::get<2>(d_framedata.front())) == 1 ||
      bytesToUint64(std::get<1>(d_framedata.front()), std::get<2>(d_framedata.front())) == 0);
 }

filedecryptor/getframe.cc

@@ -1,5 +1,5 @@
 /*
-  Copyright (C) 2019-2024  Selwin van Dijk
+  Copyright (C) 2019-2025  Selwin van Dijk
 
   This file is part of signalbackup-tools.
 
@@ -231,6 +231,15 @@ std::unique_ptr<BackupFrame> FileDecryptor::getFrame(std::ifstream &file)
 
   delete[] decodedframe;
 
+  // if (!frame->validate(d_filesize - file.tellg()))
+  // {
+  //   std::cout << std::endl << "    **************        FRAME NOT VALIDATED      ****************" << std::endl;
+  //   frame->printInfo();
+  //   std::cout << "TOTAL SIZE: " << d_filesize << std::endl;
+  //   std::cout << "POSITION  : " << file.tellg()  << std::endl;
+  //   std::cout << "AVAILABLE : " << (d_filesize - file.tellg()) << std::endl;
+  // }
+
   uint32_t attsize = frame->attachmentSize();
   if (!d_badmac && attsize > 0 &&
       (frame->frameType() == BackupFrame::FRAMETYPE::ATTACHMENT ||
@@ -420,6 +429,15 @@ std::unique_ptr<BackupFrame> FileDecryptor::getFrameOld(std::ifstream &file)
 
   delete[] decodedframe;
 
+  // if (!frame->validate(d_filesize - file.tellg()))
+  // {
+  //   std::cout << std::endl << "    **************        FRAME NOT VALIDATED      ****************" << std::endl;
+  //   frame->printInfo();
+  //   std::cout << "TOTAL SIZE: " << d_filesize << std::endl;
+  //   std::cout << "POSITION  : " << file.tellg()  << std::endl;
+  //   std::cout << "AVAILABLE : " << (d_filesize - file.tellg()) << std::endl;
+  // }
+
   uint32_t attsize = frame->attachmentSize();
   if (!d_badmac && attsize > 0 &&
       (frame->frameType() == BackupFrame::FRAMETYPE::ATTACHMENT ||

filedecryptor/getframebrute.cc

@@ -132,7 +132,7 @@ std::unique_ptr<BackupFrame> FileDecryptor::getFrameBrute(std::ifstream &file, u
     }
     else
     {
-      if (frame->validate() &&
+      if (frame->validate(d_filesize - file.tellg()) &&
           frame->frameType() != BackupFrame::FRAMETYPE::HEADER && // it is impossible to get in this function without the headerframe, and there is only one
           (frame->frameType() != BackupFrame::FRAMETYPE::END || static_cast<uint64_t>(file.tellg()) == d_filesize))
       {

headerframe/headerframe.h

@@ -57,7 +57,7 @@ class HeaderFrame : public BackupFrame
   inline virtual void printInfo() const override;
   inline std::pair<unsigned char *, uint64_t> getData() const override;
   inline std::string getHumanData() const override;
-  inline virtual bool validate() const override;
+  inline virtual bool validate(uint64_t) const override;
   inline unsigned int getField(std::string_view const &str) const;
  private:
   inline uint64_t dataSize() const override;
@@ -226,7 +226,7 @@ inline std::string HeaderFrame::getHumanData() const
   return data;
 }
 
-inline bool HeaderFrame::validate() const
+inline bool HeaderFrame::validate(uint64_t) const
 {
   if (d_framedata.empty())
     return false;

keyvalueframe/keyvalueframe.h

@@ -1,5 +1,5 @@
 /*
-  Copyright (C) 2021-2024  Selwin van Dijk
+  Copyright (C) 2021-2025  Selwin van Dijk
 
   This file is part of signalbackup-tools.
 
@@ -51,7 +51,7 @@ class KeyValueFrame : public BackupFrame
   inline virtual void printInfo() const override;
   inline virtual FRAMETYPE frameType() const override;
   inline std::pair<unsigned char *, uint64_t> getData() const override;
-  inline virtual bool validate() const override;
+  inline virtual bool validate(uint64_t) const override;
   inline std::string getHumanData() const override;
   inline unsigned int getField(std::string_view const &str) const;
   inline std::string key() const;
@@ -194,7 +194,7 @@ inline std::pair<unsigned char *, uint64_t> KeyValueFrame::getData() const
 
 // not sure about the requirements, but I'm guessing
 // 1 key and at least one value is required
-inline bool KeyValueFrame::validate() const
+inline bool KeyValueFrame::validate(uint64_t) const
 {
   if (d_framedata.empty())
     return false;

sharedprefframe/sharedprefframe.h

@@ -49,7 +49,7 @@ class SharedPrefFrame : public BackupFrame
   inline virtual void printInfo() const override;
   inline virtual FRAMETYPE frameType() const override;
   inline std::pair<unsigned char *, uint64_t> getData() const override;
-  inline virtual bool validate() const override;
+  inline virtual bool validate(uint64_t) const override;
   inline std::string getHumanData() const override;
   inline unsigned int getField(std::string_view const &str) const;
   inline std::string key() const;
@@ -179,7 +179,7 @@ inline std::pair<unsigned char *, uint64_t> SharedPrefFrame::getData() const
 
 // not sure about the requirements, but at least _a_ field should be set
 // also a key needs a value, and a value needs a key
-inline bool SharedPrefFrame::validate() const
+inline bool SharedPrefFrame::validate(uint64_t) const
 {
   if (d_framedata.empty())
     return false;

signalbackup/htmlwrite.cc

@@ -1289,6 +1289,8 @@ bool SignalBackup::HTMLwriteStart(std::ofstream &file, long long int thread_reci
       height: 24px;
       aspect-ratio: 1 / 1;
       margin-left: 8px;
+      margin-bottom: 3px;
+      vertical-align: bottom;
     }
 )";
   }
@@ -1297,11 +1299,14 @@ bool SignalBackup::HTMLwriteStart(std::ofstream &file, long long int thread_reci
   {
     file << '\n'
          << "    .blocked {\n"
+         << "      display: inline-block;\n"
          << "      background-image: url('data:image/svg+xml;utf-8,<svg version=\"1.1\" xmlns=\"http://www.w3.org/2000/svg\" viewBox=\"0 0 24 24\" fill=\"white\" stroke=\"white\"><path d=\"M12 1a11 11 0 1 0 11 11A11 11 0 0 0 12 1zm0 1.5a9.448 9.448 0 0 1 6.159 2.281L4.781 18.159A9.488 9.488 0 0 1 12 2.5zm0 19a9.448 9.448 0 0 1-6.159-2.281L19.219 5.841A9.488 9.488 0 0 1 12 21.5z\"/></svg>');\n"
          << "      height: 24px;\n"
          << "      aspect-ratio: 1 / 1;\n"
          << "      margin-right: 6px;\n"
          << "      filter: var(--icon-f);\n"
+         << "      margin-bottom: 2px;\n"
+         << "      vertical-align: bottom;\n"
          << "    }\n";
   }
 
@@ -1316,6 +1321,8 @@ bool SignalBackup::HTMLwriteStart(std::ofstream &file, long long int thread_reci
          << "      margin-top: 3px;\n"
          << "      margin-right: 8px;\n"
          << "      filter: var(--icon-f);\n"
+         << "      margin-bottom: 2px;\n"
+         << "      vertical-align: bottom;\n"
          << "    }\n";
   }
 
@@ -1783,16 +1790,17 @@ file << R"(
   }
   file << '\n'
        << "          <div id=\"thread-title\">";
+  file << "<pre class=\"threadtitle\">";
   if (isblocked)
-    file << "<div class=\"blocked\"></div>";
+    file << "<span class=\"blocked\"></span>";
   else if (ismuted)
-    file << "<div class=\"muted\"></div>";
-  file << "<pre class=\"threadtitle\">" << title;
+    file << "<span class=\"muted\"></span>";
+  file << title;
   if (expiration_timer)
     file << "<span class=\"thread-disappearing-messages-info\"></span><span class=\"threadtitle-info\">" + exptimer_short + "</span>";
-  file << "</pre>";
   if (isnotetoself || isreleasechannel)
-    file << "<div class=\"official\"></div>";
+    file << "<span class=\"official\"></span>";
+  file << "</pre>";
   file << "</div>\n";
   if (!isnotetoself && getRecipientInfoFromMap(recipient_info, thread_recipient_id).verified)
     file <<

signalplaintextbackupdatabase/signalplaintextbackupdatabase.cc

@@ -287,11 +287,13 @@ SignalPlaintextBackupDatabase::SignalPlaintextBackupDatabase(std::string const &
 
   // add group-only-contacts, as 'skip' messages, so they can be mapped...
   for (auto const &a : group_only_contacts)
-    if (!d_database.exec("INSERT INTO smses (address, skip) VALUES (?, ?)", {a, 1}))
+    if (!d_database.exec("INSERT INTO smses (address, skip) SELECT ?1, 1 WHERE NOT EXISTS (SELECT 1 FROM smses WHERE address = ?1 AND skip = 0)", a))
     {
       Logger::warning("Failed to add group-only-contact ", a);
       continue;
     }
+  //d_database.prettyPrint(false, "SELECT address FROM smses WHERE skip = 1");
+  //d_database.prettyPrint(false, "SELECT address FROM smses WHERE skip = 1 AND address IN (SELECT DISTINCT address FROM smses WHERE skip = 0)");
 
   // If contact_name IS NULL, "", or "(Unknown)", set it to MAX(contact_name) for that address,
   // If still empty (all messsages from that contact were NULL, "", OR "(Unknown)", set it

sqlitedb/sqlitedb.h

@@ -354,17 +354,36 @@ inline bool SqliteDB::exec(std::string const &q, std::vector<std::any> const &pa
     if (sqlite3_prepare_v2(d_db, q.c_str(), -1, &d_stmt, &d_error_tail) != SQLITE_OK) [[unlikely]]
     {
       Logger::error("During sqlite3_prepare_v2(): ", sqlite3_errmsg(d_db));
+      //// old way: just print the error
       //Logger::error_indent("\"", q, "\"");
+
+      //// newer way: mark the point _around_ the error posistion
+      // long long int error_pos = std::distance(q.c_str(), d_error_tail);
+      // long long int error_start = std::max(0ll, error_pos - 2);
+      // long long int error_end = std::min(error_pos + 2, static_cast<long long int>(q.size()));
+      // Logger::error_indent("-> Query: \"",
+      //                      q.substr(0, error_start),
+      //                      Logger::Control::BOLD,
+      //                      q.substr(error_start, error_end - error_start),
+      //                      Logger::Control::NORMAL,
+      //                      q.substr(error_end),
+      //                      "\"");
+
+      // attempt to mark the token that sqlite choked on
       long long int error_pos = std::distance(q.c_str(), d_error_tail);
-      long long int error_start = std::max(0ll, error_pos - 2);
-      long long int error_end = std::min(error_pos + 2, static_cast<long long int>(q.size()));
+      long long int error_start = error_pos; // find the token where the error starts...
+      while (error_start > 0 &&
+             ((q[error_start - 1] >= 'a' && q[error_start - 1] <= 'z') ||
+              (q[error_start - 1] >= 'A' && q[error_start - 1] <= 'Z') ||
+              (q[error_start - 1] >= '0' && q[error_start - 1] <= '9')))
+        --error_start;
       Logger::error_indent("-> Query: \"",
                            q.substr(0, error_start),
                            Logger::Control::BOLD,
-                           q.substr(error_start, error_end - error_start),
+                           q.substr(error_start, error_pos - error_start),
                            Logger::Control::NORMAL,
-                           q.substr(error_end),
-                           "\"");
+                           q.substr(error_pos));
+
       return false;
     }
   }