Merge pull request #1673 from advancedtelematic/feat/OTA-4882/cache-d…

…evice-data

Only send device data if it has changed

config/sql/migration/migrate.25.sql

@@ -0,0 +1,9 @@
+-- Don't modify this! Create a new migration instead--see docs/ota-client-guide/modules/ROOT/pages/schema-migrations.adoc
+SAVEPOINT MIGRATION;
+
+CREATE TABLE device_data(data_type TEXT PRIMARY KEY, hash TEXT NOT NULL);
+
+DELETE FROM version;
+INSERT INTO version VALUES(25);
+
+RELEASE MIGRATION;

config/sql/rollback/rollback.25.sql

@@ -0,0 +1,9 @@
+-- Don't modify this! Create a new migration instead--see docs/ota-client-guide/modules/ROOT/pages/schema-migrations.adoc
+SAVEPOINT ROLLBACK_MIGRATION;
+
+DROP TABLE device_data;
+
+DELETE FROM version;
+INSERT INTO version VALUES(24);
+
+RELEASE ROLLBACK_MIGRATION;

config/sql/schema.sql

@@ -1,5 +1,5 @@
 CREATE TABLE version(version INTEGER);
-INSERT INTO version(rowid,version) VALUES(1,24);
+INSERT INTO version(rowid,version) VALUES(1,25);
 CREATE TABLE device_info(unique_mark INTEGER PRIMARY KEY CHECK (unique_mark = 0), device_id TEXT, is_registered INTEGER NOT NULL DEFAULT 0 CHECK (is_registered IN (0,1)));
 CREATE TABLE ecus(id INTEGER PRIMARY KEY, serial TEXT UNIQUE, hardware_id TEXT NOT NULL, is_primary INTEGER NOT NULL DEFAULT 0 CHECK (is_primary IN (0,1)));
 CREATE TABLE secondary_ecus(serial TEXT PRIMARY KEY, sec_type TEXT, public_key_type TEXT, public_key TEXT, extra TEXT, manifest TEXT);
@@ -26,3 +26,4 @@ CREATE TABLE rollback_migrations(version_from INT PRIMARY KEY, migration TEXT NO
 CREATE TABLE delegations(meta BLOB NOT NULL, role_name TEXT NOT NULL, UNIQUE(role_name));
 CREATE TABLE ecu_report_counter(ecu_serial TEXT NOT NULL PRIMARY KEY, counter INTEGER NOT NULL DEFAULT 0);
 CREATE TABLE report_events(id INTEGER PRIMARY KEY, json_string TEXT NOT NULL);
+CREATE TABLE device_data(data_type TEXT PRIMARY KEY, hash TEXT NOT NULL);

src/libaktualizr/primary/aktualizr_test.cc

@@ -1177,7 +1177,7 @@ TEST(Aktualizr, AutoRebootAfterUpdate) {
     storage->loadPrimaryInstalledVersions(&current_target, &pending_target);
     EXPECT_TRUE(!!current_target);
     EXPECT_FALSE(!!pending_target);
-    EXPECT_EQ(http->manifest_sends, 4);
+    EXPECT_EQ(http->manifest_sends, 3);
   }
 }
 

src/libaktualizr/primary/sotauptaneclient.cc

@@ -149,10 +149,21 @@ data::InstallationResult SotaUptaneClient::PackageInstallSetResult(const Uptane:
   return result;
 }
 
+/* Hardware info is treated differently than the other device data. The default
+ * info (supplied via lshw) is only sent once and never again, even if it
+ * changes. (Unfortunately, it can change often due to CPU frequency scaling.)
+ * However, users can provide custom info via the API, and that will be sent if
+ * it has changed. */
 void SotaUptaneClient::reportHwInfo(const Json::Value &custom_hwinfo) {
   Json::Value system_info;
+  std::string stored_hash;
+  storage->loadDeviceDataHash("hardware_info", &stored_hash);
 
   if (custom_hwinfo.empty()) {
+    if (!stored_hash.empty()) {
+      LOG_TRACE << "Not reporting default hardware information because it has already been reported";
+      return;
+    }
     system_info = Utils::getHardwareInfo();
     if (system_info.empty()) {
       LOG_WARNING << "Unable to fetch hardware information from host system.";
@@ -161,43 +172,80 @@ void SotaUptaneClient::reportHwInfo(const Json::Value &custom_hwinfo) {
   }
 
   const Json::Value &hw_info = custom_hwinfo.empty() ? system_info : custom_hwinfo;
-  if (hw_info != last_hw_info_reported) {
-    if (http->put(config.tls.server + "/system_info", hw_info).isOk()) {
-      last_hw_info_reported = hw_info;
+  const Hash new_hash = Hash::generate(Hash::Type::kSha256, Utils::jsonToCanonicalStr(hw_info));
+  if (new_hash != Hash(Hash::Type::kSha256, stored_hash)) {
+    if (custom_hwinfo.empty()) {
+      LOG_DEBUG << "Reporting default hardware information";
+    } else {
+      LOG_DEBUG << "Reporting custom hardware information";
     }
+    const HttpResponse response = http->put(config.tls.server + "/system_info", hw_info);
+    if (response.isOk()) {
+      storage->storeDeviceDataHash("hardware_info", new_hash.HashString());
+    }
+  } else {
+    LOG_TRACE << "Not reporting hardware information because it has not changed";
   }
 }
 
 void SotaUptaneClient::reportInstalledPackages() {
-  http->put(config.tls.server + "/core/installed", package_manager_->getInstalledPackages());
+  const Json::Value packages = package_manager_->getInstalledPackages();
+  const Hash new_hash = Hash::generate(Hash::Type::kSha256, Utils::jsonToCanonicalStr(packages));
+  std::string stored_hash;
+  if (!(storage->loadDeviceDataHash("installed_packages", &stored_hash) &&
+        new_hash == Hash(Hash::Type::kSha256, stored_hash))) {
+    LOG_DEBUG << "Reporting installed packages";
+    const HttpResponse response = http->put(config.tls.server + "/core/installed", packages);
+    if (response.isOk()) {
+      storage->storeDeviceDataHash("installed_packages", new_hash.HashString());
+    }
+  } else {
+    LOG_TRACE << "Not reporting installed packages because they have not changed";
+  }
 }
 
 void SotaUptaneClient::reportNetworkInfo() {
-  if (config.telemetry.report_network) {
+  if (!config.telemetry.report_network) {
+    LOG_TRACE << "Not reporting network information because telemetry is disabled";
+    return;
+  }
+
+  const Json::Value network_info = Utils::getNetworkInfo();
+  const Hash new_hash = Hash::generate(Hash::Type::kSha256, Utils::jsonToCanonicalStr(network_info));
+  std::string stored_hash;
+  if (!(storage->loadDeviceDataHash("network_info", &stored_hash) &&
+        new_hash == Hash(Hash::Type::kSha256, stored_hash))) {
     LOG_DEBUG << "Reporting network information";
-    Json::Value network_info = Utils::getNetworkInfo();
-    if (network_info != last_network_info_reported) {
-      HttpResponse response = http->put(config.tls.server + "/system_info/network", network_info);
-      if (response.isOk()) {
-        last_network_info_reported = network_info;
-      }
+    const HttpResponse response = http->put(config.tls.server + "/system_info/network", network_info);
+    if (response.isOk()) {
+      storage->storeDeviceDataHash("network_info", new_hash.HashString());
     }
-
   } else {
-    LOG_DEBUG << "Not reporting network information because telemetry is disabled";
+    LOG_TRACE << "Not reporting network information because it has not changed";
   }
 }
 
 void SotaUptaneClient::reportAktualizrConfiguration() {
   if (!config.telemetry.report_config) {
-    LOG_DEBUG << "Not reporting network information because telemetry is disabled";
+    LOG_TRACE << "Not reporting libaktualizr configuration because telemetry is disabled";
     return;
   }
 
-  LOG_DEBUG << "Reporting libaktualizr configuration";
   std::stringstream conf_ss;
   config.writeToStream(conf_ss);
-  http->post(config.tls.server + "/system_info/config", "application/toml", conf_ss.str());
+  const std::string conf_str = conf_ss.str();
+  const Hash new_hash = Hash::generate(Hash::Type::kSha256, conf_str);
+  std::string stored_hash;
+  if (!(storage->loadDeviceDataHash("configuration", &stored_hash) &&
+        new_hash == Hash(Hash::Type::kSha256, stored_hash))) {
+    LOG_DEBUG << "Reporting libaktualizr configuration";
+    const HttpResponse response = http->post(config.tls.server + "/system_info/config", "application/toml", conf_str);
+    if (response.isOk()) {
+      storage->storeDeviceDataHash("configuration", new_hash.HashString());
+    }
+  } else {
+    LOG_TRACE << "Not reporting libaktualizr configuration because it has not changed";
+  }
 }
 
 Json::Value SotaUptaneClient::AssembleManifest() {
@@ -754,7 +802,6 @@ void SotaUptaneClient::sendDeviceData(const Json::Value &custom_hwinfo) {
   reportInstalledPackages();
   reportNetworkInfo();
   reportAktualizrConfiguration();
-  putManifestSimple();
   sendEvent<event::SendDeviceDataComplete>();
 }
 

src/libaktualizr/primary/sotauptaneclient.h

@@ -169,8 +169,6 @@ class SotaUptaneClient {
   std::shared_ptr<PackageManagerInterface> package_manager_;
   std::shared_ptr<Uptane::Fetcher> uptane_fetcher;
   std::unique_ptr<ReportQueue> report_queue;
-  Json::Value last_network_info_reported;
-  Json::Value last_hw_info_reported;
   std::shared_ptr<event::Channel> events_channel;
   boost::signals2::scoped_connection conn;
   std::exception_ptr last_exception;

src/libaktualizr/storage/invstorage.h

@@ -215,6 +215,10 @@ class INvStorage {
   virtual bool loadReportEvents(Json::Value* report_array, int64_t* id_max) const = 0;
   virtual void deleteReportEvents(int64_t id_max) = 0;
 
+  virtual void storeDeviceDataHash(const std::string& data_type, const std::string& hash) = 0;
+  virtual bool loadDeviceDataHash(const std::string& data_type, std::string* hash) const = 0;
+  virtual void clearDeviceData() = 0;
+
   virtual bool checkAvailableDiskSpace(uint64_t required_bytes) const = 0;
   virtual boost::optional<std::pair<uintmax_t, std::string>> checkTargetFile(const Uptane::Target& target) const = 0;
 

src/libaktualizr/storage/sqlstorage.cc

@@ -1644,6 +1644,48 @@ void SQLStorage::clearInstallationResults() {
   db.commitTransaction();
 }
 
+void SQLStorage::storeDeviceDataHash(const std::string& data_type, const std::string& hash) {
+  SQLite3Guard db = dbConnection();
+
+  auto statement = db.prepareStatement<std::string, std::string>(
+      "INSERT OR REPLACE INTO device_data(data_type,hash) VALUES (?,?);", data_type, hash);
+  if (statement.step() != SQLITE_DONE) {
+    LOG_ERROR << "Can't set " << data_type << " hash: " << db.errmsg();
+    throw std::runtime_error("Can't set " + data_type + " hash: " + db.errmsg());
+  }
+}
+
+bool SQLStorage::loadDeviceDataHash(const std::string& data_type, std::string* hash) const {
+  SQLite3Guard db = dbConnection();
+
+  auto statement =
+      db.prepareStatement<std::string>("SELECT hash FROM device_data WHERE data_type = ? LIMIT 1;", data_type);
+
+  int result = statement.step();
+  if (result == SQLITE_DONE) {
+    LOG_TRACE << data_type << " hash not present in db";
+    return false;
+  } else if (result != SQLITE_ROW) {
+    LOG_ERROR << "Can't get " << data_type << " hash: " << db.errmsg();
+    return false;
+  }
+
+  if (hash != nullptr) {
+    *hash = statement.get_result_col_str(0).value();
+  }
+
+  return true;
+}
+
+void SQLStorage::clearDeviceData() {
+  SQLite3Guard db = dbConnection();
+
+  if (db.exec("DELETE FROM device_data;", nullptr, nullptr) != SQLITE_OK) {
+    LOG_ERROR << "Can't clear device_data: " << db.errmsg();
+    return;
+  }
+}
+
 bool SQLStorage::checkAvailableDiskSpace(const uint64_t required_bytes) const {
   struct statvfs stvfsbuf {};
   const int stat_res = statvfs(dbPath().c_str(), &stvfsbuf);

src/libaktualizr/storage/sqlstorage.h

@@ -96,6 +96,10 @@ class SQLStorage : public SQLStorageBase, public INvStorage {
   void deleteReportEvents(int64_t id_max) override;
   void clearInstallationResults() override;
 
+  void storeDeviceDataHash(const std::string& data_type, const std::string& hash) override;
+  bool loadDeviceDataHash(const std::string& data_type, std::string* hash) const override;
+  void clearDeviceData() override;
+
   bool checkAvailableDiskSpace(uint64_t required_bytes) const override;
 
   std::unique_ptr<StorageTargetWHandle> allocateTargetFile(const Uptane::Target& target) override;