Add automatic removal of old installed targets

CHANGELOG.md

@@ -6,6 +6,11 @@ Our versioning scheme is `YEAR.N` where `N` is incremented whenever a new releas
 
 ## [??? (unreleased)]
 
+### Added
+
+- `GetInstallationLog` API method: [PR](https://github.com/advancedtelematic/aktualizr/pull/1318)
+- The aktualizr daemon will now automatically remove old downloaded targets to free up disk space: [PR](https://github.com/advancedtelematic/aktualizr/pull/1318)
+
 ## [2019.6] - 2019-08-21
 
 ### Added

actions.md

@@ -173,6 +173,7 @@ These are the primary actions that a user of libaktualizr can perform through th
   - [x] Store negative device installation result when an ECU installation failed (aktualizr_test.cc)
   - [x] Update is not in pending state anymore after failed installation (aktualizr_test.cc)
   - [x] Send AllInstallsComplete event after all installations are finished (aktualizr_test.cc)
+  - [x] Automatically remove old targets during installation cycles (aktualizr_test.cc)
 - [x] Send installation report
   - [x] Generate and send manifest (see below)
   - [x] Send PutManifestComplete event if send is successful (aktualizr_test.cc)

config/sql/migration/migrate.20.sql

@@ -0,0 +1,13 @@
+-- Don't modify this! Create a new migration instead--see docs/schema-migrations.adoc
+SAVEPOINT MIGRATION;
+
+ALTER TABLE installed_versions RENAME TO installed_versions_old;
+CREATE TABLE installed_versions(id INTEGER PRIMARY KEY, ecu_serial TEXT NOT NULL, sha256 TEXT NOT NULL, name TEXT NOT NULL, hashes TEXT NOT NULL, length INTEGER NOT NULL DEFAULT 0, correlation_id TEXT NOT NULL DEFAULT '', is_current INTEGER NOT NULL CHECK (is_current IN (0,1)) DEFAULT 0, is_pending INTEGER NOT NULL CHECK (is_pending IN (0,1)) DEFAULT 0, was_installed INTEGER NOT NULL CHECK (was_installed IN (0,1)) DEFAULT 0);
+INSERT INTO installed_versions(ecu_serial, sha256, name, hashes, length, correlation_id, is_current, is_pending, was_installed) SELECT installed_versions_old.ecu_serial, installed_versions_old.sha256, installed_versions_old.name, installed_versions_old.hashes, installed_versions_old.length, installed_versions_old.correlation_id, installed_versions_old.is_current, installed_versions_old.is_pending, 1 FROM installed_versions_old ORDER BY rowid;
+
+DROP TABLE installed_versions_old;
+
+DELETE FROM version;
+INSERT INTO version VALUES(20);
+
+RELEASE MIGRATION;

config/sql/rollback/rollback.20.sql

@@ -0,0 +1,13 @@
+-- Don't modify this! Create a new migration instead--see docs/schema-migrations.adoc
+SAVEPOINT ROLLBACK_MIGRATION;
+
+CREATE TABLE installed_versions_migrate(ecu_serial TEXT NOT NULL, sha256 TEXT NOT NULL, name TEXT NOT NULL, hashes TEXT NOT NULL, length INTEGER NOT NULL DEFAULT 0, correlation_id TEXT NOT NULL DEFAULT '', is_current INTEGER NOT NULL CHECK (is_current IN (0,1)) DEFAULT 0, is_pending INTEGER NOT NULL CHECK (is_pending IN (0,1)) DEFAULT 0, UNIQUE(ecu_serial, sha256, name));
+INSERT INTO installed_versions_migrate(ecu_serial, sha256, name, hashes, length, correlation_id, is_current, is_pending) SELECT installed_versions.ecu_serial, installed_versions.sha256, installed_versions.name, installed_versions.hashes, installed_versions.length, installed_versions.correlation_id, installed_versions.is_current, installed_versions.is_pending FROM installed_versions;
+
+DROP TABLE installed_versions;
+ALTER TABLE installed_versions_migrate RENAME TO installed_versions;
+
+DELETE FROM version;
+INSERT INTO version VALUES(19);
+
+RELEASE ROLLBACK_MIGRATION;

config/sql/schema.sql

@@ -1,9 +1,9 @@
 CREATE TABLE version(version INTEGER);
-INSERT INTO version(rowid,version) VALUES(1,19);
+INSERT INTO version(rowid,version) VALUES(1,20);
 CREATE TABLE device_info(unique_mark INTEGER PRIMARY KEY CHECK (unique_mark = 0), device_id TEXT, is_registered INTEGER NOT NULL DEFAULT 0 CHECK (is_registered IN (0,1)));
 CREATE TABLE ecu_serials(id INTEGER PRIMARY KEY, serial TEXT UNIQUE, hardware_id TEXT NOT NULL, is_primary INTEGER NOT NULL DEFAULT 0 CHECK (is_primary IN (0,1)));
 CREATE TABLE misconfigured_ecus(serial TEXT UNIQUE, hardware_id TEXT NOT NULL, state INTEGER NOT NULL CHECK (state IN (0,1)));
-CREATE TABLE installed_versions(ecu_serial TEXT NOT NULL, sha256 TEXT NOT NULL, name TEXT NOT NULL, hashes TEXT NOT NULL, length INTEGER NOT NULL DEFAULT 0, correlation_id TEXT NOT NULL DEFAULT '', is_current INTEGER NOT NULL CHECK (is_current IN (0,1)) DEFAULT 0, is_pending INTEGER NOT NULL CHECK (is_pending IN (0,1)) DEFAULT 0, UNIQUE(ecu_serial, sha256, name));
+CREATE TABLE installed_versions(id INTEGER PRIMARY KEY, ecu_serial TEXT NOT NULL, sha256 TEXT NOT NULL, name TEXT NOT NULL, hashes TEXT NOT NULL, length INTEGER NOT NULL DEFAULT 0, correlation_id TEXT NOT NULL DEFAULT '', is_current INTEGER NOT NULL CHECK (is_current IN (0,1)) DEFAULT 0, is_pending INTEGER NOT NULL CHECK (is_pending IN (0,1)) DEFAULT 0, was_installed INTEGER NOT NULL CHECK (was_installed IN (0,1)) DEFAULT 0);
 CREATE TABLE primary_keys(unique_mark INTEGER PRIMARY KEY CHECK (unique_mark = 0), private TEXT, public TEXT);
 CREATE TABLE tls_creds(ca_cert BLOB, ca_cert_format TEXT,
                        client_cert BLOB, client_cert_format TEXT,

src/aktualizr_info/main.cc

@@ -278,24 +278,22 @@ int main(int argc, char **argv) {
         std::cout << secondary_number++ << ") serial ID: " << it->first << std::endl;
         std::cout << "   hardware ID: " << it->second << std::endl;
 
-        std::vector<Uptane::Target> installed_targets;
-        size_t current_version = SIZE_MAX;
-        size_t pending_version = SIZE_MAX;
+        boost::optional<Uptane::Target> current_version;
+        boost::optional<Uptane::Target> pending_version;
 
-        auto load_installed_version_res = storage->loadInstalledVersions((it->first).ToString(), &installed_targets,
-                                                                         &current_version, &pending_version);
+        auto load_installed_version_res =
+            storage->loadInstalledVersions((it->first).ToString(), &current_version, &pending_version);
 
-        if (!load_installed_version_res ||
-            (current_version >= installed_targets.size() && pending_version >= installed_targets.size())) {
+        if (!load_installed_version_res || (!current_version && !pending_version)) {
           std::cout << "   no details about installed nor pending images\n";
         } else {
-          if (installed_targets.size() > current_version) {
-            std::cout << "   installed image hash: " << installed_targets[current_version].sha256Hash() << "\n";
-            std::cout << "   installed image filename: " << installed_targets[current_version].filename() << "\n";
+          if (!!current_version) {
+            std::cout << "   installed image hash: " << current_version->sha256Hash() << "\n";
+            std::cout << "   installed image filename: " << current_version->filename() << "\n";
           }
-          if (installed_targets.size() > pending_version) {
-            std::cout << "   pending image hash: " << installed_targets[pending_version].sha256Hash() << "\n";
-            std::cout << "   pending image filename: " << installed_targets[pending_version].filename() << "\n";
+          if (!!pending_version) {
+            std::cout << "   pending image hash: " << pending_version->sha256Hash() << "\n";
+            std::cout << "   pending image filename: " << pending_version->filename() << "\n";
           }
         }
       }
@@ -326,11 +324,11 @@ int main(int argc, char **argv) {
     }
 
     std::vector<Uptane::Target> installed_versions;
-    size_t pending = SIZE_MAX;
-    storage->loadInstalledVersions("", &installed_versions, nullptr, &pending);
+    boost::optional<Uptane::Target> pending;
+    storage->loadPrimaryInstalledVersions(nullptr, &pending);
 
-    if (pending != SIZE_MAX) {
-      std::cout << "Pending primary ecu version: " << installed_versions[pending].sha256Hash() << std::endl;
+    if (!!pending) {
+      std::cout << "Pending primary ecu version: " << pending->sha256Hash() << std::endl;
     }
   } catch (const po::error &o) {
     std::cout << o.what() << std::endl;

src/aktualizr_lite/main.cc

@@ -16,19 +16,18 @@
 namespace bpo = boost::program_options;
 
 static void finalizeIfNeeded(INvStorage &storage, PackageConfig &config) {
-  std::vector<Uptane::Target> installed_versions;
-  size_t pending_index = SIZE_MAX;
-  storage.loadInstalledVersions("", &installed_versions, nullptr, &pending_index);
+  boost::optional<Uptane::Target> pending_version;
+  storage.loadInstalledVersions("", nullptr, &pending_version);
 
-  if (pending_index < installed_versions.size()) {
+  if (!!pending_version) {
     GObjectUniquePtr<OstreeSysroot> sysroot_smart = OstreeManager::LoadSysroot(config.sysroot);
     OstreeDeployment *booted_deployment = ostree_sysroot_get_booted_deployment(sysroot_smart.get());
     if (booted_deployment == nullptr) {
       throw std::runtime_error("Could not get booted deployment in " + config.sysroot.string());
     }
     std::string current_hash = ostree_deployment_get_csum(booted_deployment);
 
-    const Uptane::Target &target = installed_versions[pending_index];
+    const Uptane::Target &target = *pending_version;
     if (current_hash == target.sha256Hash()) {
       LOG_INFO << "Marking target install complete for: " << target;
       storage.saveInstalledVersion("", target, InstalledVersionUpdateMode::kCurrent);

src/aktualizr_lite/ostree_mock.cc

@@ -1,4 +1,5 @@
 #include <ostree.h>
+#include <stdlib.h>
 
 extern "C" OstreeDeployment *ostree_sysroot_get_booted_deployment(OstreeSysroot *self) {
   (void)self;

src/aktualizr_primary/main.cc

@@ -8,6 +8,7 @@
 #include "config/config.h"
 #include "logging/logging.h"
 #include "primary/aktualizr.h"
+#include "primary/aktualizr_helpers.h"
 #include "secondary.h"
 #include "utilities/aktualizr_version.h"
 #include "utilities/utils.h"
@@ -87,7 +88,7 @@ bpo::variables_map parse_options(int argc, char *argv[]) {
 }
 
 void process_event(const std::shared_ptr<event::BaseEvent> &event) {
-  if (event->isTypeOf(event::DownloadProgressReport::TypeName)) {
+  if (event->isTypeOf<event::DownloadProgressReport>()) {
     // Do nothing; libaktualizr already logs it.
   } else if (event->variant == "UpdateCheckComplete") {
     // Do nothing; libaktualizr already logs it.
@@ -157,6 +158,9 @@ int main(int argc, char *argv[]) {
       aktualizr.SendDeviceData().get();
       aktualizr.UptaneCycle();
     } else {
+      boost::signals2::connection ac_conn =
+          aktualizr.SetSignalHandler(std::bind(targets_autoclean_cb, std::ref(aktualizr), std::placeholders::_1));
+
       aktualizr.RunForever().get();
     }
     r = EXIT_SUCCESS;

src/libaktualizr/package_manager/androidmanager.cc

@@ -55,7 +55,7 @@ Uptane::Target AndroidManager::getCurrent() const {
     qi::phrase_parse(getprop_output.crbegin(), getprop_output.crend(),
                      *(xdigit[push_front(boost::phoenix::ref(hash), _1)]), boost::spirit::ascii::cntrl);
     std::vector<Uptane::Target> installed_versions;
-    storage_->loadPrimaryInstalledVersions(&installed_versions, nullptr, nullptr);
+    storage_->loadPrimaryInstallationLog(&installed_versions, false);
     for (const auto& target : installed_versions) {
       if (std::equal(hash.cbegin(), hash.cend(), target.sha256Hash().cbegin())) {
         return target;

src/libaktualizr/package_manager/debianmanager.cc

@@ -31,12 +31,11 @@ data::InstallationResult DebianManager::install(const Uptane::Target &target) co
 }
 
 Uptane::Target DebianManager::getCurrent() const {
-  std::vector<Uptane::Target> installed_versions;
-  size_t current_k = SIZE_MAX;
-  storage_->loadPrimaryInstalledVersions(&installed_versions, &current_k, nullptr);
+  boost::optional<Uptane::Target> current_version;
+  storage_->loadPrimaryInstalledVersions(&current_version, nullptr);
 
-  if (current_k != SIZE_MAX) {
-    return installed_versions.at(current_k);
+  if (!!current_version) {
+    return *current_version;
   }
 
   return Uptane::Target::Unknown();

src/libaktualizr/package_manager/ostreemanager.cc

@@ -299,7 +299,7 @@ Uptane::Target OstreeManager::getCurrent() const {
   std::string current_hash = ostree_deployment_get_csum(booted_deployment);
 
   std::vector<Uptane::Target> installed_versions;
-  storage_->loadPrimaryInstalledVersions(&installed_versions, nullptr, nullptr);
+  storage_->loadPrimaryInstallationLog(&installed_versions, false);
 
   // Version should be in installed versions. Its possible that multiple
   // targets could have the same sha256Hash. In this case the safest assumption

src/libaktualizr/package_manager/packagemanagerfake.cc

@@ -12,12 +12,11 @@ Json::Value PackageManagerFake::getInstalledPackages() const {
 }
 
 Uptane::Target PackageManagerFake::getCurrent() const {
-  std::vector<Uptane::Target> installed_versions;
-  size_t current_k = SIZE_MAX;
-  storage_->loadPrimaryInstalledVersions(&installed_versions, &current_k, nullptr);
+  boost::optional<Uptane::Target> current_version;
+  storage_->loadPrimaryInstalledVersions(&current_version, nullptr);
 
-  if (current_k != SIZE_MAX) {
-    return installed_versions.at(current_k);
+  if (!!current_version) {
+    return *current_version;
   }
 
   return Uptane::Target::Unknown();
@@ -53,17 +52,16 @@ void PackageManagerFake::completeInstall() const {
 }
 
 data::InstallationResult PackageManagerFake::finalizeInstall(const Uptane::Target& target) const {
-  std::vector<Uptane::Target> targets;
-  size_t pending_version = SIZE_MAX;
-  storage_->loadPrimaryInstalledVersions(&targets, nullptr, &pending_version);
+  boost::optional<Uptane::Target> pending_version;
+  storage_->loadPrimaryInstalledVersions(nullptr, &pending_version);
 
-  if (pending_version == SIZE_MAX) {
+  if (!pending_version) {
     throw std::runtime_error("No pending update, nothing to finalize");
   }
 
   data::InstallationResult install_res;
 
-  if (target.MatchTarget(targets[pending_version])) {
+  if (target.MatchTarget(*pending_version)) {
     if (fiu_fail("fake_install_finalization_failure") != 0) {
       const std::string failure_cause = fault_injection_last_info();
       if (failure_cause.empty()) {

src/libaktualizr/primary/CMakeLists.txt

@@ -1,10 +1,12 @@
 set(SOURCES aktualizr.cc
+            aktualizr_helpers.cc
             initializer.cc
             reportqueue.cc
             sotauptaneclient.cc)
 
 set(HEADERS secondary_config.h
             aktualizr.h
+            aktualizr_helpers.h
             events.h
             initializer.h
             reportqueue.h
@@ -32,8 +34,7 @@ if (BUILD_OSTREE)
         LABELS "noptest")
     target_link_libraries(t_download_nonostree virtual_secondary)
 else (BUILD_OSTREE)
-    aktualizr_source_file_checks(aktualizr_fullostree_test.cc)
-    aktualizr_source_file_checks(download_nonostree_test.cc)
+    aktualizr_source_file_checks(aktualizr_fullostree_test.cc download_nonostree_test.cc)
 endif (BUILD_OSTREE)
 
 add_aktualizr_test(NAME reportqueue SOURCES reportqueue_test.cc PROJECT_WORKING_DIRECTORY LIBRARIES PUBLIC uptane_generator_lib)

src/libaktualizr/primary/aktualizr.cc

@@ -170,6 +170,28 @@ boost::signals2::connection Aktualizr::SetSignalHandler(
   return sig_->connect(handler);
 }
 
+Aktualizr::InstallationLog Aktualizr::GetInstallationLog() {
+  std::vector<Aktualizr::InstallationLogEntry> ilog;
+
+  EcuSerials serials;
+  if (!storage_->loadEcuSerials(&serials)) {
+    throw std::runtime_error("Could not load ecu serials");
+  }
+
+  ilog.reserve(serials.size());
+  for (const auto &s : serials) {
+    Uptane::EcuSerial serial = s.first;
+    std::vector<Uptane::Target> installs;
+
+    std::vector<Uptane::Target> log;
+    storage_->loadInstallationLog(serial.ToString(), &log, true);
+
+    ilog.emplace_back(Aktualizr::InstallationLogEntry{serial, std::move(log)});
+  }
+
+  return ilog;
+}
+
 std::vector<Uptane::Target> Aktualizr::GetStoredTargets() { return storage_->getTargetFiles(); }
 
 void Aktualizr::DeleteStoredTarget(const Uptane::Target &target) { storage_->removeTargetFile(target.filename()); }

src/libaktualizr/primary/aktualizr.h

@@ -1,7 +1,7 @@
 #ifndef AKTUALIZR_H_
 #define AKTUALIZR_H_
 
-#include <atomic>
+#include <future>
 #include <memory>
 
 #include <boost/signals2.hpp>
@@ -82,6 +82,19 @@ class Aktualizr {
    */
   std::future<result::Download> Download(const std::vector<Uptane::Target>& updates);
 
+  /**
+   * Get log of installations. The log is indexed for every ECU and contains
+   * every change of versions ordered by time. It may contain duplicates in
+   * case of rollbacks.
+   * @return installation log
+   */
+  struct InstallationLogEntry {
+    Uptane::EcuSerial ecu;
+    std::vector<Uptane::Target> installs;
+  };
+  using InstallationLog = std::vector<InstallationLogEntry>;
+  InstallationLog GetInstallationLog();
+
   /**
    * Get list of targets currently in storage. This is intended to be used with
    * DeleteStoredTarget and targets are not guaranteed to be verified and

src/libaktualizr/primary/aktualizr_helpers.cc

@@ -0,0 +1,36 @@
+#include <set>
+
+#include "aktualizr_helpers.h"
+
+void targets_autoclean_cb(Aktualizr &aktualizr, const std::shared_ptr<event::BaseEvent> &event) {
+  if (!event->isTypeOf<event::AllInstallsComplete>()) {
+    return;
+  }
+
+  std::vector<Uptane::Target> installed_targets = aktualizr.GetStoredTargets();
+  std::vector<bool> to_remove(installed_targets.size(), true);
+
+  Aktualizr::InstallationLog log = aktualizr.GetInstallationLog();
+
+  // keep the last two installed targets for each ecu
+  for (const Aktualizr::InstallationLogEntry &entry : log) {
+    auto start = entry.installs.size() >= 2 ? entry.installs.end() - 2 : entry.installs.begin();
+    for (auto it = start; it != entry.installs.end(); it++) {
+      auto fit = std::find_if(installed_targets.begin(), installed_targets.end(),
+                              [&it](const Uptane::Target &t2) { return it->sha256Hash() == t2.sha256Hash(); });
+
+      if (fit == installed_targets.end()) {
+        continue;
+      }
+
+      size_t rem_idx = static_cast<size_t>(fit - installed_targets.begin());
+      to_remove[rem_idx] = false;
+    }
+  }
+
+  for (size_t k = 0; k < installed_targets.size(); k++) {
+    if (to_remove[k]) {
+      aktualizr.DeleteStoredTarget(installed_targets[k]);
+    }
+  }
+}

src/libaktualizr/primary/aktualizr_helpers.h

@@ -0,0 +1,14 @@
+#ifndef AKTUALIZR_HELPERS_H_
+#define AKTUALIZR_HELPERS_H_
+
+#include <memory>
+#include "aktualizr.h"
+
+/*
+ * Signal handler to remove old targets just after an installation completes
+ *
+ * To be attached with Aktualizr::SetSignalHandler
+ */
+void targets_autoclean_cb(Aktualizr &aktualizr, const std::shared_ptr<event::BaseEvent> &event);
+
+#endif  // AKTUALIZR_HELPERS_H_