From 41de00290af3024f011661f8c73b6750d7b4ee16 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Wed, 4 Sep 2019 10:41:26 +0200 Subject: [PATCH 1/7] Fix a missing include Signed-off-by: Laurent Bonnans --- src/aktualizr_lite/ostree_mock.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/aktualizr_lite/ostree_mock.cc b/src/aktualizr_lite/ostree_mock.cc index 71eb50103c..44dc301241 100644 --- a/src/aktualizr_lite/ostree_mock.cc +++ b/src/aktualizr_lite/ostree_mock.cc @@ -1,4 +1,5 @@ #include +#include extern "C" OstreeDeployment *ostree_sysroot_get_booted_deployment(OstreeSysroot *self) { (void)self; From abfb0fef291826c708a011217f7ba0ec3c0666e1 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Mon, 19 Aug 2019 16:51:06 +0200 Subject: [PATCH 2/7] Systematic `isTypeOf` for objects in events.h Signed-off-by: Laurent Bonnans --- src/aktualizr_primary/main.cc | 2 +- src/libaktualizr/primary/aktualizr_test.cc | 16 +++---- src/libaktualizr/primary/events.h | 49 ++++++++++++++------- src/libaktualizr/primary/sotauptaneclient.h | 2 +- 4 files changed, 42 insertions(+), 27 deletions(-) diff --git a/src/aktualizr_primary/main.cc b/src/aktualizr_primary/main.cc index 5c60ec11bd..fa065f9890 100644 --- a/src/aktualizr_primary/main.cc +++ b/src/aktualizr_primary/main.cc @@ -87,7 +87,7 @@ bpo::variables_map parse_options(int argc, char *argv[]) { } void process_event(const std::shared_ptr &event) { - if (event->isTypeOf(event::DownloadProgressReport::TypeName)) { + if (event->isTypeOf()) { // Do nothing; libaktualizr already logs it. } else if (event->variant == "UpdateCheckComplete") { // Do nothing; libaktualizr already logs it. diff --git a/src/libaktualizr/primary/aktualizr_test.cc b/src/libaktualizr/primary/aktualizr_test.cc index c5dc840c92..f0b8a44f42 100644 --- a/src/libaktualizr/primary/aktualizr_test.cc +++ b/src/libaktualizr/primary/aktualizr_test.cc @@ -79,7 +79,7 @@ TEST(Aktualizr, FullNoUpdates) { ev_state.future = ev_state.promise.get_future(); auto f_cb = [&ev_state](const std::shared_ptr& event) { - if (event->isTypeOf(event::DownloadProgressReport::TypeName)) { + if (event->isTypeOf()) { return; } LOG_INFO << "Got " << event->variant; @@ -256,7 +256,7 @@ TEST(Aktualizr, FullWithUpdates) { ev_state.future = ev_state.promise.get_future(); auto f_cb = [&ev_state](const std::shared_ptr& event) { - if (event->isTypeOf(event::DownloadProgressReport::TypeName)) { + if (event->isTypeOf()) { return; } LOG_INFO << "Got " << event->variant; @@ -1065,7 +1065,7 @@ TEST(Aktualizr, CheckNoUpdates) { ev_state.future = ev_state.promise.get_future(); auto f_cb = [&ev_state](const std::shared_ptr& event) { - if (event->isTypeOf(event::DownloadProgressReport::TypeName)) { + if (event->isTypeOf()) { return; } LOG_INFO << "Got " << event->variant; @@ -1141,7 +1141,7 @@ TEST(Aktualizr, DownloadWithUpdates) { ev_state.future = ev_state.promise.get_future(); auto f_cb = [&ev_state](const std::shared_ptr& event) { - if (event->isTypeOf(event::DownloadProgressReport::TypeName)) { + if (event->isTypeOf()) { return; } LOG_INFO << "Got " << event->variant; @@ -1263,12 +1263,12 @@ TEST(Aktualizr, DownloadFailures) { void operator()(const std::shared_ptr& event) { ASSERT_NE(event, nullptr); - if (event->isTypeOf(event::DownloadTargetComplete::TypeName)) { + if (event->isTypeOf()) { auto download_target_complete_event = dynamic_cast(event.get()); auto target_filename = download_target_complete_event->update.filename(); download_status[target_filename] = download_target_complete_event->success; - } else if (event->isTypeOf(event::AllDownloadsComplete::TypeName)) { + } else if (event->isTypeOf()) { auto all_download_complete_event = dynamic_cast(event.get()); all_download_completed_status = all_download_complete_event->result; } @@ -1386,7 +1386,7 @@ TEST(Aktualizr, InstallWithUpdates) { auto f_cb = [&ev_state](const std::shared_ptr& event) { // Note that we do not expect a PutManifestComplete since we don't call // UptaneCycle() and that's the only function that generates that. - if (event->isTypeOf(event::DownloadProgressReport::TypeName)) { + if (event->isTypeOf()) { return; } LOG_INFO << "Got " << event->variant; @@ -1545,7 +1545,7 @@ TEST(Aktualizr, ReportDownloadProgress) { std::function event)> report_event_hdlr = [&](const std::shared_ptr& event) { ASSERT_NE(event, nullptr); - if (!event->isTypeOf(event::DownloadProgressReport::TypeName)) { + if (!event->isTypeOf()) { return; } diff --git a/src/libaktualizr/primary/events.h b/src/libaktualizr/primary/events.h index 845bfb0e53..fef0893f24 100644 --- a/src/libaktualizr/primary/events.h +++ b/src/libaktualizr/primary/events.h @@ -26,7 +26,10 @@ class BaseEvent { BaseEvent(std::string variant_in) : variant(std::move(variant_in)) {} virtual ~BaseEvent() = default; - bool isTypeOf(const std::string& type_to_cmp) { return (variant == type_to_cmp); } + template + bool isTypeOf() { + return variant == T::TypeName; + } std::string variant; }; @@ -36,7 +39,9 @@ class BaseEvent { */ class SendDeviceDataComplete : public BaseEvent { public: - SendDeviceDataComplete() { variant = "SendDeviceDataComplete"; } + static constexpr const char* TypeName{"SendDeviceDataComplete"}; + + SendDeviceDataComplete() { variant = TypeName; } }; /** @@ -44,7 +49,8 @@ class SendDeviceDataComplete : public BaseEvent { */ class PutManifestComplete : public BaseEvent { public: - explicit PutManifestComplete(bool success_in) : success(success_in) { variant = "PutManifestComplete"; } + static constexpr const char* TypeName{"PutManifestComplete"}; + explicit PutManifestComplete(bool success_in) : success(success_in) { variant = TypeName; } bool success; }; @@ -53,9 +59,8 @@ class PutManifestComplete : public BaseEvent { */ class UpdateCheckComplete : public BaseEvent { public: - explicit UpdateCheckComplete(result::UpdateCheck result_in) : result(std::move(result_in)) { - variant = "UpdateCheckComplete"; - } + static constexpr const char* TypeName{"UpdateCheckComplete"}; + explicit UpdateCheckComplete(result::UpdateCheck result_in) : result(std::move(result_in)) { variant = TypeName; } result::UpdateCheck result; }; @@ -118,7 +123,9 @@ class AllDownloadsComplete : public BaseEvent { */ class InstallStarted : public BaseEvent { public: - explicit InstallStarted(Uptane::EcuSerial serial_in) : serial(std::move(serial_in)) { variant = "InstallStarted"; } + static constexpr const char* TypeName{"InstallStarted"}; + + explicit InstallStarted(Uptane::EcuSerial serial_in) : serial(std::move(serial_in)) { variant = TypeName; } Uptane::EcuSerial serial; }; @@ -127,9 +134,11 @@ class InstallStarted : public BaseEvent { */ class InstallTargetComplete : public BaseEvent { public: + static constexpr const char* TypeName{"InstallTargetComplete"}; + InstallTargetComplete(Uptane::EcuSerial serial_in, bool success_in) : serial(std::move(serial_in)), success(success_in) { - variant = "InstallTargetComplete"; + variant = TypeName; } Uptane::EcuSerial serial; @@ -141,9 +150,9 @@ class InstallTargetComplete : public BaseEvent { */ class AllInstallsComplete : public BaseEvent { public: - explicit AllInstallsComplete(result::Install result_in) : result(std::move(result_in)) { - variant = "AllInstallsComplete"; - } + static constexpr const char* TypeName{"AllInstallsComplete"}; + + explicit AllInstallsComplete(result::Install result_in) : result(std::move(result_in)) { variant = TypeName; } result::Install result; }; @@ -153,9 +162,9 @@ class AllInstallsComplete : public BaseEvent { */ class CampaignCheckComplete : public BaseEvent { public: - explicit CampaignCheckComplete(result::CampaignCheck result_in) : result(std::move(result_in)) { - variant = "CampaignCheckComplete"; - } + static constexpr const char* TypeName{"CampaignCheckComplete"}; + + explicit CampaignCheckComplete(result::CampaignCheck result_in) : result(std::move(result_in)) { variant = TypeName; } result::CampaignCheck result; }; @@ -165,17 +174,23 @@ class CampaignCheckComplete : public BaseEvent { */ class CampaignAcceptComplete : public BaseEvent { public: - CampaignAcceptComplete() { variant = "CampaignAcceptComplete"; } + static constexpr const char* TypeName{"CampaignAcceptComplete"}; + + CampaignAcceptComplete() { variant = TypeName; } }; class CampaignDeclineComplete : public BaseEvent { public: - CampaignDeclineComplete() { variant = "CampaignDeclineComplete"; } + static constexpr const char* TypeName{"CampaignDeclineComplete"}; + + CampaignDeclineComplete() { variant = TypeName; } }; class CampaignPostponeComplete : public BaseEvent { public: - CampaignPostponeComplete() { variant = "CampaignPostponeComplete"; } + static constexpr const char* TypeName{"CampaignPostponeComplete"}; + + CampaignPostponeComplete() { variant = TypeName; } }; using Channel = boost::signals2::signal)>; diff --git a/src/libaktualizr/primary/sotauptaneclient.h b/src/libaktualizr/primary/sotauptaneclient.h index 28b11e6035..3c8979cf68 100644 --- a/src/libaktualizr/primary/sotauptaneclient.h +++ b/src/libaktualizr/primary/sotauptaneclient.h @@ -134,7 +134,7 @@ class SotaUptaneClient { std::shared_ptr event = std::make_shared(std::forward(args)...); if (events_channel) { (*events_channel)(std::move(event)); - } else if (!event->isTypeOf(event::DownloadProgressReport::TypeName)) { + } else if (!event->isTypeOf()) { LOG_INFO << "got " << event->variant << " event"; } } From 395bdc0762794e0fb8212359df72ce6f3f4b2be9 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Fri, 23 Aug 2019 15:05:30 +0200 Subject: [PATCH 3/7] Remove some zealous throws in uptane-generator This function returns empty json for failure. Throwing here ended up masking the real issue. Signed-off-by: Laurent Bonnans --- src/uptane_generator/repo.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/uptane_generator/repo.cc b/src/uptane_generator/repo.cc index 1ae3d2caf9..e6e4dc3c54 100644 --- a/src/uptane_generator/repo.cc +++ b/src/uptane_generator/repo.cc @@ -228,6 +228,9 @@ Json::Value Repo::getTarget(const std::string &target_name) { if (image_targets["targets"].isMember(target_name)) { return image_targets["targets"][target_name]; } else if (repo_type_ == Uptane::RepositoryType::Image()) { + if (!boost::filesystem::is_directory(repo_dir_ / "delegations")) { + return {}; + } for (auto &p : boost::filesystem::directory_iterator(repo_dir_ / "delegations")) { if (Uptane::Role::IsReserved(p.path().stem().string())) { continue; @@ -237,7 +240,6 @@ Json::Value Repo::getTarget(const std::string &target_name) { return targets["targets"][target_name]; } } - throw std::runtime_error(std::string("No target with name: ") + target_name); } return {}; } From 85b416294985f98323394e738ea1dc1a46dbb3d1 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Mon, 19 Aug 2019 15:28:52 +0200 Subject: [PATCH 4/7] Keep a log of installed versions Signed-off-by: Laurent Bonnans --- config/sql/migration/migrate.20.sql | 13 +++ config/sql/rollback/rollback.20.sql | 13 +++ config/sql/schema.sql | 4 +- src/libaktualizr/storage/sqlstorage.cc | 61 ++++++++++++--- src/libaktualizr/storage/sqlstorage_test.cc | 87 ++++++++++++++++++++- 5 files changed, 163 insertions(+), 15 deletions(-) create mode 100644 config/sql/migration/migrate.20.sql create mode 100644 config/sql/rollback/rollback.20.sql diff --git a/config/sql/migration/migrate.20.sql b/config/sql/migration/migrate.20.sql new file mode 100644 index 0000000000..4b49610174 --- /dev/null +++ b/config/sql/migration/migrate.20.sql @@ -0,0 +1,13 @@ +-- Don't modify this! Create a new migration instead--see docs/schema-migrations.adoc +SAVEPOINT MIGRATION; + +ALTER TABLE installed_versions RENAME TO installed_versions_old; +CREATE TABLE installed_versions(id INTEGER PRIMARY KEY, ecu_serial TEXT NOT NULL, sha256 TEXT NOT NULL, name TEXT NOT NULL, hashes TEXT NOT NULL, length INTEGER NOT NULL DEFAULT 0, correlation_id TEXT NOT NULL DEFAULT '', is_current INTEGER NOT NULL CHECK (is_current IN (0,1)) DEFAULT 0, is_pending INTEGER NOT NULL CHECK (is_pending IN (0,1)) DEFAULT 0, was_installed INTEGER NOT NULL CHECK (was_installed IN (0,1)) DEFAULT 0); +INSERT INTO installed_versions(ecu_serial, sha256, name, hashes, length, correlation_id, is_current, is_pending, was_installed) SELECT installed_versions_old.ecu_serial, installed_versions_old.sha256, installed_versions_old.name, installed_versions_old.hashes, installed_versions_old.length, installed_versions_old.correlation_id, installed_versions_old.is_current, installed_versions_old.is_pending, 1 FROM installed_versions_old ORDER BY rowid; + +DROP TABLE installed_versions_old; + +DELETE FROM version; +INSERT INTO version VALUES(20); + +RELEASE MIGRATION; diff --git a/config/sql/rollback/rollback.20.sql b/config/sql/rollback/rollback.20.sql new file mode 100644 index 0000000000..3a30813015 --- /dev/null +++ b/config/sql/rollback/rollback.20.sql @@ -0,0 +1,13 @@ +-- Don't modify this! Create a new migration instead--see docs/schema-migrations.adoc +SAVEPOINT ROLLBACK_MIGRATION; + +CREATE TABLE installed_versions_migrate(ecu_serial TEXT NOT NULL, sha256 TEXT NOT NULL, name TEXT NOT NULL, hashes TEXT NOT NULL, length INTEGER NOT NULL DEFAULT 0, correlation_id TEXT NOT NULL DEFAULT '', is_current INTEGER NOT NULL CHECK (is_current IN (0,1)) DEFAULT 0, is_pending INTEGER NOT NULL CHECK (is_pending IN (0,1)) DEFAULT 0, UNIQUE(ecu_serial, sha256, name)); +INSERT INTO installed_versions_migrate(ecu_serial, sha256, name, hashes, length, correlation_id, is_current, is_pending) SELECT installed_versions.ecu_serial, installed_versions.sha256, installed_versions.name, installed_versions.hashes, installed_versions.length, installed_versions.correlation_id, installed_versions.is_current, installed_versions.is_pending FROM installed_versions; + +DROP TABLE installed_versions; +ALTER TABLE installed_versions_migrate RENAME TO installed_versions; + +DELETE FROM version; +INSERT INTO version VALUES(19); + +RELEASE ROLLBACK_MIGRATION; diff --git a/config/sql/schema.sql b/config/sql/schema.sql index 92af5578b3..17aa216e64 100644 --- a/config/sql/schema.sql +++ b/config/sql/schema.sql @@ -1,9 +1,9 @@ CREATE TABLE version(version INTEGER); -INSERT INTO version(rowid,version) VALUES(1,19); +INSERT INTO version(rowid,version) VALUES(1,20); CREATE TABLE device_info(unique_mark INTEGER PRIMARY KEY CHECK (unique_mark = 0), device_id TEXT, is_registered INTEGER NOT NULL DEFAULT 0 CHECK (is_registered IN (0,1))); CREATE TABLE ecu_serials(id INTEGER PRIMARY KEY, serial TEXT UNIQUE, hardware_id TEXT NOT NULL, is_primary INTEGER NOT NULL DEFAULT 0 CHECK (is_primary IN (0,1))); CREATE TABLE misconfigured_ecus(serial TEXT UNIQUE, hardware_id TEXT NOT NULL, state INTEGER NOT NULL CHECK (state IN (0,1))); -CREATE TABLE installed_versions(ecu_serial TEXT NOT NULL, sha256 TEXT NOT NULL, name TEXT NOT NULL, hashes TEXT NOT NULL, length INTEGER NOT NULL DEFAULT 0, correlation_id TEXT NOT NULL DEFAULT '', is_current INTEGER NOT NULL CHECK (is_current IN (0,1)) DEFAULT 0, is_pending INTEGER NOT NULL CHECK (is_pending IN (0,1)) DEFAULT 0, UNIQUE(ecu_serial, sha256, name)); +CREATE TABLE installed_versions(id INTEGER PRIMARY KEY, ecu_serial TEXT NOT NULL, sha256 TEXT NOT NULL, name TEXT NOT NULL, hashes TEXT NOT NULL, length INTEGER NOT NULL DEFAULT 0, correlation_id TEXT NOT NULL DEFAULT '', is_current INTEGER NOT NULL CHECK (is_current IN (0,1)) DEFAULT 0, is_pending INTEGER NOT NULL CHECK (is_pending IN (0,1)) DEFAULT 0, was_installed INTEGER NOT NULL CHECK (was_installed IN (0,1)) DEFAULT 0); CREATE TABLE primary_keys(unique_mark INTEGER PRIMARY KEY CHECK (unique_mark = 0), private TEXT, public TEXT); CREATE TABLE tls_creds(ca_cert BLOB, ca_cert_format TEXT, client_cert BLOB, client_cert_format TEXT, diff --git a/src/libaktualizr/storage/sqlstorage.cc b/src/libaktualizr/storage/sqlstorage.cc index 928ec329ac..515ecf4530 100644 --- a/src/libaktualizr/storage/sqlstorage.cc +++ b/src/libaktualizr/storage/sqlstorage.cc @@ -918,6 +918,8 @@ void SQLStorage::saveInstalledVersion(const std::string& ecu_serial, const Uptan return; } + // either adds a new entry or update the last one's status + // empty serial: use primary std::string ecu_serial_real = ecu_serial; if (ecu_serial_real.empty()) { @@ -929,6 +931,30 @@ void SQLStorage::saveInstalledVersion(const std::string& ecu_serial, const Uptan } } + std::string hashes_encoded = Uptane::Hash::encodeVector(target.hashes()); + + // get the last time this version was installed on this ecu + boost::optional old_id; + bool old_was_installed = false; + { + auto statement = db.prepareStatement( + "SELECT id, sha256, name, was_installed FROM installed_versions WHERE ecu_serial = ? ORDER BY id DESC " + "LIMIT 1;", + ecu_serial_real); + + if (statement.step() == SQLITE_ROW) { + int64_t rid = statement.get_result_col_int(0); + std::string rsha256 = statement.get_result_col_str(1).value_or(""); + std::string rname = statement.get_result_col_str(2).value_or(""); + bool rwasi = statement.get_result_col_int(3) == 1; + + if (rsha256 == target.sha256Hash() && rname == target.filename()) { + old_id = rid; + old_was_installed = rwasi; + } + } + } + if (update_mode == InstalledVersionUpdateMode::kCurrent) { // unset 'current' and 'pending' on all versions for this ecu auto statement = db.prepareStatement( @@ -947,18 +973,33 @@ void SQLStorage::saveInstalledVersion(const std::string& ecu_serial, const Uptan } } - std::string hashes_encoded = Uptane::Hash::encodeVector(target.hashes()); + if (!!old_id) { + auto statement = db.prepareStatement( + "UPDATE installed_versions SET correlation_id = ?, is_current = ?, is_pending = ?, was_installed = ? WHERE id " + "= ?;", + target.correlation_id(), static_cast(update_mode == InstalledVersionUpdateMode::kCurrent), + static_cast(update_mode == InstalledVersionUpdateMode::kPending), + static_cast(update_mode == InstalledVersionUpdateMode::kCurrent || old_was_installed), old_id.value()); - auto statement = - db.prepareStatement( - "INSERT OR REPLACE INTO installed_versions VALUES (?,?,?,?,?,?,?,?);", ecu_serial_real, target.sha256Hash(), - target.filename(), hashes_encoded, static_cast(target.length()), target.correlation_id(), - static_cast(update_mode == InstalledVersionUpdateMode::kCurrent), - static_cast(update_mode == InstalledVersionUpdateMode::kPending)); + if (statement.step() != SQLITE_DONE) { + LOG_ERROR << "Can't set installed_versions: " << db.errmsg(); + return; + } + } else { + auto statement = + db.prepareStatement( + "INSERT INTO installed_versions(ecu_serial, sha256, name, hashes, length, correlation_id, " + "is_current, is_pending, was_installed) VALUES (?,?,?,?,?,?,?,?,?);", + ecu_serial_real, target.sha256Hash(), target.filename(), hashes_encoded, + static_cast(target.length()), target.correlation_id(), + static_cast(update_mode == InstalledVersionUpdateMode::kCurrent), + static_cast(update_mode == InstalledVersionUpdateMode::kPending), + static_cast(update_mode == InstalledVersionUpdateMode::kCurrent)); - if (statement.step() != SQLITE_DONE) { - LOG_ERROR << "Can't set installed_versions: " << db.errmsg(); - return; + if (statement.step() != SQLITE_DONE) { + LOG_ERROR << "Can't set installed_versions: " << db.errmsg(); + return; + } } db.commitTransaction(); diff --git a/src/libaktualizr/storage/sqlstorage_test.cc b/src/libaktualizr/storage/sqlstorage_test.cc index f44ed6989d..e9da3e8de9 100644 --- a/src/libaktualizr/storage/sqlstorage_test.cc +++ b/src/libaktualizr/storage/sqlstorage_test.cc @@ -16,7 +16,7 @@ typedef boost::tokenizer > sql_tokenizer; static std::map parseSchema() { std::map result; std::vector tokens; - enum { STATE_INIT, STATE_CREATE, STATE_INSERT, STATE_TABLE, STATE_NAME }; + enum { STATE_INIT, STATE_CREATE, STATE_INSERT, STATE_TABLE, STATE_NAME, STATE_TRIGGER, STATE_TRIGGER_END }; boost::char_separator sep(" \"\t\r\n", "(),;"); std::string schema(libaktualizr_current_schema); sql_tokenizer tok(schema, sep); @@ -42,10 +42,13 @@ static std::map parseSchema() { } break; case STATE_CREATE: - if (token != "TABLE") { + if (token == "TABLE") { + parsing_state = STATE_TABLE; + } else if (token == "TRIGGER") { + parsing_state = STATE_TRIGGER; + } else { return {}; } - parsing_state = STATE_TABLE; break; case STATE_INSERT: // do not take these into account @@ -55,6 +58,20 @@ static std::map parseSchema() { parsing_state = STATE_INIT; } break; + case STATE_TRIGGER: + // skip these + if (token == "END") { + parsing_state = STATE_TRIGGER_END; + } + break; + case STATE_TRIGGER_END: + // do not take these into account + if (token == ";") { + key.clear(); + value.clear(); + parsing_state = STATE_INIT; + } + break; case STATE_TABLE: if (token == "(" || token == ")" || token == "," || token == ";") { return {}; @@ -346,6 +363,70 @@ TEST(sqlstorage, DbMigration18to19) { FAIL() << "Too many rows"; } } + +TEST(sqlstorage, DbMigration19to20) { + // it must use raw sql primitives because the SQLStorage object does automatic + // migration + the api changes with time + auto tdb = makeDbWithVersion(DbVersion(19)); + SQLite3Guard db(tdb.db_path.c_str()); + + if (db.exec("INSERT INTO ecu_serials(serial,hardware_id,is_primary) VALUES ('primary_ecu', 'primary_hw', 1);", + nullptr, nullptr) != SQLITE_OK) { + FAIL(); + } + + if (db.exec("INSERT INTO installed_versions VALUES ('primary_ecu', 'shav1', 'v1', 'sha256:shav1', 2, 'cor1', 0, 0);", + nullptr, nullptr) != SQLITE_OK) { + FAIL(); + } + if (db.exec("INSERT INTO installed_versions VALUES ('primary_ecu', 'shav2', 'v2', 'sha256:shav2', 3, 'cor2', 1, 0);", + nullptr, nullptr) != SQLITE_OK) { + FAIL(); + } + + // run migration + if (db.exec(libaktualizr_schema_migrations.at(20), nullptr, nullptr) != SQLITE_OK) { + std::cout << db.errmsg() << "\n"; + FAIL() << "Migration 19 to 20 failed"; + } + + // check values + auto statement = db.prepareStatement( + "SELECT ecu_serial, sha256, name, hashes, length, correlation_id, is_current, is_pending, " + "was_installed FROM installed_versions ORDER BY id;"); + if (statement.step() != SQLITE_ROW) { + FAIL() << "installed_versions is empty"; + } + + EXPECT_EQ(statement.get_result_col_str(0).value(), "primary_ecu"); + EXPECT_EQ(statement.get_result_col_str(1).value(), "shav1"); + EXPECT_EQ(statement.get_result_col_str(2).value(), "v1"); + EXPECT_EQ(statement.get_result_col_str(3).value(), "sha256:shav1"); + EXPECT_EQ(statement.get_result_col_int(4), 2); + EXPECT_EQ(statement.get_result_col_str(5).value(), "cor1"); + EXPECT_EQ(statement.get_result_col_int(6), 0); + EXPECT_EQ(statement.get_result_col_int(7), 0); + EXPECT_EQ(statement.get_result_col_int(8), 1); + + if (statement.step() != SQLITE_ROW) { + FAIL() << "installed_versions contains only one element"; + } + + EXPECT_EQ(statement.get_result_col_str(0).value(), "primary_ecu"); + EXPECT_EQ(statement.get_result_col_str(1).value(), "shav2"); + EXPECT_EQ(statement.get_result_col_str(2).value(), "v2"); + EXPECT_EQ(statement.get_result_col_str(3).value(), "sha256:shav2"); + EXPECT_EQ(statement.get_result_col_int(4), 3); + EXPECT_EQ(statement.get_result_col_str(5).value(), "cor2"); + EXPECT_EQ(statement.get_result_col_int(6), 1); + EXPECT_EQ(statement.get_result_col_int(7), 0); + EXPECT_EQ(statement.get_result_col_int(8), 1); + + if (statement.step() != SQLITE_DONE) { + FAIL() << "Too many rows"; + } +} + /** * Check that old metadata is still valid */ From 0ea8674de387e1ce759c561a8bec78f7e64137be Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Tue, 3 Sep 2019 13:52:56 +0200 Subject: [PATCH 5/7] Split loadInstalledVersions into two Most of the time it was used to get current and pending versions. For the other cases, provide loadInstallationLog which also returns the ordered log of installation. Signed-off-by: Laurent Bonnans --- src/aktualizr_info/main.cc | 32 ++-- src/aktualizr_lite/main.cc | 9 +- .../package_manager/androidmanager.cc | 2 +- .../package_manager/debianmanager.cc | 9 +- .../package_manager/ostreemanager.cc | 2 +- .../package_manager/packagemanagerfake.cc | 18 +- src/libaktualizr/primary/aktualizr_test.cc | 173 +++++++----------- src/libaktualizr/primary/sotauptaneclient.cc | 24 ++- src/libaktualizr/storage/invstorage.cc | 2 +- src/libaktualizr/storage/invstorage.h | 15 +- src/libaktualizr/storage/sqlstorage.cc | 155 +++++++++++----- src/libaktualizr/storage/sqlstorage.h | 6 +- src/libaktualizr/storage/sqlstorage_test.cc | 6 +- .../storage/storage_common_test.cc | 98 ++++++---- src/libaktualizr/uptane/uptane_test.cc | 32 ++-- 15 files changed, 320 insertions(+), 263 deletions(-) diff --git a/src/aktualizr_info/main.cc b/src/aktualizr_info/main.cc index 78bf6e63ee..70c67fba3b 100644 --- a/src/aktualizr_info/main.cc +++ b/src/aktualizr_info/main.cc @@ -278,24 +278,22 @@ int main(int argc, char **argv) { std::cout << secondary_number++ << ") serial ID: " << it->first << std::endl; std::cout << " hardware ID: " << it->second << std::endl; - std::vector installed_targets; - size_t current_version = SIZE_MAX; - size_t pending_version = SIZE_MAX; + boost::optional current_version; + boost::optional pending_version; - auto load_installed_version_res = storage->loadInstalledVersions((it->first).ToString(), &installed_targets, - ¤t_version, &pending_version); + auto load_installed_version_res = + storage->loadInstalledVersions((it->first).ToString(), ¤t_version, &pending_version); - if (!load_installed_version_res || - (current_version >= installed_targets.size() && pending_version >= installed_targets.size())) { + if (!load_installed_version_res || (!current_version && !pending_version)) { std::cout << " no details about installed nor pending images\n"; } else { - if (installed_targets.size() > current_version) { - std::cout << " installed image hash: " << installed_targets[current_version].sha256Hash() << "\n"; - std::cout << " installed image filename: " << installed_targets[current_version].filename() << "\n"; + if (!!current_version) { + std::cout << " installed image hash: " << current_version->sha256Hash() << "\n"; + std::cout << " installed image filename: " << current_version->filename() << "\n"; } - if (installed_targets.size() > pending_version) { - std::cout << " pending image hash: " << installed_targets[pending_version].sha256Hash() << "\n"; - std::cout << " pending image filename: " << installed_targets[pending_version].filename() << "\n"; + if (!!pending_version) { + std::cout << " pending image hash: " << pending_version->sha256Hash() << "\n"; + std::cout << " pending image filename: " << pending_version->filename() << "\n"; } } } @@ -326,11 +324,11 @@ int main(int argc, char **argv) { } std::vector installed_versions; - size_t pending = SIZE_MAX; - storage->loadInstalledVersions("", &installed_versions, nullptr, &pending); + boost::optional pending; + storage->loadPrimaryInstalledVersions(nullptr, &pending); - if (pending != SIZE_MAX) { - std::cout << "Pending primary ecu version: " << installed_versions[pending].sha256Hash() << std::endl; + if (!!pending) { + std::cout << "Pending primary ecu version: " << pending->sha256Hash() << std::endl; } } catch (const po::error &o) { std::cout << o.what() << std::endl; diff --git a/src/aktualizr_lite/main.cc b/src/aktualizr_lite/main.cc index 09417e267c..c0fa7506ce 100644 --- a/src/aktualizr_lite/main.cc +++ b/src/aktualizr_lite/main.cc @@ -16,11 +16,10 @@ namespace bpo = boost::program_options; static void finalizeIfNeeded(INvStorage &storage, PackageConfig &config) { - std::vector installed_versions; - size_t pending_index = SIZE_MAX; - storage.loadInstalledVersions("", &installed_versions, nullptr, &pending_index); + boost::optional pending_version; + storage.loadInstalledVersions("", nullptr, &pending_version); - if (pending_index < installed_versions.size()) { + if (!!pending_version) { GObjectUniquePtr sysroot_smart = OstreeManager::LoadSysroot(config.sysroot); OstreeDeployment *booted_deployment = ostree_sysroot_get_booted_deployment(sysroot_smart.get()); if (booted_deployment == nullptr) { @@ -28,7 +27,7 @@ static void finalizeIfNeeded(INvStorage &storage, PackageConfig &config) { } std::string current_hash = ostree_deployment_get_csum(booted_deployment); - const Uptane::Target &target = installed_versions[pending_index]; + const Uptane::Target &target = *pending_version; if (current_hash == target.sha256Hash()) { LOG_INFO << "Marking target install complete for: " << target; storage.saveInstalledVersion("", target, InstalledVersionUpdateMode::kCurrent); diff --git a/src/libaktualizr/package_manager/androidmanager.cc b/src/libaktualizr/package_manager/androidmanager.cc index fe6b5cca0d..6aa364db55 100644 --- a/src/libaktualizr/package_manager/androidmanager.cc +++ b/src/libaktualizr/package_manager/androidmanager.cc @@ -55,7 +55,7 @@ Uptane::Target AndroidManager::getCurrent() const { qi::phrase_parse(getprop_output.crbegin(), getprop_output.crend(), *(xdigit[push_front(boost::phoenix::ref(hash), _1)]), boost::spirit::ascii::cntrl); std::vector installed_versions; - storage_->loadPrimaryInstalledVersions(&installed_versions, nullptr, nullptr); + storage_->loadPrimaryInstallationLog(&installed_versions, false); for (const auto& target : installed_versions) { if (std::equal(hash.cbegin(), hash.cend(), target.sha256Hash().cbegin())) { return target; diff --git a/src/libaktualizr/package_manager/debianmanager.cc b/src/libaktualizr/package_manager/debianmanager.cc index 394bd72c31..c65a567446 100644 --- a/src/libaktualizr/package_manager/debianmanager.cc +++ b/src/libaktualizr/package_manager/debianmanager.cc @@ -31,12 +31,11 @@ data::InstallationResult DebianManager::install(const Uptane::Target &target) co } Uptane::Target DebianManager::getCurrent() const { - std::vector installed_versions; - size_t current_k = SIZE_MAX; - storage_->loadPrimaryInstalledVersions(&installed_versions, ¤t_k, nullptr); + boost::optional current_version; + storage_->loadPrimaryInstalledVersions(¤t_version, nullptr); - if (current_k != SIZE_MAX) { - return installed_versions.at(current_k); + if (!!current_version) { + return *current_version; } return Uptane::Target::Unknown(); diff --git a/src/libaktualizr/package_manager/ostreemanager.cc b/src/libaktualizr/package_manager/ostreemanager.cc index 936401f6b1..db7c820645 100644 --- a/src/libaktualizr/package_manager/ostreemanager.cc +++ b/src/libaktualizr/package_manager/ostreemanager.cc @@ -299,7 +299,7 @@ Uptane::Target OstreeManager::getCurrent() const { std::string current_hash = ostree_deployment_get_csum(booted_deployment); std::vector installed_versions; - storage_->loadPrimaryInstalledVersions(&installed_versions, nullptr, nullptr); + storage_->loadPrimaryInstallationLog(&installed_versions, false); // Version should be in installed versions. Its possible that multiple // targets could have the same sha256Hash. In this case the safest assumption diff --git a/src/libaktualizr/package_manager/packagemanagerfake.cc b/src/libaktualizr/package_manager/packagemanagerfake.cc index e53a950170..b0c108703e 100644 --- a/src/libaktualizr/package_manager/packagemanagerfake.cc +++ b/src/libaktualizr/package_manager/packagemanagerfake.cc @@ -12,12 +12,11 @@ Json::Value PackageManagerFake::getInstalledPackages() const { } Uptane::Target PackageManagerFake::getCurrent() const { - std::vector installed_versions; - size_t current_k = SIZE_MAX; - storage_->loadPrimaryInstalledVersions(&installed_versions, ¤t_k, nullptr); + boost::optional current_version; + storage_->loadPrimaryInstalledVersions(¤t_version, nullptr); - if (current_k != SIZE_MAX) { - return installed_versions.at(current_k); + if (!!current_version) { + return *current_version; } return Uptane::Target::Unknown(); @@ -53,17 +52,16 @@ void PackageManagerFake::completeInstall() const { } data::InstallationResult PackageManagerFake::finalizeInstall(const Uptane::Target& target) const { - std::vector targets; - size_t pending_version = SIZE_MAX; - storage_->loadPrimaryInstalledVersions(&targets, nullptr, &pending_version); + boost::optional pending_version; + storage_->loadPrimaryInstalledVersions(nullptr, &pending_version); - if (pending_version == SIZE_MAX) { + if (!pending_version) { throw std::runtime_error("No pending update, nothing to finalize"); } data::InstallationResult install_res; - if (target.MatchTarget(targets[pending_version])) { + if (target.MatchTarget(*pending_version)) { if (fiu_fail("fake_install_finalization_failure") != 0) { const std::string failure_cause = fault_injection_last_info(); if (failure_cause.empty()) { diff --git a/src/libaktualizr/primary/aktualizr_test.cc b/src/libaktualizr/primary/aktualizr_test.cc index f0b8a44f42..0d9a53b3fc 100644 --- a/src/libaktualizr/primary/aktualizr_test.cc +++ b/src/libaktualizr/primary/aktualizr_test.cc @@ -425,10 +425,9 @@ TEST(Aktualizr, FullWithUpdatesNeedReboot) { // check that a version is here, set to pending - size_t pending_target = SIZE_MAX; - std::vector targets; - storage->loadPrimaryInstalledVersions(&targets, nullptr, &pending_target); - EXPECT_NE(pending_target, SIZE_MAX); + boost::optional pending_target; + storage->loadPrimaryInstalledVersions(nullptr, &pending_target); + EXPECT_TRUE(!!pending_target); } // check that no manifest has been sent after the update application @@ -444,10 +443,9 @@ TEST(Aktualizr, FullWithUpdatesNeedReboot) { aktualizr.Initialize(); // check that everything is still pending - size_t pending_target = SIZE_MAX; - std::vector targets; - storage->loadPrimaryInstalledVersions(&targets, nullptr, &pending_target); - EXPECT_LT(pending_target, targets.size()); + boost::optional pending_target; + storage->loadPrimaryInstalledVersions(nullptr, &pending_target); + EXPECT_TRUE(!!pending_target); result::UpdateCheck update_res = aktualizr.CheckUpdates().get(); EXPECT_EQ(update_res.status, result::UpdateStatus::kError); @@ -470,20 +468,18 @@ TEST(Aktualizr, FullWithUpdatesNeedReboot) { EXPECT_EQ(update_res.status, result::UpdateStatus::kNoUpdatesAvailable); // primary is installed, nothing pending - size_t current_target = SIZE_MAX; - size_t pending_target = SIZE_MAX; - std::vector targets; - storage->loadPrimaryInstalledVersions(&targets, ¤t_target, &pending_target); - EXPECT_LT(current_target, targets.size()); - EXPECT_EQ(pending_target, SIZE_MAX); + boost::optional current_target; + boost::optional pending_target; + storage->loadPrimaryInstalledVersions(¤t_target, &pending_target); + EXPECT_TRUE(!!current_target); + EXPECT_FALSE(!!pending_target); // secondary is installed, nothing pending - size_t sec_current_target = SIZE_MAX; - size_t sec_pending_target = SIZE_MAX; - std::vector sec_targets; - storage->loadInstalledVersions("secondary_ecu_serial", &sec_targets, &sec_current_target, &sec_pending_target); - EXPECT_LT(sec_current_target, sec_targets.size()); - EXPECT_EQ(sec_pending_target, SIZE_MAX); + boost::optional sec_current_target; + boost::optional sec_pending_target; + storage->loadInstalledVersions("secondary_ecu_serial", &sec_current_target, &sec_pending_target); + EXPECT_TRUE(!!sec_current_target); + EXPECT_FALSE(!!sec_pending_target); } // check that the manifest has been sent @@ -585,7 +581,7 @@ class EventHandler { private: std::function)> functor_; - std::vector received_events_ = {}; + std::vector received_events_{}; }; /* @@ -626,17 +622,14 @@ TEST(Aktualizr, FinalizationFailure) { aktualizr.Initialize(); // verify currently installed version - std::vector installed_versions; - size_t current_version{SIZE_MAX}; - size_t pending_version{SIZE_MAX}; - ASSERT_TRUE( - storage->loadInstalledVersions(primary_ecu_id, &installed_versions, ¤t_version, &pending_version)); + boost::optional current_version; + boost::optional pending_version; + ASSERT_TRUE(storage->loadInstalledVersions(primary_ecu_id, ¤t_version, &pending_version)); // for some reason there is no any installed version at initial Aktualizr boot/run // IMHO it should return currently installed version - EXPECT_TRUE(installed_versions.empty()); - EXPECT_EQ(pending_version, SIZE_MAX); - EXPECT_EQ(current_version, SIZE_MAX); + EXPECT_FALSE(!!pending_version); + EXPECT_FALSE(!!current_version); auto aktualizr_cycle_thread = aktualizr.RunForever(); auto aktualizr_cycle_thread_status = aktualizr_cycle_thread.wait_for(std::chrono::seconds(20)); @@ -679,27 +672,21 @@ TEST(Aktualizr, FinalizationFailure) { } } - pending_version = SIZE_MAX; - current_version = SIZE_MAX; - - ASSERT_TRUE( - storage->loadInstalledVersions(primary_ecu_id, &installed_versions, ¤t_version, &pending_version)); - ASSERT_EQ(installed_versions.size(), 1); - EXPECT_TRUE(installed_versions[0].IsValid()); - // if pending_version equals 0 then it means that this is a pending version - EXPECT_EQ(pending_version, 0); - EXPECT_EQ(current_version, SIZE_MAX); - - pending_version = SIZE_MAX; - current_version = SIZE_MAX; - - ASSERT_TRUE( - storage->loadInstalledVersions(secondary_ecu_id, &installed_versions, ¤t_version, &pending_version)); - ASSERT_EQ(installed_versions.size(), 1); - EXPECT_TRUE(installed_versions[0].IsValid()); - EXPECT_EQ(pending_version, SIZE_MAX); - // if current_version equals 0 then it means that this is a current version - EXPECT_EQ(current_version, 0); + pending_version = boost::none; + current_version = boost::none; + + ASSERT_TRUE(storage->loadInstalledVersions(primary_ecu_id, ¤t_version, &pending_version)); + EXPECT_FALSE(!!current_version); + EXPECT_TRUE(!!pending_version); + EXPECT_TRUE(pending_version->IsValid()); + + pending_version = boost::none; + current_version = boost::none; + + ASSERT_TRUE(storage->loadInstalledVersions(secondary_ecu_id, ¤t_version, &pending_version)); + EXPECT_TRUE(!!current_version); + EXPECT_TRUE(current_version->IsValid()); + EXPECT_FALSE(!!pending_version); } { @@ -733,26 +720,19 @@ TEST(Aktualizr, FinalizationFailure) { EXPECT_FALSE(storage->loadEcuInstallationResults(&ecu_installation_res)); // verify currently installed version - std::vector installed_versions; - size_t current_version{SIZE_MAX}; - size_t pending_version{SIZE_MAX}; - - ASSERT_TRUE( - storage->loadInstalledVersions(primary_ecu_id, &installed_versions, ¤t_version, &pending_version)); - ASSERT_EQ(installed_versions.size(), 1); - EXPECT_TRUE(installed_versions[0].IsValid()); - EXPECT_EQ(pending_version, SIZE_MAX); - EXPECT_EQ(current_version, SIZE_MAX); - - current_version = SIZE_MAX; - pending_version = SIZE_MAX; - - ASSERT_TRUE( - storage->loadInstalledVersions(secondary_ecu_id, &installed_versions, ¤t_version, &pending_version)); - ASSERT_EQ(installed_versions.size(), 1); - EXPECT_TRUE(installed_versions[0].IsValid()); - EXPECT_EQ(pending_version, SIZE_MAX); - EXPECT_EQ(current_version, 0); + boost::optional current_version; + boost::optional pending_version; + + ASSERT_TRUE(storage->loadInstalledVersions(primary_ecu_id, ¤t_version, &pending_version)); + EXPECT_FALSE(!!current_version); + EXPECT_FALSE(!!pending_version); + + current_version = boost::none; + pending_version = boost::none; + + ASSERT_TRUE(storage->loadInstalledVersions(secondary_ecu_id, ¤t_version, &pending_version)); + EXPECT_TRUE(!!current_version); + EXPECT_FALSE(!!pending_version); } } @@ -793,16 +773,13 @@ TEST(Aktualizr, InstallationFailure) { aktualizr.Initialize(); // verify currently installed version - std::vector installed_versions; - size_t current_version{SIZE_MAX}; - size_t pending_version{SIZE_MAX}; + boost::optional current_version; + boost::optional pending_version; - ASSERT_TRUE( - storage->loadInstalledVersions(primary_ecu_id, &installed_versions, ¤t_version, &pending_version)); + ASSERT_TRUE(storage->loadInstalledVersions(primary_ecu_id, ¤t_version, &pending_version)); - EXPECT_TRUE(installed_versions.empty()); - EXPECT_EQ(pending_version, SIZE_MAX); - EXPECT_EQ(current_version, SIZE_MAX); + EXPECT_FALSE(!!pending_version); + EXPECT_FALSE(!!current_version); aktualizr.UptaneCycle(); aktualizr.uptane_client()->completeInstall(); @@ -825,14 +802,12 @@ TEST(Aktualizr, InstallationFailure) { EXPECT_FALSE(storage->loadEcuInstallationResults(&ecu_installation_res)); EXPECT_EQ(ecu_installation_res.size(), 0); - ASSERT_TRUE( - storage->loadInstalledVersions(primary_ecu_id, &installed_versions, ¤t_version, &pending_version)); + ASSERT_TRUE(storage->loadInstalledVersions(primary_ecu_id, ¤t_version, &pending_version)); // it says that no any installed version found, // which is, on one hand is correct since installation of the found update failed hence nothing was installed, // on the other hand some version should have been installed prior to the failed update - EXPECT_EQ(installed_versions.size(), 0); - EXPECT_EQ(current_version, SIZE_MAX); - EXPECT_EQ(pending_version, SIZE_MAX); + EXPECT_FALSE(!!current_version); + EXPECT_FALSE(!!pending_version); fiu_disable("fake_package_install"); } @@ -854,16 +829,13 @@ TEST(Aktualizr, InstallationFailure) { aktualizr.Initialize(); // verify currently installed version - std::vector installed_versions; - size_t current_version{SIZE_MAX}; - size_t pending_version{SIZE_MAX}; + boost::optional current_version; + boost::optional pending_version; - ASSERT_TRUE( - storage->loadInstalledVersions(primary_ecu_id, &installed_versions, ¤t_version, &pending_version)); + ASSERT_TRUE(storage->loadInstalledVersions(primary_ecu_id, ¤t_version, &pending_version)); - EXPECT_TRUE(installed_versions.empty()); - EXPECT_EQ(pending_version, SIZE_MAX); - EXPECT_EQ(current_version, SIZE_MAX); + EXPECT_FALSE(!!pending_version); + EXPECT_FALSE(!!current_version); aktualizr.UptaneCycle(); aktualizr.uptane_client()->completeInstall(); @@ -892,14 +864,12 @@ TEST(Aktualizr, InstallationFailure) { EXPECT_FALSE(storage->loadEcuInstallationResults(&ecu_installation_res)); EXPECT_EQ(ecu_installation_res.size(), 0); - ASSERT_TRUE( - storage->loadInstalledVersions(primary_ecu_id, &installed_versions, ¤t_version, &pending_version)); + ASSERT_TRUE(storage->loadInstalledVersions(primary_ecu_id, ¤t_version, &pending_version)); // it says that no any installed version found, // which is, on one hand is correct since installation of the found update failed hence nothing was installed, // on the other hand some version should have been installed prior to the failed update - EXPECT_EQ(installed_versions.size(), 0); - EXPECT_EQ(current_version, SIZE_MAX); - EXPECT_EQ(pending_version, SIZE_MAX); + EXPECT_FALSE(!!current_version); + EXPECT_FALSE(!!pending_version); fault_injection_disable("fake_package_install"); fault_injection_disable(sec_fault_name.c_str()); @@ -952,12 +922,11 @@ TEST(Aktualizr, AutoRebootAfterUpdate) { EXPECT_EQ(update_res.status, result::UpdateStatus::kNoUpdatesAvailable); // primary is installed, nothing pending - size_t current_target = SIZE_MAX; - size_t pending_target = SIZE_MAX; - std::vector targets; - storage->loadPrimaryInstalledVersions(&targets, ¤t_target, &pending_target); - EXPECT_LT(current_target, targets.size()); - EXPECT_EQ(pending_target, SIZE_MAX); + boost::optional current_target; + boost::optional pending_target; + storage->loadPrimaryInstalledVersions(¤t_target, &pending_target); + EXPECT_TRUE(!!current_target); + EXPECT_FALSE(!!pending_target); EXPECT_EQ(http->manifest_sends, 4); } } diff --git a/src/libaktualizr/primary/sotauptaneclient.cc b/src/libaktualizr/primary/sotauptaneclient.cc index 381ccb4a19..83f58e9827 100644 --- a/src/libaktualizr/primary/sotauptaneclient.cc +++ b/src/libaktualizr/primary/sotauptaneclient.cc @@ -124,23 +124,22 @@ void SotaUptaneClient::finalizeAfterReboot() { const Uptane::EcuSerial &ecu_serial = uptane_manifest.getPrimaryEcuSerial(); std::vector installed_versions; - size_t pending_index = SIZE_MAX; - storage->loadInstalledVersions(ecu_serial.ToString(), &installed_versions, nullptr, &pending_index); + boost::optional pending_target; + storage->loadInstalledVersions(ecu_serial.ToString(), nullptr, &pending_target); - if (pending_index < installed_versions.size()) { - const Uptane::Target &target = installed_versions[pending_index]; - const std::string correlation_id = target.correlation_id(); + if (!!pending_target) { + const std::string correlation_id = pending_target->correlation_id(); - data::InstallationResult install_res = package_manager_->finalizeInstall(target); + data::InstallationResult install_res = package_manager_->finalizeInstall(*pending_target); storage->saveEcuInstallationResult(ecu_serial, install_res); if (install_res.success) { - storage->saveInstalledVersion(ecu_serial.ToString(), target, InstalledVersionUpdateMode::kCurrent); + storage->saveInstalledVersion(ecu_serial.ToString(), *pending_target, InstalledVersionUpdateMode::kCurrent); report_queue->enqueue(std_::make_unique(ecu_serial, correlation_id, true)); } else { // finalize failed // unset pending flag so that the rest of the uptane process can // go forward again - storage->saveInstalledVersion(ecu_serial.ToString(), target, InstalledVersionUpdateMode::kNone); + storage->saveInstalledVersion(ecu_serial.ToString(), *pending_target, InstalledVersionUpdateMode::kNone); report_queue->enqueue(std_::make_unique(ecu_serial, correlation_id, false)); director_repo.dropTargets(*storage); // fix for OTA-2587, listen to backend again after end of install } @@ -420,17 +419,16 @@ bool SotaUptaneClient::getNewTargets(std::vector *new_targets, u return false; } - std::vector installed_versions; - size_t current_version = SIZE_MAX; - if (!storage->loadInstalledVersions(ecu_serial.ToString(), &installed_versions, ¤t_version, nullptr)) { + boost::optional current_version; + if (!storage->loadInstalledVersions(ecu_serial.ToString(), ¤t_version, nullptr)) { LOG_WARNING << "Could not load currently installed version for ECU ID: " << ecu_serial.ToString(); break; } - if (current_version > installed_versions.size()) { + if (!current_version) { LOG_WARNING << "Current version for ECU ID: " << ecu_serial.ToString() << " is unknown"; is_new = true; - } else if (installed_versions[current_version].filename() != target.filename()) { + } else if (current_version->filename() != target.filename()) { is_new = true; } diff --git a/src/libaktualizr/storage/invstorage.cc b/src/libaktualizr/storage/invstorage.cc index 94703e3aff..a4c592852d 100644 --- a/src/libaktualizr/storage/invstorage.cc +++ b/src/libaktualizr/storage/invstorage.cc @@ -70,7 +70,7 @@ void INvStorage::importPrimaryKeys(const boost::filesystem::path& base_path, con void INvStorage::importInstalledVersions(const boost::filesystem::path& base_path) { std::vector installed_versions; const boost::filesystem::path file_path = BasedPath("installed_versions").get(base_path); - loadPrimaryInstalledVersions(&installed_versions, nullptr, nullptr); + loadPrimaryInstallationLog(&installed_versions, false); if (!installed_versions.empty()) { return; } diff --git a/src/libaktualizr/storage/invstorage.h b/src/libaktualizr/storage/invstorage.h index 59305b5125..9eee761fb3 100644 --- a/src/libaktualizr/storage/invstorage.h +++ b/src/libaktualizr/storage/invstorage.h @@ -164,8 +164,10 @@ class INvStorage { virtual void saveInstalledVersion(const std::string& ecu_serial, const Uptane::Target& target, InstalledVersionUpdateMode update_mode) = 0; - virtual bool loadInstalledVersions(const std::string& ecu_serial, std::vector* installed_versions, - size_t* current_version, size_t* pending_version) = 0; + virtual bool loadInstalledVersions(const std::string& ecu_serial, boost::optional* current_version, + boost::optional* pending_version) = 0; + virtual bool loadInstallationLog(const std::string& ecu_serial, std::vector* log, + bool only_installed) = 0; virtual bool hasPendingInstall() = 0; virtual void clearInstalledVersions() = 0; @@ -199,13 +201,16 @@ class INvStorage { // Not purely virtual void importData(const ImportConfig& import_config); - bool loadPrimaryInstalledVersions(std::vector* installed_versions, size_t* current_version, - size_t* pending_version) { - return loadInstalledVersions("", installed_versions, current_version, pending_version); + bool loadPrimaryInstalledVersions(boost::optional* current_version, + boost::optional* pending_version) { + return loadInstalledVersions("", current_version, pending_version); } void savePrimaryInstalledVersion(const Uptane::Target& target, InstalledVersionUpdateMode update_mode) { return saveInstalledVersion("", target, update_mode); } + bool loadPrimaryInstallationLog(std::vector* log, bool only_installed) { + return loadInstallationLog("", log, only_installed); + } private: void importSimple(const boost::filesystem::path& base_path, store_data_t store_func, load_data_t load_func, diff --git a/src/libaktualizr/storage/sqlstorage.cc b/src/libaktualizr/storage/sqlstorage.cc index 515ecf4530..171467ca72 100644 --- a/src/libaktualizr/storage/sqlstorage.cc +++ b/src/libaktualizr/storage/sqlstorage.cc @@ -1005,49 +1005,59 @@ void SQLStorage::saveInstalledVersion(const std::string& ecu_serial, const Uptan db.commitTransaction(); } -bool SQLStorage::loadInstalledVersions(const std::string& ecu_serial, std::vector* installed_versions, - size_t* current_version, size_t* pending_version) { - SQLite3Guard db = dbConnection(); - - // empty serial: use primary - std::string ecu_serial_real = ecu_serial; - if (ecu_serial_real.empty()) { +static void loadEcuMap(SQLite3Guard& db, std::string& ecu_serial, Uptane::EcuMap& ecu_map) { + if (ecu_serial.empty()) { auto statement = db.prepareStatement("SELECT serial FROM ecu_serials WHERE is_primary = 1;"); if (statement.step() == SQLITE_ROW) { - ecu_serial_real = statement.get_result_col_str(0).value(); + ecu_serial = statement.get_result_col_str(0).value(); } else { LOG_WARNING << "Could not find primary ecu serial, defaulting to empty serial: " << db.errmsg(); } } + { + auto statement = + db.prepareStatement("SELECT hardware_id FROM ecu_serials WHERE serial = ?;", ecu_serial); + if (statement.step() == SQLITE_ROW) { + ecu_map.insert( + {Uptane::EcuSerial(ecu_serial), Uptane::HardwareIdentifier(statement.get_result_col_str(0).value())}); + } else { + LOG_WARNING << "Could not find hardware_id for serial " << ecu_serial << ": " << db.errmsg(); + } + } +} + +bool SQLStorage::loadInstallationLog(const std::string& ecu_serial, std::vector* log, + bool only_installed) { + SQLite3Guard db = dbConnection(); + + std::string ecu_serial_real = ecu_serial; Uptane::EcuMap ecu_map; - auto statement = - db.prepareStatement("SELECT hardware_id FROM ecu_serials WHERE serial = ?;", ecu_serial_real); - if (statement.step() == SQLITE_ROW) { - ecu_map.insert( - {Uptane::EcuSerial(ecu_serial_real), Uptane::HardwareIdentifier(statement.get_result_col_str(0).value())}); - } else { - LOG_WARNING << "Could not find hardware_id for serial " << ecu_serial_real << ": " << db.errmsg(); + loadEcuMap(db, ecu_serial_real, ecu_map); + + std::string query = + "SELECT id, sha256, name, hashes, length, correlation_id FROM installed_versions WHERE " + "ecu_serial = ? ORDER BY id;"; + if (only_installed) { + query = + "SELECT id, sha256, name, hashes, length, correlation_id FROM installed_versions WHERE " + "ecu_serial = ? AND was_installed = 1 ORDER BY id;"; } - size_t current_index = SIZE_MAX; - size_t pending_index = SIZE_MAX; - statement = db.prepareStatement( - "SELECT sha256, name, hashes, length, correlation_id, is_current, is_pending FROM installed_versions WHERE " - "ecu_serial = ?;", - ecu_serial_real); + auto statement = db.prepareStatement(query, ecu_serial_real); int statement_state; - std::vector new_installed_versions; + std::vector new_log; + std::map ids_map; + size_t k = 0; while ((statement_state = statement.step()) == SQLITE_ROW) { try { - auto sha256 = statement.get_result_col_str(0).value(); - auto filename = statement.get_result_col_str(1).value(); - auto hashes_str = statement.get_result_col_str(2).value(); - auto length = statement.get_result_col_int(3); - auto correlation_id = statement.get_result_col_str(4).value(); - auto is_current = statement.get_result_col_int(5) != 0; - auto is_pending = statement.get_result_col_int(6) != 0; + auto id = statement.get_result_col_int(0); + auto sha256 = statement.get_result_col_str(1).value(); + auto filename = statement.get_result_col_str(2).value(); + auto hashes_str = statement.get_result_col_str(3).value(); + auto length = statement.get_result_col_int(4); + auto correlation_id = statement.get_result_col_str(5).value(); // note: sha256 should always be present and is used to uniquely identify // a version. It should normally be part of the hash list as well. @@ -1060,13 +1070,10 @@ bool SQLStorage::loadInstalledVersions(const std::string& ecu_serial, std::vecto hashes.emplace_back(Uptane::Hash::Type::kSha256, sha256); } - new_installed_versions.emplace_back(filename, ecu_map, hashes, length, correlation_id); - if (is_current) { - current_index = new_installed_versions.size() - 1; - } - if (is_pending) { - pending_index = new_installed_versions.size() - 1; - } + new_log.emplace_back(filename, ecu_map, hashes, static_cast(length), correlation_id); + + ids_map[id] = k; + k++; } catch (const boost::bad_optional_access&) { LOG_ERROR << "Incompleted installed version, keeping old one"; return false; @@ -1078,16 +1085,80 @@ bool SQLStorage::loadInstalledVersions(const std::string& ecu_serial, std::vecto return false; } - if (current_version != nullptr && current_index != SIZE_MAX) { - *current_version = current_index; + if (log == nullptr) { + return true; } - if (pending_version != nullptr && pending_index != SIZE_MAX) { - *pending_version = pending_index; + *log = std::move(new_log); + + return true; +} + +bool SQLStorage::loadInstalledVersions(const std::string& ecu_serial, boost::optional* current_version, + boost::optional* pending_version) { + SQLite3Guard db = dbConnection(); + + std::string ecu_serial_real = ecu_serial; + Uptane::EcuMap ecu_map; + loadEcuMap(db, ecu_serial_real, ecu_map); + + auto read_target = [&ecu_map](SQLiteStatement& statement) -> Uptane::Target { + auto sha256 = statement.get_result_col_str(0).value(); + auto filename = statement.get_result_col_str(1).value(); + auto hashes_str = statement.get_result_col_str(2).value(); + auto length = statement.get_result_col_int(3); + auto correlation_id = statement.get_result_col_str(4).value(); + + // note: sha256 should always be present and is used to uniquely identify + // a version. It should normally be part of the hash list as well. + std::vector hashes = Uptane::Hash::decodeVector(hashes_str); + + auto find_sha256 = std::find_if(hashes.cbegin(), hashes.cend(), + [](const Uptane::Hash& h) { return h.type() == Uptane::Hash::Type::kSha256; }); + if (find_sha256 == hashes.cend()) { + LOG_WARNING << "No sha256 in hashes list"; + hashes.emplace_back(Uptane::Hash::Type::kSha256, sha256); + } + + return Uptane::Target(filename, ecu_map, hashes, static_cast(length), correlation_id); + }; + + if (current_version != nullptr) { + auto statement = db.prepareStatement( + "SELECT sha256, name, hashes, length, correlation_id FROM installed_versions WHERE " + "ecu_serial = ? AND is_current = 1 LIMIT 1;", + ecu_serial_real); + + if (statement.step() == SQLITE_ROW) { + try { + *current_version = read_target(statement); + } catch (const boost::bad_optional_access&) { + LOG_ERROR << "Could not read current installed version"; + return false; + } + } else { + LOG_TRACE << "Cannot get current installed version: " << db.errmsg(); + *current_version = boost::none; + } } - if (installed_versions != nullptr) { - *installed_versions = std::move(new_installed_versions); + if (pending_version != nullptr) { + auto statement = db.prepareStatement( + "SELECT sha256, name, hashes, length, correlation_id FROM installed_versions WHERE " + "ecu_serial = ? AND is_pending = 1 LIMIT 1;", + ecu_serial_real); + + if (statement.step() == SQLITE_ROW) { + try { + *pending_version = read_target(statement); + } catch (const boost::bad_optional_access&) { + LOG_ERROR << "Could not read pending installed version"; + return false; + } + } else { + LOG_TRACE << "Cannot get pending installed version: " << db.errmsg(); + *pending_version = boost::none; + } } return true; diff --git a/src/libaktualizr/storage/sqlstorage.h b/src/libaktualizr/storage/sqlstorage.h index 83f51100dd..2b7bfe9082 100644 --- a/src/libaktualizr/storage/sqlstorage.h +++ b/src/libaktualizr/storage/sqlstorage.h @@ -66,8 +66,10 @@ class SQLStorage : public SQLStorageBase, public INvStorage { void clearNeedReboot() override; void saveInstalledVersion(const std::string& ecu_serial, const Uptane::Target& target, InstalledVersionUpdateMode update_mode) override; - bool loadInstalledVersions(const std::string& ecu_serial, std::vector* installed_versions, - size_t* current_version, size_t* pending_version) override; + bool loadInstalledVersions(const std::string& ecu_serial, boost::optional* current_version, + boost::optional* pending_version) override; + bool loadInstallationLog(const std::string& ecu_serial, std::vector* log, + bool only_installed) override; bool hasPendingInstall() override; void clearInstalledVersions() override; diff --git a/src/libaktualizr/storage/sqlstorage_test.cc b/src/libaktualizr/storage/sqlstorage_test.cc index e9da3e8de9..57b0a294ac 100644 --- a/src/libaktualizr/storage/sqlstorage_test.cc +++ b/src/libaktualizr/storage/sqlstorage_test.cc @@ -496,9 +496,9 @@ TEST(sqlstorage, migrate_from_fs) { EXPECT_TRUE(storage->loadEcuRegistered()); - std::vector installed; - storage->loadPrimaryInstalledVersions(&installed, nullptr, nullptr); - EXPECT_NE(installed.size(), 0); + boost::optional installed; + storage->loadPrimaryInstalledVersions(&installed, nullptr); + EXPECT_TRUE(!!installed); } // note: installation result is not migrated anymore diff --git a/src/libaktualizr/storage/storage_common_test.cc b/src/libaktualizr/storage/storage_common_test.cc index 0735fccae8..2095387ec5 100644 --- a/src/libaktualizr/storage/storage_common_test.cc +++ b/src/libaktualizr/storage/storage_common_test.cc @@ -302,6 +302,12 @@ TEST(storage, load_store_installed_versions) { Uptane::EcuMap primary_ecu{{Uptane::EcuSerial("primary"), Uptane::HardwareIdentifier("primary_hw")}}; Uptane::Target t1{"update.bin", primary_ecu, hashes, 1, "corrid"}; storage->savePrimaryInstalledVersion(t1, InstalledVersionUpdateMode::kCurrent); + { + std::vector log; + storage->loadPrimaryInstallationLog(&log, true); + EXPECT_EQ(log.size(), 1); + EXPECT_EQ(log[0].filename(), "update.bin"); + } EcuSerials serials{{Uptane::EcuSerial("primary"), Uptane::HardwareIdentifier("primary_hw")}, {Uptane::EcuSerial("secondary_1"), Uptane::HardwareIdentifier("secondary_hw")}, @@ -309,16 +315,15 @@ TEST(storage, load_store_installed_versions) { storage->storeEcuSerials(serials); { - std::vector installed_versions; - size_t current = SIZE_MAX; - EXPECT_TRUE(storage->loadInstalledVersions("primary", &installed_versions, ¤t, nullptr)); - EXPECT_EQ(installed_versions.size(), 1); - EXPECT_EQ(installed_versions.at(current).filename(), "update.bin"); - EXPECT_EQ(installed_versions.at(current).sha256Hash(), "2561"); - EXPECT_EQ(installed_versions.at(current).hashes(), hashes); - EXPECT_EQ(installed_versions.at(current).ecus(), primary_ecu); - EXPECT_EQ(installed_versions.at(current).correlation_id(), "corrid"); - EXPECT_EQ(installed_versions.at(current).length(), 1); + boost::optional current; + EXPECT_TRUE(storage->loadInstalledVersions("primary", ¤t, nullptr)); + EXPECT_TRUE(!!current); + EXPECT_EQ(current->filename(), "update.bin"); + EXPECT_EQ(current->sha256Hash(), "2561"); + EXPECT_EQ(current->hashes(), hashes); + EXPECT_EQ(current->ecus(), primary_ecu); + EXPECT_EQ(current->correlation_id(), "corrid"); + EXPECT_EQ(current->length(), 1); } // Set t2 as a pending version @@ -326,11 +331,10 @@ TEST(storage, load_store_installed_versions) { storage->savePrimaryInstalledVersion(t2, InstalledVersionUpdateMode::kPending); { - std::vector installed_versions; - size_t pending = SIZE_MAX; - EXPECT_TRUE(storage->loadInstalledVersions("primary", &installed_versions, nullptr, &pending)); - EXPECT_EQ(installed_versions.size(), 2); - EXPECT_EQ(installed_versions.at(pending).filename(), "update2.bin"); + boost::optional pending; + EXPECT_TRUE(storage->loadInstalledVersions("primary", nullptr, &pending)); + EXPECT_TRUE(!!pending); + EXPECT_EQ(pending->filename(), "update2.bin"); } // Set t3 as the new pending @@ -338,25 +342,36 @@ TEST(storage, load_store_installed_versions) { storage->savePrimaryInstalledVersion(t3, InstalledVersionUpdateMode::kPending); { - std::vector installed_versions; - size_t pending = SIZE_MAX; - EXPECT_TRUE(storage->loadInstalledVersions("primary", &installed_versions, nullptr, &pending)); - EXPECT_EQ(installed_versions.size(), 3); - EXPECT_EQ(installed_versions.at(pending).filename(), "update3.bin"); + boost::optional pending; + EXPECT_TRUE(storage->loadInstalledVersions("primary", nullptr, &pending)); + EXPECT_TRUE(!!pending); + EXPECT_EQ(pending->filename(), "update3.bin"); } // Set t3 as current: should replace the pending flag but not create a new // version storage->savePrimaryInstalledVersion(t3, InstalledVersionUpdateMode::kCurrent); + { + boost::optional current; + boost::optional pending; + EXPECT_TRUE(storage->loadInstalledVersions("primary", ¤t, &pending)); + EXPECT_TRUE(!!current); + EXPECT_EQ(current->filename(), "update3.bin"); + EXPECT_FALSE(!!pending); + + std::vector log; + storage->loadInstallationLog("primary", &log, true); + EXPECT_EQ(log.size(), 2); + EXPECT_EQ(log.back().filename(), "update3.bin"); + } + // Set t1 as current: the log should have grown even though we rolled back { - std::vector installed_versions; - size_t current = SIZE_MAX; - size_t pending = SIZE_MAX; - EXPECT_TRUE(storage->loadInstalledVersions("primary", &installed_versions, ¤t, &pending)); - EXPECT_EQ(installed_versions.size(), 3); - EXPECT_EQ(installed_versions.at(current).filename(), "update3.bin"); - EXPECT_EQ(pending, SIZE_MAX); + storage->savePrimaryInstalledVersion(t1, InstalledVersionUpdateMode::kCurrent); + std::vector log; + storage->loadInstallationLog("primary", &log, true); + EXPECT_EQ(log.size(), 3); + EXPECT_EQ(log.back().filename(), "update.bin"); } // Set t2 as the new pending and t3 as current afterwards: the pending flag @@ -365,13 +380,17 @@ TEST(storage, load_store_installed_versions) { storage->savePrimaryInstalledVersion(t3, InstalledVersionUpdateMode::kCurrent); { - std::vector installed_versions; - size_t current = SIZE_MAX; - size_t pending = SIZE_MAX; - EXPECT_TRUE(storage->loadInstalledVersions("primary", &installed_versions, ¤t, &pending)); - EXPECT_EQ(installed_versions.size(), 3); - EXPECT_EQ(installed_versions.at(current).filename(), "update3.bin"); - EXPECT_EQ(pending, SIZE_MAX); + boost::optional current; + boost::optional pending; + EXPECT_TRUE(storage->loadInstalledVersions("primary", ¤t, &pending)); + EXPECT_TRUE(!!current); + EXPECT_EQ(current->filename(), "update3.bin"); + EXPECT_FALSE(!!pending); + + std::vector log; + storage->loadInstallationLog("primary", &log, true); + EXPECT_EQ(log.size(), 4); + EXPECT_EQ(log.back().filename(), "update3.bin"); } // Add a secondary installed version @@ -380,12 +399,13 @@ TEST(storage, load_store_installed_versions) { storage->saveInstalledVersion("secondary_1", tsec, InstalledVersionUpdateMode::kCurrent); { - std::vector installed_versions; - EXPECT_TRUE(storage->loadInstalledVersions("primary", &installed_versions, nullptr, nullptr)); - EXPECT_EQ(installed_versions.size(), 3); + EXPECT_TRUE(storage->loadInstalledVersions("primary", nullptr, nullptr)); + EXPECT_TRUE(storage->loadInstalledVersions("secondary_1", nullptr, nullptr)); - EXPECT_TRUE(storage->loadInstalledVersions("secondary_1", &installed_versions, nullptr, nullptr)); - EXPECT_EQ(installed_versions.size(), 1); + std::vector log; + storage->loadInstallationLog("secondary_1", &log, true); + EXPECT_EQ(log.size(), 1); + EXPECT_EQ(log.back().filename(), "secondary.bin"); } } diff --git a/src/libaktualizr/uptane/uptane_test.cc b/src/libaktualizr/uptane/uptane_test.cc index 4299b87e6f..ba555fe89f 100644 --- a/src/libaktualizr/uptane/uptane_test.cc +++ b/src/libaktualizr/uptane/uptane_test.cc @@ -1071,7 +1071,7 @@ TEST(Uptane, FsToSqlFull) { bool sql_ecu_registered = sql_storage->loadEcuRegistered(); std::vector sql_installed_versions; - sql_storage->loadPrimaryInstalledVersions(&sql_installed_versions, nullptr, nullptr); + sql_storage->loadPrimaryInstallationLog(&sql_installed_versions, true); std::string sql_director_root; std::string sql_director_targets; @@ -1122,10 +1122,10 @@ TEST(Uptane, InstalledVersionImport) { auto storage = INvStorage::newStorage(config.storage); storage->importData(config.import); - std::vector installed_versions; - storage->loadPrimaryInstalledVersions(&installed_versions, nullptr, nullptr); - EXPECT_EQ(installed_versions.at(0).filename(), - "master-863de625f305413dc3be306afab7c3f39d8713045cfff812b3af83f9722851f0"); + boost::optional current_version; + storage->loadPrimaryInstalledVersions(¤t_version, nullptr); + EXPECT_TRUE(!!current_version); + EXPECT_EQ(current_version->filename(), "master-863de625f305413dc3be306afab7c3f39d8713045cfff812b3af83f9722851f0"); // check that data is not re-imported later: store new data, reload a new // storage with import and see that the new data is still there @@ -1138,10 +1138,10 @@ TEST(Uptane, InstalledVersionImport) { auto new_storage = INvStorage::newStorage(config.storage); new_storage->importData(config.import); - size_t current_index = SIZE_MAX; - new_storage->loadPrimaryInstalledVersions(&installed_versions, ¤t_index, nullptr); - EXPECT_LT(current_index, installed_versions.size()); - EXPECT_EQ(installed_versions.at(current_index).filename(), "filename"); + current_version = boost::none; + new_storage->loadPrimaryInstalledVersions(¤t_version, nullptr); + EXPECT_TRUE(!!current_version); + EXPECT_EQ(current_version->filename(), "filename"); } /* Store a list of installed package versions. */ @@ -1160,15 +1160,13 @@ TEST(Uptane, SaveAndLoadVersion) { Uptane::Target t("target_name", target_json); storage->savePrimaryInstalledVersion(t, InstalledVersionUpdateMode::kCurrent); - std::vector installed_versions; - storage->loadPrimaryInstalledVersions(&installed_versions, nullptr, nullptr); + boost::optional current_version; + storage->loadPrimaryInstalledVersions(¤t_version, nullptr); - auto f = std::find_if(installed_versions.begin(), installed_versions.end(), - [](const Uptane::Target &t_) { return t_.filename() == "target_name"; }); - EXPECT_NE(f, installed_versions.end()); - EXPECT_EQ(f->sha256Hash(), "a0fb2e119cf812f1aa9e993d01f5f07cb41679096cb4492f1265bff5ac901d0d"); - EXPECT_EQ(f->length(), 123); - EXPECT_TRUE(f->MatchTarget(t)); + EXPECT_TRUE(!!current_version); + EXPECT_EQ(current_version->sha256Hash(), "a0fb2e119cf812f1aa9e993d01f5f07cb41679096cb4492f1265bff5ac901d0d"); + EXPECT_EQ(current_version->length(), 123); + EXPECT_TRUE(current_version->MatchTarget(t)); } class HttpFakeUnstable : public HttpFake { From a2a8a40744d4217e5a83efae1e417fda10824f53 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Mon, 19 Aug 2019 17:20:01 +0200 Subject: [PATCH 6/7] Add a simple autoclean feature for aktualizr daemon Put it in a separate source so it can be more easily tested and won't pollute aktualizr_primary's main too much. Only deletes until the second last version. Signed-off-by: Laurent Bonnans --- src/aktualizr_primary/main.cc | 4 + src/libaktualizr/primary/CMakeLists.txt | 5 +- src/libaktualizr/primary/aktualizr.cc | 22 ++++ src/libaktualizr/primary/aktualizr.h | 13 +- src/libaktualizr/primary/aktualizr_helpers.cc | 36 ++++++ src/libaktualizr/primary/aktualizr_helpers.h | 11 ++ src/libaktualizr/primary/aktualizr_test.cc | 111 ++++++++++++++++++ 7 files changed, 199 insertions(+), 3 deletions(-) create mode 100644 src/libaktualizr/primary/aktualizr_helpers.cc create mode 100644 src/libaktualizr/primary/aktualizr_helpers.h diff --git a/src/aktualizr_primary/main.cc b/src/aktualizr_primary/main.cc index fa065f9890..f72d79afa6 100644 --- a/src/aktualizr_primary/main.cc +++ b/src/aktualizr_primary/main.cc @@ -8,6 +8,7 @@ #include "config/config.h" #include "logging/logging.h" #include "primary/aktualizr.h" +#include "primary/aktualizr_helpers.h" #include "secondary.h" #include "utilities/aktualizr_version.h" #include "utilities/utils.h" @@ -157,6 +158,9 @@ int main(int argc, char *argv[]) { aktualizr.SendDeviceData().get(); aktualizr.UptaneCycle(); } else { + boost::signals2::connection ac_conn = + aktualizr.SetSignalHandler(std::bind(targets_autoclean_cb, std::ref(aktualizr), std::placeholders::_1)); + aktualizr.RunForever().get(); } r = EXIT_SUCCESS; diff --git a/src/libaktualizr/primary/CMakeLists.txt b/src/libaktualizr/primary/CMakeLists.txt index 25a62908ea..1a972b9c53 100644 --- a/src/libaktualizr/primary/CMakeLists.txt +++ b/src/libaktualizr/primary/CMakeLists.txt @@ -1,10 +1,12 @@ set(SOURCES aktualizr.cc + aktualizr_helpers.cc initializer.cc reportqueue.cc sotauptaneclient.cc) set(HEADERS secondary_config.h aktualizr.h + aktualizr_helpers.h events.h initializer.h reportqueue.h @@ -32,8 +34,7 @@ if (BUILD_OSTREE) LABELS "noptest") target_link_libraries(t_download_nonostree virtual_secondary) else (BUILD_OSTREE) - aktualizr_source_file_checks(aktualizr_fullostree_test.cc) - aktualizr_source_file_checks(download_nonostree_test.cc) + aktualizr_source_file_checks(aktualizr_fullostree_test.cc download_nonostree_test.cc) endif (BUILD_OSTREE) add_aktualizr_test(NAME reportqueue SOURCES reportqueue_test.cc PROJECT_WORKING_DIRECTORY LIBRARIES PUBLIC uptane_generator_lib) diff --git a/src/libaktualizr/primary/aktualizr.cc b/src/libaktualizr/primary/aktualizr.cc index c568fc5562..9d368c323f 100644 --- a/src/libaktualizr/primary/aktualizr.cc +++ b/src/libaktualizr/primary/aktualizr.cc @@ -170,6 +170,28 @@ boost::signals2::connection Aktualizr::SetSignalHandler( return sig_->connect(handler); } +Aktualizr::InstallationLog Aktualizr::GetInstallationLog() { + std::vector ilog; + + EcuSerials serials; + if (!storage_->loadEcuSerials(&serials)) { + throw std::runtime_error("Could not load ecu serials"); + } + + ilog.reserve(serials.size()); + for (const auto &s : serials) { + Uptane::EcuSerial serial = s.first; + std::vector installs; + + std::vector log; + storage_->loadInstallationLog(serial.ToString(), &log, true); + + ilog.emplace_back(Aktualizr::InstallationLogEntry{serial, std::move(log)}); + } + + return ilog; +} + std::vector Aktualizr::GetStoredTargets() { return storage_->getTargetFiles(); } void Aktualizr::DeleteStoredTarget(const Uptane::Target &target) { storage_->removeTargetFile(target.filename()); } diff --git a/src/libaktualizr/primary/aktualizr.h b/src/libaktualizr/primary/aktualizr.h index 43d4a6b412..278226f555 100644 --- a/src/libaktualizr/primary/aktualizr.h +++ b/src/libaktualizr/primary/aktualizr.h @@ -1,7 +1,7 @@ #ifndef AKTUALIZR_H_ #define AKTUALIZR_H_ -#include +#include #include #include @@ -82,6 +82,17 @@ class Aktualizr { */ std::future Download(const std::vector& updates); + /** + * Get log of installations + * @return installation log + */ + struct InstallationLogEntry { + Uptane::EcuSerial ecu; + std::vector installs; + }; + using InstallationLog = std::vector; + InstallationLog GetInstallationLog(); + /** * Get list of targets currently in storage. This is intended to be used with * DeleteStoredTarget and targets are not guaranteed to be verified and diff --git a/src/libaktualizr/primary/aktualizr_helpers.cc b/src/libaktualizr/primary/aktualizr_helpers.cc new file mode 100644 index 0000000000..0697f33d64 --- /dev/null +++ b/src/libaktualizr/primary/aktualizr_helpers.cc @@ -0,0 +1,36 @@ +#include + +#include "aktualizr_helpers.h" + +void targets_autoclean_cb(Aktualizr &aktualizr, const std::shared_ptr &event) { + if (!event->isTypeOf()) { + return; + } + + std::vector installed_targets = aktualizr.GetStoredTargets(); + std::vector to_remove(installed_targets.size(), true); + + Aktualizr::InstallationLog log = aktualizr.GetInstallationLog(); + + // keep the last two installed targets for each ecu + for (const Aktualizr::InstallationLogEntry &entry : log) { + auto start = entry.installs.size() >= 2 ? entry.installs.end() - 2 : entry.installs.begin(); + for (auto it = start; it != entry.installs.end(); it++) { + auto fit = std::find_if(installed_targets.begin(), installed_targets.end(), + [&it](const Uptane::Target &t2) { return it->sha256Hash() == t2.sha256Hash(); }); + + if (fit == installed_targets.end()) { + continue; + } + + size_t rem_idx = static_cast(fit - installed_targets.begin()); + to_remove[rem_idx] = false; + } + } + + for (size_t k = 0; k < installed_targets.size(); k++) { + if (to_remove[k]) { + aktualizr.DeleteStoredTarget(installed_targets[k]); + } + } +} diff --git a/src/libaktualizr/primary/aktualizr_helpers.h b/src/libaktualizr/primary/aktualizr_helpers.h new file mode 100644 index 0000000000..fb68a6be46 --- /dev/null +++ b/src/libaktualizr/primary/aktualizr_helpers.h @@ -0,0 +1,11 @@ +#ifndef AKTUALIZR_HELPERS_H_ +#define AKTUALIZR_HELPERS_H_ + +#include +#include "aktualizr.h" + +// add as a signal handler to remove old targets just after an installation +// completes +void targets_autoclean_cb(Aktualizr &aktualizr, const std::shared_ptr &event); + +#endif // AKTUALIZR_HELPERS_H_ diff --git a/src/libaktualizr/primary/aktualizr_test.cc b/src/libaktualizr/primary/aktualizr_test.cc index 0d9a53b3fc..bd85c49862 100644 --- a/src/libaktualizr/primary/aktualizr_test.cc +++ b/src/libaktualizr/primary/aktualizr_test.cc @@ -12,6 +12,7 @@ #include "config/config.h" #include "httpfake.h" #include "primary/aktualizr.h" +#include "primary/aktualizr_helpers.h" #include "primary/events.h" #include "primary/sotauptaneclient.h" #include "uptane_test_common.h" @@ -1331,6 +1332,116 @@ TEST(Aktualizr, DownloadListRemove) { EXPECT_EQ(targets.size(), 0); } +TEST(Aktualizr, TargetAutoremove) { + TemporaryDirectory temp_dir; + const boost::filesystem::path local_metadir = temp_dir / "metadir"; + Utils::createDirectories(local_metadir, S_IRWXU); + auto http = std::make_shared(temp_dir.Path(), "", local_metadir / "repo"); + + UptaneRepo repo{local_metadir, "2021-07-04T16:33:27Z", "id0"}; + repo.generateRepo(KeyType::kED25519); + const std::string hwid = "primary_hw"; + repo.addImage(fake_meta_dir / "fake_meta/primary_firmware.txt", "primary_firmware.txt", hwid, "", {}); + repo.addTarget("primary_firmware.txt", hwid, "CA:FE:A6:D2:84:9D", ""); + repo.signTargets(); + + Config conf = UptaneTestCommon::makeTestConfig(temp_dir, http->tls_server); + auto storage = INvStorage::newStorage(conf.storage); + UptaneTestCommon::TestAktualizr aktualizr(conf, storage, http); + + // attach the autoclean handler + boost::signals2::connection ac_conn = + aktualizr.SetSignalHandler(std::bind(targets_autoclean_cb, std::ref(aktualizr), std::placeholders::_1)); + aktualizr.Initialize(); + + { + result::UpdateCheck update_result = aktualizr.CheckUpdates().get(); + aktualizr.Download(update_result.updates).get(); + + EXPECT_EQ(aktualizr.GetStoredTargets().size(), 1); + + result::Install install_result = aktualizr.Install(update_result.updates).get(); + EXPECT_TRUE(install_result.dev_report.success); + + std::vector targets = aktualizr.GetStoredTargets(); + ASSERT_EQ(targets.size(), 1); + EXPECT_EQ(targets[0].filename(), "primary_firmware.txt"); + } + + // second install + repo.emptyTargets(); + repo.addImage(fake_meta_dir / "fake_meta/dummy_firmware.txt", "dummy_firmware.txt", hwid, "", {}); + repo.addTarget("dummy_firmware.txt", hwid, "CA:FE:A6:D2:84:9D", ""); + repo.signTargets(); + + { + result::UpdateCheck update_result = aktualizr.CheckUpdates().get(); + aktualizr.Download(update_result.updates).get(); + + EXPECT_EQ(aktualizr.GetStoredTargets().size(), 2); + + result::Install install_result = aktualizr.Install(update_result.updates).get(); + EXPECT_TRUE(install_result.dev_report.success); + + // all targets are kept (current and previous) + std::vector targets = aktualizr.GetStoredTargets(); + ASSERT_EQ(targets.size(), 2); + } + + // third install (first firmware again) + repo.emptyTargets(); + repo.addImage(fake_meta_dir / "fake_meta/primary_firmware.txt", "primary_firmware.txt", hwid, "", {}); + repo.addTarget("primary_firmware.txt", hwid, "CA:FE:A6:D2:84:9D", ""); + repo.signTargets(); + + { + result::UpdateCheck update_result = aktualizr.CheckUpdates().get(); + aktualizr.Download(update_result.updates).get(); + + EXPECT_EQ(aktualizr.GetStoredTargets().size(), 2); + + result::Install install_result = aktualizr.Install(update_result.updates).get(); + EXPECT_TRUE(install_result.dev_report.success); + + // all targets are kept again (current and previous) + std::vector targets = aktualizr.GetStoredTargets(); + ASSERT_EQ(targets.size(), 2); + } + + // fourth install (some new third firmware) + repo.emptyTargets(); + repo.addImage(fake_meta_dir / "fake_meta/secondary_firmware.txt", "secondary_firmware.txt", hwid, "", {}); + repo.addTarget("secondary_firmware.txt", hwid, "CA:FE:A6:D2:84:9D", ""); + repo.signTargets(); + + { + result::UpdateCheck update_result = aktualizr.CheckUpdates().get(); + aktualizr.Download(update_result.updates).get(); + + EXPECT_EQ(aktualizr.GetStoredTargets().size(), 3); + + result::Install install_result = aktualizr.Install(update_result.updates).get(); + EXPECT_TRUE(install_result.dev_report.success); + + // only two targets are left: dummy_firmware has been cleaned up + std::vector targets = aktualizr.GetStoredTargets(); + ASSERT_EQ(targets.size(), 2); + } + + Aktualizr::InstallationLog log = aktualizr.GetInstallationLog(); + ASSERT_EQ(log.size(), 2); + + EXPECT_EQ(log[0].installs.size(), 4); + EXPECT_EQ(log[1].installs.size(), 0); + + std::vector fws{"primary_firmware.txt", "dummy_firmware.txt", "primary_firmware.txt", + "secondary_firmware.txt"}; + for (auto it = log[0].installs.begin(); it < log[0].installs.end(); it++) { + auto idx = static_cast(it - log[0].installs.begin()); + EXPECT_EQ(it->filename(), fws[idx]); + } +} + /* * Initialize -> Install -> nothing to install. * From 33d1973e2bb2f7aec17b2c5db5218bc91fa354be Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Fri, 6 Sep 2019 15:44:54 +0200 Subject: [PATCH 7/7] Document autoclean and release log features Signed-off-by: Laurent Bonnans --- CHANGELOG.md | 5 +++++ actions.md | 1 + src/libaktualizr/primary/aktualizr.h | 4 +++- src/libaktualizr/primary/aktualizr_helpers.h | 7 +++++-- src/libaktualizr/primary/aktualizr_test.cc | 4 ++++ 5 files changed, 18 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6429c6a997..cd3b3c3239 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ Our versioning scheme is `YEAR.N` where `N` is incremented whenever a new releas ## [??? (unreleased)] +### Added + +- `GetInstallationLog` API method: [PR](https://github.com/advancedtelematic/aktualizr/pull/1318) +- The aktualizr daemon will now automatically remove old downloaded targets to free up disk space: [PR](https://github.com/advancedtelematic/aktualizr/pull/1318) + ## [2019.6] - 2019-08-21 ### Added diff --git a/actions.md b/actions.md index 3611fa94db..87bad28895 100644 --- a/actions.md +++ b/actions.md @@ -173,6 +173,7 @@ These are the primary actions that a user of libaktualizr can perform through th - [x] Store negative device installation result when an ECU installation failed (aktualizr_test.cc) - [x] Update is not in pending state anymore after failed installation (aktualizr_test.cc) - [x] Send AllInstallsComplete event after all installations are finished (aktualizr_test.cc) + - [x] Automatically remove old targets during installation cycles (aktualizr_test.cc) - [x] Send installation report - [x] Generate and send manifest (see below) - [x] Send PutManifestComplete event if send is successful (aktualizr_test.cc) diff --git a/src/libaktualizr/primary/aktualizr.h b/src/libaktualizr/primary/aktualizr.h index 278226f555..91398e81d5 100644 --- a/src/libaktualizr/primary/aktualizr.h +++ b/src/libaktualizr/primary/aktualizr.h @@ -83,7 +83,9 @@ class Aktualizr { std::future Download(const std::vector& updates); /** - * Get log of installations + * Get log of installations. The log is indexed for every ECU and contains + * every change of versions ordered by time. It may contain duplicates in + * case of rollbacks. * @return installation log */ struct InstallationLogEntry { diff --git a/src/libaktualizr/primary/aktualizr_helpers.h b/src/libaktualizr/primary/aktualizr_helpers.h index fb68a6be46..30e72b2143 100644 --- a/src/libaktualizr/primary/aktualizr_helpers.h +++ b/src/libaktualizr/primary/aktualizr_helpers.h @@ -4,8 +4,11 @@ #include #include "aktualizr.h" -// add as a signal handler to remove old targets just after an installation -// completes +/* + * Signal handler to remove old targets just after an installation completes + * + * To be attached with Aktualizr::SetSignalHandler + */ void targets_autoclean_cb(Aktualizr &aktualizr, const std::shared_ptr &event); #endif // AKTUALIZR_HELPERS_H_ diff --git a/src/libaktualizr/primary/aktualizr_test.cc b/src/libaktualizr/primary/aktualizr_test.cc index bd85c49862..07e13e6bc6 100644 --- a/src/libaktualizr/primary/aktualizr_test.cc +++ b/src/libaktualizr/primary/aktualizr_test.cc @@ -1332,6 +1332,10 @@ TEST(Aktualizr, DownloadListRemove) { EXPECT_EQ(targets.size(), 0); } +/* + * Automatically remove old targets during installation cycles. + * Get log of installation. + */ TEST(Aktualizr, TargetAutoremove) { TemporaryDirectory temp_dir; const boost::filesystem::path local_metadir = temp_dir / "metadir";