Feat/backward migrations

actions.md

@@ -223,6 +223,7 @@ These are internal requirements that are relatively opaque to the user and/or co
 - [x] Store state in an SQL database
   - [x] Migrate forward through SQL schemas (sqlstorage_test.cc)
     - [x] Automatically use latest SQL schema version when initializing database (sqlstorage_test.cc)
+  - [x] Migrate backward through SQL schemas (sqlstorage_test.cc)
   - [x] Reject invalid SQL databases (sqlstorage_test.cc)
   - [x] Migrate from the legacy filesystem storage (sqlstorage_test.cc, uptane_test.cc)
   - [x] Load and store primary keys in an SQL database (storage_common_test.cc)

config/sql/migration/migrate.00.sql

@@ -1,5 +1,5 @@
 -- Don't modify this! Create a new migration instead--see docs/schema-migrations.adoc
-BEGIN TRANSACTION;
+SAVEPOINT MIGRATION;
 
 CREATE TABLE version(version INTEGER);
 
@@ -22,4 +22,4 @@ CREATE TABLE meta(director_root BLOB NOT NULL,
 CREATE TABLE primary_image(filepath TEXT NOT NULL);
 
 INSERT INTO version VALUES(0);
-COMMIT TRANSACTION;
+RELEASE MIGRATION;

config/sql/migration/migrate.01.sql

@@ -1,5 +1,5 @@
 -- Don't modify this! Create a new migration instead--see docs/schema-migrations.adoc
-BEGIN TRANSACTION;
+SAVEPOINT MIGRATION;
 
 ALTER TABLE primary_image ADD COLUMN installed_versions TEXT NOT NULL DEFAULT '';
 
@@ -11,4 +11,4 @@ ALTER TABLE device_info_migrate RENAME TO device_info;
 DELETE FROM version;
 INSERT INTO version VALUES(1);
 
-COMMIT TRANSACTION;
+RELEASE MIGRATION;

config/sql/migration/migrate.02.sql

@@ -1,5 +1,5 @@
 -- Don't modify this! Create a new migration instead--see docs/schema-migrations.adoc
-BEGIN TRANSACTION;
+SAVEPOINT MIGRATION;
 
 
 DROP TABLE primary_image;
@@ -9,4 +9,4 @@ CREATE TABLE target_images(filename TEXT UNIQUE, image_data BLOB NOT NULL);
 DELETE FROM version;
 INSERT INTO version VALUES(2);
 
-COMMIT TRANSACTION;
+RELEASE MIGRATION;

config/sql/migration/migrate.03.sql

@@ -1,8 +1,8 @@
-BEGIN TRANSACTION;
+SAVEPOINT MIGRATION;
 
 ALTER TABLE installed_versions ADD COLUMN is_current INTEGER NOT NULL CHECK ( is_current IN ( 0 , 1 ) ) DEFAULT 0;
 
 DELETE FROM version;
 INSERT INTO version VALUES(3);
 
-COMMIT TRANSACTION;
+RELEASE MIGRATION;

config/sql/migration/migrate.04.sql

@@ -1,8 +1,8 @@
-BEGIN TRANSACTION;
+SAVEPOINT MIGRATION;
 
 ALTER TABLE installed_versions ADD COLUMN length INTEGER NOT NULL DEFAULT 0;
 
 DELETE FROM version;
 INSERT INTO version VALUES(4);
 
-COMMIT TRANSACTION;
+RELEASE MIGRATION;

config/sql/migration/migrate.05.sql

@@ -1,4 +1,4 @@
-BEGIN TRANSACTION;
+SAVEPOINT MIGRATION;
 
 CREATE TABLE rawmeta(director_root BLOB NOT NULL,
                   director_targets BLOB NOT NULL,
@@ -15,4 +15,4 @@ CREATE TABLE root_rawmeta(root BLOB NOT NULL, director INTEGER NOT NULL CHECK (d
 DELETE FROM version;
 INSERT INTO version VALUES(5);
 
-COMMIT TRANSACTION;
+RELEASE MIGRATION;

config/sql/migration/migrate.06.sql

@@ -1,5 +1,5 @@
 -- Don't modify this! Create a new migration instead--see docs/schema-migrations.adoc
-BEGIN TRANSACTION;
+SAVEPOINT MIGRATION;
 
 DROP TABLE root_meta;
 DROP TABLE rawmeta;
@@ -47,4 +47,4 @@ ALTER TABLE meta_migrate RENAME TO meta;
 DELETE FROM version;
 INSERT INTO version VALUES(6);
 
-COMMIT TRANSACTION;
+RELEASE MIGRATION;

config/sql/migration/migrate.07.sql

@@ -1,9 +1,9 @@
 -- Don't modify this! Create a new migration instead--see docs/schema-migrations.adoc
-BEGIN TRANSACTION;
+SAVEPOINT MIGRATION;
 
 CREATE TABLE installation_result(unique_mark INTEGER PRIMARY KEY CHECK (unique_mark = 0), id TEXT, result_code INTEGER NOT NULL DEFAULT 0, result_text TEXT);
 
 DELETE FROM version;
 INSERT INTO version VALUES(7);
 
-COMMIT TRANSACTION;
+RELEASE MIGRATION;

config/sql/migration/migrate.08.sql

@@ -1,5 +1,5 @@
 -- Don't modify this! Create a new migration instead--see docs/schema-migrations.adoc
-BEGIN TRANSACTION;
+SAVEPOINT MIGRATION;
 
 CREATE TABLE primary_keys_migrate(unique_mark INTEGER PRIMARY KEY CHECK (unique_mark = 0), private TEXT, public TEXT);
 INSERT INTO primary_keys_migrate SELECT 0,private,public FROM primary_keys LIMIT 1;
@@ -14,4 +14,4 @@ ALTER TABLE device_info_migrate RENAME TO device_info;
 DELETE FROM version;
 INSERT INTO version VALUES(8);
 
-COMMIT TRANSACTION;
+RELEASE MIGRATION;

config/sql/migration/migrate.09.sql

@@ -1,5 +1,5 @@
 -- Don't modify this! Create a new migration instead--see docs/schema-migrations.adoc
-BEGIN TRANSACTION;
+SAVEPOINT MIGRATION;
 
 CREATE TABLE installed_versions_migrate(hash TEXT, name TEXT NOT NULL, is_current INTEGER NOT NULL CHECK (is_current IN (0,1)) DEFAULT 0, length INTEGER NOT NULL DEFAULT 0, UNIQUE(hash, name));
 INSERT INTO installed_versions_migrate SELECT * FROM installed_versions;
@@ -9,4 +9,4 @@ ALTER TABLE installed_versions_migrate RENAME TO installed_versions;
 DELETE FROM version;
 INSERT INTO version VALUES(9);
 
-COMMIT TRANSACTION;
+RELEASE MIGRATION;

config/sql/migration/migrate.10.sql

@@ -1,9 +1,9 @@
 -- Don't modify this! Create a new migration instead--see docs/schema-migrations.adoc
-BEGIN TRANSACTION;
+SAVEPOINT MIGRATION;
 
 ALTER TABLE target_images ADD real_size INTEGER NOT NULL DEFAULT 0;
 
 DELETE FROM version;
 INSERT INTO version VALUES(10);
 
-COMMIT TRANSACTION;
+RELEASE MIGRATION;

config/sql/migration/migrate.11.sql

@@ -1,10 +1,10 @@
 -- Don't modify this! Create a new migration instead--see docs/schema-migrations.adoc
-BEGIN TRANSACTION;
+SAVEPOINT MIGRATION;
 
 ALTER TABLE target_images ADD sha256 TEXT NOT NULL DEFAULT "";
 ALTER TABLE target_images ADD sha512 TEXT NOT NULL DEFAULT "";
 
 DELETE FROM version;
 INSERT INTO version VALUES(11);
 
-COMMIT TRANSACTION;
+RELEASE MIGRATION;

config/sql/migration/migrate.12.sql

@@ -1,9 +1,9 @@
 -- Don't modify this! Create a new migration instead--see docs/schema-migrations.adoc
-BEGIN TRANSACTION;
+SAVEPOINT MIGRATION;
 
 CREATE TABLE need_reboot(unique_mark INTEGER PRIMARY KEY CHECK (unique_mark = 0), flag INTEGER NOT NULL DEFAULT 0);
 
 DELETE FROM version;
 INSERT INTO version VALUES(12);
 
-COMMIT TRANSACTION;
+RELEASE MIGRATION;

config/sql/migration/migrate.13.sql

@@ -1,5 +1,5 @@
 -- Don't modify this! Create a new migration instead--see docs/schema-migrations.adoc
-BEGIN TRANSACTION;
+SAVEPOINT MIGRATION;
 
 CREATE TABLE installed_versions_migrate(ecu_serial TEXT NOT NULL, sha256 TEXT NOT NULL, name TEXT NOT NULL, hashes TEXT NOT NULL, length INTEGER NOT NULL DEFAULT 0, correlation_id TEXT NOT NULL DEFAULT '', is_current INTEGER NOT NULL CHECK (is_current IN (0,1)) DEFAULT 0, is_pending INTEGER NOT NULL CHECK (is_pending IN (0,1)) DEFAULT 0, UNIQUE(ecu_serial, sha256, name));
 
@@ -11,4 +11,4 @@ ALTER TABLE installed_versions_migrate RENAME TO installed_versions;
 DELETE FROM version;
 INSERT INTO version VALUES(13);
 
-COMMIT TRANSACTION;
+RELEASE MIGRATION;

config/sql/migration/migrate.14.sql

@@ -1,5 +1,5 @@
 -- Don't modify this! Create a new migration instead--see docs/schema-migrations.adoc
-BEGIN TRANSACTION;
+SAVEPOINT MIGRATION;
 
 CREATE TABLE target_images_data(filename TEXT PRIMARY KEY, image_data BLOB NOT NULL);
 CREATE TABLE target_images_info(filename TEXT PRIMARY KEY, real_size INTEGER NOT NULL DEFAULT 0, sha256 TEXT NOT NULL DEFAULT "", sha512 TEXT NOT NULL DEFAULT "");
@@ -16,4 +16,4 @@ ALTER TABLE target_images_info RENAME TO target_images;
 DELETE FROM version;
 INSERT INTO version VALUES(14);
 
-COMMIT TRANSACTION;
+RELEASE MIGRATION;

config/sql/migration/migrate.15.sql

@@ -0,0 +1,9 @@
+-- Don't modify this! Create a new migration instead--see docs/schema-migrations.adoc
+SAVEPOINT MIGRATION;
+
+CREATE TABLE rollback_migrations(version_from INT PRIMARY KEY, migration TEXT NOT NULL);
+
+DELETE FROM version;
+INSERT INTO version VALUES(15);
+
+RELEASE MIGRATION;

config/sql/rollback/rollback.15.sql

@@ -0,0 +1,9 @@
+-- Don't modify this! Create a new migration instead--see docs/schema-migrations.adoc
+SAVEPOINT ROLLBACK_MIGRATION;
+
+DROP TABLE rollback_migrations;
+
+DELETE FROM version;
+INSERT INTO version VALUES(14);
+
+RELEASE ROLLBACK_MIGRATION;

config/sql/schema.sql

@@ -1,5 +1,5 @@
 CREATE TABLE version(version INTEGER);
-INSERT INTO version(rowid,version) VALUES(1,14);
+INSERT INTO version(rowid,version) VALUES(1,15);
 CREATE TABLE device_info(unique_mark INTEGER PRIMARY KEY CHECK (unique_mark = 0), device_id TEXT, is_registered INTEGER NOT NULL DEFAULT 0 CHECK (is_registered IN (0,1)));
 CREATE TABLE ecu_serials(serial TEXT UNIQUE, hardware_id TEXT NOT NULL, is_primary INTEGER NOT NULL CHECK (is_primary IN (0,1)));
 CREATE TABLE misconfigured_ecus(serial TEXT UNIQUE, hardware_id TEXT NOT NULL, state INTEGER NOT NULL CHECK (state IN (0,1)));
@@ -21,3 +21,4 @@ INSERT INTO repo_types(rowid,repo,repo_string) VALUES(1,0,'images');
 INSERT INTO repo_types(rowid,repo,repo_string) VALUES(2,1,'director');
 CREATE TABLE installation_result(unique_mark INTEGER PRIMARY KEY CHECK (unique_mark = 0), id TEXT, result_code INTEGER NOT NULL DEFAULT 0, result_text TEXT);
 CREATE TABLE need_reboot(unique_mark INTEGER PRIMARY KEY CHECK (unique_mark = 0), flag INTEGER NOT NULL DEFAULT 0);
+CREATE TABLE rollback_migrations(version_from INT PRIMARY KEY, migration TEXT NOT NULL);

src/libaktualizr/storage/embed_schemas.py

@@ -9,6 +9,7 @@
 //Latest schema timestamp: %d
 #include <string>
 #include <vector>
+#include <map>
 '''
 
 def escape_string(sql):
@@ -32,7 +33,9 @@ def apend_migration(migration_path, header_file):
     schemas_header = sys.argv[2]
     prefix = sys.argv[3]
     migration_folder = os.path.join(sql_folder, 'migration')
+    rollback_folder = os.path.join(sql_folder, 'rollback')
     migration_list = sorted(os.listdir(migration_folder))
+    rollback_migrations_list = sorted(os.listdir(rollback_folder))
 
     file_depends_list = [os.path.join(migration_folder, p) for p in migration_list]
     file_depends_list.append(__file__)  # set dependency on itself (this script)
@@ -54,6 +57,18 @@ def apend_migration(migration_path, header_file):
             header_file.write("\",\n")
         apend_migration(os.path.join(migration_folder, migration_list[-1]), header_file)
         header_file.write("\"\n};\n")
+
+        header_file.write("extern const std::vector<std::string> {}_schema_rollback_migrations = {{".format(prefix))
+        ver = int(rollback_migrations_list[0].split(".")[1])
+        for i in range(ver):
+            header_file.write("\"\",\n")
+        for migration in rollback_migrations_list[:-1]:
+            apend_migration(os.path.join(rollback_folder, migration), header_file)
+            header_file.write("\",\n")
+        version = int(rollback_migrations_list[-1].split(".")[1])
+        apend_migration(os.path.join(rollback_folder, rollback_migrations_list[-1]), header_file)
+        header_file.write("\"\n};\n")
+
         current_schema = open(os.path.join(sql_folder, "schema.sql"), 'r').read()
         current_schema_escaped = escape_string(current_schema)
         header_file.write('extern const std::string %s_current_schema = "%s";' % (prefix, current_schema_escaped));

src/libaktualizr/storage/sqlstorage.cc

@@ -62,7 +62,8 @@ void SQLStorage::cleanMetaVersion(Uptane::RepositoryType repo, Uptane::Role role
 
 SQLStorage::SQLStorage(const StorageConfig& config, bool readonly)
     : SQLStorageBase(config.sqldb_path.get(config.path), readonly, libaktualizr_schema_migrations,
-                     libaktualizr_current_schema, libaktualizr_current_schema_version),
+                     libaktualizr_schema_rollback_migrations, libaktualizr_current_schema,
+                     libaktualizr_current_schema_version),
       INvStorage(config) {
   try {
     cleanMetaVersion(Uptane::RepositoryType::Director(), Uptane::Role::Root());

src/libaktualizr/storage/sqlstorage.h

@@ -10,6 +10,7 @@
 #include "sqlstorage_base.h"
 
 extern const std::vector<std::string> libaktualizr_schema_migrations;
+extern const std::vector<std::string> libaktualizr_schema_rollback_migrations;
 extern const std::string libaktualizr_current_schema;
 extern const int libaktualizr_current_schema_version;
 

src/libaktualizr/storage/sqlstorage_base.cc

@@ -6,11 +6,13 @@
 boost::filesystem::path SQLStorageBase::dbPath() const { return sqldb_path_; }
 
 SQLStorageBase::SQLStorageBase(boost::filesystem::path sqldb_path, bool readonly,
-                               std::vector<std::string> schema_migrations, std::string current_schema,
+                               std::vector<std::string> schema_migrations,
+                               std::vector<std::string> schema_rollback_migrations, std::string current_schema,
                                int current_schema_version)
     : sqldb_path_(std::move(sqldb_path)),
       readonly_(readonly),
       schema_migrations_(std::move(schema_migrations)),
+      schema_rollback_migrations_(std::move(schema_rollback_migrations)),
       current_schema_(std::move(current_schema)),
       current_schema_version_(current_schema_version) {
   boost::filesystem::path db_parent_path = dbPath().parent_path();
@@ -61,6 +63,59 @@ std::string SQLStorageBase::getTableSchemaFromDb(const std::string& tablename) {
   return schema.value() + ";";
 }
 
+bool SQLStorageBase::dbMigrateForward(int version_from) {
+  SQLite3Guard db = dbConnection();
+
+  if (!db.beginTransaction()) {
+    LOG_ERROR << "Can't start transaction: " << db.errmsg();
+    return false;
+  }
+
+  for (int32_t k = version_from + 1; k <= current_schema_version_; k++) {
+    auto result_code = db.exec(schema_migrations_.at(static_cast<size_t>(k)), nullptr, nullptr);
+    if (result_code != SQLITE_OK) {
+      LOG_ERROR << "Can't migrate db from version " << (k - 1) << " to version " << k << ": " << db.errmsg();
+      return false;
+    }
+    if (schema_rollback_migrations_.at(static_cast<uint32_t>(k)).empty()) {
+      continue;
+    }
+    auto statement = db.prepareStatement("INSERT OR REPLACE INTO rollback_migrations VALUES (?,?);", k,
+                                         schema_rollback_migrations_.at(static_cast<uint32_t>(k)));
+    if (statement.step() != SQLITE_DONE) {
+      LOG_ERROR << "Can't insert rollback migration script: " << db.errmsg();
+      return false;
+    }
+  }
+
+  db.commitTransaction();
+
+  return true;
+}
+
+bool SQLStorageBase::dbMigrateBackward(int version_from) {
+  SQLite3Guard db = dbConnection();
+  for (int ver = version_from; ver > current_schema_version_; --ver) {
+    auto statement = db.prepareStatement("SELECT migration FROM rollback_migrations WHERE version_from=?;", ver);
+    if (statement.step() != SQLITE_ROW) {
+      LOG_ERROR << "Can't extract migration script: " << db.errmsg();
+      return false;
+    }
+    auto migration = statement.get_result_col_str(0);
+    statement.step();
+    if (db.exec(*migration, nullptr, nullptr) != SQLITE_OK) {
+      LOG_ERROR << "Can't migrate db from version " << (ver) << " to version " << ver - 1 << ": " << db.errmsg();
+      return false;
+    }
+    if (db.exec(std::string("DELETE FROM rollback_migrations WHERE version_from=") + std::to_string(ver) + ";", nullptr,
+                nullptr) != SQLITE_OK) {
+      LOG_ERROR << "Can't clear old migration script: " << db.errmsg();
+      return false;
+    }
+  }
+  return true;
+}
+
 bool SQLStorageBase::dbMigrate() {
   DbVersion schema_version = getVersion();
 
@@ -79,17 +134,10 @@ bool SQLStorageBase::dbMigrate() {
     return false;
   }
 
-  if (schema_num_version > current_schema_version_) {
-    LOG_ERROR << "Only forward migrations are supported. You cannot migrate to an older schema.";
-    return false;
-  }
-
-  SQLite3Guard db = dbConnection();
-  for (int32_t k = schema_num_version + 1; k <= current_schema_version_; k++) {
-    if (db.exec(schema_migrations_.at(static_cast<size_t>(k)), nullptr, nullptr) != SQLITE_OK) {
-      LOG_ERROR << "Can't migrate db from version " << (k - 1) << " to version " << k << ": " << db.errmsg();
-      return false;
-    }
+  if (schema_version != DbVersion::kEmpty && schema_num_version > current_schema_version_) {
+    dbMigrateBackward(schema_num_version);
+  } else {
+    dbMigrateForward(schema_num_version);
   }
 
   return true;