Store image files on the filesystem

Signed-off-by: Serhiy Stetskovych <[email protected]>

config/sql/migration/migrate.18.sql

@@ -0,0 +1,9 @@
+-- Don't modify this! Create a new migration instead--see docs/schema-migrations.adoc
+SAVEPOINT MIGRATION;
+
+DROP TABLE target_images_data;
+
+DELETE FROM version;
+INSERT INTO version VALUES(18);
+
+RELEASE MIGRATION;

config/sql/rollback/rollback.18.sql

@@ -0,0 +1,9 @@
+-- Don't modify this! Create a new migration instead--see docs/schema-migrations.adoc
+SAVEPOINT ROLLBACK_MIGRATION;
+
+CREATE TABLE target_images_data(filename TEXT PRIMARY KEY, image_data BLOB NOT NULL);
+
+DELETE FROM version;
+INSERT INTO version VALUES(17);
+
+RELEASE ROLLBACK_MIGRATION;

config/sql/schema.sql

@@ -1,5 +1,5 @@
 CREATE TABLE version(version INTEGER);
-INSERT INTO version(rowid,version) VALUES(1,17);
+INSERT INTO version(rowid,version) VALUES(1,18);
 CREATE TABLE device_info(unique_mark INTEGER PRIMARY KEY CHECK (unique_mark = 0), device_id TEXT, is_registered INTEGER NOT NULL DEFAULT 0 CHECK (is_registered IN (0,1)));
 CREATE TABLE ecu_serials(serial TEXT UNIQUE, hardware_id TEXT NOT NULL, is_primary INTEGER NOT NULL CHECK (is_primary IN (0,1)));
 CREATE TABLE misconfigured_ecus(serial TEXT UNIQUE, hardware_id TEXT NOT NULL, state INTEGER NOT NULL CHECK (state IN (0,1)));
@@ -10,7 +10,6 @@ CREATE TABLE tls_creds(ca_cert BLOB, ca_cert_format TEXT,
                        client_pkey BLOB, client_pkey_format TEXT);
 CREATE TABLE meta(meta BLOB NOT NULL, repo INTEGER NOT NULL, meta_type INTEGER NOT NULL, version INTEGER NOT NULL, UNIQUE(repo, meta_type, version));
 CREATE TABLE target_images(filename TEXT PRIMARY KEY, real_size INTEGER NOT NULL DEFAULT 0, sha256 TEXT NOT NULL DEFAULT "", sha512 TEXT NOT NULL DEFAULT "");
-CREATE TABLE target_images_data(filename TEXT PRIMARY KEY, image_data BLOB NOT NULL);
 CREATE TABLE repo_types(repo INTEGER NOT NULL, repo_string TEXT NOT NULL);
 CREATE TABLE meta_types(meta INTEGER NOT NULL, meta_string TEXT NOT NULL);
 INSERT INTO meta_types(rowid,meta,meta_string) VALUES(1,0,'root');

src/libaktualizr/storage/sqlstorage.cc

@@ -1192,8 +1192,8 @@ boost::optional<size_t> SQLStorage::checkTargetFile(const Uptane::Target& target
 class SQLTargetWHandle : public StorageTargetWHandle {
  public:
   SQLTargetWHandle(const SQLStorage& storage, Uptane::Target target)
-      : db_(storage.dbPath()), target_(std::move(target)), closed_(false), row_id_(0) {
-    StorageTargetWHandle::WriteError exc("could not save file " + target_.filename() + " to sql storage");
+      : db_(storage.dbPath()), target_(std::move(target)), closed_(false) {
+    StorageTargetWHandle::WriteError exc("could not save file " + target_.filename() + " to the filesystem");
     if (!db_.beginTransaction()) {
       throw exc;
     }
@@ -1207,25 +1207,20 @@ class SQLTargetWHandle : public StorageTargetWHandle {
         sha512Hash = hash.HashString();
       }
     }
-
-    auto statement = db_.prepareStatement<std::string, SQLZeroBlob>(
-        "INSERT OR REPLACE INTO target_images_data (filename, image_data) VALUES (?,?);", target_.filename(),
-        SQLZeroBlob{target_.length()});
-
-    if (statement.step() != SQLITE_DONE) {
-      LOG_ERROR << "Statement step failure: " << db_.errmsg();
-      throw exc;
-    }
-
-    row_id_ = sqlite3_last_insert_rowid(db_.get());
-    statement = db_.prepareStatement<std::string, std::string, std::string>(
+    auto statement = db_.prepareStatement<std::string, std::string, std::string>(
         "INSERT OR REPLACE INTO target_images (filename, sha256, sha512) VALUES ( ?, ?, ?);", target_.filename(),
         sha256Hash, sha512Hash);
 
     if (statement.step() != SQLITE_DONE) {
       LOG_ERROR << "Statement step failure: " << db_.errmsg();
       throw exc;
     }
+    boost::filesystem::create_directories((storage.images_path_ / target_.filename()).parent_path());
+    stream_.open((storage.images_path_ / target_.filename()).string());
+    if (!stream_.good()) {
+      LOG_ERROR << "Could not open image for write: " << storage.images_path_ / target_.filename();
+      throw exc;
+    }
 
     if (!db_.commitTransaction()) {
       throw exc;
@@ -1234,35 +1229,24 @@ class SQLTargetWHandle : public StorageTargetWHandle {
 
   ~SQLTargetWHandle() override {
     if (!closed_) {
+      std::cout << "commiting\n";
       SQLTargetWHandle::wcommit();
     }
+    stream_.close();
   }
 
   size_t wfeed(const uint8_t* buf, size_t size) override {
     StorageTargetWHandle::WriteError exc("could not save file " + target_.filename() + " to sql storage");
 
-    if (blob_ == nullptr) {
-      if (sqlite3_blob_open(db_.get(), "main", "target_images_data", "image_data", row_id_, 1, &blob_) != SQLITE_OK) {
-        LOG_ERROR << "Could not open blob " << db_.errmsg();
-        wabort();
-        throw exc;
-      }
-    }
-
-    if (sqlite3_blob_write(blob_, buf, static_cast<int>(size), static_cast<int>(written_size_)) != SQLITE_OK) {
-      LOG_ERROR << "Could not write in blob: " << db_.errmsg();
-      wabort();
-      return 0;
-    }
+    stream_.write(reinterpret_cast<const char*>(buf), static_cast<std::streamsize>(size));
     written_size_ += size;
 
     return size;
   }
 
   void wcommit() override {
-    if (blob_ != nullptr) {
-      sqlite3_blob_close(blob_);
-      blob_ = nullptr;
+    if (stream_) {
+      stream_.close();
       auto statement =
           db_.prepareStatement<int64_t, std::string>("UPDATE target_images SET real_size = ? WHERE filename = ?;",
                                                      static_cast<int64_t>(written_size_), target_.filename());
@@ -1277,9 +1261,8 @@ class SQLTargetWHandle : public StorageTargetWHandle {
   }
 
   void wabort() noexcept override {
-    if (blob_ != nullptr) {
-      sqlite3_blob_close(blob_);
-      blob_ = nullptr;
+    if (stream_) {
+      stream_.close();
     }
     if (sqlite3_changes(db_.get()) > 0) {
       auto statement =
@@ -1292,21 +1275,25 @@ class SQLTargetWHandle : public StorageTargetWHandle {
   friend class SQLTargetRHandle;
 
  private:
-  SQLTargetWHandle(const boost::filesystem::path& db_path, Uptane::Target target, const sqlite3_int64& row_id,
-                   const size_t& start_from = 0)
-      : db_(db_path), target_(std::move(target)), closed_(false), row_id_(row_id) {
+  SQLTargetWHandle(const boost::filesystem::path& db_path, Uptane::Target target,
+                   const boost::filesystem::path& image_path, const size_t& start_from = 0)
+      : db_(db_path), target_(std::move(target)), closed_(false) {
     if (db_.get_rc() != SQLITE_OK) {
       LOG_ERROR << "Can't open database: " << db_.errmsg();
-      throw StorageTargetWHandle::WriteError("could not save file " + target_.filename() + " to sql storage");
+      throw StorageTargetWHandle::WriteError("could not open sql storage");
+    }
+    stream_.open(image_path.string(), std::ofstream::out | std::ofstream::app);
+    if (!stream_.good()) {
+      LOG_ERROR << "Could not open image for write: " << image_path;
+      throw StorageTargetWHandle::WriteError("could not open file for write: " + image_path.string());
     }
 
     written_size_ = start_from;
   }
   SQLite3Guard db_;
   Uptane::Target target_;
   bool closed_;
-  sqlite3_int64 row_id_;
-  sqlite3_blob* blob_{nullptr};
+  std::ofstream stream_;
 };
 
 std::unique_ptr<StorageTargetWHandle> SQLStorage::allocateTargetFile(bool from_director, const Uptane::Target& target) {
@@ -1317,32 +1304,20 @@ std::unique_ptr<StorageTargetWHandle> SQLStorage::allocateTargetFile(bool from_d
 class SQLTargetRHandle : public StorageTargetRHandle {
  public:
   SQLTargetRHandle(const SQLStorage& storage, Uptane::Target target)
-      : db_path_(storage.dbPath()),
-        db_(db_path_),
-        target_(std::move(target)),
-        size_(0),
-        read_size_(0),
-        closed_(false),
-        blob_(nullptr) {
+      : db_path_(storage.dbPath()), db_(db_path_), target_(std::move(target)), size_(0), closed_(false) {
     StorageTargetRHandle::ReadError exc("could not read file " + target_.filename() + " from sql storage");
 
     auto exists = storage.checkTargetFile(target_);
     if (!exists) {
       throw exc;
     }
-    auto statement = db_.prepareStatement<std::string>("SELECT rowid FROM target_images_data WHERE filename = ?;",
-                                                       target_.filename());
 
-    if (statement.step() != SQLITE_ROW) {
-      LOG_ERROR << "Statement step failure: " << db_.errmsg();
-      throw std::runtime_error("Could not find target file");
-    }
-
-    row_id_ = statement.get_result_col_int(0);
     size_ = *exists;
     partial_ = size_ < target_.length();
-    if (sqlite3_blob_open(db_.get(), "main", "target_images_data", "image_data", row_id_, 0, &blob_) != SQLITE_OK) {
-      LOG_ERROR << "Could not open blob: " << db_.errmsg();
+    image_path_ = storage.images_path_ / target_.filename();
+    stream_.open(image_path_.string());
+    if (!stream_.good()) {
+      LOG_ERROR << "Could not open image: " << storage.images_path_ / target_.filename();
       throw exc;
     }
   }
@@ -1356,42 +1331,33 @@ class SQLTargetRHandle : public StorageTargetRHandle {
   size_t rsize() const override { return size_; }
 
   size_t rread(uint8_t* buf, size_t size) override {
-    if (read_size_ + size > size_) {
-      size = size_ - read_size_;
-    }
-    if (size == 0) {
-      return 0;
-    }
-
-    if (sqlite3_blob_read(blob_, buf, static_cast<int>(size), static_cast<int>(read_size_)) != SQLITE_OK) {
-      LOG_ERROR << "Could not read from blob: " << db_.errmsg();
-      return 0;
+    auto red_bytes = stream_.readsome(reinterpret_cast<char*>(buf), static_cast<std::streamsize>(size));
+    if (!stream_.good()) {
+      LOG_ERROR << "Could not read from image file";
+      throw StorageTargetRHandle::ReadError("Could not read from image file");
     }
-    read_size_ += size;
-
-    return size;
+    return red_bytes;
   }
 
   void rclose() noexcept override {
-    sqlite3_blob_close(blob_);
+    stream_.close();
     closed_ = true;
   }
 
   bool isPartial() const noexcept override { return partial_; }
   std::unique_ptr<StorageTargetWHandle> toWriteHandle() override {
-    return std::unique_ptr<StorageTargetWHandle>(new SQLTargetWHandle(db_path_, target_, row_id_, size_));
+    return std::unique_ptr<StorageTargetWHandle>(new SQLTargetWHandle(db_path_, target_, image_path_, size_));
   }
 
  private:
   boost::filesystem::path db_path_;
   SQLite3Guard db_;
   Uptane::Target target_;
   size_t size_;
-  size_t read_size_;
   bool closed_;
-  sqlite3_blob* blob_;
   bool partial_{false};
-  sqlite3_int64 row_id_{0};
+  boost::filesystem::path image_path_;
+  std::ifstream stream_;
 };
 
 std::unique_ptr<StorageTargetRHandle> SQLStorage::openTargetFile(const Uptane::Target& target) {
@@ -1419,17 +1385,7 @@ void SQLStorage::removeTargetFile(const std::string& filename) {
     LOG_ERROR << "Statement step failure: " << db.errmsg();
     throw std::runtime_error("Could not remove target file");
   }
-
-  statement = db.prepareStatement<std::string>("DELETE FROM target_images_data WHERE filename=?;", filename);
-
-  if (statement.step() != SQLITE_DONE) {
-    LOG_ERROR << "Statement step failure: " << db.errmsg();
-    throw std::runtime_error("Could not remove target file");
-  }
-
-  if (sqlite3_changes(db.get()) != 1) {
-    throw std::runtime_error("Target file " + filename + " not found");
-  }
+  boost::filesystem::remove(images_path_ / filename);
 
   db.commitTransaction();
 }

src/libaktualizr/storage/sqlstorage.h

@@ -14,6 +14,7 @@ extern const std::vector<std::string> libaktualizr_schema_rollback_migrations;
 extern const std::string libaktualizr_current_schema;
 extern const int libaktualizr_current_schema_version;
 
+class SQLTargetRHandle;
 class SQLStorage : public SQLStorageBase, public INvStorage {
  public:
   friend class SQLTargetWHandle;
@@ -86,6 +87,7 @@ class SQLStorage : public SQLStorageBase, public INvStorage {
 
  private:
   void cleanMetaVersion(Uptane::RepositoryType repo, const Uptane::Role& role);
+  friend SQLTargetRHandle;
 };
 
 #endif  // SQLSTORAGE_H_

src/libaktualizr/storage/sqlstorage_base.cc

@@ -77,6 +77,42 @@ bool SQLStorageBase::dbMigrateForward(int version_from, int version_to) {
   }
 
   for (int32_t k = version_from + 1; k <= version_to; k++) {
+    if (k == 18) {  // Hack, because we can't write to the file from sql scripts.
+
+      auto statement = db.prepareStatement("SELECT rowid, filename FROM target_images_data;");
+
+      while (statement.step() == SQLITE_ROW) {
+        sqlite3_blob* blob{nullptr};
+        if (sqlite3_blob_open(db.get(), "main", "target_images_data", "image_data", statement.get_result_col_int(0), 1,
+                              &blob) != SQLITE_OK) {
+          LOG_ERROR << "Can't open blob: " << db.errmsg();
+          return false;
+        }
+        int buff_size = 1024;
+        char buf[buff_size];
+        int total_red_bytes = 0;
+        int blob_size = sqlite3_blob_bytes(blob);
+        boost::filesystem::create_directories(images_path_);
+
+        std::ofstream image_file((images_path_ / (*statement.get_result_col_str(1))).string());
+        if (!image_file.good()) {
+          throw std::runtime_error(std::string("Error opening image file for writting"));
+        }
+
+        while (total_red_bytes < blob_size) {
+          int read_bytes = ((blob_size - total_red_bytes) > buff_size) ? buff_size : (blob_size - total_red_bytes);
+          if (sqlite3_blob_read(blob, buf, read_bytes, static_cast<int>(total_red_bytes)) != SQLITE_OK) {
+            LOG_ERROR << "Could not read from blob: " << db.errmsg();
+            sqlite3_blob_close(blob);
+            return false;
+          }
+          image_file.write(buf, static_cast<std::streamsize>(read_bytes));
+          total_red_bytes += read_bytes;
+        }
+        sqlite3_blob_close(blob);
+        image_file.close();
+      }
+    }
     auto result_code = db.exec(schema_migrations_.at(static_cast<size_t>(k)), nullptr, nullptr);
     if (result_code != SQLITE_OK) {
       LOG_ERROR << "Can't migrate db from version " << (k - 1) << " to version " << k << ": " << db.errmsg();

src/libaktualizr/storage/sqlstorage_base.h

@@ -24,6 +24,7 @@ class SQLStorageBase {
  protected:
   SQLite3Guard dbConnection() const;
   boost::filesystem::path sqldb_path_;
+  boost::filesystem::path images_path_{sqldb_path_.parent_path() / "images"};
   bool readonly_{false};
   const std::vector<std::string> schema_migrations_;
   std::vector<std::string> schema_rollback_migrations_;

src/libaktualizr/storage/storage_common_test.cc

@@ -477,7 +477,6 @@ TEST(storage, store_target) {
   // write stream
   {
     std::unique_ptr<StorageTargetWHandle> fhandle = storage->allocateTargetFile(false, target);
-
     std::stringstream("ab") >> *fhandle;
   }