diff --git a/engine/controllers/hardware.cc b/engine/controllers/hardware.cc index 39a109750..8b7884710 100644 --- a/engine/controllers/hardware.cc +++ b/engine/controllers/hardware.cc @@ -38,7 +38,7 @@ void Hardware::Activate( ahc.gpus.push_back(g.asInt()); } } - std::sort(ahc.gpus.begin(), ahc.gpus.end()); + if (!hw_svc_->IsValidConfig(ahc)) { Json::Value ret; ret["message"] = "Invalid GPU index provided."; diff --git a/engine/database/hardware.cc b/engine/database/hardware.cc index ff2eb853a..2ee1db968 100644 --- a/engine/database/hardware.cc +++ b/engine/database/hardware.cc @@ -5,14 +5,15 @@ namespace cortex::db { -Hardwares::Hardwares() : db_(cortex::db::Database::GetInstance().db()) {} +Hardware::Hardware() : db_(cortex::db::Database::GetInstance().db()) {} -Hardwares::Hardwares(SQLite::Database& db) : db_(db) {} +Hardware::Hardware(SQLite::Database& db) : db_(db) {} -Hardwares::~Hardwares() {} + +Hardware::~Hardware() {} cpp::result, std::string> -Hardwares::LoadHardwareList() const { +Hardware::LoadHardwareList() const { try { db_.exec("BEGIN TRANSACTION;"); cortex::utils::ScopeExit se([this] { db_.exec("COMMIT;"); }); @@ -20,7 +21,7 @@ Hardwares::LoadHardwareList() const { SQLite::Statement query( db_, "SELECT uuid, type, " - "hardware_id, software_id, activated FROM hardware"); + "hardware_id, software_id, activated, priority FROM hardware"); while (query.executeStep()) { HardwareEntry entry; @@ -29,6 +30,7 @@ Hardwares::LoadHardwareList() const { entry.hardware_id = query.getColumn(2).getInt(); entry.software_id = query.getColumn(3).getInt(); entry.activated = query.getColumn(4).getInt(); + entry.priority = query.getColumn(5).getInt(); entries.push_back(entry); } return entries; @@ -37,19 +39,20 @@ Hardwares::LoadHardwareList() const { return cpp::fail(e.what()); } } -cpp::result Hardwares::AddHardwareEntry( +cpp::result Hardware::AddHardwareEntry( const HardwareEntry& new_entry) { try { SQLite::Statement insert( db_, "INSERT INTO hardware (uuid, type, " - "hardware_id, software_id, activated) VALUES (?, ?, " - "?, ?, ?)"); + "hardware_id, software_id, activated, priority) VALUES (?, ?, " + "?, ?, ?, ?)"); insert.bind(1, new_entry.uuid); insert.bind(2, new_entry.type); insert.bind(3, new_entry.hardware_id); insert.bind(4, new_entry.software_id); insert.bind(5, new_entry.activated); + insert.bind(6, new_entry.priority); insert.exec(); CTL_INF("Inserted: " << new_entry.ToJsonString()); return true; @@ -58,17 +61,19 @@ cpp::result Hardwares::AddHardwareEntry( return cpp::fail(e.what()); } } -cpp::result Hardwares::UpdateHardwareEntry( +cpp::result Hardware::UpdateHardwareEntry( const std::string& id, const HardwareEntry& updated_entry) { try { - SQLite::Statement upd(db_, - "UPDATE hardware " - "SET hardware_id = ?, software_id = ?, activated = ? " - "WHERE uuid = ?"); + SQLite::Statement upd( + db_, + "UPDATE hardware " + "SET hardware_id = ?, software_id = ?, activated = ?, priority = ? " + "WHERE uuid = ?"); upd.bind(1, updated_entry.hardware_id); upd.bind(2, updated_entry.software_id); upd.bind(3, updated_entry.activated); - upd.bind(4, id); + upd.bind(4, updated_entry.priority); + upd.bind(5, id); if (upd.exec() == 1) { CTL_INF("Updated: " << updated_entry.ToJsonString()); return true; @@ -79,7 +84,7 @@ cpp::result Hardwares::UpdateHardwareEntry( } } -cpp::result Hardwares::DeleteHardwareEntry( +cpp::result Hardware::DeleteHardwareEntry( const std::string& id) { try { SQLite::Statement del(db_, "DELETE from hardware WHERE uuid = ?"); diff --git a/engine/database/hardware.h b/engine/database/hardware.h index 0966d58a3..04d0bbda1 100644 --- a/engine/database/hardware.h +++ b/engine/database/hardware.h @@ -4,8 +4,8 @@ #include #include #include -#include "utils/result.hpp" #include "utils/json_helper.h" +#include "utils/result.hpp" namespace cortex::db { struct HardwareEntry { @@ -14,6 +14,7 @@ struct HardwareEntry { int hardware_id; int software_id; bool activated; + int priority; std::string ToJsonString() const { Json::Value root; root["uuid"] = uuid; @@ -21,26 +22,26 @@ struct HardwareEntry { root["hardware_id"] = hardware_id; root["software_id"] = software_id; root["activated"] = activated; + root["priority"] = priority; return json_helper::DumpJsonString(root); } }; -class Hardwares { +class Hardware { private: SQLite::Database& db_; - public: - Hardwares(); - Hardwares(SQLite::Database& db); - ~Hardwares(); + Hardware(); + Hardware(SQLite::Database& db); + ~Hardware(); cpp::result, std::string> LoadHardwareList() const; - cpp::result AddHardwareEntry(const HardwareEntry& new_entry); + cpp::result AddHardwareEntry( + const HardwareEntry& new_entry); cpp::result UpdateHardwareEntry( const std::string& id, const HardwareEntry& updated_entry); - cpp::result DeleteHardwareEntry( - const std::string& id); + cpp::result DeleteHardwareEntry(const std::string& id); }; } // namespace cortex::db \ No newline at end of file diff --git a/engine/migrations/db_helper.h b/engine/migrations/db_helper.h index 0990426bf..867e871ff 100644 --- a/engine/migrations/db_helper.h +++ b/engine/migrations/db_helper.h @@ -4,23 +4,28 @@ namespace cortex::mgr { #include #include -#include #include +#include -inline bool ColumnExists(SQLite::Database& db, const std::string& table_name, const std::string& column_name) { - try { - SQLite::Statement query(db, "SELECT " + column_name + " FROM " + table_name + " LIMIT 0"); - return true; - } catch (std::exception&) { - return false; - } +inline bool ColumnExists(SQLite::Database& db, const std::string& table_name, + const std::string& column_name) { + try { + SQLite::Statement query( + db, "SELECT " + column_name + " FROM " + table_name + " LIMIT 0"); + return true; + } catch (std::exception&) { + return false; + } } -inline void AddColumnIfNotExists(SQLite::Database& db, const std::string& table_name, - const std::string& column_name, const std::string& column_type) { - if (!ColumnExists(db, table_name, column_name)) { - std::string sql = "ALTER TABLE " + table_name + " ADD COLUMN " + column_name + " " + column_type; - db.exec(sql); - } +inline void AddColumnIfNotExists(SQLite::Database& db, + const std::string& table_name, + const std::string& column_name, + const std::string& column_type) { + if (!ColumnExists(db, table_name, column_name)) { + std::string sql = "ALTER TABLE " + table_name + " ADD COLUMN " + + column_name + " " + column_type; + db.exec(sql); + } } -} \ No newline at end of file +} // namespace cortex::mgr diff --git a/engine/migrations/migration_manager.cc b/engine/migrations/migration_manager.cc index 0e2e41e4e..6936f45a0 100644 --- a/engine/migrations/migration_manager.cc +++ b/engine/migrations/migration_manager.cc @@ -7,7 +7,7 @@ #include "utils/widechar_conv.h" #include "v0/migration.h" #include "v1/migration.h" - +#include "v2/migration.h" namespace cortex::migr { namespace { @@ -141,9 +141,11 @@ cpp::result MigrationManager::DoUpFolderStructure( switch (version) { case 0: return v0::MigrateFolderStructureUp(); - break; case 1: return v1::MigrateFolderStructureUp(); + case 2: + return v2::MigrateFolderStructureUp(); + break; default: @@ -155,9 +157,10 @@ cpp::result MigrationManager::DoDownFolderStructure( switch (version) { case 0: return v0::MigrateFolderStructureDown(); - break; case 1: return v1::MigrateFolderStructureDown(); + case 2: + return v2::MigrateFolderStructureDown(); break; default: @@ -191,9 +194,10 @@ cpp::result MigrationManager::DoUpDB(int version) { switch (version) { case 0: return v0::MigrateDBUp(db_); - break; case 1: return v1::MigrateDBUp(db_); + case 2: + return v2::MigrateDBUp(db_); break; default: @@ -205,9 +209,10 @@ cpp::result MigrationManager::DoDownDB(int version) { switch (version) { case 0: return v0::MigrateDBDown(db_); - break; case 1: return v1::MigrateDBDown(db_); + case 2: + return v2::MigrateDBDown(db_); break; default: diff --git a/engine/migrations/schema_version.h b/engine/migrations/schema_version.h index 1e64110e3..5739040d0 100644 --- a/engine/migrations/schema_version.h +++ b/engine/migrations/schema_version.h @@ -1,4 +1,5 @@ #pragma once //Track the current schema version -#define SCHEMA_VERSION 1 \ No newline at end of file +#define SCHEMA_VERSION 2 + diff --git a/engine/migrations/v2/migration.h b/engine/migrations/v2/migration.h new file mode 100644 index 000000000..54b79f666 --- /dev/null +++ b/engine/migrations/v2/migration.h @@ -0,0 +1,210 @@ +#pragma once +#include +#include +#include +#include "migrations/db_helper.h" +#include "utils/file_manager_utils.h" +#include "utils/logging_utils.h" +#include "utils/result.hpp" + +namespace cortex::migr::v2 { +// Data folder +namespace fmu = file_manager_utils; + +// cortexcpp +// |__ models +// | |__ cortex.so +// | |__ tinyllama +// | |__ gguf +// |__ engines +// | |__ cortex.llamacpp +// | |__ deps +// | |__ windows-amd64-avx +// |__ logs +// +inline cpp::result MigrateFolderStructureUp() { + if (!std::filesystem::exists(fmu::GetCortexDataPath() / "models")) { + std::filesystem::create_directory(fmu::GetCortexDataPath() / "models"); + } + + if (!std::filesystem::exists(fmu::GetCortexDataPath() / "engines")) { + std::filesystem::create_directory(fmu::GetCortexDataPath() / "engines"); + } + + if (!std::filesystem::exists(fmu::GetCortexDataPath() / "logs")) { + std::filesystem::create_directory(fmu::GetCortexDataPath() / "logs"); + } + + return true; +} + +inline cpp::result MigrateFolderStructureDown() { + // CTL_INF("Folder structure already up to date!"); + return true; +} + +// Database +inline cpp::result MigrateDBUp(SQLite::Database& db) { + try { + db.exec( + "CREATE TABLE IF NOT EXISTS schema_version ( version INTEGER PRIMARY " + "KEY);"); + + // models + { + // Check if the table exists + SQLite::Statement query(db, + "SELECT name FROM sqlite_master WHERE " + "type='table' AND name='models'"); + auto table_exists = query.executeStep(); + + if (table_exists) { + // Alter existing table + cortex::mgr::AddColumnIfNotExists(db, "models", "metadata", "TEXT"); + } else { + // Create new table + db.exec( + "CREATE TABLE models (" + "model_id TEXT PRIMARY KEY," + "author_repo_id TEXT," + "branch_name TEXT," + "path_to_model_yaml TEXT," + "model_alias TEXT," + "model_format TEXT," + "model_source TEXT," + "status TEXT," + "engine TEXT," + "metadata TEXT" + ")"); + } + } + + // Check if the table exists + SQLite::Statement hw_query(db, + "SELECT name FROM sqlite_master WHERE " + "type='table' AND name='hardware'"); + auto hw_table_exists = hw_query.executeStep(); + + if (hw_table_exists) { + // Alter existing table + cortex::mgr::AddColumnIfNotExists(db, "hardware", "priority", "INTEGER"); + } else { + db.exec( + "CREATE TABLE IF NOT EXISTS hardware (" + "uuid TEXT PRIMARY KEY, " + "type TEXT NOT NULL, " + "hardware_id INTEGER NOT NULL, " + "software_id INTEGER NOT NULL, " + "activated INTEGER NOT NULL CHECK (activated IN (0, 1)), " + "priority INTEGER); "); + } + + // engines + db.exec( + "CREATE TABLE IF NOT EXISTS engines (" + "id INTEGER PRIMARY KEY AUTOINCREMENT," + "engine_name TEXT," + "type TEXT," + "api_key TEXT," + "url TEXT," + "version TEXT," + "variant TEXT," + "status TEXT," + "metadata TEXT," + "date_created TEXT DEFAULT CURRENT_TIMESTAMP," + "date_updated TEXT DEFAULT CURRENT_TIMESTAMP," + "UNIQUE(engine_name, variant));"); + + // CTL_INF("Database migration up completed successfully."); + return true; + } catch (const std::exception& e) { + CTL_WRN("Migration up failed: " << e.what()); + return cpp::fail(e.what()); + } +}; + +inline cpp::result MigrateDBDown(SQLite::Database& db) { + try { + // models + { + SQLite::Statement query(db, + "SELECT name FROM sqlite_master WHERE " + "type='table' AND name='models'"); + auto table_exists = query.executeStep(); + if (table_exists) { + // Create a new table with the old schema + db.exec( + "CREATE TABLE models_old (" + "model_id TEXT PRIMARY KEY," + "author_repo_id TEXT," + "branch_name TEXT," + "path_to_model_yaml TEXT," + "model_alias TEXT," + "model_format TEXT," + "model_source TEXT," + "status TEXT," + "engine TEXT" + ")"); + + // Copy data from the current table to the new table + db.exec( + "INSERT INTO models_old (model_id, author_repo_id, branch_name, " + "path_to_model_yaml, model_alias, model_format, model_source, " + "status, engine) " + "SELECT model_id, author_repo_id, branch_name, path_to_model_yaml, " + "model_alias, model_format, model_source, status, engine FROM " + "models"); + + // Drop the current table + db.exec("DROP TABLE models"); + + // Rename the new table to the original name + db.exec("ALTER TABLE models_old RENAME TO models"); + } + } + + // hardware + { + SQLite::Statement query(db, + "SELECT name FROM sqlite_master WHERE " + "type='table' AND name='hardware'"); + auto table_exists = query.executeStep(); + if (table_exists) { + // Create a new table with the old schema + db.exec( + "CREATE TABLE hardware_old (" + "uuid TEXT PRIMARY KEY, " + "type TEXT NOT NULL, " + "hardware_id INTEGER NOT NULL, " + "software_id INTEGER NOT NULL, " + "activated INTEGER NOT NULL CHECK (activated IN (0, 1))" + ")"); + + // Copy data from the current table to the new table + db.exec( + "INSERT INTO hardware_old (uuid, type, hardware_id, " + "software_id, activated) " + "SELECT uuid, type, hardware_id, software_id, " + "activated FROM hardware"); + + // Drop the current table + db.exec("DROP TABLE hardware"); + + // Rename the new table to the original name + db.exec("ALTER TABLE hardware_old RENAME TO hardware"); + } + } + + // engines + { + // do nothing + } + // CTL_INF("Migration down completed successfully."); + return true; + } catch (const std::exception& e) { + CTL_WRN("Migration down failed: " << e.what()); + return cpp::fail(e.what()); + } +} + +}; // namespace cortex::migr::v2 \ No newline at end of file diff --git a/engine/services/hardware_service.cc b/engine/services/hardware_service.cc index 681ca7578..25be78873 100644 --- a/engine/services/hardware_service.cc +++ b/engine/services/hardware_service.cc @@ -34,7 +34,7 @@ bool TryConnectToServer(const std::string& host, int port) { HardwareInfo HardwareService::GetHardwareInfo() { // append active state - cortex::db::Hardwares hw_db; + cortex::db::Hardware hw_db; auto gpus = cortex::hw::GetGPUInfo(); auto res = hw_db.LoadHardwareList(); if (res.has_value()) { @@ -191,31 +191,61 @@ bool HardwareService::Restart(const std::string& host, int port) { return true; } +// GPU identifiers are given as integer indices or as UUID strings. GPU UUID strings +// should follow the same format as given by nvidia-smi, such as GPU-8932f937-d72c-4106-c12f-20bd9faed9f6. +// However, for convenience, abbreviated forms are allowed; simply specify enough digits +// from the beginning of the GPU UUID to uniquely identify that GPU in the target system. +// For example, CUDA_VISIBLE_DEVICES=GPU-8932f937 may be a valid way to refer to the above GPU UUID, +// assuming no other GPU in the system shares this prefix. Only the devices whose index +// is present in the sequence are visible to CUDA applications and they are enumerated +// in the order of the sequence. If one of the indices is invalid, only the devices whose +// index precedes the invalid index are visible to CUDA applications. For example, setting +// CUDA_VISIBLE_DEVICES to 2,1 causes device 0 to be invisible and device 2 to be enumerated +// before device 1. Setting CUDA_VISIBLE_DEVICES to 0,2,-1,1 causes devices 0 and 2 to be +// visible and device 1 to be invisible. MIG format starts with MIG keyword and GPU UUID +// should follow the same format as given by nvidia-smi. +// For example, MIG-GPU-8932f937-d72c-4106-c12f-20bd9faed9f6/1/2. +// Only single MIG instance enumeration is supported. bool HardwareService::SetActivateHardwareConfig( const cortex::hw::ActivateHardwareConfig& ahc) { // Note: need to map software_id and hardware_id // Update to db - cortex::db::Hardwares hw_db; + cortex::db::Hardware hw_db; + // copy all gpu information to new vector + auto ahc_gpus = ahc.gpus; auto activate = [&ahc](int software_id) { return std::count(ahc.gpus.begin(), ahc.gpus.end(), software_id) > 0; }; + auto priority = [&ahc](int software_id) -> int { + for (size_t i = 0; i < ahc.gpus.size(); i++) { + if (ahc.gpus[i] == software_id) + return i; + break; + } + return INT_MAX; + }; + auto res = hw_db.LoadHardwareList(); if (res.has_value()) { bool need_update = false; - std::vector activated_ids; + std::vector> activated_ids; // Check if need to update for (auto const& e : res.value()) { if (e.activated) { - activated_ids.push_back(e.software_id); + activated_ids.push_back(std::pair(e.software_id, e.priority)); } } std::sort(activated_ids.begin(), activated_ids.end()); - if (ahc.gpus.size() != activated_ids.size()) { + std::sort(ahc_gpus.begin(), ahc_gpus.end()); + if (ahc_gpus.size() != activated_ids.size()) { need_update = true; } else { - for (size_t i = 0; i < ahc.gpus.size(); i++) { - if (ahc.gpus[i] != activated_ids[i]) + for (size_t i = 0; i < ahc_gpus.size(); i++) { + // if activated id or priority changes + if (ahc_gpus[i] != activated_ids[i].first || + i != activated_ids[i].second) need_update = true; + break; } } @@ -227,6 +257,7 @@ bool HardwareService::SetActivateHardwareConfig( // Need to update, proceed for (auto& e : res.value()) { e.activated = activate(e.software_id); + e.priority = priority(e.software_id); auto res = hw_db.UpdateHardwareEntry(e.uuid, e); if (res.has_error()) { CTL_WRN(res.error()); @@ -240,14 +271,14 @@ bool HardwareService::SetActivateHardwareConfig( void HardwareService::UpdateHardwareInfos() { using HwEntry = cortex::db::HardwareEntry; auto gpus = cortex::hw::GetGPUInfo(); - cortex::db::Hardwares hw_db; + cortex::db::Hardware hw_db; auto b = hw_db.LoadHardwareList(); - std::vector activated_gpu_bf; + std::vector> activated_gpu_bf; std::string debug_b; for (auto const& he : b.value()) { if (he.type == "gpu" && he.activated) { debug_b += std::to_string(he.software_id) + " "; - activated_gpu_bf.push_back(he.software_id); + activated_gpu_bf.push_back(std::pair(he.software_id, he.priority)); } } CTL_INF("Activated GPUs before: " << debug_b); @@ -258,7 +289,8 @@ void HardwareService::UpdateHardwareInfos() { .type = "gpu", .hardware_id = std::stoi(gpu.id), .software_id = std::stoi(gpu.id), - .activated = true}); + .activated = true, + .priority = INT_MAX}); if (res.has_error()) { CTL_WRN(res.error()); } @@ -266,24 +298,26 @@ void HardwareService::UpdateHardwareInfos() { auto a = hw_db.LoadHardwareList(); std::vector a_gpu; - std::vector activated_gpu_af; + std::vector> activated_gpu_af; std::string debug_a; for (auto const& he : a.value()) { if (he.type == "gpu" && he.activated) { debug_a += std::to_string(he.software_id) + " "; - activated_gpu_af.push_back(he.software_id); + activated_gpu_af.push_back(std::pair(he.software_id, he.priority)); } } CTL_INF("Activated GPUs after: " << debug_a); // if hardware list changes, need to restart - std::sort(activated_gpu_bf.begin(), activated_gpu_bf.end()); - std::sort(activated_gpu_af.begin(), activated_gpu_af.end()); + std::sort(activated_gpu_bf.begin(), activated_gpu_bf.end(), + [](auto& p1, auto& p2) { return p1.second < p2.second; }); + std::sort(activated_gpu_af.begin(), activated_gpu_af.end(), + [](auto& p1, auto& p2) { return p1.second < p2.second; }); bool need_restart = false; if (activated_gpu_bf.size() != activated_gpu_af.size()) { need_restart = true; } else { for (size_t i = 0; i < activated_gpu_bf.size(); i++) { - if (activated_gpu_bf[i] != activated_gpu_af[i]) { + if (activated_gpu_bf[i].first != activated_gpu_af[i].first) { need_restart = true; break; } @@ -291,7 +325,8 @@ void HardwareService::UpdateHardwareInfos() { } #if defined(_WIN32) || defined(_WIN64) || defined(__linux__) - if (!gpus.empty()) { + bool has_deactivated_gpu = a.value().size() != activated_gpu_af.size(); + if (!gpus.empty() && has_deactivated_gpu) { const char* value = std::getenv("CUDA_VISIBLE_DEVICES"); if (value) { LOG_INFO << "CUDA_VISIBLE_DEVICES: " << value; @@ -303,7 +338,11 @@ void HardwareService::UpdateHardwareInfos() { if (need_restart) { CTL_INF("Need restart"); - ahc_ = {.gpus = activated_gpu_af}; + std::vector gpus; + for (auto const& p : activated_gpu_af) { + gpus.push_back(p.first); + } + ahc_ = {.gpus = gpus}; } } @@ -311,7 +350,7 @@ bool HardwareService::IsValidConfig( const cortex::hw::ActivateHardwareConfig& ahc) { if (ahc.gpus.empty()) return true; - cortex::db::Hardwares hw_db; + cortex::db::Hardware hw_db; auto is_valid = [&ahc](int software_id) { return std::count(ahc.gpus.begin(), ahc.gpus.end(), software_id) > 0; };