From 55739c355e7a42716645254835527c2b890d69c3 Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Sun, 29 Dec 2024 09:36:07 +0700 Subject: [PATCH] chore: db: models to service --- engine/cli/command_line_parser.cc | 9 ++- engine/cli/commands/chat_completion_cmd.cc | 3 +- engine/cli/commands/chat_completion_cmd.h | 4 + engine/cli/commands/model_start_cmd.cc | 2 +- engine/cli/commands/model_start_cmd.h | 9 ++- engine/cli/commands/run_cmd.cc | 18 ++--- engine/cli/commands/run_cmd.h | 7 +- engine/controllers/models.cc | 15 ++-- engine/controllers/models.h | 7 +- engine/main.cc | 8 +- engine/services/database_service.cc | 57 +++++++++++++- engine/services/database_service.h | 21 +++++ engine/services/model_service.cc | 92 +++++++++------------- engine/services/model_service.h | 11 +-- engine/services/model_source_service.cc | 62 +++++++-------- engine/services/model_source_service.h | 15 ++-- 16 files changed, 204 insertions(+), 136 deletions(-) diff --git a/engine/cli/command_line_parser.cc b/engine/cli/command_line_parser.cc index d275dbf57..6f8f227e6 100644 --- a/engine/cli/command_line_parser.cc +++ b/engine/cli/command_line_parser.cc @@ -178,7 +178,7 @@ void CommandLineParser::SetupCommonCommands() { return; commands::RunCmd rc(cml_data_.config.apiServerHost, std::stoi(cml_data_.config.apiServerPort), - cml_data_.model_id, engine_service_); + cml_data_.model_id, db_service_, engine_service_); rc.Exec(cml_data_.run_detach, run_settings_); }); } @@ -217,9 +217,10 @@ void CommandLineParser::SetupModelCommands() { CLI_LOG(model_start_cmd->help()); return; }; - commands::ModelStartCmd().Exec(cml_data_.config.apiServerHost, - std::stoi(cml_data_.config.apiServerPort), - cml_data_.model_id, run_settings_); + commands::ModelStartCmd(db_service_) + .Exec(cml_data_.config.apiServerHost, + std::stoi(cml_data_.config.apiServerPort), cml_data_.model_id, + run_settings_); }); auto stop_model_cmd = diff --git a/engine/cli/commands/chat_completion_cmd.cc b/engine/cli/commands/chat_completion_cmd.cc index 77d222176..77ee4fca3 100644 --- a/engine/cli/commands/chat_completion_cmd.cc +++ b/engine/cli/commands/chat_completion_cmd.cc @@ -56,10 +56,9 @@ void ChatCompletionCmd::Exec(const std::string& host, int port, const std::string& model_handle, std::string msg) { namespace fs = std::filesystem; namespace fmu = file_manager_utils; - cortex::db::Models modellist_handler; config::YamlHandler yaml_handler; try { - auto model_entry = modellist_handler.GetModelInfo(model_handle); + auto model_entry = db_service_->GetModelInfo(model_handle); if (model_entry.has_error()) { CLI_LOG("Error: " + model_entry.error()); return; diff --git a/engine/cli/commands/chat_completion_cmd.h b/engine/cli/commands/chat_completion_cmd.h index a784b4604..44de5d256 100644 --- a/engine/cli/commands/chat_completion_cmd.h +++ b/engine/cli/commands/chat_completion_cmd.h @@ -3,16 +3,20 @@ #include #include #include "config/model_config.h" +#include "services/database_service.h" namespace commands { class ChatCompletionCmd { public: + explicit ChatCompletionCmd(std::shared_ptr db_service) + : db_service_(db_service) {} void Exec(const std::string& host, int port, const std::string& model_handle, std::string msg); void Exec(const std::string& host, int port, const std::string& model_handle, const config::ModelConfig& mc, std::string msg); private: + std::shared_ptr db_service_; std::vector histories_; }; } // namespace commands diff --git a/engine/cli/commands/model_start_cmd.cc b/engine/cli/commands/model_start_cmd.cc index 12aec944d..ef5d5c1f2 100644 --- a/engine/cli/commands/model_start_cmd.cc +++ b/engine/cli/commands/model_start_cmd.cc @@ -13,7 +13,7 @@ bool ModelStartCmd::Exec( const std::unordered_map& options, bool print_success_log) { std::optional model_id = - SelectLocalModel(host, port, model_handle); + SelectLocalModel(host, port, model_handle, *db_service_); if (!model_id.has_value()) { return false; diff --git a/engine/cli/commands/model_start_cmd.h b/engine/cli/commands/model_start_cmd.h index 124ef463d..c69bfc32a 100644 --- a/engine/cli/commands/model_start_cmd.h +++ b/engine/cli/commands/model_start_cmd.h @@ -3,16 +3,23 @@ #include #include #include "json/json.h" +#include "services/database_service.h" namespace commands { class ModelStartCmd { public: + explicit ModelStartCmd(std::shared_ptr db_service) + : db_service_(db_service) {} bool Exec(const std::string& host, int port, const std::string& model_handle, const std::unordered_map& options, bool print_success_log = true); - private: + + private: bool UpdateConfig(Json::Value& data, const std::string& key, const std::string& value); + + private: + std::shared_ptr db_service_; }; } // namespace commands diff --git a/engine/cli/commands/run_cmd.cc b/engine/cli/commands/run_cmd.cc index 91a813d64..c01d3d806 100644 --- a/engine/cli/commands/run_cmd.cc +++ b/engine/cli/commands/run_cmd.cc @@ -14,12 +14,11 @@ namespace commands { std::optional SelectLocalModel(std::string host, int port, - const std::string& model_handle) { + const std::string& model_handle, + DatabaseService& db_service) { std::optional model_id = model_handle; - cortex::db::Models modellist_handler; - if (model_handle.empty()) { - auto all_local_models = modellist_handler.LoadModelList(); + auto all_local_models = db_service.LoadModelList(); if (all_local_models.has_error() || all_local_models.value().empty()) { CLI_LOG("No local models available!"); return std::nullopt; @@ -42,7 +41,7 @@ std::optional SelectLocalModel(std::string host, int port, CLI_LOG("Selected: " << selection.value()); } } else { - auto related_models_ids = modellist_handler.FindRelatedModel(model_handle); + auto related_models_ids = db_service.FindRelatedModel(model_handle); if (related_models_ids.has_error() || related_models_ids.value().empty()) { auto result = ModelPullCmd().Exec(host, port, model_handle); if (!result) { @@ -69,19 +68,18 @@ std::optional SelectLocalModel(std::string host, int port, void RunCmd::Exec(bool run_detach, const std::unordered_map& options) { std::optional model_id = - SelectLocalModel(host_, port_, model_handle_); + SelectLocalModel(host_, port_, model_handle_, *db_service_); if (!model_id.has_value()) { return; } - cortex::db::Models modellist_handler; config::YamlHandler yaml_handler; auto address = host_ + ":" + std::to_string(port_); try { namespace fs = std::filesystem; namespace fmu = file_manager_utils; - auto model_entry = modellist_handler.GetModelInfo(*model_id); + auto model_entry = db_service_->GetModelInfo(*model_id); if (model_entry.has_error()) { CLI_LOG("Error: " + model_entry.error()); return; @@ -128,7 +126,7 @@ void RunCmd::Exec(bool run_detach, mc.engine.find(kLlamaEngine) == std::string::npos) || !commands::ModelStatusCmd().IsLoaded(host_, port_, *model_id)) { - auto res = commands::ModelStartCmd() + auto res = commands::ModelStartCmd(db_service_) .Exec(host_, port_, *model_id, options, false /*print_success_log*/); if (!res) { @@ -144,7 +142,7 @@ void RunCmd::Exec(bool run_detach, << commands::GetCortexBinary() << " run " << *model_id << "` for interactive chat shell"); } else { - ChatCompletionCmd().Exec(host_, port_, *model_id, mc, ""); + ChatCompletionCmd(db_service_).Exec(host_, port_, *model_id, mc, ""); } } } catch (const std::exception& e) { diff --git a/engine/cli/commands/run_cmd.h b/engine/cli/commands/run_cmd.h index b22b064f9..ec5c61fd3 100644 --- a/engine/cli/commands/run_cmd.h +++ b/engine/cli/commands/run_cmd.h @@ -2,20 +2,24 @@ #include #include +#include "services/database_service.h" #include "services/engine_service.h" namespace commands { std::optional SelectLocalModel(std::string host, int port, - const std::string& model_handle); + const std::string& model_handle, + DatabaseService& db_service); class RunCmd { public: explicit RunCmd(std::string host, int port, std::string model_handle, + std::shared_ptr db_service, std::shared_ptr engine_service) : host_{std::move(host)}, port_{port}, model_handle_{std::move(model_handle)}, + db_service_(db_service), engine_service_{engine_service} {}; void Exec(bool chat_flag, @@ -25,6 +29,7 @@ class RunCmd { std::string host_; int port_; std::string model_handle_; + std::shared_ptr db_service_; std::shared_ptr engine_service_; }; } // namespace commands diff --git a/engine/controllers/models.cc b/engine/controllers/models.cc index 1c33ab1dc..1a501287d 100644 --- a/engine/controllers/models.cc +++ b/engine/controllers/models.cc @@ -165,10 +165,9 @@ void Models::ListModel( model_service_->ForceIndexingModelList(); // Iterate through directory - cortex::db::Models modellist_handler; config::YamlHandler yaml_handler; - auto list_entry = modellist_handler.LoadModelList(); + auto list_entry = db_service_->LoadModelList(); if (list_entry) { for (const auto& model_entry : list_entry.value()) { try { @@ -256,9 +255,8 @@ void Models::GetModel(const HttpRequestPtr& req, Json::Value ret; try { - cortex::db::Models modellist_handler; config::YamlHandler yaml_handler; - auto model_entry = modellist_handler.GetModelInfo(model_id); + auto model_entry = db_service_->GetModelInfo(model_id); if (model_entry.has_error()) { ret["id"] = model_id; ret["object"] = "model"; @@ -337,8 +335,7 @@ void Models::UpdateModel(const HttpRequestPtr& req, namespace fmu = file_manager_utils; auto json_body = *(req->getJsonObject()); try { - cortex::db::Models model_list_utils; - auto model_entry = model_list_utils.GetModelInfo(model_id); + auto model_entry = db_service_->GetModelInfo(model_id); config::YamlHandler yaml_handler; auto yaml_fp = fmu::ToAbsoluteCortexDataPath( fs::path(model_entry.value().path_to_model_yaml)); @@ -401,7 +398,6 @@ void Models::ImportModel( auto option = (*(req->getJsonObject())).get("option", "symlink").asString(); config::GGUFHandler gguf_handler; config::YamlHandler yaml_handler; - cortex::db::Models modellist_utils_obj; std::string model_yaml_path = (file_manager_utils::GetModelsContainerPath() / std::filesystem::path("imported") / std::filesystem::path(modelHandle + ".yml")) @@ -440,7 +436,7 @@ void Models::ImportModel( model_config.name = modelName.empty() ? model_config.name : modelName; yaml_handler.UpdateModelConfig(model_config); - if (modellist_utils_obj.AddModelEntry(model_entry).value()) { + if (db_service_->AddModelEntry(model_entry).value()) { yaml_handler.WriteYamlFile(model_yaml_path); std::string success_message = "Model is imported successfully!"; LOG_INFO << success_message; @@ -667,7 +663,6 @@ void Models::AddRemoteModel( config::RemoteModelConfig model_config; model_config.LoadFromJson(*(req->getJsonObject())); - cortex::db::Models modellist_utils_obj; std::string model_yaml_path = (file_manager_utils::GetModelsContainerPath() / std::filesystem::path("remote") / std::filesystem::path(model_handle + ".yml")) @@ -683,7 +678,7 @@ void Models::AddRemoteModel( "openai"}; std::filesystem::create_directories( std::filesystem::path(model_yaml_path).parent_path()); - if (modellist_utils_obj.AddModelEntry(model_entry).value()) { + if (db_service_->AddModelEntry(model_entry).value()) { model_config.SaveToYamlFile(model_yaml_path); std::string success_message = "Model is imported successfully!"; LOG_INFO << success_message; diff --git a/engine/controllers/models.h b/engine/controllers/models.h index 775562445..60053acdb 100644 --- a/engine/controllers/models.h +++ b/engine/controllers/models.h @@ -45,10 +45,12 @@ class Models : public drogon::HttpController { ADD_METHOD_TO(Models::GetModelSources, "/v1/models/sources", Get); METHOD_LIST_END - explicit Models(std::shared_ptr model_service, + explicit Models(std::shared_ptr db_service, + std::shared_ptr model_service, std::shared_ptr engine_service, std::shared_ptr mss) - : model_service_{model_service}, + : db_service_(db_service), + model_service_{model_service}, engine_service_{engine_service}, model_src_svc_(mss) {} @@ -105,6 +107,7 @@ class Models : public drogon::HttpController { std::function&& callback); private: + std::shared_ptr db_service_; std::shared_ptr model_service_; std::shared_ptr engine_service_; std::shared_ptr model_src_svc_; diff --git a/engine/main.cc b/engine/main.cc index 957049922..77f51c7fa 100644 --- a/engine/main.cc +++ b/engine/main.cc @@ -162,9 +162,9 @@ void RunServer(std::optional host, std::optional port, auto engine_service = std::make_shared( download_service, dylib_path_manager, db_service); auto inference_svc = std::make_shared(engine_service); - auto model_src_svc = std::make_shared(); + auto model_src_svc = std::make_shared(db_service); auto model_service = std::make_shared( - hw_service, download_service, inference_svc, engine_service); + db_service, hw_service, download_service, inference_svc, engine_service); inference_svc->SetModelService(model_service); auto file_watcher_srv = std::make_shared( @@ -179,8 +179,8 @@ void RunServer(std::optional host, std::optional port, auto thread_ctl = std::make_shared(thread_srv, message_srv); auto message_ctl = std::make_shared(message_srv); auto engine_ctl = std::make_shared(engine_service); - auto model_ctl = - std::make_shared(model_service, engine_service, model_src_svc); + auto model_ctl = std::make_shared(db_service, model_service, + engine_service, model_src_svc); auto event_ctl = std::make_shared(event_queue_ptr); auto pm_ctl = std::make_shared(); auto hw_ctl = std::make_shared(engine_service, hw_service); diff --git a/engine/services/database_service.cc b/engine/services/database_service.cc index d0ba2940e..d4cd977a9 100644 --- a/engine/services/database_service.cc +++ b/engine/services/database_service.cc @@ -71,5 +71,60 @@ cpp::result DatabaseService::DeleteHardwareEntry( const std::string& id) { return cortex::db::Hardware().DeleteHardwareEntry(id); } +// end hardware -// end hardware \ No newline at end of file +// begin models +cpp::result, std::string> +DatabaseService::LoadModelList() const { + return cortex::db::Models().LoadModelList(); +} + +cpp::result DatabaseService::GetModelInfo( + const std::string& identifier) const { + return cortex::db::Models().GetModelInfo(identifier); +} + +cpp::result DatabaseService::AddModelEntry( + ModelEntry new_entry) { + return cortex::db::Models().AddModelEntry(new_entry); +} + +cpp::result DatabaseService::UpdateModelEntry( + const std::string& identifier, const ModelEntry& updated_entry) { + return cortex::db::Models().UpdateModelEntry(identifier, updated_entry); +} + +cpp::result DatabaseService::DeleteModelEntry( + const std::string& identifier) { + return cortex::db::Models().DeleteModelEntry(identifier); +} + +cpp::result DatabaseService::DeleteModelEntryWithOrg( + const std::string& src) { + return cortex::db::Models().DeleteModelEntryWithOrg(src); +} + +cpp::result DatabaseService::DeleteModelEntryWithRepo( + const std::string& src) { + return cortex::db::Models().DeleteModelEntryWithRepo(src); +} + +cpp::result, std::string> +DatabaseService::FindRelatedModel(const std::string& identifier) const { + return cortex::db::Models().FindRelatedModel(identifier); +} + +bool DatabaseService::HasModel(const std::string& identifier) const { + return cortex::db::Models().HasModel(identifier); +} + +cpp::result, std::string> +DatabaseService::GetModelSources() const { + return cortex::db::Models().GetModelSources(); +} + +cpp::result, std::string> DatabaseService::GetModels( + const std::string& model_src) const { + return cortex::db::Models().GetModels(model_src); +} +// end models \ No newline at end of file diff --git a/engine/services/database_service.h b/engine/services/database_service.h index 21767b358..4fb4f7be0 100644 --- a/engine/services/database_service.h +++ b/engine/services/database_service.h @@ -6,6 +6,7 @@ using EngineEntry = cortex::db::EngineEntry; using HardwareEntry = cortex::db::HardwareEntry; +using ModelEntry = cortex::db::ModelEntry; class DatabaseService { public: @@ -43,5 +44,25 @@ class DatabaseService { cpp::result DeleteHardwareEntry(const std::string& id); // models + cpp::result, std::string> LoadModelList() const; + cpp::result GetModelInfo( + const std::string& identifier) const; + void PrintModelInfo(const ModelEntry& entry) const; + cpp::result AddModelEntry(ModelEntry new_entry); + cpp::result UpdateModelEntry( + const std::string& identifier, const ModelEntry& updated_entry); + cpp::result DeleteModelEntry( + const std::string& identifier); + cpp::result DeleteModelEntryWithOrg( + const std::string& src); + cpp::result DeleteModelEntryWithRepo( + const std::string& src); + cpp::result, std::string> FindRelatedModel( + const std::string& identifier) const; + bool HasModel(const std::string& identifier) const; + cpp::result, std::string> GetModelSources() const; + cpp::result, std::string> GetModels( + const std::string& model_src) const; + private: }; \ No newline at end of file diff --git a/engine/services/model_service.cc b/engine/services/model_service.cc index 7eddb7567..2d69e0f17 100644 --- a/engine/services/model_service.cc +++ b/engine/services/model_service.cc @@ -21,7 +21,8 @@ #include "utils/widechar_conv.h" namespace { -void ParseGguf(const DownloadItem& ggufDownloadItem, +void ParseGguf(DatabaseService& db_service, + const DownloadItem& ggufDownloadItem, std::optional author, std::optional name, std::optional size) { @@ -64,8 +65,7 @@ void ParseGguf(const DownloadItem& ggufDownloadItem, CTL_INF("path_to_model_yaml: " << rel.string()); auto author_id = author.has_value() ? author.value() : "cortexso"; - cortex::db::Models modellist_utils_obj; - if (!modellist_utils_obj.HasModel(ggufDownloadItem.id)) { + if (!db_service.HasModel(ggufDownloadItem.id)) { cortex::db::ModelEntry model_entry{ .model = ggufDownloadItem.id, .author_repo_id = author_id, @@ -73,18 +73,17 @@ void ParseGguf(const DownloadItem& ggufDownloadItem, .path_to_model_yaml = rel.string(), .model_alias = ggufDownloadItem.id, .status = cortex::db::ModelStatus::Downloaded}; - auto result = modellist_utils_obj.AddModelEntry(model_entry); + auto result = db_service.AddModelEntry(model_entry); if (result.has_error()) { CTL_ERR("Error adding model to modellist: " + result.error()); } } else { - if (auto m = modellist_utils_obj.GetModelInfo(ggufDownloadItem.id); + if (auto m = db_service.GetModelInfo(ggufDownloadItem.id); m.has_value()) { auto upd_m = m.value(); upd_m.status = cortex::db::ModelStatus::Downloaded; - if (auto r = - modellist_utils_obj.UpdateModelEntry(ggufDownloadItem.id, upd_m); + if (auto r = db_service.UpdateModelEntry(ggufDownloadItem.id, upd_m); r.has_error()) { CTL_ERR(r.error()); } @@ -137,10 +136,9 @@ cpp::result GetDownloadTask( void ModelService::ForceIndexingModelList() { CTL_INF("Force indexing model list"); - cortex::db::Models modellist_handler; config::YamlHandler yaml_handler; - auto list_entry = modellist_handler.LoadModelList(); + auto list_entry = db_service_->LoadModelList(); if (list_entry.has_error()) { CTL_ERR("Failed to load model list: " << list_entry.error()); return; @@ -164,8 +162,7 @@ void ModelService::ForceIndexingModelList() { yaml_handler.Reset(); } catch (const std::exception& e) { // remove in db - auto remove_result = - modellist_handler.DeleteModelEntry(model_entry.model); + auto remove_result = db_service_->DeleteModelEntry(model_entry.model); // silently ignore result } } @@ -218,10 +215,8 @@ cpp::result ModelService::HandleCortexsoModel( auto default_model_branch = huggingface_utils::GetDefaultBranch(modelName); - cortex::db::Models modellist_handler; - auto downloaded_model_ids = - modellist_handler.FindRelatedModel(modelName).value_or( - std::vector{}); + auto downloaded_model_ids = db_service_->FindRelatedModel(modelName).value_or( + std::vector{}); std::vector avai_download_opts{}; for (const auto& branch : branches.value()) { @@ -261,9 +256,8 @@ cpp::result ModelService::HandleCortexsoModel( std::optional ModelService::GetDownloadedModel( const std::string& modelId) const { - cortex::db::Models modellist_handler; config::YamlHandler yaml_handler; - auto model_entry = modellist_handler.GetModelInfo(modelId); + auto model_entry = db_service_->GetModelInfo(modelId); if (!model_entry.has_value()) { return std::nullopt; } @@ -310,7 +304,6 @@ cpp::result ModelService::HandleDownloadUrlAsync( } std::string huggingFaceHost{kHuggingFaceHost}; - cortex::db::Models modellist_handler; std::string unique_model_id = ""; if (temp_model_id.has_value()) { unique_model_id = temp_model_id.value(); @@ -318,7 +311,7 @@ cpp::result ModelService::HandleDownloadUrlAsync( unique_model_id = author + ":" + model_id + ":" + file_name; } - auto model_entry = modellist_handler.GetModelInfo(unique_model_id); + auto model_entry = db_service_->GetModelInfo(unique_model_id); if (model_entry.has_value() && model_entry->status == cortex::db::ModelStatus::Downloaded) { CLI_LOG("Model already downloaded: " << unique_model_id); @@ -346,14 +339,15 @@ cpp::result ModelService::HandleDownloadUrlAsync( .localPath = local_path, }}}}; - auto on_finished = [author, temp_name](const DownloadTask& finishedTask) { + auto on_finished = [this, author, + temp_name](const DownloadTask& finishedTask) { // Sum downloadedBytes from all items uint64_t model_size = 0; for (const auto& item : finishedTask.items) { model_size = model_size + item.bytes.value_or(0); } auto gguf_download_item = finishedTask.items[0]; - ParseGguf(gguf_download_item, author, temp_name, model_size); + ParseGguf(*db_service_, gguf_download_item, author, temp_name, model_size); }; downloadTask.id = unique_model_id; @@ -366,11 +360,10 @@ ModelService::GetEstimation(const std::string& model_handle, int n_ubatch) { namespace fs = std::filesystem; namespace fmu = file_manager_utils; - cortex::db::Models modellist_handler; config::YamlHandler yaml_handler; try { - auto model_entry = modellist_handler.GetModelInfo(model_handle); + auto model_entry = db_service_->GetModelInfo(model_handle); if (model_entry.has_error()) { CTL_WRN("Error: " + model_entry.error()); return cpp::fail(model_entry.error()); @@ -438,8 +431,7 @@ cpp::result ModelService::HandleUrl( std::string huggingFaceHost{kHuggingFaceHost}; std::string unique_model_id{author + ":" + model_id + ":" + file_name}; - cortex::db::Models modellist_handler; - auto model_entry = modellist_handler.GetModelInfo(unique_model_id); + auto model_entry = db_service_->GetModelInfo(unique_model_id); if (model_entry.has_value()) { CLI_LOG("Model already downloaded: " << unique_model_id); @@ -467,14 +459,14 @@ cpp::result ModelService::HandleUrl( .localPath = local_path, }}}}; - auto on_finished = [author](const DownloadTask& finishedTask) { + auto on_finished = [this, author](const DownloadTask& finishedTask) { // Sum downloadedBytes from all items uint64_t model_size = 0; for (const auto& item : finishedTask.items) { model_size = model_size + item.bytes.value_or(0); } auto gguf_download_item = finishedTask.items[0]; - ParseGguf(gguf_download_item, author, std::nullopt, model_size); + ParseGguf(*db_service_, gguf_download_item, author, std::nullopt, model_size); }; auto result = download_service_->AddDownloadTask(downloadTask, on_finished); @@ -488,7 +480,7 @@ cpp::result ModelService::HandleUrl( } bool ModelService::HasModel(const std::string& id) const { - return cortex::db::Models().HasModel(id); + return db_service_->HasModel(id); } cpp::result @@ -501,7 +493,6 @@ ModelService::DownloadModelFromCortexsoAsync( return cpp::fail(download_task.error()); } - cortex::db::Models modellist_handler; std::string unique_model_id = ""; if (temp_model_id.has_value()) { unique_model_id = temp_model_id.value(); @@ -509,13 +500,13 @@ ModelService::DownloadModelFromCortexsoAsync( unique_model_id = name + ":" + branch; } - auto model_entry = modellist_handler.GetModelInfo(unique_model_id); + auto model_entry = db_service_->GetModelInfo(unique_model_id); if (model_entry.has_value() && model_entry->status == cortex::db::ModelStatus::Downloaded) { return cpp::fail("Please delete the model before downloading again"); } - auto on_finished = [unique_model_id, + auto on_finished = [this, unique_model_id, branch](const DownloadTask& finishedTask) { const DownloadItem* model_yml_item = nullptr; auto need_parse_gguf = true; @@ -551,8 +542,7 @@ ModelService::DownloadModelFromCortexsoAsync( file_manager_utils::ToRelativeCortexDataPath(model_yml_item->localPath); CTL_INF("path_to_model_yaml: " << rel.string()); - cortex::db::Models modellist_utils_obj; - if (!modellist_utils_obj.HasModel(unique_model_id)) { + if (!db_service_->HasModel(unique_model_id)) { cortex::db::ModelEntry model_entry{ .model = unique_model_id, .author_repo_id = "cortexso", @@ -560,18 +550,16 @@ ModelService::DownloadModelFromCortexsoAsync( .path_to_model_yaml = rel.string(), .model_alias = unique_model_id, .status = cortex::db::ModelStatus::Downloaded}; - auto result = modellist_utils_obj.AddModelEntry(model_entry); + auto result = db_service_->AddModelEntry(model_entry); if (result.has_error()) { CTL_ERR("Error adding model to modellist: " + result.error()); } } else { - if (auto m = modellist_utils_obj.GetModelInfo(unique_model_id); - m.has_value()) { + if (auto m = db_service_->GetModelInfo(unique_model_id); m.has_value()) { auto upd_m = m.value(); upd_m.status = cortex::db::ModelStatus::Downloaded; - if (auto r = - modellist_utils_obj.UpdateModelEntry(unique_model_id, upd_m); + if (auto r = db_service_->UpdateModelEntry(unique_model_id, upd_m); r.has_error()) { CTL_ERR(r.error()); } @@ -595,7 +583,7 @@ cpp::result ModelService::DownloadModelFromCortexso( } std::string model_id{name + ":" + branch}; - auto on_finished = [branch, model_id](const DownloadTask& finishedTask) { + auto on_finished = [this, branch, model_id](const DownloadTask& finishedTask) { const DownloadItem* model_yml_item = nullptr; auto need_parse_gguf = true; @@ -622,8 +610,7 @@ cpp::result ModelService::DownloadModelFromCortexso( file_manager_utils::ToRelativeCortexDataPath(model_yml_item->localPath); CTL_INF("path_to_model_yaml: " << rel.string()); - cortex::db::Models modellist_utils_obj; - if (!modellist_utils_obj.HasModel(model_id)) { + if (!db_service_->HasModel(model_id)) { cortex::db::ModelEntry model_entry{ .model = model_id, .author_repo_id = "cortexso", @@ -631,16 +618,16 @@ cpp::result ModelService::DownloadModelFromCortexso( .path_to_model_yaml = rel.string(), .model_alias = model_id, .status = cortex::db::ModelStatus::Downloaded}; - auto result = modellist_utils_obj.AddModelEntry(model_entry); + auto result = db_service_->AddModelEntry(model_entry); if (result.has_error()) { CTL_ERR("Error adding model to modellist: " + result.error()); } } else { - if (auto m = modellist_utils_obj.GetModelInfo(model_id); m.has_value()) { + if (auto m = db_service_->GetModelInfo(model_id); m.has_value()) { auto upd_m = m.value(); upd_m.status = cortex::db::ModelStatus::Downloaded; - if (auto r = modellist_utils_obj.UpdateModelEntry(model_id, upd_m); + if (auto r = db_service_->UpdateModelEntry(model_id, upd_m); r.has_error()) { CTL_ERR(r.error()); } @@ -694,7 +681,6 @@ cpp::result ModelService::DeleteModel( const std::string& model_handle) { namespace fs = std::filesystem; namespace fmu = file_manager_utils; - cortex::db::Models modellist_handler; config::YamlHandler yaml_handler; auto result = StopModel(model_handle); @@ -706,7 +692,7 @@ cpp::result ModelService::DeleteModel( } try { - auto model_entry = modellist_handler.GetModelInfo(model_handle); + auto model_entry = db_service_->GetModelInfo(model_handle); if (model_entry.has_error()) { CTL_WRN("Error: " + model_entry.error()); return cpp::fail(model_entry.error()); @@ -737,7 +723,7 @@ cpp::result ModelService::DeleteModel( } // update model.list - if (modellist_handler.DeleteModelEntry(model_handle)) { + if (db_service_->DeleteModelEntry(model_handle)) { return {}; } else { return cpp::fail("Could not delete model: " + model_handle); @@ -753,7 +739,6 @@ cpp::result ModelService::StartModel( bool bypass_model_check) { namespace fs = std::filesystem; namespace fmu = file_manager_utils; - cortex::db::Models modellist_handler; config::YamlHandler yaml_handler; std::optional custom_prompt_template; std::optional ctx_len; @@ -771,7 +756,7 @@ cpp::result ModelService::StartModel( Json::Value json_data; // Currently we don't support download vision models, so we need to bypass check if (!bypass_model_check) { - auto model_entry = modellist_handler.GetModelInfo(model_handle); + auto model_entry = db_service_->GetModelInfo(model_handle); if (model_entry.has_error()) { CTL_WRN("Error: " + model_entry.error()); return cpp::fail(model_entry.error()); @@ -910,7 +895,6 @@ cpp::result ModelService::StopModel( const std::string& model_handle) { namespace fs = std::filesystem; namespace fmu = file_manager_utils; - cortex::db::Models modellist_handler; config::YamlHandler yaml_handler; try { @@ -918,7 +902,7 @@ cpp::result ModelService::StopModel( bypass_stop_check_set_.end()); std::string engine_name = ""; if (!bypass_check) { - auto model_entry = modellist_handler.GetModelInfo(model_handle); + auto model_entry = db_service_->GetModelInfo(model_handle); if (model_entry.has_error()) { CTL_WRN("Error: " + model_entry.error()); return cpp::fail(model_entry.error()); @@ -958,11 +942,10 @@ cpp::result ModelService::GetModelStatus( const std::string& model_handle) { namespace fs = std::filesystem; namespace fmu = file_manager_utils; - cortex::db::Models modellist_handler; config::YamlHandler yaml_handler; try { - auto model_entry = modellist_handler.GetModelInfo(model_handle); + auto model_entry = db_service_->GetModelInfo(model_handle); if (model_entry.has_error()) { CTL_WRN("Error: " + model_entry.error()); return cpp::fail(model_entry.error()); @@ -1083,8 +1066,7 @@ cpp::result ModelService::GetModelPullInfo( auto default_model_branch = huggingface_utils::GetDefaultBranch(model_name); - cortex::db::Models modellist_handler; - auto downloaded_model_ids = modellist_handler.FindRelatedModel(model_name) + auto downloaded_model_ids = db_service_->FindRelatedModel(model_name) .value_or(std::vector{}); std::vector avai_download_opts{}; diff --git a/engine/services/model_service.h b/engine/services/model_service.h index f5ea2dddf..cc659fea5 100644 --- a/engine/services/model_service.h +++ b/engine/services/model_service.h @@ -6,6 +6,7 @@ #include "common/engine_servicei.h" #include "common/model_metadata.h" #include "config/model_config.h" +#include "services/database_service.h" #include "services/download_service.h" #include "services/hardware_service.h" #include "utils/hardware/gguf/gguf_file_estimate.h" @@ -30,14 +31,13 @@ class ModelService { public: void ForceIndexingModelList(); - explicit ModelService(std::shared_ptr download_service) - : download_service_{download_service} {}; - - explicit ModelService(std::shared_ptr hw_service, + explicit ModelService(std::shared_ptr db_service, + std::shared_ptr hw_service, std::shared_ptr download_service, std::shared_ptr inference_service, std::shared_ptr engine_svc) - : hw_service_(hw_service), + : db_service_(db_service), + hw_service_(hw_service), download_service_{download_service}, inference_svc_(inference_service), engine_svc_(engine_svc) {}; @@ -115,6 +115,7 @@ class ModelService { const std::string& model_path, int ngl, int ctx_len, int n_batch = 2048, int n_ubatch = 2048, const std::string& kv_cache_type = "f16"); + std::shared_ptr db_service_; std::shared_ptr hw_service_; std::shared_ptr download_service_; std::shared_ptr inference_svc_; diff --git a/engine/services/model_source_service.cc b/engine/services/model_source_service.cc index 49362d3d5..7fc0ef5b2 100644 --- a/engine/services/model_source_service.cc +++ b/engine/services/model_source_service.cc @@ -60,10 +60,13 @@ std::vector ParseJsonString(const std::string& json_str) { } // namespace -ModelSourceService::ModelSourceService() { +ModelSourceService::ModelSourceService( + std::shared_ptr db_service) + : db_service_(db_service) { sync_db_thread_ = std::thread(&ModelSourceService::SyncModelSource, this); running_ = true; } + ModelSourceService::~ModelSourceService() { running_ = false; if (sync_db_thread_.joinable()) { @@ -105,8 +108,7 @@ cpp::result ModelSourceService::AddModelSource( cpp::result ModelSourceService::RemoveModelSource( const std::string& model_source) { - cortex::db::Models model_db; - auto srcs = model_db.GetModelSources(); + auto srcs = db_service_->GetModelSources(); if (srcs.has_error()) { return cpp::fail(srcs.error()); } else { @@ -126,13 +128,13 @@ cpp::result ModelSourceService::RemoveModelSource( } if (r.pathParams.size() == 1) { - if (auto del_res = model_db.DeleteModelEntryWithOrg(model_source); + if (auto del_res = db_service_->DeleteModelEntryWithOrg(model_source); del_res.has_error()) { CTL_INF(del_res.error()); return cpp::fail(del_res.error()); } } else { - if (auto del_res = model_db.DeleteModelEntryWithRepo(model_source); + if (auto del_res = db_service_->DeleteModelEntryWithRepo(model_source); del_res.has_error()) { CTL_INF(del_res.error()); return cpp::fail(del_res.error()); @@ -144,8 +146,7 @@ cpp::result ModelSourceService::RemoveModelSource( cpp::result, std::string> ModelSourceService::GetModelSources() { - cortex::db::Models model_db; - return model_db.GetModelSources(); + return db_service_->GetModelSources(); } cpp::result ModelSourceService::AddHfOrg( @@ -155,10 +156,9 @@ cpp::result ModelSourceService::AddHfOrg( if (res.has_value()) { auto models = ParseJsonString(res.value()); // Get models from db - cortex::db::Models model_db; - auto model_list_before = - model_db.GetModels(model_source).value_or(std::vector{}); + auto model_list_before = db_service_->GetModels(model_source) + .value_or(std::vector{}); std::unordered_set updated_model_list; // Add new models for (auto const& m : models) { @@ -178,7 +178,7 @@ cpp::result ModelSourceService::AddHfOrg( // Clean up for (auto const& mid : model_list_before) { if (updated_model_list.find(mid) == updated_model_list.end()) { - if (auto del_res = model_db.DeleteModelEntry(mid); + if (auto del_res = db_service_->DeleteModelEntry(mid); del_res.has_error()) { CTL_INF(del_res.error()); } @@ -194,10 +194,9 @@ cpp::result ModelSourceService::AddHfRepo( const std::string& model_source, const std::string& author, const std::string& model_name) { // Get models from db - cortex::db::Models model_db; auto model_list_before = - model_db.GetModels(model_source).value_or(std::vector{}); + db_service_->GetModels(model_source).value_or(std::vector{}); std::unordered_set updated_model_list; auto add_res = AddRepoSiblings(model_source, author, model_name); if (add_res.has_error()) { @@ -207,7 +206,8 @@ cpp::result ModelSourceService::AddHfRepo( } for (auto const& mid : model_list_before) { if (updated_model_list.find(mid) == updated_model_list.end()) { - if (auto del_res = model_db.DeleteModelEntry(mid); del_res.has_error()) { + if (auto del_res = db_service_->DeleteModelEntry(mid); + del_res.has_error()) { CTL_INF(del_res.error()); } } @@ -233,7 +233,6 @@ ModelSourceService::AddRepoSiblings(const std::string& model_source, for (const auto& sibling : repo_info->siblings) { if (string_utils::EndsWith(sibling.rfilename, ".gguf")) { - cortex::db::Models model_db; std::string model_id = author + ":" + model_name + ":" + sibling.rfilename; cortex::db::ModelEntry e = { @@ -247,15 +246,15 @@ ModelSourceService::AddRepoSiblings(const std::string& model_source, .status = cortex::db::ModelStatus::Downloadable, .engine = "llama-cpp", .metadata = repo_info->metadata}; - if (!model_db.HasModel(model_id)) { - if (auto add_res = model_db.AddModelEntry(e); add_res.has_error()) { + if (!db_service_->HasModel(model_id)) { + if (auto add_res = db_service_->AddModelEntry(e); add_res.has_error()) { CTL_INF(add_res.error()); } } else { - if (auto m = model_db.GetModelInfo(model_id); + if (auto m = db_service_->GetModelInfo(model_id); m.has_value() && m->status == cortex::db::ModelStatus::Downloadable) { - if (auto upd_res = model_db.UpdateModelEntry(model_id, e); + if (auto upd_res = db_service_->UpdateModelEntry(model_id, e); upd_res.has_error()) { CTL_INF(upd_res.error()); } @@ -275,10 +274,9 @@ cpp::result ModelSourceService::AddCortexsoOrg( if (res.has_value()) { auto models = ParseJsonString(res.value()); // Get models from db - cortex::db::Models model_db; - auto model_list_before = - model_db.GetModels(model_source).value_or(std::vector{}); + auto model_list_before = db_service_->GetModels(model_source) + .value_or(std::vector{}); std::unordered_set updated_model_list; for (auto const& m : models) { CTL_INF(m.id); @@ -312,7 +310,7 @@ cpp::result ModelSourceService::AddCortexsoOrg( // Clean up for (auto const& mid : model_list_before) { if (updated_model_list.find(mid) == updated_model_list.end()) { - if (auto del_res = model_db.DeleteModelEntry(mid); + if (auto del_res = db_service_->DeleteModelEntry(mid); del_res.has_error()) { CTL_INF(del_res.error()); } @@ -339,10 +337,9 @@ cpp::result ModelSourceService::AddCortexsoRepo( return cpp::fail(repo_info.error()); } // Get models from db - cortex::db::Models model_db; auto model_list_before = - model_db.GetModels(model_source).value_or(std::vector{}); + db_service_->GetModels(model_source).value_or(std::vector{}); std::unordered_set updated_model_list; for (auto const& [branch, _] : branches.value()) { @@ -358,7 +355,8 @@ cpp::result ModelSourceService::AddCortexsoRepo( // Clean up for (auto const& mid : model_list_before) { if (updated_model_list.find(mid) == updated_model_list.end()) { - if (auto del_res = model_db.DeleteModelEntry(mid); del_res.has_error()) { + if (auto del_res = db_service_->DeleteModelEntry(mid); + del_res.has_error()) { CTL_INF(del_res.error()); } } @@ -396,7 +394,6 @@ ModelSourceService::AddCortexsoRepoBranch(const std::string& model_source, CTL_INF("Only support gguf file format! - branch: " << branch); return {}; } else { - cortex::db::Models model_db; std::string model_id = model_name + ":" + branch; cortex::db::ModelEntry e = {.model = model_id, .author_repo_id = author, @@ -408,16 +405,16 @@ ModelSourceService::AddCortexsoRepoBranch(const std::string& model_source, .status = cortex::db::ModelStatus::Downloadable, .engine = "llama-cpp", .metadata = metadata}; - if (!model_db.HasModel(model_id)) { + if (!db_service_->HasModel(model_id)) { CTL_INF("Adding model to db: " << model_name << ":" << branch); - if (auto res = model_db.AddModelEntry(e); + if (auto res = db_service_->AddModelEntry(e); res.has_error() || !res.value()) { CTL_DBG("Cannot add model to db: " << model_id); } } else { - if (auto m = model_db.GetModelInfo(model_id); + if (auto m = db_service_->GetModelInfo(model_id); m.has_value() && m->status == cortex::db::ModelStatus::Downloadable) { - if (auto upd_res = model_db.UpdateModelEntry(model_id, e); + if (auto upd_res = db_service_->UpdateModelEntry(model_id, e); upd_res.has_error()) { CTL_INF(upd_res.error()); } @@ -443,8 +440,7 @@ void ModelSourceService::SyncModelSource() { CTL_DBG("Start to sync cortex.db"); start_time = current_time; - cortex::db::Models model_db; - auto res = model_db.GetModelSources(); + auto res = db_service_->GetModelSources(); if (res.has_error()) { CTL_INF(res.error()); } else { diff --git a/engine/services/model_source_service.h b/engine/services/model_source_service.h index 45ec2aabf..7227267d3 100644 --- a/engine/services/model_source_service.h +++ b/engine/services/model_source_service.h @@ -2,13 +2,14 @@ #include #include #include +#include "services/database_service.h" #include "utils/result.hpp" class ModelSourceService { public: - explicit ModelSourceService(); + explicit ModelSourceService(std::shared_ptr db_service); ~ModelSourceService(); - + cpp::result AddModelSource( const std::string& model_source); @@ -21,9 +22,9 @@ class ModelSourceService { cpp::result AddHfOrg(const std::string& model_source, const std::string& author); - cpp::result AddHfRepo( - const std::string& model_source, const std::string& author, - const std::string& model_name); + cpp::result AddHfRepo(const std::string& model_source, + const std::string& author, + const std::string& model_name); cpp::result, std::string> AddRepoSiblings( const std::string& model_source, const std::string& author, @@ -40,12 +41,12 @@ class ModelSourceService { AddCortexsoRepoBranch(const std::string& model_source, const std::string& author, const std::string& model_name, - const std::string& branch, - const std::string& metadata); + const std::string& branch, const std::string& metadata); void SyncModelSource(); private: + std::shared_ptr db_service_ = nullptr; std::thread sync_db_thread_; std::atomic running_; }; \ No newline at end of file