From f606e66f07c07cdfb87b6b2bbb181b482f4371bf Mon Sep 17 00:00:00 2001 From: Thuandz Date: Wed, 18 Sep 2024 22:49:17 +0700 Subject: [PATCH 1/7] add model import command --- engine/commands/model_import_cmd.cc | 52 +++++++++++++++++++++++ engine/commands/model_import_cmd.h | 16 +++++++ engine/controllers/command_line_parser.cc | 17 +++++++- 3 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 engine/commands/model_import_cmd.cc create mode 100644 engine/commands/model_import_cmd.h diff --git a/engine/commands/model_import_cmd.cc b/engine/commands/model_import_cmd.cc new file mode 100644 index 000000000..d43d8ddf4 --- /dev/null +++ b/engine/commands/model_import_cmd.cc @@ -0,0 +1,52 @@ +#include "model_import_cmd.h" +#include +#include +#include +#include "config/gguf_parser.h" +#include "config/yaml_config.h" +#include "trantor/utils/Logger.h" +#include "utils/file_manager_utils.h" +#include "utils/logging_utils.h" +#include "utils/modellist_utils.h" + +namespace commands { + +ModelImportCmd::ModelImportCmd(std::string model_handle, std::string model_path) + : model_handle_(std::move(model_handle)), + model_path_(std::move(model_path)) {} + +void ModelImportCmd::Exec() { + config::GGUFHandler gguf_handler; + config::YamlHandler yaml_handler; + modellist_utils::ModelListUtils modellist_utils_obj; + + std::string model_yaml_path = (file_manager_utils::GetModelsContainerPath() / + std::filesystem::path("imported") / + std::filesystem::path(model_handle_ + ".yml")) + .string(); + modellist_utils::ModelEntry model_entry{ + model_handle_, "local", "imported", + model_yaml_path, model_handle_, modellist_utils::ModelStatus::READY}; + try { + std::filesystem::create_directories( + std::filesystem::path(model_yaml_path).parent_path()); + gguf_handler.Parse(model_path_); + config::ModelConfig model_config = gguf_handler.GetModelConfig(); + model_config.files.push_back(model_path_); + yaml_handler.UpdateModelConfig(model_config); + + if(modellist_utils_obj.AddModelEntry(model_entry)){ + yaml_handler.WriteYamlFile(model_yaml_path); + CLI_LOG("Model is imported successfully!"); + } + else{ + CLI_LOG("Fail to import model, model_id '"+model_handle_+"' already exists!" ); + } + + } catch (const std::exception& e) { + std::remove(model_yaml_path.c_str()); + CTL_ERR("Error importing model '" << model_path_ << "' with model_id '" + << model_handle_ << "': " << e.what()); + } +} +} // namespace commands \ No newline at end of file diff --git a/engine/commands/model_import_cmd.h b/engine/commands/model_import_cmd.h new file mode 100644 index 000000000..b1441a281 --- /dev/null +++ b/engine/commands/model_import_cmd.h @@ -0,0 +1,16 @@ +#pragma once + +#include // For std::isnan +#include +namespace commands { + +class ModelImportCmd { + public: + ModelImportCmd(std::string model_handle, std::string model_path); + void Exec(); + + private: + std::string model_handle_; + std::string model_path_; +}; +} // namespace commands \ No newline at end of file diff --git a/engine/controllers/command_line_parser.cc b/engine/controllers/command_line_parser.cc index 9c4b5713f..06751ae5f 100644 --- a/engine/controllers/command_line_parser.cc +++ b/engine/controllers/command_line_parser.cc @@ -8,6 +8,7 @@ #include "commands/engine_uninstall_cmd.h" #include "commands/model_del_cmd.h" #include "commands/model_get_cmd.h" +#include "commands/model_import_cmd.h" #include "commands/model_list_cmd.h" #include "commands/model_pull_cmd.h" #include "commands/model_start_cmd.h" @@ -155,6 +156,20 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) { auto model_update_cmd = models_cmd->add_subcommand("update", "Update configuration of a model"); + std::string model_path; + auto model_import_cmd = models_cmd->add_subcommand( + "import", "Import a gguf model from local file"); + model_import_cmd->add_option("--model_id", model_id, ""); + model_import_cmd->require_option(); + model_import_cmd->add_option("--model_path", model_path, + "Absolute path to .gguf model, the path should " + "include the gguf file name"); + model_import_cmd->require_option(); + model_import_cmd->callback([&model_id,&model_path]() { + commands::ModelImportCmd command(model_id, model_path); + command.Exec(); + }); + // Default version is latest std::string version{"latest"}; // engines group commands @@ -238,7 +253,7 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) { auto ps_cmd = app_.add_subcommand("ps", "Show running models and their status"); ps_cmd->group(kSystemGroup); - + CLI11_PARSE(app_, argc, argv); if (argc == 1) { CLI_LOG(app_.help()); From cc4200b9dc8e1026cd153de2d45e0dfbf378506b Mon Sep 17 00:00:00 2001 From: Thuandz Date: Thu, 19 Sep 2024 08:15:44 +0700 Subject: [PATCH 2/7] Add name to model.yml --- engine/commands/model_import_cmd.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/engine/commands/model_import_cmd.cc b/engine/commands/model_import_cmd.cc index d43d8ddf4..07079fc3b 100644 --- a/engine/commands/model_import_cmd.cc +++ b/engine/commands/model_import_cmd.cc @@ -33,6 +33,7 @@ void ModelImportCmd::Exec() { gguf_handler.Parse(model_path_); config::ModelConfig model_config = gguf_handler.GetModelConfig(); model_config.files.push_back(model_path_); + model_config.name = model_handle_; yaml_handler.UpdateModelConfig(model_config); if(modellist_utils_obj.AddModelEntry(model_entry)){ From f9af68042eb1b2931ef8ee52af2f0e492339469b Mon Sep 17 00:00:00 2001 From: Thuandz Date: Thu, 19 Sep 2024 09:47:48 +0700 Subject: [PATCH 3/7] add e2e test --- engine/commands/model_import_cmd.cc | 19 ++++++++++--------- engine/e2e-test/main.py | 1 + engine/e2e-test/test_cli_model_import.py | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 9 deletions(-) create mode 100644 engine/e2e-test/test_cli_model_import.py diff --git a/engine/commands/model_import_cmd.cc b/engine/commands/model_import_cmd.cc index 07079fc3b..dfc9997f8 100644 --- a/engine/commands/model_import_cmd.cc +++ b/engine/commands/model_import_cmd.cc @@ -36,18 +36,19 @@ void ModelImportCmd::Exec() { model_config.name = model_handle_; yaml_handler.UpdateModelConfig(model_config); - if(modellist_utils_obj.AddModelEntry(model_entry)){ - yaml_handler.WriteYamlFile(model_yaml_path); - CLI_LOG("Model is imported successfully!"); + if (modellist_utils_obj.AddModelEntry(model_entry)) { + yaml_handler.WriteYamlFile(model_yaml_path); + CLI_LOG("Model is imported successfully!"); + } else { + CLI_LOG("Fail to import model, model_id '" + model_handle_ + + "' already exists!"); } - else{ - CLI_LOG("Fail to import model, model_id '"+model_handle_+"' already exists!" ); - } - + } catch (const std::exception& e) { std::remove(model_yaml_path.c_str()); - CTL_ERR("Error importing model '" << model_path_ << "' with model_id '" - << model_handle_ << "': " << e.what()); + throw std::runtime_error("Error importing model '" + model_path_ + + "' with model_id '" + model_handle_ + + "': " + e.what()); } } } // namespace commands \ No newline at end of file diff --git a/engine/e2e-test/main.py b/engine/e2e-test/main.py index 1df424e65..f5a1c65ff 100644 --- a/engine/e2e-test/main.py +++ b/engine/e2e-test/main.py @@ -9,6 +9,7 @@ from test_cli_server_start import TestCliServerStart from test_cortex_update import TestCortexUpdate from test_create_log_folder import TestCreateLogFolder +from test_cli_model_import import TestCliModelImport if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/engine/e2e-test/test_cli_model_import.py b/engine/e2e-test/test_cli_model_import.py new file mode 100644 index 000000000..1f54ae511 --- /dev/null +++ b/engine/e2e-test/test_cli_model_import.py @@ -0,0 +1,14 @@ +import pytest +from test_runner import run + +class TestCliModelImport: + + @pytest.mark.skipif(True, reason="Expensive test. Only test when you have local gguf file.") + def test_model_import_should_be_success(self): + + exit_code, output, error = run( + "Pull model", ["models", "import", "--model_id","test_model","--model_path","/path/to/local/gguf"], + timeout=None + ) + assert exit_code == 0, f"Model import failed failed with error: {error}" + # TODO: skip this test. since download model is taking too long \ No newline at end of file From c8bd76e57e1b974d3ac968b0b07b6aa4d90ac439 Mon Sep 17 00:00:00 2001 From: Thuandz Date: Thu, 19 Sep 2024 12:08:54 +0700 Subject: [PATCH 4/7] Add API for import model --- engine/controllers/models.cc | 78 ++++++++++++++++++++++++++++++++++-- engine/controllers/models.h | 3 ++ 2 files changed, 77 insertions(+), 4 deletions(-) diff --git a/engine/controllers/models.cc b/engine/controllers/models.cc index 1d3157fcb..4bfbded61 100644 --- a/engine/controllers/models.cc +++ b/engine/controllers/models.cc @@ -5,11 +5,11 @@ #include "utils/cortex_utils.h" #include "utils/file_manager_utils.h" #include "utils/model_callback_utils.h" +#include "utils/modellist_utils.h" - void - Models::PullModel( - const HttpRequestPtr& req, - std::function&& callback) const { +void Models::PullModel( + const HttpRequestPtr& req, + std::function&& callback) const { if (!http_util::HasFieldInReq(req, callback, "modelId")) { return; } @@ -192,4 +192,74 @@ void Models::DeleteModel(const HttpRequestPtr& req, resp->setStatusCode(k404NotFound); callback(resp); } +} + +void Models::ImportModel( + const HttpRequestPtr& req, + std::function&& callback) const { + if (!http_util::HasFieldInReq(req, callback, "modelId") || + !http_util::HasFieldInReq(req, callback, "modelPath")) { + return; + } + auto modelHandle = (*(req->getJsonObject())).get("modelId", "").asString(); + auto modelPath = (*(req->getJsonObject())).get("modelPath", "").asString(); + config::GGUFHandler gguf_handler; + config::YamlHandler yaml_handler; + modellist_utils::ModelListUtils modellist_utils_obj; + + std::string model_yaml_path = (file_manager_utils::GetModelsContainerPath() / + std::filesystem::path("imported") / + std::filesystem::path(modelHandle + ".yml")) + .string(); + modellist_utils::ModelEntry model_entry{ + modelHandle, "local", "imported", + model_yaml_path, modelHandle, modellist_utils::ModelStatus::READY}; + try { + std::filesystem::create_directories( + std::filesystem::path(model_yaml_path).parent_path()); + gguf_handler.Parse(modelPath); + config::ModelConfig model_config = gguf_handler.GetModelConfig(); + model_config.files.push_back(modelPath); + model_config.name = modelHandle; + yaml_handler.UpdateModelConfig(model_config); + + if (modellist_utils_obj.AddModelEntry(model_entry)) { + yaml_handler.WriteYamlFile(model_yaml_path); + std::string success_message = "Model is imported successfully!"; + LOG_INFO << success_message; + Json::Value ret; + ret["result"] = "OK"; + ret["modelHandle"] = modelHandle; + ret["message"] = success_message; + auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret); + resp->setStatusCode(k200OK); + callback(resp); + + } else { + std::string error_message = "Fail to import model, model_id '" + + modelHandle + "' already exists!"; + LOG_ERROR << error_message; + Json::Value ret; + ret["result"] = "Import failed!"; + ret["modelHandle"] = modelHandle; + ret["message"] = error_message; + auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret); + resp->setStatusCode(k400BadRequest); + callback(resp); + } + + } catch (const std::exception& e) { + std::remove(model_yaml_path.c_str()); + std::string error_message = "Error importing model '" + modelPath + + "' with model_id '" + modelHandle + + "': " + e.what(); + LOG_ERROR << error_message; + Json::Value ret; + ret["result"] = "Import failed!"; + ret["modelHandle"] = modelHandle; + ret["message"] = error_message; + auto resp = cortex_utils::CreateCortexHttpJsonResponse(ret); + resp->setStatusCode(k400BadRequest); + callback(resp); + } } \ No newline at end of file diff --git a/engine/controllers/models.h b/engine/controllers/models.h index 907ab3917..2fd7c9b2d 100644 --- a/engine/controllers/models.h +++ b/engine/controllers/models.h @@ -15,6 +15,7 @@ class Models : public drogon::HttpController { METHOD_ADD(Models::PullModel, "/pull", Post); METHOD_ADD(Models::ListModel, "/list", Get); METHOD_ADD(Models::GetModel, "/get", Post); + METHOD_ADD(Models::ImportModel, "/import", Post); METHOD_ADD(Models::DeleteModel, "/{1}", Delete); METHOD_LIST_END @@ -24,6 +25,8 @@ class Models : public drogon::HttpController { std::function&& callback) const; void GetModel(const HttpRequestPtr& req, std::function&& callback) const; + void ImportModel(const HttpRequestPtr& req, + std::function&& callback) const; void DeleteModel(const HttpRequestPtr& req, std::function&& callback, const std::string& model_id) const; From 4a6d08db5335c9331cbd7c694070efa59b7de091 Mon Sep 17 00:00:00 2001 From: nguyenhoangthuan99 <35255081+nguyenhoangthuan99@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:54:18 +0700 Subject: [PATCH 5/7] Update model_import_cmd.cc --- engine/commands/model_import_cmd.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/commands/model_import_cmd.cc b/engine/commands/model_import_cmd.cc index dfc9997f8..e4e8878ec 100644 --- a/engine/commands/model_import_cmd.cc +++ b/engine/commands/model_import_cmd.cc @@ -33,7 +33,7 @@ void ModelImportCmd::Exec() { gguf_handler.Parse(model_path_); config::ModelConfig model_config = gguf_handler.GetModelConfig(); model_config.files.push_back(model_path_); - model_config.name = model_handle_; + model_config.model = model_handle_; yaml_handler.UpdateModelConfig(model_config); if (modellist_utils_obj.AddModelEntry(model_entry)) { @@ -51,4 +51,4 @@ void ModelImportCmd::Exec() { "': " + e.what()); } } -} // namespace commands \ No newline at end of file +} // namespace commands From 009139d9f0addf89814dc3221cf81527bfbc964a Mon Sep 17 00:00:00 2001 From: Thuandz Date: Thu, 19 Sep 2024 16:35:17 +0700 Subject: [PATCH 6/7] Fix comment --- engine/commands/model_import_cmd.cc | 7 +++---- engine/controllers/command_line_parser.cc | 3 +-- engine/controllers/models.cc | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/engine/commands/model_import_cmd.cc b/engine/commands/model_import_cmd.cc index dfc9997f8..d0125a03f 100644 --- a/engine/commands/model_import_cmd.cc +++ b/engine/commands/model_import_cmd.cc @@ -31,7 +31,7 @@ void ModelImportCmd::Exec() { std::filesystem::create_directories( std::filesystem::path(model_yaml_path).parent_path()); gguf_handler.Parse(model_path_); - config::ModelConfig model_config = gguf_handler.GetModelConfig(); + auto model_config = gguf_handler.GetModelConfig(); model_config.files.push_back(model_path_); model_config.name = model_handle_; yaml_handler.UpdateModelConfig(model_config); @@ -46,9 +46,8 @@ void ModelImportCmd::Exec() { } catch (const std::exception& e) { std::remove(model_yaml_path.c_str()); - throw std::runtime_error("Error importing model '" + model_path_ + - "' with model_id '" + model_handle_ + - "': " + e.what()); + CLI_LOG("Error importing model path '" + model_path_ + "' with model_id '" + + model_handle_ + "': " + e.what()); } } } // namespace commands \ No newline at end of file diff --git a/engine/controllers/command_line_parser.cc b/engine/controllers/command_line_parser.cc index 0043180db..72015bdfd 100644 --- a/engine/controllers/command_line_parser.cc +++ b/engine/controllers/command_line_parser.cc @@ -158,11 +158,10 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) { auto model_import_cmd = models_cmd->add_subcommand( "import", "Import a gguf model from local file"); model_import_cmd->add_option("--model_id", model_id, ""); - model_import_cmd->require_option(); model_import_cmd->add_option("--model_path", model_path, "Absolute path to .gguf model, the path should " "include the gguf file name"); - model_import_cmd->require_option(); + model_import_cmd->require_option(2); model_import_cmd->callback([&model_id,&model_path]() { commands::ModelImportCmd command(model_id, model_path); command.Exec(); diff --git a/engine/controllers/models.cc b/engine/controllers/models.cc index 4bfbded61..08f2c0a20 100644 --- a/engine/controllers/models.cc +++ b/engine/controllers/models.cc @@ -250,7 +250,7 @@ void Models::ImportModel( } catch (const std::exception& e) { std::remove(model_yaml_path.c_str()); - std::string error_message = "Error importing model '" + modelPath + + std::string error_message = "Error importing model path '" + modelPath + "' with model_id '" + modelHandle + "': " + e.what(); LOG_ERROR << error_message; From ad72ea40b808ad844ad004be5fe7c1f87a918d61 Mon Sep 17 00:00:00 2001 From: Thuandz Date: Fri, 20 Sep 2024 09:00:30 +0700 Subject: [PATCH 7/7] Fix comment --- engine/commands/model_import_cmd.h | 1 - 1 file changed, 1 deletion(-) diff --git a/engine/commands/model_import_cmd.h b/engine/commands/model_import_cmd.h index b1441a281..d4248281f 100644 --- a/engine/commands/model_import_cmd.h +++ b/engine/commands/model_import_cmd.h @@ -1,6 +1,5 @@ #pragma once -#include // For std::isnan #include namespace commands {