Skip to content

Commit

Permalink
Merge pull request #1034 from janhq/feat/add-engine-init
Browse files Browse the repository at this point in the history
feat: add engine install cli
  • Loading branch information
namchuai authored Aug 28, 2024
2 parents 95f7d67 + 287c750 commit de5716c
Show file tree
Hide file tree
Showing 10 changed files with 696 additions and 37 deletions.
47 changes: 38 additions & 9 deletions engine/commands/engine_init_cmd.cc
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include "utils/archive_utils.h"
#include "utils/system_info_utils.h"
// clang-format on
#include "utils/engine_matcher_utils.h"

namespace commands {

Expand All @@ -27,6 +28,7 @@ void EngineInitCmd::Exec() const {
<< system_info.arch;
return;
}
LOG_INFO << "OS: " << system_info.os << ", Arch: " << system_info.arch;

// check if engine is supported
if (std::find(supportedEngines_.begin(), supportedEngines_.end(),
Expand All @@ -36,11 +38,11 @@ void EngineInitCmd::Exec() const {
}

constexpr auto gitHubHost = "https://api.github.com";

std::string version = version_.empty() ? "latest" : version_;
std::ostringstream engineReleasePath;
engineReleasePath << "/repos/janhq/" << engineName_ << "/releases/"
<< version_;

<< version;
LOG_INFO << "Engine release path: " << gitHubHost << engineReleasePath.str();
using namespace nlohmann;

httplib::Client cli(gitHubHost);
Expand All @@ -51,9 +53,37 @@ void EngineInitCmd::Exec() const {
auto assets = jsonResponse["assets"];
auto os_arch{system_info.os + "-" + system_info.arch};

std::vector<std::string> variants;
for (auto& asset : assets) {
auto asset_name = asset["name"].get<std::string>();
variants.push_back(asset_name);
}

auto cuda_version = system_info_utils::GetCudaVersion();
LOG_INFO << "engineName_: " << engineName_;
LOG_INFO << "CUDA version: " << cuda_version;
std::string matched_variant = "";
if (engineName_ == "cortex.tensorrt-llm") {
matched_variant = engine_matcher_utils::ValidateTensorrtLlm(
variants, system_info.os, cuda_version);
} else if (engineName_ == "cortex.onnx") {
matched_variant = engine_matcher_utils::ValidateOnnx(
variants, system_info.os, system_info.arch);
} else if (engineName_ == "cortex.llamacpp") {
auto suitable_avx = engine_matcher_utils::GetSuitableAvxVariant();
matched_variant = engine_matcher_utils::Validate(
variants, system_info.os, system_info.arch, suitable_avx,
cuda_version);
}
LOG_INFO << "Matched variant: " << matched_variant;
if (matched_variant.empty()) {
LOG_ERROR << "No variant found for " << os_arch;
return;
}

for (auto& asset : assets) {
auto assetName = asset["name"].get<std::string>();
if (assetName.find(os_arch) != std::string::npos) {
if (assetName == matched_variant) {
std::string host{"https://github.com"};

auto full_url = asset["browser_download_url"].get<std::string>();
Expand All @@ -74,8 +104,7 @@ void EngineInitCmd::Exec() const {
}}};

DownloadService().AddDownloadTask(
downloadTask,
[&downloadTask](const std::string& absolute_path) {
downloadTask, [](const std::string& absolute_path) {
// try to unzip the downloaded file
std::filesystem::path downloadedEnginePath{absolute_path};
LOG_INFO << "Downloaded engine path: "
Expand All @@ -95,15 +124,15 @@ void EngineInitCmd::Exec() const {
return;
}
}
LOG_ERROR << "No asset found for " << os_arch;
} catch (const json::parse_error& e) {
std::cerr << "JSON parse error: " << e.what() << std::endl;
}
} else {
LOG_ERROR << "HTTP error: " << res->status;
}
} else {
auto err = res.error();
LOG_ERROR << "HTTP error: " << httplib::to_string(err);
}
}

}; // namespace commands
}; // namespace commands
4 changes: 2 additions & 2 deletions engine/commands/engine_init_cmd.h
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ class EngineInitCmd {
std::string engineName_;
std::string version_;

static constexpr std::array<const char*, 1> supportedEngines_ = {
"cortex.llamacpp"};
static constexpr std::array<const char*, 3> supportedEngines_ = {
"cortex.llamacpp", "cortex.onnx", "cortex.tensorrt-llm"};
};
} // namespace commands
41 changes: 23 additions & 18 deletions engine/controllers/command_line_parser.cc
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#include "command_line_parser.h"
#include "commands/engine_init_cmd.h"
#include "commands/model_pull_cmd.h"
#include "commands/model_list_cmd.h"
#include "commands/model_pull_cmd.h"
#include "commands/start_model_cmd.h"
#include "commands/stop_model_cmd.h"
#include "commands/stop_server_cmd.h"
Expand Down Expand Up @@ -45,7 +45,7 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) {

auto list_models_cmd =
models_cmd->add_subcommand("list", "List all models locally");
list_models_cmd->callback([](){
list_models_cmd->callback([]() {
commands::ModelListCmd command;
command.Exec();
});
Expand Down Expand Up @@ -92,27 +92,15 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) {
auto embeddings_cmd = app_.add_subcommand(
"embeddings", "Creates an embedding vector representing the input text");

// engines group commands
{
{ // engines group commands
auto engines_cmd = app_.add_subcommand("engines", "Get cortex engines");
auto list_engines_cmd =
engines_cmd->add_subcommand("list", "List all cortex engines");
auto get_engine_cmd = engines_cmd->add_subcommand("get", "Get an engine");

{ // Engine init command
auto init_cmd = engines_cmd->add_subcommand("init", "Initialize engine");
std::string engine_name;
std::string version = "latest";

init_cmd->add_option("-n,--name", engine_name,
"Engine name. E.g: cortex.llamacpp");
init_cmd->add_option("-v,--version", version,
"Engine version. Default will be latest");
init_cmd->callback([&engine_name, &version]() {
commands::EngineInitCmd eic(engine_name, version);
eic.Exec();
});
}
EngineInstall(engines_cmd, "cortex.llamacpp");
EngineInstall(engines_cmd, "cortex.onnx");
EngineInstall(engines_cmd, "cortex.tensorrt-llm");
}

auto run_cmd =
Expand All @@ -128,4 +116,21 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) {

CLI11_PARSE(app_, argc, argv);
return true;
}

void CommandLineParser::EngineInstall(CLI::App* parent,
const std::string& engine_name) {
auto engine_cmd =
parent->add_subcommand(engine_name, "Manage " + engine_name + " engine");

// Default version is latest
std::string version{"latest"};
auto install_cmd = engine_cmd->add_subcommand(
"install", "Install " + engine_name + " engine");
install_cmd->add_option("-v, --version", version,
"Engine version. Default will be latest");
install_cmd->callback([&engine_name, &version] {
commands::EngineInitCmd eic(engine_name, version);
eic.Exec();
});
}
2 changes: 2 additions & 0 deletions engine/controllers/command_line_parser.h
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,7 @@ class CommandLineParser {
bool SetupCommand(int argc, char** argv);

private:
void EngineInstall(CLI::App* parent, const std::string& engine_name);

CLI::App app_;
};
111 changes: 111 additions & 0 deletions engine/controllers/engines.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
#include "engines.h"
#include "utils/archive_utils.h"
#include "utils/file_manager_utils.h"
#include "utils/system_info_utils.h"

void Engines::InitEngine(const HttpRequestPtr& req,
std::function<void(const HttpResponsePtr&)>&& callback,
const std::string& engine) const {
LOG_DEBUG << "InitEngine, Engine: " << engine;
if (engine.empty()) {
Json::Value res;
res["message"] = "Engine name is required";
auto resp = cortex_utils::CreateCortexHttpJsonResponse(res);
resp->setStatusCode(k409Conflict);
callback(resp);
LOG_WARN << "No engine field in path param";
return;
}

auto system_info = system_info_utils::GetSystemInfo();
if (system_info.arch == system_info_utils::kUnsupported ||
system_info.os == system_info_utils::kUnsupported) {
Json::Value res;
res["message"] = "Unsupported OS or architecture";
auto resp = cortex_utils::CreateCortexHttpJsonResponse(res);
resp->setStatusCode(k409Conflict);
callback(resp);
LOG_ERROR << "Unsupported OS or architecture: " << system_info.os << ", "
<< system_info.arch;
return;
}

auto version{"latest"};
constexpr auto gitHubHost = "https://api.github.com";

std::ostringstream engineReleasePath;
engineReleasePath << "/repos/janhq/" << engine << "/releases/" << version;

httplib::Client cli(gitHubHost);
using namespace nlohmann;
if (auto res = cli.Get(engineReleasePath.str())) {
if (res->status == httplib::StatusCode::OK_200) {
try {
auto jsonResponse = json::parse(res->body);
auto assets = jsonResponse["assets"];

auto os_arch{system_info.os + "-" + system_info.arch};
for (auto& asset : assets) {
auto assetName = asset["name"].get<std::string>();
if (assetName.find(os_arch) != std::string::npos) {
std::string host{"https://github.com"};

auto full_url = asset["browser_download_url"].get<std::string>();
std::string path = full_url.substr(host.length());

auto fileName = asset["name"].get<std::string>();
LOG_INFO << "URL: " << full_url;

auto downloadTask = DownloadTask{.id = engine,
.type = DownloadType::Engine,
.error = std::nullopt,
.items = {DownloadItem{
.id = engine,
.host = host,
.fileName = fileName,
.type = DownloadType::Engine,
.path = path,
}}};

DownloadService().AddAsyncDownloadTask(
downloadTask, [](const std::string& absolute_path) {
// try to unzip the downloaded file
std::filesystem::path downloadedEnginePath{absolute_path};
LOG_INFO << "Downloaded engine path: "
<< downloadedEnginePath.string();

archive_utils::ExtractArchive(
downloadedEnginePath.string(),
downloadedEnginePath.parent_path()
.parent_path()
.string());

// remove the downloaded file
std::filesystem::remove(absolute_path);
LOG_INFO << "Finished!";
});

Json::Value res;
res["message"] = "Engine download started";
res["result"] = "OK";
auto resp = cortex_utils::CreateCortexHttpJsonResponse(res);
resp->setStatusCode(k200OK);
callback(resp);
return;
}
}
Json::Value res;
res["message"] = "Engine not found";
res["result"] = "Error";
auto resp = cortex_utils::CreateCortexHttpJsonResponse(res);
resp->setStatusCode(k404NotFound);
callback(resp);
} catch (const json::parse_error& e) {
std::cerr << "JSON parse error: " << e.what() << std::endl;
}
}
} else {
auto err = res.error();
LOG_ERROR << "HTTP error: " << httplib::to_string(err);
}
}
21 changes: 21 additions & 0 deletions engine/controllers/engines.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#pragma once

#include <drogon/HttpController.h>
#include <trantor/utils/Logger.h>
#include "services/download_service.h"
#include "utils/cortex_utils.h"
#include "utils/cortexso_parser.h"
#include "utils/http_util.h"

using namespace drogon;

class Engines : public drogon::HttpController<Engines> {
public:
METHOD_LIST_BEGIN
METHOD_ADD(Engines::InitEngine, "/{1}/init", Post);
METHOD_LIST_END

void InitEngine(const HttpRequestPtr& req,
std::function<void(const HttpResponsePtr&)>&& callback,
const std::string& engine) const;
};
5 changes: 2 additions & 3 deletions engine/main.cc
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
#include <drogon/HttpAppFramework.h>
#include <drogon/drogon.h>
#include <climits> // for PATH_MAX
#include <iostream>
#include "controllers/command_line_parser.h"
#include "cortex-common/cortexpythoni.h"
#include "utils/archive_utils.h"
#include "utils/cortex_utils.h"
#include "utils/dylib.h"
#include "utils/archive_utils.h"

#if defined(__APPLE__) && defined(__MACH__)
#include <libgen.h> // for dirname()
Expand Down Expand Up @@ -99,4 +98,4 @@ int main(int argc, char* argv[]) {
drogon::app().run();

return 0;
}
}
49 changes: 49 additions & 0 deletions engine/utils/command_executor.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
#include <array>
#include <cstdio>
#include <iostream>
#include <memory>
#include <stdexcept>
#include <string>

#ifdef _WIN32
#define POPEN _popen
#define PCLOSE _pclose
#else
#define POPEN popen
#define PCLOSE pclose
#endif

class CommandExecutor {
public:
CommandExecutor(const std::string& command) {
FILE* pipe = POPEN(command.c_str(), "r");
if (!pipe) {
throw std::runtime_error("popen() failed!");
}
m_pipe = std::unique_ptr<FILE, decltype(&PCLOSE)>(pipe, PCLOSE);
}

CommandExecutor(const CommandExecutor&) = delete;
CommandExecutor& operator=(const CommandExecutor&) = delete;
CommandExecutor(CommandExecutor&&) = default;
CommandExecutor& operator=(CommandExecutor&&) = default;
~CommandExecutor() = default;

std::string execute() {
if (!m_pipe) {
throw std::runtime_error("Command not initialized!");
}

std::array<char, 128> buffer;
std::string result;

while (fgets(buffer.data(), buffer.size(), m_pipe.get()) != nullptr) {
result += buffer.data();
}

return result;
}

private:
std::unique_ptr<FILE, decltype(&PCLOSE)> m_pipe{nullptr, PCLOSE};
};
Loading

0 comments on commit de5716c

Please sign in to comment.