From 5c64b0a91d91f2cd093f2bd4ce711bc7e2d91fdb Mon Sep 17 00:00:00 2001 From: James Date: Thu, 26 Sep 2024 17:55:28 +0700 Subject: [PATCH] feat: cortex ps --- engine/commands/model_stop_cmd.cc | 3 +- engine/commands/model_stop_cmd.h | 6 +-- engine/commands/ps_cmd.cc | 63 +++++++++++++++++++++++ engine/commands/ps_cmd.h | 27 ++++++++++ engine/controllers/command_line_parser.cc | 8 ++- engine/controllers/command_line_parser.h | 1 - engine/utils/string_utils.h | 46 +++++++++++++++++ 7 files changed, 147 insertions(+), 7 deletions(-) create mode 100644 engine/commands/ps_cmd.cc create mode 100644 engine/commands/ps_cmd.h diff --git a/engine/commands/model_stop_cmd.cc b/engine/commands/model_stop_cmd.cc index 0f4816dad..f9a43141d 100644 --- a/engine/commands/model_stop_cmd.cc +++ b/engine/commands/model_stop_cmd.cc @@ -1,7 +1,6 @@ #include "model_stop_cmd.h" #include "httplib.h" #include "nlohmann/json.hpp" -#include "trantor/utils/Logger.h" #include "utils/logging_utils.h" namespace commands { @@ -30,4 +29,4 @@ void ModelStopCmd::Exec() { } } -}; // namespace commands \ No newline at end of file +}; // namespace commands diff --git a/engine/commands/model_stop_cmd.h b/engine/commands/model_stop_cmd.h index 9ac36e36d..2a817aaa7 100644 --- a/engine/commands/model_stop_cmd.h +++ b/engine/commands/model_stop_cmd.h @@ -1,11 +1,11 @@ #pragma once + #include -#include #include "config/model_config.h" namespace commands { -class ModelStopCmd{ +class ModelStopCmd { public: ModelStopCmd(std::string host, int port, const config::ModelConfig& mc); void Exec(); @@ -15,4 +15,4 @@ class ModelStopCmd{ int port_; const config::ModelConfig& mc_; }; -} // namespace commands \ No newline at end of file +} // namespace commands diff --git a/engine/commands/ps_cmd.cc b/engine/commands/ps_cmd.cc new file mode 100644 index 000000000..fab19b9b3 --- /dev/null +++ b/engine/commands/ps_cmd.cc @@ -0,0 +1,63 @@ +#include "ps_cmd.h" +#include +#include +#include +#include +#include "nlohmann/json.hpp" +#include "utils/logging_utils.h" +#include "utils/string_utils.h" + +namespace commands { + +void PsCmd::Exec(const std::string& host, int port) { + auto host_and_port{host + ":" + std::to_string(port)}; + httplib::Client cli(host_and_port); + + auto res = cli.Get("/inferences/server/models"); + if (!res || res->status != httplib::StatusCode::OK_200) { + CLI_LOG("No model loaded!"); + return; + } + + auto body = nlohmann::json::parse(res->body); + auto data = body["data"]; + std::vector model_status_list; + try { + for (const auto& item : data) { + ModelLoadedStatus model_status; + model_status.engine = item["engine"]; + model_status.model = item["id"]; + model_status.ram = item["ram"]; + model_status.start_time = item["start_time"]; + model_status.vram = item["vram"]; + model_status_list.push_back(model_status); + } + } catch (const std::exception& e) { + CLI_LOG("Fail to get list model information: " + std::string(e.what())); + } + + PrintModelStatusList(model_status_list); +} + +void PsCmd::PrintModelStatusList( + const std::vector& model_status_list) const { + if (model_status_list.empty()) { + CLI_LOG("No model loaded!"); + return; + } + + tabulate::Table table; + table.add_row({"Model", "Engine", "RAM", "VRAM", "Up time"}); + for (const auto& model_status : model_status_list) { + table.add_row({ + model_status.model, + model_status.engine, + model_status.ram, + model_status.vram, + string_utils::FormatTimeElapsed(model_status.start_time), + }); + } + std::cout << table << std::endl; +} + +}; // namespace commands diff --git a/engine/commands/ps_cmd.h b/engine/commands/ps_cmd.h new file mode 100644 index 000000000..c48d3d97f --- /dev/null +++ b/engine/commands/ps_cmd.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include + +namespace commands { + +struct ModelLoadedStatus { + std::string engine; + std::string model; + std::string ram; + uint64_t start_time; + std::string vram; +}; + +class PsCmd { + public: + explicit PsCmd() = default; + + void Exec(const std::string& host, int port); + + private: + void PrintModelStatusList( + const std::vector& model_status_list) const; +}; + +} // namespace commands diff --git a/engine/controllers/command_line_parser.cc b/engine/controllers/command_line_parser.cc index 3925ad502..61abf7182 100644 --- a/engine/controllers/command_line_parser.cc +++ b/engine/controllers/command_line_parser.cc @@ -15,6 +15,7 @@ #include "commands/model_start_cmd.h" #include "commands/model_stop_cmd.h" #include "commands/model_upd_cmd.h" +#include "commands/ps_cmd.h" #include "commands/run_cmd.h" #include "commands/server_start_cmd.h" #include "commands/server_stop_cmd.h" @@ -73,7 +74,7 @@ bool CommandLineParser::SetupCommand(int argc, char** argv) { #ifdef CORTEX_CPP_VERSION if (cml_data_.check_upd) { // TODO(sang) find a better way to handle - // This is an extremely ungly way to deal with connection + // This is an extremely ungly way to deal with connection // hang when network down std::atomic done = false; std::thread t([&]() { @@ -387,6 +388,11 @@ void CommandLineParser::SetupSystemCommands() { auto ps_cmd = app_.add_subcommand("ps", "Show running models and their status"); ps_cmd->group(kSystemGroup); + ps_cmd->usage("Usage:\n" + commands::GetCortexBinary() + "ps"); + ps_cmd->callback([&]() { + commands::PsCmd().Exec(cml_data_.config.apiServerHost, + std::stoi(cml_data_.config.apiServerPort)); + }); auto update_cmd = app_.add_subcommand("update", "Update cortex version"); update_cmd->group(kSystemGroup); diff --git a/engine/controllers/command_line_parser.h b/engine/controllers/command_line_parser.h index aaa24e064..1ca308eef 100644 --- a/engine/controllers/command_line_parser.h +++ b/engine/controllers/command_line_parser.h @@ -1,7 +1,6 @@ #pragma once #include "CLI/CLI.hpp" -#include "commands/model_upd_cmd.h" #include "services/engine_service.h" #include "utils/config_yaml_utils.h" class CommandLineParser { diff --git a/engine/utils/string_utils.h b/engine/utils/string_utils.h index 150b8a61f..8da860631 100644 --- a/engine/utils/string_utils.h +++ b/engine/utils/string_utils.h @@ -1,3 +1,5 @@ +#include +#include #include #include @@ -29,4 +31,48 @@ inline std::vector SplitBy(const std::string& str, } while (pos < str.length() && prev < str.length()); return tokens; } + +inline uint64_t getCurrentTimeInMilliseconds() { + using namespace std::chrono; + return duration_cast(system_clock::now().time_since_epoch()) + .count(); +} + +inline std::string FormatTimeElapsed(uint64_t pastTimestamp) { + uint64_t currentTimestamp = getCurrentTimeInMilliseconds(); + uint64_t milliseconds = currentTimestamp - pastTimestamp; + + // Constants for time units + const uint64_t millisInSecond = 1000; + const uint64_t millisInMinute = millisInSecond * 60; + const uint64_t millisInHour = millisInMinute * 60; + const uint64_t millisInDay = millisInHour * 24; + + uint64_t days = milliseconds / millisInDay; + milliseconds %= millisInDay; + + uint64_t hours = milliseconds / millisInHour; + milliseconds %= millisInHour; + + uint64_t minutes = milliseconds / millisInMinute; + milliseconds %= millisInMinute; + + uint64_t seconds = milliseconds / millisInSecond; + + std::ostringstream oss; + + if (days > 0) { + oss << days << " day" << (days > 1 ? "s" : "") << ", "; + } + if (hours > 0 || days > 0) { + oss << hours << " hour" << (hours > 1 ? "s" : "") << ", "; + } + if (minutes > 0 || hours > 0 || days > 0) { + oss << minutes << " minute" << (minutes > 1 ? "s" : "") << ", "; + } + + oss << seconds << " second" << (seconds > 1 ? "s" : ""); + + return oss.str(); +} } // namespace string_utils