Skip to content

Commit

Permalink
Merge pull request #1702 from janhq/dev
Browse files Browse the repository at this point in the history
Sync dev to main
hiento09 authored Nov 19, 2024
2 parents 8d60c95 + 70e25b7 commit 3932539
Showing 16 changed files with 1,010 additions and 735 deletions.
929 changes: 283 additions & 646 deletions docs/static/openapi/cortex.json

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions engine/cli/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -73,6 +73,7 @@ add_executable(${TARGET_NAME} main.cc
${CMAKE_CURRENT_SOURCE_DIR}/../utils/cpuid/cpu_info.cc
${CMAKE_CURRENT_SOURCE_DIR}/../utils/file_logger.cc
${CMAKE_CURRENT_SOURCE_DIR}/command_line_parser.cc
${CMAKE_CURRENT_SOURCE_DIR}/../services/config_service.cc
${CMAKE_CURRENT_SOURCE_DIR}/../services/download_service.cc
${CMAKE_CURRENT_SOURCE_DIR}/../services/engine_service.cc
${CMAKE_CURRENT_SOURCE_DIR}/../services/model_service.cc
34 changes: 14 additions & 20 deletions engine/cli/command_line_parser.cc
Original file line number Diff line number Diff line change
@@ -326,14 +326,8 @@ void CommandLineParser::SetupModelCommands() {
void CommandLineParser::SetupConfigsCommands() {
auto config_cmd =
app_.add_subcommand("config", "Subcommands for managing configurations");
config_cmd->usage(
"Usage:\n" + commands::GetCortexBinary() +
" config status for listing all API server configuration.\n" +
commands::GetCortexBinary() +
" config --cors [on/off] to toggle CORS.\n" +
commands::GetCortexBinary() +
" config --allowed_origins [comma separated origin] to set a list of "
"allowed origin");
config_cmd->usage("Usage:\n" + commands::GetCortexBinary() +
" config [option] [value]");
config_cmd->group(kConfigGroup);
auto config_status_cmd =
config_cmd->add_subcommand("status", "Print all configurations");
@@ -344,18 +338,18 @@ void CommandLineParser::SetupConfigsCommands() {
std::stoi(cml_data_.config.apiServerPort));
});

// TODO: this can be improved
std::vector<std::string> avai_opts{"cors", "allowed_origins"};
std::unordered_map<std::string, std::string> description{
{"cors", "[on/off] Toggling CORS."},
{"allowed_origins",
"Allowed origins for CORS. Comma separated. E.g. "
"http://localhost,https://cortex.so"}};
for (const auto& opt : avai_opts) {
std::string option = "--" + opt;
config_cmd->add_option(option, config_update_opts_[opt], description[opt])
->expected(0, 1)
->default_str("*");
for (const auto& [key, opt] : CONFIGURATIONS) {
std::string option = "--" + opt.name;
auto option_cmd =
config_cmd->add_option(option, config_update_opts_[opt.name], opt.desc)
->group(opt.group)
->default_str(opt.default_value);

if (opt.allow_empty) {
option_cmd->expected(0, 1);
} else {
option_cmd->expected(1);
}
}

config_cmd->callback([this, config_cmd] {
30 changes: 21 additions & 9 deletions engine/cli/commands/config_upd_cmd.cc
Original file line number Diff line number Diff line change
@@ -1,35 +1,39 @@
#include "config_upd_cmd.h"
#include "commands/server_start_cmd.h"
#include "common/api_server_configuration.h"
#include "utils/curl_utils.h"
#include "utils/logging_utils.h"
#include "utils/string_utils.h"
#include "utils/url_parser.h"

namespace {
const std::vector<std::string> config_keys{"cors", "allowed_origins"};

inline Json::Value NormalizeJson(
const std::unordered_map<std::string, std::string> options) {
Json::Value root;
for (const auto& [key, value] : options) {
if (std::find(config_keys.begin(), config_keys.end(), key) ==
config_keys.end()) {
if (CONFIGURATIONS.find(key) == CONFIGURATIONS.end()) {
continue;
}
auto config = CONFIGURATIONS.at(key);

if (key == "cors") {
if (config.accept_value == "[on|off]") {
if (string_utils::EqualsIgnoreCase("on", value)) {
root["cors"] = true;
root[key] = true;
} else if (string_utils::EqualsIgnoreCase("off", value)) {
root["cors"] = false;
root[key] = false;
}
} else if (key == "allowed_origins") {
} else if (config.accept_value == "comma separated") {
auto origins = string_utils::SplitBy(value, ",");
Json::Value origin_array(Json::arrayValue);
for (const auto& origin : origins) {
origin_array.append(origin);
}
root[key] = origin_array;
} else if (config.accept_value == "string") {
root[key] = value;
} else {
CTL_ERR("Not support configuration type: " << config.accept_value
<< " for config key: " << key);
}
}

@@ -50,13 +54,21 @@ void commands::ConfigUpdCmd::Exec(
}
}

auto non_null_opts = std::unordered_map<std::string, std::string>();
for (const auto& [key, value] : options) {
if (value.empty()) {
continue;
}
non_null_opts[key] = value;
}

auto url = url_parser::Url{
.protocol = "http",
.host = host + ":" + std::to_string(port),
.pathParams = {"v1", "configs"},
};

auto json = NormalizeJson(options);
auto json = NormalizeJson(non_null_opts);
if (json.empty()) {
CLI_LOG_ERROR("Invalid configuration options provided");
return;
200 changes: 197 additions & 3 deletions engine/common/api_server_configuration.h
Original file line number Diff line number Diff line change
@@ -5,22 +5,144 @@
#include <unordered_map>
#include <vector>

// current only support basic auth
enum class ProxyAuthMethod {
Basic,
Digest,
DigestIe,
Bearer,
Negotiate,
Ntlm,
NtlmWb,
Any,
AnySafe,
AuthOnly,
AwsSigV4
};

struct ApiConfigurationMetadata {
std::string name;
std::string desc;
std::string group;
std::string accept_value;
std::string default_value;

bool allow_empty = false;
};

static const std::unordered_map<std::string, ApiConfigurationMetadata>
CONFIGURATIONS = {
{"cors",
ApiConfigurationMetadata{
.name = "cors",
.desc = "Cross-Origin Resource Sharing configuration.",
.group = "CORS",
.accept_value = "[on|off]",
.default_value = "on"}},
{"allowed_origins",
ApiConfigurationMetadata{
.name = "allowed_origins",
.desc = "Allowed origins for CORS. Comma separated. E.g. "
"http://localhost,https://cortex.so",
.group = "CORS",
.accept_value = "comma separated",
.default_value = "*",
.allow_empty = true}},
{"proxy_url", ApiConfigurationMetadata{.name = "proxy_url",
.desc = "Proxy URL",
.group = "Proxy",
.accept_value = "string",
.default_value = ""}},
{"proxy_username", ApiConfigurationMetadata{.name = "proxy_username",
.desc = "Proxy Username",
.group = "Proxy",
.accept_value = "string",
.default_value = ""}},
{"proxy_password", ApiConfigurationMetadata{.name = "proxy_password",
.desc = "Proxy Password",
.group = "Proxy",
.accept_value = "string",
.default_value = ""}},
{"verify_proxy_ssl",
ApiConfigurationMetadata{.name = "verify_proxy_ssl",
.desc = "Verify SSL for proxy",
.group = "Proxy",
.accept_value = "[on|off]",
.default_value = "on"}},
{"verify_proxy_host_ssl",
ApiConfigurationMetadata{.name = "verify_proxy_host_ssl",
.desc = "Verify SSL for proxy",
.group = "Proxy",
.accept_value = "[on|off]",
.default_value = "on"}},
{"no_proxy", ApiConfigurationMetadata{.name = "no_proxy",
.desc = "No proxy for hosts",
.group = "Proxy",
.accept_value = "string",
.default_value = ""}},
{"verify_peer_ssl", ApiConfigurationMetadata{.name = "verify_peer_ssl",
.desc = "Verify peer SSL",
.group = "Proxy",
.accept_value = "[on|off]",
.default_value = "on"}},
{"verify_host_ssl", ApiConfigurationMetadata{.name = "verify_host_ssl",
.desc = "Verify host SSL",
.group = "Proxy",
.accept_value = "[on|off]",
.default_value = "on"}},
};

class ApiServerConfiguration {
public:
ApiServerConfiguration(bool cors = true,
std::vector<std::string> allowed_origins = {})
: cors{cors}, allowed_origins{allowed_origins} {}
ApiServerConfiguration(
bool cors = true, std::vector<std::string> allowed_origins = {},
bool verify_proxy_ssl = true, bool verify_proxy_host_ssl = true,
const std::string& proxy_url = "", const std::string& proxy_username = "",
const std::string& proxy_password = "", const std::string& no_proxy = "",
bool verify_peer_ssl = true, bool verify_host_ssl = true)
: cors{cors},
allowed_origins{allowed_origins},
verify_proxy_ssl{verify_proxy_ssl},
verify_proxy_host_ssl{verify_proxy_host_ssl},
proxy_url{proxy_url},
proxy_username{proxy_username},
proxy_password{proxy_password},
no_proxy{no_proxy},
verify_peer_ssl{verify_peer_ssl},
verify_host_ssl{verify_host_ssl} {}

// cors
bool cors{true};
std::vector<std::string> allowed_origins;

// proxy
bool verify_proxy_ssl{true};
bool verify_proxy_host_ssl{true};
ProxyAuthMethod proxy_auth_method{ProxyAuthMethod::Basic};
std::string proxy_url{""};
std::string proxy_username{""};
std::string proxy_password{""};
std::string no_proxy{""};

bool verify_peer_ssl{true};
bool verify_host_ssl{true};

Json::Value ToJson() const {
Json::Value root;
root["cors"] = cors;
root["allowed_origins"] = Json::Value(Json::arrayValue);
for (const auto& origin : allowed_origins) {
root["allowed_origins"].append(origin);
}
root["verify_proxy_ssl"] = verify_proxy_ssl;
root["verify_proxy_host_ssl"] = verify_proxy_host_ssl;
root["proxy_url"] = proxy_url;
root["proxy_username"] = proxy_username;
root["proxy_password"] = proxy_password;
root["no_proxy"] = no_proxy;
root["verify_peer_ssl"] = verify_peer_ssl;
root["verify_host_ssl"] = verify_host_ssl;

return root;
}

@@ -31,6 +153,78 @@ class ApiServerConfiguration {
const std::unordered_map<std::string,
std::function<bool(const Json::Value&)>>
field_updater{
{"verify_peer_ssl",
[this](const Json::Value& value) -> bool {
if (!value.isBool()) {
return false;
}
verify_peer_ssl = value.asBool();
return true;
}},

{"verify_host_ssl",
[this](const Json::Value& value) -> bool {
if (!value.isBool()) {
return false;
}
verify_host_ssl = value.asBool();
return true;
}},

{"verify_proxy_host_ssl",
[this](const Json::Value& value) -> bool {
if (!value.isBool()) {
return false;
}
verify_proxy_host_ssl = value.asBool();
return true;
}},

{"verify_proxy_ssl",
[this](const Json::Value& value) -> bool {
if (!value.isBool()) {
return false;
}
verify_proxy_ssl = value.asBool();
return true;
}},

{"no_proxy",
[this](const Json::Value& value) -> bool {
if (!value.isString()) {
return false;
}
no_proxy = value.asString();
return true;
}},

{"proxy_url",
[this](const Json::Value& value) -> bool {
if (!value.isString()) {
return false;
}
proxy_url = value.asString();
return true;
}},

{"proxy_username",
[this](const Json::Value& value) -> bool {
if (!value.isString()) {
return false;
}
proxy_username = value.asString();
return true;
}},

{"proxy_password",
[this](const Json::Value& value) -> bool {
if (!value.isString()) {
return false;
}
proxy_password = value.asString();
return true;
}},

{"cors",
[this](const Json::Value& value) -> bool {
if (!value.isBool()) {
7 changes: 5 additions & 2 deletions engine/config/yaml_config.cc
Original file line number Diff line number Diff line change
@@ -8,6 +8,7 @@
#include "utils/file_manager_utils.h"
#include "utils/format_utils.h"
#include "yaml_config.h"

namespace config {
// Method to read YAML file
void YamlHandler::Reset() {
@@ -44,6 +45,7 @@ void YamlHandler::ReadYamlFile(const std::string& file_path) {
throw;
}
}

void YamlHandler::SplitPromptTemplate(ModelConfig& mc) {
if (mc.prompt_template.size() > 0) {
auto& pt = mc.prompt_template;
@@ -220,7 +222,7 @@ void YamlHandler::UpdateModelConfig(ModelConfig new_model_config) {
yaml_node_["ngl"] = model_config_.ngl;
if (!std::isnan(static_cast<double>(model_config_.ctx_len)))
yaml_node_["ctx_len"] = model_config_.ctx_len;
if (!std::isnan(static_cast<double>(model_config_.n_parallel)))
if (!std::isnan(static_cast<double>(model_config_.n_parallel)))
yaml_node_["n_parallel"] = model_config_.n_parallel;
if (!std::isnan(static_cast<double>(model_config_.tp)))
yaml_node_["tp"] = model_config_.tp;
@@ -377,7 +379,8 @@ void YamlHandler::WriteYamlFile(const std::string& file_path) const {
outFile << format_utils::writeKeyValue(
"ctx_len", yaml_node_["ctx_len"],
"llama.context_length | 0 or undefined = loaded from model");
outFile << format_utils::writeKeyValue("n_parallel", yaml_node_["n_parallel"]);
outFile << format_utils::writeKeyValue("n_parallel",
yaml_node_["n_parallel"]);
outFile << format_utils::writeKeyValue("ngl", yaml_node_["ngl"],
"Undefined = loaded from model");
outFile << "# END OPTIONAL\n";
5 changes: 3 additions & 2 deletions engine/main.cc
Original file line number Diff line number Diff line change
@@ -106,13 +106,14 @@ void RunServer(std::optional<int> port, bool ignore_cout) {
auto event_queue_ptr = std::make_shared<EventQueue>();
cortex::event::EventProcessor event_processor(event_queue_ptr);

auto download_service = std::make_shared<DownloadService>(event_queue_ptr);
auto config_service = std::make_shared<ConfigService>();
auto download_service =
std::make_shared<DownloadService>(event_queue_ptr, config_service);
auto engine_service = std::make_shared<EngineService>(download_service);
auto inference_svc =
std::make_shared<services::InferenceService>(engine_service);
auto model_service = std::make_shared<ModelService>(
download_service, inference_svc, engine_service);
auto config_service = std::make_shared<ConfigService>();

// initialize custom controllers
auto engine_ctl = std::make_shared<Engines>(engine_service);
26 changes: 22 additions & 4 deletions engine/services/config_service.cc
Original file line number Diff line number Diff line change
@@ -5,8 +5,12 @@
cpp::result<ApiServerConfiguration, std::string>
ConfigService::UpdateApiServerConfiguration(const Json::Value& json) {
auto config = file_manager_utils::GetCortexConfig();
ApiServerConfiguration api_server_config{config.enableCors,
config.allowedOrigins};
ApiServerConfiguration api_server_config{
config.enableCors, config.allowedOrigins, config.verifyProxySsl,
config.verifyProxyHostSsl, config.proxyUrl, config.proxyUsername,
config.proxyPassword, config.noProxy, config.verifyPeerSsl,
config.verifyHostSsl};

std::vector<std::string> updated_fields;
std::vector<std::string> invalid_fields;
std::vector<std::string> unknown_fields;
@@ -20,13 +24,27 @@ ConfigService::UpdateApiServerConfiguration(const Json::Value& json) {

config.enableCors = api_server_config.cors;
config.allowedOrigins = api_server_config.allowed_origins;
config.verifyProxySsl = api_server_config.verify_proxy_ssl;
config.verifyProxyHostSsl = api_server_config.verify_proxy_host_ssl;

config.proxyUrl = api_server_config.proxy_url;
config.proxyUsername = api_server_config.proxy_username;
config.proxyPassword = api_server_config.proxy_password;
config.noProxy = api_server_config.no_proxy;

config.verifyPeerSsl = api_server_config.verify_peer_ssl;
config.verifyHostSsl = api_server_config.verify_host_ssl;

auto result = file_manager_utils::UpdateCortexConfig(config);
return api_server_config;
}

cpp::result<ApiServerConfiguration, std::string>
ConfigService::GetApiServerConfiguration() const {
ConfigService::GetApiServerConfiguration() {
auto config = file_manager_utils::GetCortexConfig();
return ApiServerConfiguration{config.enableCors, config.allowedOrigins};
return ApiServerConfiguration{
config.enableCors, config.allowedOrigins, config.verifyProxySsl,
config.verifyProxyHostSsl, config.proxyUrl, config.proxyUsername,
config.proxyPassword, config.noProxy, config.verifyPeerSsl,
config.verifyHostSsl};
}
3 changes: 1 addition & 2 deletions engine/services/config_service.h
Original file line number Diff line number Diff line change
@@ -8,6 +8,5 @@ class ConfigService {
cpp::result<ApiServerConfiguration, std::string> UpdateApiServerConfiguration(
const Json::Value& json);

cpp::result<ApiServerConfiguration, std::string> GetApiServerConfiguration()
const;
cpp::result<ApiServerConfiguration, std::string> GetApiServerConfiguration();
};
46 changes: 46 additions & 0 deletions engine/services/download_service.cc
Original file line number Diff line number Diff line change
@@ -52,6 +52,48 @@ cpp::result<void, std::string> ProcessCompletedTransfers(CURLM* multi_handle) {
}
return {};
}

void SetUpProxy(CURL* handle, std::shared_ptr<ConfigService> config_service) {
auto configuration = config_service->GetApiServerConfiguration();
if (configuration.has_value()) {
if (!configuration->proxy_url.empty()) {
auto proxy_url = configuration->proxy_url;
auto verify_proxy_ssl = configuration->verify_proxy_ssl;
auto verify_proxy_host_ssl = configuration->verify_proxy_host_ssl;

auto verify_ssl = configuration->verify_peer_ssl;
auto verify_host_ssl = configuration->verify_host_ssl;

auto proxy_username = configuration->proxy_username;
auto proxy_password = configuration->proxy_password;

CTL_INF("=== Proxy configuration ===");
CTL_INF("Proxy url: " << proxy_url);
CTL_INF("Verify proxy ssl: " << verify_proxy_ssl);
CTL_INF("Verify proxy host ssl: " << verify_proxy_host_ssl);
CTL_INF("Verify ssl: " << verify_ssl);
CTL_INF("Verify host ssl: " << verify_host_ssl);

curl_easy_setopt(handle, CURLOPT_PROXY, proxy_url.c_str());
curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, verify_ssl ? 1L : 0L);
curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST,
verify_host_ssl ? 2L : 0L);

curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYPEER,
verify_proxy_ssl ? 1L : 0L);
curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYHOST,
verify_proxy_host_ssl ? 2L : 0L);

auto proxy_auth = proxy_username + ":" + proxy_password;
curl_easy_setopt(handle, CURLOPT_PROXYUSERPWD, proxy_auth.c_str());

curl_easy_setopt(handle, CURLOPT_NOPROXY,
configuration->no_proxy.c_str());
}
} else {
CTL_ERR("Failed to get configuration");
}
}
} // namespace

cpp::result<bool, std::string> DownloadService::AddDownloadTask(
@@ -87,6 +129,7 @@ cpp::result<uint64_t, std::string> DownloadService::GetFileSize(
return cpp::fail(static_cast<std::string>("Failed to init CURL"));
}

SetUpProxy(curl, config_service_);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
@@ -189,6 +232,8 @@ cpp::result<bool, std::string> DownloadService::Download(

curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers);
}

SetUpProxy(curl, config_service_);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &WriteCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, file);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
@@ -407,6 +452,7 @@ cpp::result<void, ProcessDownloadFailed> DownloadService::ProcessMultiDownload(

void DownloadService::SetUpCurlHandle(CURL* handle, const DownloadItem& item,
FILE* file, DownloadingData* dl_data) {
SetUpProxy(handle, config_service_);
curl_easy_setopt(handle, CURLOPT_URL, item.downloadUrl.c_str());
curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, WriteCallback);
curl_easy_setopt(handle, CURLOPT_WRITEDATA, file);
8 changes: 6 additions & 2 deletions engine/services/download_service.h
Original file line number Diff line number Diff line change
@@ -8,6 +8,7 @@
#include <unordered_set>
#include "common/download_task_queue.h"
#include "common/event.h"
#include "services/config_service.h"
#include "utils/result.hpp"

struct ProcessDownloadFailed {
@@ -20,6 +21,8 @@ class DownloadService {
private:
static constexpr int MAX_CONCURRENT_TASKS = 4;

std::shared_ptr<ConfigService> config_service_;

struct DownloadingData {
std::string task_id;
std::string item_id;
@@ -82,8 +85,9 @@ class DownloadService {

explicit DownloadService() = default;

explicit DownloadService(std::shared_ptr<EventQueue> event_queue)
: event_queue_{event_queue} {
explicit DownloadService(std::shared_ptr<EventQueue> event_queue,
std::shared_ptr<ConfigService> config_service)
: event_queue_{event_queue}, config_service_{config_service} {
InitializeWorkers();
};

281 changes: 281 additions & 0 deletions engine/templates/linux/install.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,281 @@
#!/bin/bash -e

# Check for root privileges
if [ "$(id -u)" != "0" ]; then
echo "This script must be run as root. Please run again with sudo."
exit 1
fi

# Determine the home directory based on the user
USER_TO_RUN_AS=${SUDO_USER:-$(whoami)}
if [ "$USER_TO_RUN_AS" = "root" ]; then
USER_HOME="/root"
else
USER_HOME="/home/$USER_TO_RUN_AS"
fi

# Check and suggest installing jq and tar if not present
check_install_jq_tar() {
RED='\033[0;31m'
GREEN='\033[0;32m'
NC='\033[0m' # No Color

if ! command -v jq &> /dev/null; then
echo -e "${RED}jq could not be found ...${NC}"
echo -e "${GREEN}Please install jq then rerun this script${NC}"
exit 1
fi

if ! command -v tar &> /dev/null; then
echo -e "${RED}tar could not be found ...${NC}"
echo -e "${GREEN}Please install tar then rerun this script${NC}"
exit 1
fi
}

# Function to fetch the latest version based on channel
get_latest_version() {
local channel=$1
local tag_name
case $channel in
stable)
tag_name=$(curl -s "https://api.github.com/repos/janhq/cortex.cpp/releases/latest" | grep -oP '"tag_name": "\K(.*)(?=")')
;;
beta)
tag_name=$(curl -s "https://api.github.com/repos/janhq/cortex.cpp/releases" | jq -r '.[] | select(.prerelease) | .tag_name' | head -n 1)
;;
nightly)
tag_name=$(curl -s "https://delta.jan.ai/cortex/latest/version.json" | jq -r '.tag_name')
;;
*)
echo "Invalid channel specified."
exit 1
;;
esac
echo "${tag_name#v}"
}

# Default values
CHANNEL="stable"
VERSION=""
IS_UPDATE="false"
DEB_LOCAL="false"

# Function to parse command-line arguments
parse_args() {
while [[ "$#" -gt 0 ]]; do
case $1 in
--channel)
CHANNEL="$2"
shift 2
;;
--version)
VERSION="$2"
shift 2
;;
--deb_local)
DEB_LOCAL="true"
shift 1
;;
--is_update)
IS_UPDATE="true"
shift 1
;;
*)
echo "Unknown option: $1"
exit 1
;;
esac
done
}

# Call parse_args function to handle options
parse_args "$@"

# Check if VERSION is empty and fetch latest if necessary
if [ -z "$VERSION" ]; then
VERSION=$(get_latest_version $CHANNEL)
fi

# Set paths based on channel
case $CHANNEL in
stable)
CLI_BINARY_NAME="cortex"
SERVER_BINARY_NAME="cortex-server"
DATA_DIR="$USER_HOME/cortexcpp"
UNINSTALL_SCRIPT="/usr/bin/cortex-uninstall.sh"
CONFIGURATION_FILE="$USER_HOME/.cortexrc"
DEB_APP_NAME="cortexcpp"
;;
beta)
CLI_BINARY_NAME="cortex-beta"
SERVER_BINARY_NAME="cortex-server-beta"
DATA_DIR="$USER_HOME/cortexcpp-beta"
UNINSTALL_SCRIPT="/usr/bin/cortex-beta-uninstall.sh"
CONFIGURATION_FILE="$USER_HOME/.cortexrc-beta"
DEB_APP_NAME="cortexcpp-beta"
;;
nightly)
CLI_BINARY_NAME="cortex-nightly"
SERVER_BINARY_NAME="cortex-server-nightly"
DATA_DIR="$USER_HOME/cortexcpp-nightly"
UNINSTALL_SCRIPT="/usr/bin/cortex-nightly-uninstall.sh"
CONFIGURATION_FILE="$USER_HOME/.cortexrc-nightly"
DEB_APP_NAME="cortexcpp-nightly"
;;
*)
echo "Invalid channel specified."
exit 1
;;
esac

INSTALL_DIR="/usr/bin"

# Function to download and extract cortex
install_cortex() {
local channel=$1
local version=$2
local is_deb=$3
local url_binary=""
local url_deb_local=""
local url_deb_network=""

case $channel in
stable)
url_binary="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-amd64.tar.gz"
url_deb_local="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-amd64-local-installer.deb"
url_deb_network="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-amd64-network-installer.deb"
;;
beta)
url_binary="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-amd64.tar.gz"
url_deb_local="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-amd64-local-installer.deb"
url_deb_network="https://github.com/janhq/cortex.cpp/releases/download/v${version}/cortex-${version}-linux-amd64-network-installer.deb"
;;
nightly)
url_binary="https://delta.jan.ai/cortex/v${version}/linux-amd64/cortex-nightly.tar.gz"
url_deb_local="https://delta.jan.ai/cortex/v${version}/linux-amd64/cortex-${version}-linux-amd64-local-installer.deb"
url_deb_network="https://delta.jan.ai/cortex/v${version}/linux-amd64/cortex-${version}-linux-amd64-network-installer.deb"
;;
esac

if [ "$is_deb" = "true" ]; then
# Download the deb package
if [ "$DEB_LOCAL" = "true" ]; then
echo "Downloading cortex $channel version $version from $url_deb_local"
curl -L $url_deb_local -o /tmp/cortex.deb
else
echo "Downloading cortex $channel version $version from $url_deb_network"
curl -L $url_deb_network -o /tmp/cortex.deb
fi

# Install the deb package
if [ "$IS_UPDATE" = "false" ]; then
apt-get install -y /tmp/cortex.deb
else
echo -e "n\n" | SKIP_POSTINSTALL=true apt-get install -y --allow-downgrades /tmp/cortex.deb
fi
rm -f /tmp/cortex.deb
else
echo "Downloading cortex $channel version $version from $url_binary"
curl -L $url_binary -o /tmp/cortex.tar.gz
tar -xzvf /tmp/cortex.tar.gz -C /tmp
chmod +x /tmp/cortex/*
cp /tmp/cortex/* /usr/bin/
# Check is update or not
if [ "$IS_UPDATE" = "false" ]; then
su -c "$INSTALL_DIR/$CLI_BINARY_NAME engines install llama-cpp" $USER_TO_RUN_AS
su -c "$INSTALL_DIR/$CLI_BINARY_NAME stop > /dev/null 2>&1" $USER_TO_RUN_AS
fi
rm -rf /tmp/cortex
rm -f /tmp/cortex.tar.gz
fi
}

# Function to create uninstall script
create_uninstall_script() {
local is_deb=$1
if [ "$is_deb" = "false" ]; then
cat << EOF > $UNINSTALL_SCRIPT
#!/bin/bash
# Check for root privileges
if [ "\$(id -u)" != "0" ]; then
echo "This script must be run as root. Please run again with sudo."
exit 1
fi
echo "Stopping cortex..."
su -c "$INSTALL_DIR/$CLI_BINARY_NAME stop > /dev/null 2>&1" $USER_TO_RUN_AS
rm -f $INSTALL_DIR/$CLI_BINARY_NAME
rm -f $INSTALL_DIR/$SERVER_BINARY_NAME
rm -f $UNINSTALL_SCRIPT
echo "Do you want to delete the $DATA_DIR data folder and file $CONFIGURATION_FILE? (yes/no) [default: no]"
read -r answer
while true; do
case "\$answer" in
[yY][eE][sS]|[yY])
echo "Deleting cortex data folders..."
if [ -d "$DATA_DIR" ]; then
echo "Removing $DATA_DIR"
rm -rf "$DATA_DIR" > /dev/null 2>&1
fi
if [ -f "$CONFIGURATION_FILE" ]; then
echo "Removing $CONFIGURATION_FILE"
rm -f "$CONFIGURATION_FILE" > /dev/null 2>&1
fi
break
;;
[nN][oO]|[nN]|"")
echo "Keeping the 'cortex' data folders."
break
;;
*)
echo "Invalid response. Please type 'yes', 'no', 'y', or 'n' (case-insensitive)."
read -r answer
;;
esac
done
EOF

else
cat << EOF > $UNINSTALL_SCRIPT
#!/bin/bash
# Check for root privileges
if [ "\$(id -u)" != "0" ]; then
echo "This script must be run as root. Please run again with sudo."
exit 1
fi
apt-get remove -y $DEB_APP_NAME
rm -f $UNINSTALL_SCRIPT
EOF
fi

chmod +x $UNINSTALL_SCRIPT
echo "Uninstall script created at $UNINSTALL_SCRIPT"
}

# Run installation
check_install_jq_tar

IS_DEB="false"

# Check if apt-get command is available
if command -v apt-get &> /dev/null; then
if [ "$IS_UPDATE" = "true" ]; then
# check if cortexcpp deb package is installed
if dpkg -l | grep -q $DEB_APP_NAME; then
IS_DEB="true"
else
IS_DEB="false"
fi
else
IS_DEB="true"
fi
fi

install_cortex $CHANNEL $VERSION $IS_DEB
create_uninstall_script $IS_DEB

echo "Installation complete. Run cortex-uninstall.sh to uninstall."
1 change: 1 addition & 0 deletions engine/test/components/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@ add_executable(${PROJECT_NAME}
${CMAKE_CURRENT_SOURCE_DIR}/../../config/gguf_parser.cc
${CMAKE_CURRENT_SOURCE_DIR}/../../cli/commands/cortex_upd_cmd.cc
${CMAKE_CURRENT_SOURCE_DIR}/../../cli/commands/server_stop_cmd.cc
${CMAKE_CURRENT_SOURCE_DIR}/../../services/config_service.cc
${CMAKE_CURRENT_SOURCE_DIR}/../../services/download_service.cc
${CMAKE_CURRENT_SOURCE_DIR}/../../database/models.cc
)
67 changes: 58 additions & 9 deletions engine/utils/config_yaml_utils.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#pragma once

#include <filesystem>
#include <fstream>
#include <iostream>
@@ -34,6 +35,16 @@ struct CortexConfig {

bool enableCors;
std::vector<std::string> allowedOrigins;

std::string proxyUrl;
bool verifyProxySsl;
bool verifyProxyHostSsl;
std::string proxyUsername;
std::string proxyPassword;
std::string noProxy;

bool verifyPeerSsl;
bool verifyHostSsl;
};

const std::string kDefaultHost{"127.0.0.1"};
@@ -46,6 +57,7 @@ constexpr const auto kDefaultLatestLlamacppRelease = "";
constexpr const auto kDefaultCorsEnabled = true;
const std::vector<std::string> kDefaultEnabledOrigins{
"http://localhost:39281", "http://127.0.0.1:39281", "http://0.0.0.0:39281"};
constexpr const auto kDefaultNoProxy = "localhost,127.0.0.1";

inline cpp::result<void, std::string> DumpYamlConfig(const CortexConfig& config,
const std::string& path) {
@@ -76,6 +88,14 @@ inline cpp::result<void, std::string> DumpYamlConfig(const CortexConfig& config,
node["llamacppVersion"] = config.llamacppVersion;
node["enableCors"] = config.enableCors;
node["allowedOrigins"] = config.allowedOrigins;
node["proxyUrl"] = config.proxyUrl;
node["verifyProxySsl"] = config.verifyProxySsl;
node["verifyProxyHostSsl"] = config.verifyProxyHostSsl;
node["proxyUsername"] = config.proxyUsername;
node["proxyPassword"] = config.proxyPassword;
node["noProxy"] = config.noProxy;
node["verifyPeerSsl"] = config.verifyPeerSsl;
node["verifyHostSsl"] = config.verifyHostSsl;

out_file << node;
out_file.close();
@@ -105,7 +125,11 @@ inline CortexConfig FromYaml(const std::string& path,
!node["huggingFaceToken"] || !node["gitHubUserAgent"] ||
!node["gitHubToken"] || !node["llamacppVariant"] ||
!node["llamacppVersion"] || !node["enableCors"] ||
!node["allowedOrigins"]);
!node["allowedOrigins"] || !node["proxyUrl"] ||
!node["proxyUsername"] || !node["proxyPassword"] ||
!node["verifyPeerSsl"] || !node["verifyHostSsl"] ||
!node["verifyProxySsl"] || !node["verifyProxyHostSsl"] ||
!node["noProxy"]);

CortexConfig config = {
.logFolderPath = node["logFolderPath"]
@@ -147,23 +171,48 @@ inline CortexConfig FromYaml(const std::string& path,
: default_cfg.latestLlamacppRelease,
.huggingFaceToken = node["huggingFaceToken"]
? node["huggingFaceToken"].as<std::string>()
: "",
: default_cfg.huggingFaceToken,
.gitHubUserAgent = node["gitHubUserAgent"]
? node["gitHubUserAgent"].as<std::string>()
: "",
.gitHubToken =
node["gitHubToken"] ? node["gitHubToken"].as<std::string>() : "",
: default_cfg.gitHubUserAgent,
.gitHubToken = node["gitHubToken"]
? node["gitHubToken"].as<std::string>()
: default_cfg.gitHubToken,
.llamacppVariant = node["llamacppVariant"]
? node["llamacppVariant"].as<std::string>()
: "",
: default_cfg.llamacppVariant,
.llamacppVersion = node["llamacppVersion"]
? node["llamacppVersion"].as<std::string>()
: "",
.enableCors = node["enableCors"] ? node["enableCors"].as<bool>() : true,
: default_cfg.llamacppVersion,
.enableCors = node["enableCors"] ? node["enableCors"].as<bool>()
: default_cfg.enableCors,
.allowedOrigins =
node["allowedOrigins"]
? node["allowedOrigins"].as<std::vector<std::string>>()
: std::vector<std::string>{}};
: default_cfg.allowedOrigins,
.proxyUrl = node["proxyUrl"] ? node["proxyUrl"].as<std::string>()
: default_cfg.proxyUrl,
.verifyProxySsl = node["verifyProxySsl"]
? node["verifyProxySsl"].as<bool>()
: default_cfg.verifyProxySsl,
.verifyProxyHostSsl = node["verifyProxyHostSsl"]
? node["verifyProxyHostSsl"].as<bool>()
: default_cfg.verifyProxyHostSsl,
.proxyUsername = node["proxyUsername"]
? node["proxyUsername"].as<std::string>()
: default_cfg.proxyUsername,
.proxyPassword = node["proxyPassword"]
? node["proxyPassword"].as<std::string>()
: default_cfg.proxyPassword,
.noProxy = node["noProxy"] ? node["noProxy"].as<std::string>()
: default_cfg.noProxy,
.verifyPeerSsl = node["verifyPeerSsl"]
? node["verifyPeerSsl"].as<bool>()
: default_cfg.verifyPeerSsl,
.verifyHostSsl = node["verifyHostSsl"]
? node["verifyHostSsl"].as<bool>()
: default_cfg.verifyHostSsl,
};
if (should_update_config) {
auto result = DumpYamlConfig(config, path);
if (result.has_error()) {
41 changes: 39 additions & 2 deletions engine/utils/curl_utils.h
Original file line number Diff line number Diff line change
@@ -23,6 +23,43 @@ size_t WriteCallback(void* contents, size_t size, size_t nmemb,
output->append((char*)contents, totalSize);
return totalSize;
}

void SetUpProxy(CURL* handle) {
auto config = file_manager_utils::GetCortexConfig();
if (!config.proxyUrl.empty()) {
auto proxy_url = config.proxyUrl;
auto verify_proxy_ssl = config.verifyProxySsl;
auto verify_proxy_host_ssl = config.verifyProxyHostSsl;

auto verify_ssl = config.verifyPeerSsl;
auto verify_host_ssl = config.verifyHostSsl;

auto proxy_username = config.proxyUsername;
auto proxy_password = config.proxyPassword;
auto no_proxy = config.noProxy;

CTL_INF("=== Proxy configuration ===");
CTL_INF("Proxy url: " << proxy_url);
CTL_INF("Verify proxy ssl: " << verify_proxy_ssl);
CTL_INF("Verify proxy host ssl: " << verify_proxy_host_ssl);
CTL_INF("Verify ssl: " << verify_ssl);
CTL_INF("Verify host ssl: " << verify_host_ssl);

curl_easy_setopt(handle, CURLOPT_PROXY, proxy_url.c_str());
curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, verify_ssl ? 1L : 0L);
curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, verify_host_ssl ? 2L : 0L);

curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYPEER,
verify_proxy_ssl ? 1L : 0L);
curl_easy_setopt(handle, CURLOPT_PROXY_SSL_VERIFYHOST,
verify_proxy_host_ssl ? 2L : 0L);

auto proxy_auth = proxy_username + ":" + proxy_password;
curl_easy_setopt(handle, CURLOPT_PROXYUSERPWD, proxy_auth.c_str());

curl_easy_setopt(handle, CURLOPT_NOPROXY, no_proxy.c_str());
}
}
} // namespace

inline std::optional<std::unordered_map<std::string, std::string>> GetHeaders(
@@ -51,6 +88,7 @@ inline cpp::result<std::string, std::string> SimpleGet(const std::string& url,

std::string readBuffer;

SetUpProxy(curl);
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
@@ -101,6 +139,7 @@ inline cpp::result<std::string, std::string> SimpleRequest(
}
std::string readBuffer;

SetUpProxy(curl);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers);
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
if (request_type == RequestType::PATCH) {
@@ -111,8 +150,6 @@ inline cpp::result<std::string, std::string> SimpleRequest(
} else if (request_type == RequestType::DEL) {
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE");
}
// enable below line for debugging
// curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
66 changes: 32 additions & 34 deletions engine/utils/file_manager_utils.h
Original file line number Diff line number Diff line change
@@ -150,29 +150,18 @@ inline cpp::result<void, std::string> UpdateCortexConfig(
return DumpYamlConfig(config, config_path.string());
}

inline cpp::result<void, std::string> CreateConfigFileIfNotExist() {
inline config_yaml_utils::CortexConfig GetDefaultConfig() {
auto config_path = GetConfigurationPath();
if (std::filesystem::exists(config_path)) {
// already exists, no need to create
return {};
}

auto default_data_folder_name = GetDefaultDataFolderName();
auto default_data_folder_path =
file_manager_utils::GetHomeDirectoryPath() / default_data_folder_name;

CLI_LOG("Config file not found. Creating one at " + config_path.string());
auto defaultDataFolderPath =
cortex_data_folder_path.empty()
? file_manager_utils::GetHomeDirectoryPath() /
default_data_folder_name
: std::filesystem::path(cortex_data_folder_path);
CLI_LOG("Default data folder path: " + defaultDataFolderPath.string());

auto config = config_yaml_utils::CortexConfig{
.logFolderPath = defaultDataFolderPath.string(),
return config_yaml_utils::CortexConfig{
.logFolderPath = default_data_folder_path.string(),
.logLlamaCppPath = kLogsLlamacppBaseName,
.logTensorrtLLMPath = kLogsTensorrtllmBaseName,
.logOnnxPath = kLogsOnnxBaseName,
.dataFolderPath = defaultDataFolderPath.string(),
.dataFolderPath = default_data_folder_path.string(),
.maxLogLines = config_yaml_utils::kDefaultMaxLines,
.apiServerHost = config_yaml_utils::kDefaultHost,
.apiServerPort = config_yaml_utils::kDefaultPort,
@@ -182,7 +171,30 @@ inline cpp::result<void, std::string> CreateConfigFileIfNotExist() {
.latestRelease = config_yaml_utils::kDefaultLatestRelease,
.latestLlamacppRelease = config_yaml_utils::kDefaultLatestLlamacppRelease,
.enableCors = config_yaml_utils::kDefaultCorsEnabled,
.allowedOrigins = config_yaml_utils::kDefaultEnabledOrigins};
.allowedOrigins = config_yaml_utils::kDefaultEnabledOrigins,
.proxyUrl = "",
.verifyProxySsl = true,
.verifyProxyHostSsl = true,
.proxyUsername = "",
.proxyPassword = "",
.noProxy = config_yaml_utils::kDefaultNoProxy,
.verifyPeerSsl = true,
.verifyHostSsl = true,
};
}

inline cpp::result<void, std::string> CreateConfigFileIfNotExist() {
auto config_path = GetConfigurationPath();
if (std::filesystem::exists(config_path)) {
// already exists, no need to create
return {};
}

auto default_data_folder_name = GetDefaultDataFolderName();

CLI_LOG("Config file not found. Creating one at " + config_path.string());
auto config = GetDefaultConfig();
CLI_LOG("Default data folder path: " + config.dataFolderPath);
return DumpYamlConfig(config, config_path.string());
}

@@ -191,23 +203,8 @@ inline config_yaml_utils::CortexConfig GetCortexConfig() {
auto default_data_folder_name = GetDefaultDataFolderName();
auto default_data_folder_path =
file_manager_utils::GetHomeDirectoryPath() / default_data_folder_name;
auto default_cfg = config_yaml_utils::CortexConfig{
.logFolderPath = default_data_folder_path.string(),
.logLlamaCppPath = kLogsLlamacppBaseName,
.logTensorrtLLMPath = kLogsTensorrtllmBaseName,
.logOnnxPath = kLogsOnnxBaseName,
.dataFolderPath = default_data_folder_path.string(),
.maxLogLines = config_yaml_utils::kDefaultMaxLines,
.apiServerHost = config_yaml_utils::kDefaultHost,
.apiServerPort = config_yaml_utils::kDefaultPort,
.checkedForUpdateAt = config_yaml_utils::kDefaultCheckedForUpdateAt,
.checkedForLlamacppUpdateAt =
config_yaml_utils::kDefaultCheckedForLlamacppUpdateAt,
.latestRelease = config_yaml_utils::kDefaultLatestRelease,
.latestLlamacppRelease = config_yaml_utils::kDefaultLatestLlamacppRelease,
.enableCors = config_yaml_utils::kDefaultCorsEnabled,
.allowedOrigins = config_yaml_utils::kDefaultEnabledOrigins};

auto default_cfg = GetDefaultConfig();
return config_yaml_utils::FromYaml(config_path.string(), default_cfg);
}

@@ -238,6 +235,7 @@ inline std::filesystem::path GetCortexDataPath() {
inline std::filesystem::path GetCortexLogPath() {
// TODO: We will need to support user to move the data folder to other place.
// TODO: get the variant of cortex. As discussed, we will have: prod, beta, nightly

// currently we will store cortex data at ~/cortexcpp
auto config = GetCortexConfig();
std::filesystem::path log_folder_path;

0 comments on commit 3932539

Please sign in to comment.