From 8922896c7355620fb52a75dc161043f8b7541a5c Mon Sep 17 00:00:00 2001 From: vansangpfiev Date: Mon, 30 Dec 2024 16:46:55 +0700 Subject: [PATCH] fix: add remote engine to /v1/engines GET --- engine/common/engine_servicei.h | 7 ++- engine/controllers/engines.cc | 18 ++++++ .../extensions/remote-engine/remote_engine.cc | 55 ++++++++----------- engine/services/engine_service.cc | 5 +- engine/services/engine_service.h | 4 +- 5 files changed, 49 insertions(+), 40 deletions(-) diff --git a/engine/common/engine_servicei.h b/engine/common/engine_servicei.h index 17f5d3de2..ceb9b2fec 100644 --- a/engine/common/engine_servicei.h +++ b/engine/common/engine_servicei.h @@ -25,13 +25,14 @@ struct EngineVariantResponse { std::string name; std::string version; std::string engine; + std::string type; Json::Value ToJson() const { Json::Value root; root["name"] = name; root["version"] = version; root["engine"] = engine; - root["type"] = "local"; + root["type"] = type.empty() ? "local" : type; return root; } }; @@ -58,7 +59,7 @@ class EngineServiceI { virtual cpp::result GetEngineByNameAndVariant( const std::string& engine_name, - const std::optional variant = std::nullopt) = 0; + const std::optional variant = std::nullopt) const = 0; - virtual bool IsRemoteEngine(const std::string& engine_name) = 0; + virtual bool IsRemoteEngine(const std::string& engine_name) const = 0; }; diff --git a/engine/controllers/engines.cc b/engine/controllers/engines.cc index 3b3725089..8cf98785e 100644 --- a/engine/controllers/engines.cc +++ b/engine/controllers/engines.cc @@ -307,6 +307,24 @@ void Engines::GetInstalledEngineVariants( const HttpRequestPtr& req, std::function&& callback, const std::string& engine) const { + + if (engine_service_->IsRemoteEngine(engine)) { + auto remote_engines = engine_service_->GetEngines(); + Json::Value releases(Json::arrayValue); + if (remote_engines.has_value()) { + for (auto e : remote_engines.value()) { + if (e.type == kRemote && e.engine_name == engine) { + releases.append(e.ToJson()); + break; + } + } + } + auto resp = cortex_utils::CreateCortexHttpJsonResponse(releases); + resp->setStatusCode(k200OK); + callback(resp); + return; + } + auto result = engine_service_->GetInstalledEngineVariants(engine); if (result.has_error()) { Json::Value res; diff --git a/engine/extensions/remote-engine/remote_engine.cc b/engine/extensions/remote-engine/remote_engine.cc index 6c7b7881c..83d81c9c5 100644 --- a/engine/extensions/remote-engine/remote_engine.cc +++ b/engine/extensions/remote-engine/remote_engine.cc @@ -69,8 +69,6 @@ size_t StreamWriteCallback(char* ptr, size_t size, size_t nmemb, continue; } - Json::Reader reader; - Json::Value status; status["is_done"] = false; status["has_error"] = false; @@ -114,6 +112,12 @@ CurlResponse RemoteEngine::MakeStreamingChatCompletionRequest( headers = curl_slist_append(headers, "Connection: keep-alive"); std::string stream_template = chat_res_template_; + if (!config.transform_resp["chat_completions"] && + !config.transform_resp["chat_completions"]["template"]) { + // Model level overrides engine level + stream_template = + config.transform_resp["chat_completions"]["template"].as(); + } StreamContext context{ std::make_shared>( @@ -522,12 +526,16 @@ void RemoteEngine::HandleChatCompletion( // Get template string with error check std::string template_str; - try { + if (!chat_req_template_.empty()) { + CTL_DBG("Use engine transform request template: " << chat_req_template_); + template_str = chat_req_template_; + } + if (!model_config->transform_req["chat_completions"] && + !model_config->transform_req["chat_completions"]["template"]) { + // Model level overrides engine level template_str = model_config->transform_req["chat_completions"]["template"] .as(); - } catch (const YAML::BadConversion& e) { - throw std::runtime_error("Failed to convert template node to string: " + - std::string(e.what())); + CTL_DBG("Use model transform request template: " << template_str); } // Render with error handling @@ -586,33 +594,14 @@ void RemoteEngine::HandleChatCompletion( CTL_DBG( "Use engine transform response template: " << chat_res_template_); template_str = chat_res_template_; - } else { - // Check if required YAML nodes exist - if (!model_config->transform_resp["chat_completions"]) { - throw std::runtime_error( - "Missing 'chat_completions' node in transform_resp"); - } - if (!model_config->transform_resp["chat_completions"]["template"]) { - throw std::runtime_error( - "Missing 'template' node in chat_completions"); - } - - // Validate JSON body - if (!response_json || response_json.isNull()) { - throw std::runtime_error("Invalid or null JSON body"); - } - - // Get template string with error check - - try { - template_str = - model_config->transform_resp["chat_completions"]["template"] - .as(); - } catch (const YAML::BadConversion& e) { - throw std::runtime_error( - "Failed to convert template node to string: " + - std::string(e.what())); - } + } + if (!model_config->transform_resp["chat_completions"] && + !model_config->transform_resp["chat_completions"]["template"]) { + // Model level overrides engine level + template_str = + model_config->transform_resp["chat_completions"]["template"] + .as(); + CTL_DBG("Use model transform request template: " << template_str); } try { diff --git a/engine/services/engine_service.cc b/engine/services/engine_service.cc index 73212a048..c9bb8da56 100644 --- a/engine/services/engine_service.cc +++ b/engine/services/engine_service.cc @@ -1057,7 +1057,8 @@ cpp::result EngineService::GetEngineById( cpp::result EngineService::GetEngineByNameAndVariant( - const std::string& engine_name, const std::optional variant) { + const std::string& engine_name, + const std::optional variant) const { cortex::db::Engines engines; auto get_res = engines.GetEngineByNameAndVariant(engine_name, variant); @@ -1130,7 +1131,7 @@ cpp::result EngineService::GetRemoteModels( } } -bool EngineService::IsRemoteEngine(const std::string& engine_name) { +bool EngineService::IsRemoteEngine(const std::string& engine_name) const { auto ne = Repo2Engine(engine_name); auto local_engines = file_manager_utils::GetCortexConfig().supportedEngines; for (auto const& le : local_engines) { diff --git a/engine/services/engine_service.h b/engine/services/engine_service.h index 8ead4f6d6..d3d1180dd 100644 --- a/engine/services/engine_service.h +++ b/engine/services/engine_service.h @@ -134,7 +134,7 @@ class EngineService : public EngineServiceI { cpp::result GetEngineByNameAndVariant( const std::string& engine_name, - const std::optional variant = std::nullopt) override; + const std::optional variant = std::nullopt) const override; cpp::result UpsertEngine( const std::string& engine_name, const std::string& type, @@ -150,7 +150,7 @@ class EngineService : public EngineServiceI { void RegisterEngineLibPath(); - bool IsRemoteEngine(const std::string& engine_name) override; + bool IsRemoteEngine(const std::string& engine_name) const override; private: bool IsEngineLoaded(const std::string& engine);