fix: engine Issues & API Issues (#1811)

* chore: convention * fix: correct get remote model list * feat: auto generate remote model config * feat: support update remote engine * fix: do not generate remote model * chore: change engine_name to engine * fix: api key template on engine level * fix: add type for local engine * chore: cleanup * fix: add remote engine to /v1/engines GET * fix: build * fix: load engine when start model * chore: add log * fix: ignore chat_completions in model * fix: delete remote model * fix: replace api_key_template by header_template * fix: use engine from model yaml * fix: better error handling in stream mode * chore: cleanup * chore: unit test for anthropic response --------- Co-authored-by: vansangpfiev <[email protected]>
janhq · Jan 13, 2025 · f8c1df6 · f8c1df6
1 parent c508e68
commit f8c1df6
Show file tree

Hide file tree

Showing 20 changed files with 663 additions and 322 deletions.
diff --git a/docs/static/openapi/cortex.json b/docs/static/openapi/cortex.json
@@ -5388,18 +5388,18 @@
           "engine",
           "version",
           "inference_params",
-          "TransformReq",
-          "TransformResp",
+          "transform_req",
+          "transform_resp",
           "metadata"
         ],
         "properties": {
           "model": {
             "type": "string",
             "description": "The identifier of the model."
           },
-          "api_key_template": {
+          "header_template": {
             "type": "string",
-            "description": "Template for the API key header."
+            "description": "Template for the header."
           },
           "engine": {
             "type": "string",
@@ -5432,7 +5432,7 @@
               }
             }
           },
-          "TransformReq": {
+          "transform_req": {
             "type": "object",
             "properties": {
               "get_models": {
@@ -5454,7 +5454,7 @@
               }
             }
           },
-          "TransformResp": {
+          "transform_resp": {
             "type": "object",
             "properties": {
               "chat_completions": {
@@ -6162,9 +6162,9 @@
             "description": "Number of GPU layers.",
             "example": 33
           },
-          "api_key_template": {
+          "header_template": {
             "type": "string",
-            "description": "Template for the API key header."
+            "description": "Template for the header."
           },
           "version": {
             "type": "string",

diff --git a/engine/common/engine_servicei.h b/engine/common/engine_servicei.h
@@ -25,12 +25,14 @@ struct EngineVariantResponse {
   std::string name;
   std::string version;
   std::string engine;
+  std::string type;
 
   Json::Value ToJson() const {
     Json::Value root;
     root["name"] = name;
     root["version"] = version;
     root["engine"] = engine;
+    root["type"] = type.empty() ? "local" : type;
     return root;
   }
 };
@@ -57,7 +59,7 @@ class EngineServiceI {
   virtual cpp::result<cortex::db::EngineEntry, std::string>
   GetEngineByNameAndVariant(
       const std::string& engine_name,
-      const std::optional<std::string> variant = std::nullopt) = 0;
-  
-  virtual bool IsRemoteEngine(const std::string& engine_name) = 0;
+      const std::optional<std::string> variant = std::nullopt) const = 0;
+
+  virtual bool IsRemoteEngine(const std::string& engine_name) const = 0;
 };
diff --git a/engine/config/model_config.h b/engine/config/model_config.h
@@ -11,23 +11,22 @@
 #include <stdexcept>
 #include <string>
 #include <vector>
-#include "config/remote_template.h"
 #include "utils/format_utils.h"
 #include "utils/remote_models_utils.h"
 
 namespace config {
 
 struct RemoteModelConfig {
   std::string model;
-  std::string api_key_template;
+  std::string header_template;
   std::string engine;
   std::string version;
-  std::size_t created;
+  size_t created;
   std::string object = "model";
   std::string owned_by = "";
   Json::Value inference_params;
-  Json::Value TransformReq;
-  Json::Value TransformResp;
+  Json::Value transform_req;
+  Json::Value transform_resp;
   Json::Value metadata;
   void LoadFromJson(const Json::Value& json) {
     if (!json.isObject()) {
@@ -36,8 +35,8 @@ struct RemoteModelConfig {
 
     // Load basic string fields
     model = json.get("model", model).asString();
-    api_key_template =
-        json.get("api_key_template", api_key_template).asString();
+    header_template =
+        json.get("header_template", header_template).asString();
     engine = json.get("engine", engine).asString();
     version = json.get("version", version).asString();
     created =
@@ -47,31 +46,8 @@ struct RemoteModelConfig {
 
     // Load JSON object fields directly
     inference_params = json.get("inference_params", inference_params);
-    TransformReq = json.get("TransformReq", TransformReq);
-    // Use default template if it is empty, currently we only support 2 remote engines
-    auto is_anthropic = [](const std::string& model) {
-      return model.find("claude") != std::string::npos;
-    };
-    if (TransformReq["chat_completions"]["template"].isNull()) {
-      if (is_anthropic(model)) {
-        TransformReq["chat_completions"]["template"] =
-            kAnthropicTransformReqTemplate;
-      } else {
-        TransformReq["chat_completions"]["template"] =
-            kOpenAITransformReqTemplate;
-      }
-    }
-    TransformResp = json.get("TransformResp", TransformResp);
-    if (TransformResp["chat_completions"]["template"].isNull()) {
-      if (is_anthropic(model)) {
-        TransformResp["chat_completions"]["template"] =
-            kAnthropicTransformRespTemplate;
-      } else {
-        TransformResp["chat_completions"]["template"] =
-            kOpenAITransformRespTemplate;
-      }
-    }
-
+    transform_req = json.get("transform_req", transform_req);
+    transform_resp = json.get("transform_resp", transform_resp);
     metadata = json.get("metadata", metadata);
   }
 
@@ -80,7 +56,7 @@ struct RemoteModelConfig {
 
     // Add basic string fields
     json["model"] = model;
-    json["api_key_template"] = api_key_template;
+    json["header_template"] = header_template;
     json["engine"] = engine;
     json["version"] = version;
     json["created"] = static_cast<Json::UInt64>(created);
@@ -89,8 +65,8 @@ struct RemoteModelConfig {
 
     // Add JSON object fields directly
     json["inference_params"] = inference_params;
-    json["TransformReq"] = TransformReq;
-    json["TransformResp"] = TransformResp;
+    json["transform_req"] = transform_req;
+    json["transform_resp"] = transform_resp;
     json["metadata"] = metadata;
 
     return json;
@@ -101,7 +77,7 @@ struct RemoteModelConfig {
 
     // Convert basic fields
     root["model"] = model;
-    root["api_key_template"] = api_key_template;
+    root["header_template"] = header_template;
     root["engine"] = engine;
     root["version"] = version;
     root["object"] = object;
@@ -111,8 +87,8 @@ struct RemoteModelConfig {
     // Convert Json::Value to YAML::Node using utility function
     root["inference_params"] =
         remote_models_utils::jsonToYaml(inference_params);
-    root["TransformReq"] = remote_models_utils::jsonToYaml(TransformReq);
-    root["TransformResp"] = remote_models_utils::jsonToYaml(TransformResp);
+    root["transform_req"] = remote_models_utils::jsonToYaml(transform_req);
+    root["transform_resp"] = remote_models_utils::jsonToYaml(transform_resp);
     root["metadata"] = remote_models_utils::jsonToYaml(metadata);
 
     // Save to file
@@ -134,7 +110,7 @@ struct RemoteModelConfig {
 
     // Load basic fields
     model = root["model"].as<std::string>("");
-    api_key_template = root["api_key_template"].as<std::string>("");
+    header_template = root["header_template"].as<std::string>("");
     engine = root["engine"].as<std::string>("");
     version = root["version"] ? root["version"].as<std::string>() : "";
     created = root["created"] ? root["created"].as<std::size_t>() : 0;
@@ -144,8 +120,8 @@ struct RemoteModelConfig {
     // Load complex fields using utility function
     inference_params =
         remote_models_utils::yamlToJson(root["inference_params"]);
-    TransformReq = remote_models_utils::yamlToJson(root["TransformReq"]);
-    TransformResp = remote_models_utils::yamlToJson(root["TransformResp"]);
+    transform_req = remote_models_utils::yamlToJson(root["transform_req"]);
+    transform_resp = remote_models_utils::yamlToJson(root["transform_resp"]);
     metadata = remote_models_utils::yamlToJson(root["metadata"]);
   }
 };

diff --git a/engine/config/remote_template.h b/engine/config/remote_template.h