fix: remote engine: mistral error on stream mode (#1871)

Co-authored-by: vansangpfiev <[email protected]>
janhq · Jan 17, 2025 · c2d8f61 · c2d8f61
1 parent d847779
commit c2d8f61
Show file tree

Hide file tree

Showing 3 changed files with 63 additions and 61 deletions.
diff --git a/engine/controllers/models.cc b/engine/controllers/models.cc
@@ -526,6 +526,7 @@ void Models::StartModel(
   if (auto& o = (*(req->getJsonObject()))["llama_model_path"]; !o.isNull()) {
     auto model_path = o.asString();
     if (auto& mp = (*(req->getJsonObject()))["model_path"]; mp.isNull()) {
+      mp = model_path;
       // Bypass if model does not exist in DB and llama_model_path exists
       if (std::filesystem::exists(model_path) &&
           !model_service_->HasModel(model_handle)) {

diff --git a/engine/extensions/remote-engine/remote_engine.cc b/engine/extensions/remote-engine/remote_engine.cc
@@ -27,8 +27,9 @@ size_t StreamWriteCallback(char* ptr, size_t size, size_t nmemb,
   auto* context = static_cast<StreamContext*>(userdata);
   std::string chunk(ptr, size * nmemb);
   CTL_DBG(chunk);
-  auto check_error = json_helper::ParseJsonString(chunk);
-  if (check_error.isMember("error")) {
+  Json::Value check_error;
+  Json::Reader reader;
+  if (reader.parse(chunk, check_error)) {
     CTL_WRN(chunk);
     Json::Value status;
     status["is_done"] = true;

diff --git a/engine/services/model_service.cc b/engine/services/model_service.cc
@@ -819,75 +819,75 @@ cpp::result<StartModelResult, std::string> ModelService::StartModel(
     constexpr const int kDefautlContextLength = 8192;
     int max_model_context_length = kDefautlContextLength;
     Json::Value json_data;
-    auto model_entry = db_service_->GetModelInfo(model_handle);
-    if (model_entry.has_error()) {
-      CTL_WRN("Error: " + model_entry.error());
-      return cpp::fail(model_entry.error());
-    }
-    yaml_handler.ModelConfigFromFile(
-        fmu::ToAbsoluteCortexDataPath(
-            fs::path(model_entry.value().path_to_model_yaml))
-            .string());
-    auto mc = yaml_handler.GetModelConfig();
-
-    // Check if Python model first
-    if (mc.engine == kPythonEngine) {
-
-      config::PythonModelConfig python_model_config;
-      python_model_config.ReadFromYaml(
-
+    // Currently we don't support download vision models, so we need to bypass check
+    if (!bypass_model_check) {
+      auto model_entry = db_service_->GetModelInfo(model_handle);
+      if (model_entry.has_error()) {
+        CTL_WRN("Error: " + model_entry.error());
+        return cpp::fail(model_entry.error());
+      }
+      yaml_handler.ModelConfigFromFile(
           fmu::ToAbsoluteCortexDataPath(
               fs::path(model_entry.value().path_to_model_yaml))
               .string());
-      // Start all depends model
-      auto depends = python_model_config.depends;
-      for (auto& depend : depends) {
-        Json::Value temp;
-        auto res = StartModel(depend, temp, false);
-        if (res.has_error()) {
-          CTL_WRN("Error: " + res.error());
-          for (auto& depend : depends) {
-            if (depend != model_handle) {
-              StopModel(depend);
+      auto mc = yaml_handler.GetModelConfig();
+
+      // Check if Python model first
+      if (mc.engine == kPythonEngine) {
+
+        config::PythonModelConfig python_model_config;
+        python_model_config.ReadFromYaml(
+
+            fmu::ToAbsoluteCortexDataPath(
+                fs::path(model_entry.value().path_to_model_yaml))
+                .string());
+        // Start all depends model
+        auto depends = python_model_config.depends;
+        for (auto& depend : depends) {
+          Json::Value temp;
+          auto res = StartModel(depend, temp, false);
+          if (res.has_error()) {
+            CTL_WRN("Error: " + res.error());
+            for (auto& depend : depends) {
+              if (depend != model_handle) {
+                StopModel(depend);
+              }
             }
+            return cpp::fail("Model failed to start dependency '" + depend +
+                             "' : " + res.error());
           }
-          return cpp::fail("Model failed to start dependency '" + depend +
-                           "' : " + res.error());
         }
-      }
 
-      json_data["model"] = model_handle;
-      json_data["model_path"] =
-          fmu::ToAbsoluteCortexDataPath(
-              fs::path(model_entry.value().path_to_model_yaml))
-              .string();
-      json_data["engine"] = mc.engine;
-      assert(!!inference_svc_);
-      // Check if python engine
-
-      auto ir =
-          inference_svc_->LoadModel(std::make_shared<Json::Value>(json_data));
-      auto status = std::get<0>(ir)["status_code"].asInt();
-      auto data = std::get<1>(ir);
-
-      if (status == drogon::k200OK) {
-        return StartModelResult{.success = true, .warning = ""};
-      } else if (status == drogon::k409Conflict) {
-        CTL_INF("Model '" + model_handle + "' is already loaded");
-        return StartModelResult{.success = true, .warning = ""};
-      } else {
-        // only report to user the error
-        for (auto& depend : depends) {
+        json_data["model"] = model_handle;
+        json_data["model_path"] =
+            fmu::ToAbsoluteCortexDataPath(
+                fs::path(model_entry.value().path_to_model_yaml))
+                .string();
+        json_data["engine"] = mc.engine;
+        assert(!!inference_svc_);
+        // Check if python engine
+
+        auto ir =
+            inference_svc_->LoadModel(std::make_shared<Json::Value>(json_data));
+        auto status = std::get<0>(ir)["status_code"].asInt();
+        auto data = std::get<1>(ir);
 
-          StopModel(depend);
+        if (status == drogon::k200OK) {
+          return StartModelResult{.success = true, .warning = ""};
+        } else if (status == drogon::k409Conflict) {
+          CTL_INF("Model '" + model_handle + "' is already loaded");
+          return StartModelResult{.success = true, .warning = ""};
+        } else {
+          // only report to user the error
+          for (auto& depend : depends) {
+
+            StopModel(depend);
+          }
         }
+        CTL_ERR("Model failed to start with status code: " << status);
+        return cpp::fail("Model failed to start: " +
+                         data["message"].asString());
       }
-      CTL_ERR("Model failed to start with status code: " << status);
-      return cpp::fail("Model failed to start: " + data["message"].asString());
-    }
-
-    // Currently we don't support download vision models, so we need to bypass check
-    if (!bypass_model_check) {
 
       // Running remote model
       if (engine_svc_->IsRemoteEngine(mc.engine)) {