fix: auto-reload model for remote engine (#1971)

Co-authored-by: sangjanai <[email protected]>
janhq · Feb 14, 2025 · c18c650 · c18c650
1 parent 0991133
commit c18c650
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 7 deletions.
diff --git a/engine/controllers/engines.cc b/engine/controllers/engines.cc
@@ -379,19 +379,21 @@ void Engines::UpdateEngine(
           metadata = (*exist_engine).metadata;
         }
 
-        (void) engine_service_->UnloadEngine(engine);
+        (void)engine_service_->UnloadEngine(engine);
 
         auto upd_res =
             engine_service_->UpsertEngine(engine, type, api_key, url, version,
                                           "all-platforms", status, metadata);
         if (upd_res.has_error()) {
           Json::Value res;
           res["message"] = upd_res.error();
+          CTL_WRN("Error: " << upd_res.error());
           auto resp = cortex_utils::CreateCortexHttpJsonResponse(res);
           resp->setStatusCode(k400BadRequest);
           callback(resp);
         } else {
           Json::Value res;
+          CTL_INF("Remote Engine update successfully!");
           res["message"] = "Remote Engine update successfully!";
           auto resp = cortex_utils::CreateCortexHttpJsonResponse(res);
           resp->setStatusCode(k200OK);
@@ -400,6 +402,7 @@ void Engines::UpdateEngine(
       } else {
         Json::Value res;
         res["message"] = "Request body is empty!";
+        CTL_WRN("Error: Request body is empty!");
         auto resp = cortex_utils::CreateCortexHttpJsonResponse(res);
         resp->setStatusCode(k400BadRequest);
         callback(resp);

diff --git a/engine/services/inference_service.cc b/engine/services/inference_service.cc
@@ -38,7 +38,7 @@ cpp::result<void, InferResult> InferenceService::HandleChatCompletion(
     LOG_WARN << "Engine is not loaded yet";
     return cpp::fail(std::make_pair(stt, res));
   }
- 
+
   if (!model_id.empty()) {
     if (auto model_service = model_service_.lock()) {
       auto metadata_ptr = model_service->GetCachedModelMetadata(model_id);
@@ -72,7 +72,6 @@ cpp::result<void, InferResult> InferenceService::HandleChatCompletion(
     }
   }
 
-
   CTL_DBG("Json body inference: " + json_body->toStyledString());
 
   auto cb = [q, tool_choice](Json::Value status, Json::Value res) {
@@ -217,10 +216,9 @@ InferResult InferenceService::LoadModel(
     std::get<RemoteEngineI*>(engine_result.value())
         ->LoadModel(json_body, std::move(cb));
   }
-  if (!engine_service_->IsRemoteEngine(engine_type)) {
-    auto model_id = json_body->get("model", "").asString();
-    saved_models_[model_id] = json_body;
-  }
+  // Save model config to reload if needed
+  auto model_id = json_body->get("model", "").asString();
+  saved_models_[model_id] = json_body;
   return std::make_pair(stt, r);
 }