fix: use updated configuration for remote model when reload (#1972)

Co-authored-by: sangjanai <[email protected]>
janhq · Feb 15, 2025 · df4b1a2 · df4b1a2
1 parent c18c650
commit df4b1a2
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/engine/services/inference_service.cc b/engine/services/inference_service.cc
@@ -24,8 +24,12 @@ cpp::result<void, InferResult> InferenceService::HandleChatCompletion(
     auto status = std::get<0>(ir)["status_code"].asInt();
     if (status != drogon::k200OK) {
       CTL_INF("Model is not loaded, start loading it: " << model_id);
-      auto res = LoadModel(saved_models_.at(model_id));
-      // ignore return result
+      // For remote engine, we use the updated configuration
+      if (engine_service_->IsRemoteEngine(engine_type)) {
+        (void)model_service_.lock()->StartModel(model_id, {}, false);
+      } else {
+        (void)LoadModel(saved_models_.at(model_id));
+      }
     }
   }