Fix/max context length (#1370)

* fix/mistral-nemo-chat-template * Fix: set max context length to 8192 * Fix: CI build window * Fix: CI build window * Fix: log download and CI build window * Fix: CI build window * Fix: CI build window * Fix: download log for model and engines
janhq · Oct 1, 2024 · 50f3419 · 50f3419
1 parent 5c2cae1
commit 50f3419
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 10 deletions.
diff --git a/engine/config/gguf_parser.cc b/engine/config/gguf_parser.cc
@@ -1,3 +1,4 @@
+#include <algorithm>
 #include <cstdint>
 #include <cstring>
 #include <ctime>
@@ -12,6 +13,7 @@
 #ifdef _WIN32
 #include <io.h>
 #include <windows.h>
+#include <limits>
 #else
 #include <sys/mman.h>  // For memory-mapped file
 #include <unistd.h>    // For file descriptors
@@ -25,6 +27,9 @@
 #include "trantor/utils/Logger.h"
 
 namespace config {
+#define NOMINMAX
+constexpr int kDefaultMaxContextLength = 8192;
+
 void GGUFHandler::OpenFile(const std::string& file_path) {
 #ifdef _WIN32
   HANDLE file_handle_ = INVALID_HANDLE_VALUE;
@@ -582,8 +587,8 @@ void GGUFHandler::ModelConfigFromMetadata() {
   model_config_.model = name;
   model_config_.id = name;
   model_config_.version = std::to_string(version);
-  model_config_.max_tokens = max_tokens;
-  model_config_.ctx_len = max_tokens;
+  model_config_.max_tokens = std::min<int>(kDefaultMaxContextLength, max_tokens);
+  model_config_.ctx_len = std::min<int>(kDefaultMaxContextLength, max_tokens);
   model_config_.ngl = ngl;
 }
 

diff --git a/engine/services/download_service.cc b/engine/services/download_service.cc
@@ -74,7 +74,7 @@ cpp::result<void, std::string> DownloadService::AddDownloadTask(
     }
   }
   if (dl_err_msg.has_value()) {
-    CTL_ERR(dl_err_msg.value());
+    // CTL_ERR(dl_err_msg.value());
     return cpp::fail(dl_err_msg.value());
   }
 
@@ -183,7 +183,7 @@ cpp::result<void, std::string> DownloadService::Download(
           CLI_LOG("Resuming download..");
         } else {
           CLI_LOG("Start over..");
-          return {};
+          return cpp::fail("Cancelled Resume download!");
         }
       } else {
         CLI_LOG(download_item.localPath.filename().string()
@@ -195,7 +195,7 @@ cpp::result<void, std::string> DownloadService::Download(
         if (answer == "Y" || answer == "y" || answer.empty()) {
           CLI_LOG("Re-downloading..");
         } else {
-          return {};
+          return cpp::fail("Cancelled Re-download!");
         }
       }
     }
@@ -232,7 +232,6 @@ cpp::result<void, std::string> DownloadService::Download(
 
   fclose(file);
   curl_easy_cleanup(curl);
-  CLI_LOG("Model " << download_id << " downloaded successfully!")
   return {};
 }
 

diff --git a/engine/services/download_service.h b/engine/services/download_service.h
@@ -77,9 +77,9 @@ class DownloadService {
   cpp::result<void, std::string> VerifyDownloadTask(
       DownloadTask& task) const noexcept;
 
-  cpp::result<void, std::string> Download(const std::string& download_id,
-                                          const DownloadItem& download_item,
-                                          bool allow_resume) noexcept;
+  cpp::result<void, std::string> Download(
+      const std::string& download_id, const DownloadItem& download_item,
+      bool allow_resume) noexcept;
 
   curl_off_t GetLocalFileSize(const std::filesystem::path& path) const;
 };
diff --git a/engine/services/model_service.cc b/engine/services/model_service.cc
@@ -239,8 +239,10 @@ cpp::result<std::string, std::string> ModelService::HandleUrl(
   } else {
     auto result = download_service_.AddDownloadTask(downloadTask, on_finished);
     if (result.has_error()) {
-      CTL_ERR(result.error());
+      // CTL_ERR(result.error());
       return cpp::fail(result.error());
+    } else {
+      CLI_LOG("Model " << model_id << " downloaded successfully!")
     }
     return unique_model_id;
   }
@@ -292,6 +294,8 @@ cpp::result<std::string, std::string> ModelService::DownloadModelFromCortexso(
 
   if (result.has_error()) {
     return cpp::fail(result.error());
+  } else {
+    CLI_LOG("Model " << model_id << " downloaded successfully!")
   }
 
   return model_id;