Merge branch 'dev' of github.com:janhq/nitro into feat/format-output

cortex-js/src/domain/config/config.interface.ts

@@ -5,4 +5,5 @@ export interface Config {
   // todo: will remove optional when all command request api server
   apiServerPort?: number;
   apiServerHost?: string;
+  logPath?: string;
 }

cortex-js/src/index.ts

@@ -21,7 +21,11 @@ export async function start(
   portNumber?: number,
   enginePortNumber?: number,
   dataFolder?: string,
+  logPath?: string,
 ) {
+  if (logPath) {
+    fileManagerService.setLogPath(logPath);
+  }
   if (name) {
     fileManagerService.setConfigProfile(name);
     const isProfileConfigExists = fileManagerService.profileConfigExists(name);
@@ -49,10 +53,10 @@ export async function start(
     Number(enginePortNumber) || configCortexCppPort || defaultCortexCppPort;
   const dataFolderPath = dataFolder;
 
-  return startServer(dataFolderPath);
+  return startServer(dataFolderPath, logPath);
 }
 
-async function startServer(dataFolderPath?: string) {
+async function startServer(dataFolderPath?: string, logPath?: string) {
   const config = await fileManagerService.getConfig();
   try {
     if (dataFolderPath) {
@@ -80,6 +84,7 @@ async function startServer(dataFolderPath?: string) {
       apiServerHost: host,
       apiServerPort: port,
       dataFolderPath: dataFolderPath || config.dataFolderPath,
+      logPath: logPath || config.logPath,
       cortexCppPort: enginePort,
     });
     return app;

cortex-js/src/infrastructure/commanders/cortex-command.commander.ts

@@ -34,6 +34,7 @@ type ServeOptions = {
   name?: string;
   configPath?: string;
   enginePort?: string;
+  logPath?: string;
 };
 
 @RootCommand({
@@ -74,6 +75,9 @@ export class CortexCommand extends CommandRunner {
     if (options?.name) {
       fileManagerService.setConfigProfile(options.name);
     }
+    if (options?.logPath) {
+      fileManagerService.setLogPath(options.logPath);
+    }
     if (options?.name) {
       const isProfileConfigExists = fileManagerService.profileConfigExists(
         options.name,
@@ -114,10 +118,14 @@ export class CortexCommand extends CommandRunner {
       console.log(chalk.blue(`Github: ${pkg.homepage}`));
       return;
     }
-    return this.startServer(showLogs, dataFolderPath);
+    return this.startServer(showLogs, dataFolderPath, options?.logPath);
   }
 
-  private async startServer(attach: boolean, dataFolderPath?: string) {
+  private async startServer(
+    attach: boolean,
+    dataFolderPath?: string,
+    logPath?: string,
+  ) {
     const config = await fileManagerService.getConfig();
     try {
       const startEngineSpinner = ora('Starting Cortex engine...');
@@ -155,11 +163,13 @@ export class CortexCommand extends CommandRunner {
           `API Playground available at http://${this.host}:${this.port}/api`,
         ),
       );
+
       await fileManagerService.writeConfigFile({
         ...config,
         apiServerHost: this.host,
         apiServerPort: this.port,
         dataFolderPath: dataFolderPath || config.dataFolderPath,
+        logPath: logPath || config.logPath,
         cortexCppPort: this.enginePort,
       });
       if (!attach) process.exit(0);
@@ -237,4 +247,12 @@ export class CortexCommand extends CommandRunner {
   parseEnginePort(value: string) {
     return value;
   }
+
+  @Option({
+    flags: '-lp, --logPath <logPath>',
+    description: 'Path to the logs folder',
+  })
+  parseLogPath(value: string) {
+    return value;
+  }
 }

cortex-js/src/infrastructure/services/file-manager/file-manager.service.ts

@@ -35,9 +35,10 @@ export class FileManagerService {
   private benchmarkFoldername = 'benchmark';
   private cortexEnginesFolderName = 'engines';
   private cortexTelemetryFolderName = 'telemetry';
+  private logFileName = 'cortex.log';
   private configProfile = process.env.CORTEX_PROFILE || 'default';
   private configPath = process.env.CORTEX_CONFIG_PATH || os.homedir();
-
+  private customLogPath = process.env.CORTEX_LOG_PATH || '';
   /**
    * Get cortex configs
    * @returns the config object
@@ -309,7 +310,10 @@ export class FileManagerService {
    * @returns the path to the cortex engines folder
    */
   async getLogPath(): Promise<string> {
-    return join(await this.getDataFolderPath(), 'cortex.log');
+    if (this.customLogPath) {
+      return this.customLogPath + `/${this.logFileName}`;
+    }
+    return join(await this.getDataFolderPath(), this.logFileName);
   }
 
   async createFolderIfNotExistInDataFolder(folderName: string): Promise<void> {
@@ -401,6 +405,10 @@ export class FileManagerService {
   public getConfigPath(): string {
     return this.configPath;
   }
+
+  public setLogPath(logPath: string) {
+    this.customLogPath = logPath;
+  }
 }
 
 export const fileManagerService = new FileManagerService();

engine/commands/engine_init_cmd.cc

@@ -9,6 +9,9 @@
 // clang-format on
 #include "utils/cuda_toolkit_utils.h"
 #include "utils/engine_matcher_utils.h"
+#if defined(_WIN32) || defined(__linux__)
+#include "utils/file_manager_utils.h"
+#endif
 
 namespace commands {
 
@@ -60,21 +63,22 @@ bool EngineInitCmd::Exec() const {
           variants.push_back(asset_name);
         }
 
-        auto cuda_version = system_info_utils::GetCudaVersion();
+        auto cuda_driver_version = system_info_utils::GetCudaVersion();
         CTLOG_INFO("engineName_: " << engineName_);
-        CTLOG_INFO("CUDA version: " << cuda_version);
+        CTLOG_INFO("CUDA version: " << cuda_driver_version);
         std::string matched_variant = "";
+
         if (engineName_ == "cortex.tensorrt-llm") {
           matched_variant = engine_matcher_utils::ValidateTensorrtLlm(
-              variants, system_info.os, cuda_version);
+              variants, system_info.os, cuda_driver_version);
         } else if (engineName_ == "cortex.onnx") {
           matched_variant = engine_matcher_utils::ValidateOnnx(
               variants, system_info.os, system_info.arch);
         } else if (engineName_ == "cortex.llamacpp") {
           auto suitable_avx = engine_matcher_utils::GetSuitableAvxVariant();
           matched_variant = engine_matcher_utils::Validate(
               variants, system_info.os, system_info.arch, suitable_avx,
-              cuda_version);
+              cuda_driver_version);
         }
         CTLOG_INFO("Matched variant: " << matched_variant);
         if (matched_variant.empty()) {
@@ -105,17 +109,46 @@ bool EngineInitCmd::Exec() const {
                                              }}};
 
             DownloadService download_service;
-            download_service.AddDownloadTask(downloadTask, [](const std::string&
-                                                                  absolute_path,
-                                                              bool unused) {
+            download_service.AddDownloadTask(downloadTask, [this](
+                                                               const std::string&
+                                                                   absolute_path,
+                                                               bool unused) {
               // try to unzip the downloaded file
               std::filesystem::path downloadedEnginePath{absolute_path};
-              CTLOG_INFO("Downloaded engine path: "
-                       << downloadedEnginePath.string());
+              CTLOG_INFO(
+                  "Downloaded engine path: " << downloadedEnginePath.string());
+
+              std::filesystem::path extract_path =
+                  downloadedEnginePath.parent_path().parent_path();
+
+              archive_utils::ExtractArchive(downloadedEnginePath.string(),
+                                            extract_path.string());
+#if defined(_WIN32) || defined(__linux__)
+              // FIXME: hacky try to copy the file. Remove this when we are able to set the library path
+              auto engine_path = extract_path / engineName_;
+              LOG_INFO << "Source path: " << engine_path.string();
+              auto executable_path =
+                  file_manager_utils::GetExecutableFolderContainerPath();
+              for (const auto& entry :
+                   std::filesystem::recursive_directory_iterator(engine_path)) {
+                if (entry.is_regular_file() &&
+                    entry.path().extension() != ".gz") {
+                  std::filesystem::path relative_path =
+                      std::filesystem::relative(entry.path(), engine_path);
+                  std::filesystem::path destFile =
+                      executable_path / relative_path;
 
-              archive_utils::ExtractArchive(
-                  downloadedEnginePath.string(),
-                  downloadedEnginePath.parent_path().parent_path().string());
+                  std::filesystem::create_directories(destFile.parent_path());
+                  std::filesystem::copy_file(
+                      entry.path(), destFile,
+                      std::filesystem::copy_options::overwrite_existing);
+
+                  std::cout << "Copied: " << entry.path().filename().string()
+                            << " to " << destFile.string() << std::endl;
+                }
+              }
+              std::cout << "DLL copying completed successfully." << std::endl;
+#endif
 
               // remove the downloaded file
               // TODO(any) Could not delete file on Windows because it is currently hold by httplib(?)
@@ -128,24 +161,47 @@ bool EngineInitCmd::Exec() const {
               CTLOG_INFO("Finished!");
             });
             if (system_info.os == "mac" || engineName_ == "cortex.onnx") {
-              return false;
+              // mac and onnx engine does not require cuda toolkit
+              return true;
             }
+
             // download cuda toolkit
             const std::string jan_host = "https://catalog.jan.ai";
             const std::string cuda_toolkit_file_name = "cuda.tar.gz";
             const std::string download_id = "cuda";
 
-            auto gpu_driver_version = system_info_utils::GetDriverVersion();       
-            if(gpu_driver_version.empty()) return true;     
+            // TODO: we don't have API to retrieve list of cuda toolkit dependencies atm because we hosting it at jan
+            // will have better logic after https://github.com/janhq/cortex/issues/1046 finished
+            // for now, assume that we have only 11.7 and 12.4
+            auto suitable_toolkit_version = "";
+            if (engineName_ == "cortex.tensorrt-llm") {
+              // for tensorrt-llm, we need to download cuda toolkit v12.4
+              suitable_toolkit_version = "12.4";
+            } else {
+              // llamacpp
+              auto cuda_driver_semver =
+                  semantic_version_utils::SplitVersion(cuda_driver_version);
+              if (cuda_driver_semver.major == 11) {
+                suitable_toolkit_version = "11.7";
+              } else if (cuda_driver_semver.major == 12) {
+                suitable_toolkit_version = "12.4";
+              }
+            }
+
+            // compare cuda driver version with cuda toolkit version
+            // cuda driver version should be greater than toolkit version to ensure compatibility
+            if (semantic_version_utils::CompareSemanticVersion(
+                    cuda_driver_version, suitable_toolkit_version) < 0) {
+              LOG_ERROR << "Your Cuda driver version " << cuda_driver_version
+                        << " is not compatible with cuda toolkit version "
+                        << suitable_toolkit_version;
+              return false;
+            }
 
-            auto cuda_runtime_version =
-                cuda_toolkit_utils::GetCompatibleCudaToolkitVersion(
-                    gpu_driver_version, system_info.os, engineName_);
-            LOG_INFO << "abc";
             std::ostringstream cuda_toolkit_path;
-            cuda_toolkit_path << "dist/cuda-dependencies/" << 11.7 << "/"
-                              << system_info.os << "/"
-                              << cuda_toolkit_file_name;
+            cuda_toolkit_path << "dist/cuda-dependencies/"
+                              << cuda_driver_version << "/" << system_info.os
+                              << "/" << cuda_toolkit_file_name;
 
             LOG_DEBUG << "Cuda toolkit download url: " << jan_host
                       << cuda_toolkit_path.str();

engine/utils/engine_matcher_utils.h

@@ -1,5 +1,5 @@
+#include <trantor/utils/Logger.h>
 #include <algorithm>
-#include <iostream>
 #include <iterator>
 #include <regex>
 #include <string>
@@ -94,9 +94,19 @@ inline std::string GetSuitableCudaVariant(
           bestMatchMinor = variantMinor;
         }
       }
-    } else if (cuda_version.empty() && selectedVariant.empty()) {
-      // If no CUDA version is provided, select the variant without any CUDA in the name
-      selectedVariant = variant;
+    }
+  }
+
+  // If no CUDA version is provided, select the variant without any CUDA in the name
+  if (selectedVariant.empty()) {
+    LOG_WARN
+        << "No suitable CUDA variant found, selecting a variant without CUDA";
+    for (const auto& variant : variants) {
+      if (variant.find("cuda") == std::string::npos) {
+        selectedVariant = variant;
+        LOG_INFO << "Found variant without CUDA: " << selectedVariant << "\n";
+        break;
+      }
     }
   }
 
@@ -178,4 +188,4 @@ inline std::string Validate(const std::vector<std::string>& variants,
 
   return cuda_compatible;
 }
-}  // namespace engine_matcher_utils
+}  // namespace engine_matcher_utils