Merge branch 'dev' of github.com:janhq/nitro into fix/add-dll-search-…

…path

.github/workflows/cortex-build.yml

@@ -2,7 +2,7 @@ name: CI Cortex Release
 
 on:
   push:
-    tags: ["v[0-9]+.[0-9]+.[0-9]+", "v[0-9]+.[0-9]+.[0-9]+-*", "!v[0-9]+.[0-9]+.[0-9]+-cortex-js"]
+    tags: ["v[0-9]+.[0-9]+.[0-9]+", "v[0-9]+.[0-9]+.[0-9]+-*", "!v[0-9]+.[0-9]+.[0-9]+-cortex-js", "!v[0-9]+.[0-9]+.[0-9]+-[0-9]+-cortex-js"]
     paths: ["cortex-cpp/**", "cortex-js/**"]
   workflow_dispatch:
 
@@ -187,6 +187,13 @@ jobs:
           rm package.json
           mv package-tmp.json package.json
 
+      - name: Remove build build-deps and build folder for windows
+        if: runner.os == 'Windows'
+        run: |
+          cd cortex-cpp
+          Remove-Item -Recurse -Force build
+          Remove-Item -Recurse -Force build-deps
+
       # build prebuilds
       - name: Build Prebuilds
         working-directory: cortex-cpp
@@ -207,7 +214,7 @@ jobs:
           asset_content_type: application/gzip
 
       # Setup .npmrc file to publish to npm - upload only once
-      - run: npm publish --access public
+      - run: rm -rf build-deps && rm -rf build && rm -rf prebuilds && npm publish --access public
         continue-on-error: true
         if: runner.os == 'linux'
         env:
@@ -219,14 +226,14 @@ jobs:
   build-cortex-single-binary:
     runs-on: ${{ matrix.runs-on }}
     needs: [create-draft-release, build-and-test]
-    timeout-minutes: 20
+    timeout-minutes: 40
     strategy:
       fail-fast: false
       matrix:
         include:
           - os: "linux"
             name: "amd64"
-            runs-on: "ubuntu-latest"
+            runs-on: "ubuntu-20-04"
 
           - os: "windows"
             name: "amd64"
@@ -315,14 +322,14 @@ jobs:
           which cp
           which mv
           npm install -g cpx
-          npx cpx ./dist/cortexso-macos ./
-          mv cortexso-macos cortex
+          npx cpx ./dist/cortexso ./
+          mv cortexso cortex
 
       - name: Code Signing macOS
         if: runner.os == 'macOS'
         run: |
           cd cortex-js
-          ./dist/cortexso-macos --help
+          ./dist/cortexso --help
           echo "--------"
           ./cortex --help
           make codesign-binary CODE_SIGN=true DEVELOPER_ID="${{ secrets.DEVELOPER_ID }}"

.github/workflows/cortex-js.yml

@@ -1,7 +1,7 @@
 name: Publish cortex js Package to npmjs
 on:
   push:
-    tags: ["v[0-9]+.[0-9]+.[0-9]+-cortex-js"]
+    tags: ["v[0-9]+.[0-9]+.[0-9]+-cortex-js", "v[0-9]+.[0-9]+.[0-9]+-[0-9]+-cortex-js"]
     paths:
       [
         "cortex-js/**",

cortex-cpp/addon.cc

@@ -27,7 +27,6 @@ static Napi::Env* s_env = nullptr;
 void start(const int port = 3929) {
   int thread_num = 1;
   std::string host = "127.0.0.1";
-  std::string uploads_folder_path;
   int logical_cores = std::thread::hardware_concurrency();
   int drogon_thread_num = std::max(thread_num, logical_cores);
 #ifdef CORTEX_CPP_VERSION
@@ -40,10 +39,6 @@ void start(const int port = 3929) {
   LOG_INFO << "Please load your model";
   drogon::app().addListener(host, port);
   drogon::app().setThreadNum(drogon_thread_num);
-  if (!uploads_folder_path.empty()) {
-    LOG_INFO << "Drogon uploads folder is at: " << uploads_folder_path;
-    drogon::app().setUploadPath(uploads_folder_path);
-  }
   LOG_INFO << "Number of thread is:" << drogon::app().getThreadNum();
 
   drogon::app().run();

cortex-cpp/controllers/server.cc

@@ -20,7 +20,6 @@ constexpr static auto kTensorrtLlmEngine = "cortex.tensorrt-llm";
 }  // namespace
 
 server::server() {
-
 // Some default values for now below
 // log_disable();  // Disable the log to file feature, reduce bloat for
 // target
@@ -297,7 +296,7 @@ void server::LoadModel(const HttpRequestPtr& req,
           get_engine_path(engine_type);
 #if defined(WIN32)
       auto ws = std::wstring(abs_path.begin(), abs_path.end());
-      if(AddDllDirectory(ws.c_str()) == 0) {
+      if (AddDllDirectory(ws.c_str()) == 0) {
         LOG_WARN << "Could not add dll directory: " << abs_path;
       }
 #endif
@@ -335,6 +334,36 @@ void server::LoadModel(const HttpRequestPtr& req,
   LOG_TRACE << "Done load model";
 }
 
+void server::UnloadEngine(
+    const HttpRequestPtr& req,
+    std::function<void(const HttpResponsePtr&)>&& callback) {
+  if (!HasFieldInReq(req, callback, "engine")) {
+    return;
+  }
+
+  auto engine_type =
+      (*(req->getJsonObject())).get("engine", cur_engine_type_).asString();
+  if (!IsEngineLoaded(engine_type)) {
+    Json::Value res;
+    res["message"] = "Engine is not loaded yet";
+    auto resp = cortex_utils::CreateCortexHttpJsonResponse(res);
+    resp->setStatusCode(k409Conflict);
+    callback(resp);
+    LOG_WARN << "Engine is not loaded yet";
+    return;
+  }
+
+  EngineI* e = std::get<EngineI*>(engines_[engine_type].engine);
+  delete e;
+  engines_.erase(engine_type);
+  LOG_INFO << "Unloaded engine " + engine_type;
+  Json::Value res;
+  res["message"] = "Unloaded engine " + engine_type;
+  auto resp = cortex_utils::CreateCortexHttpJsonResponse(res);
+  resp->setStatusCode(k200OK);
+  callback(resp);
+}
+
 void server::ProcessStreamRes(std::function<void(const HttpResponsePtr&)> cb,
                               std::shared_ptr<SyncQueue> q) {
   auto err_or_done = std::make_shared<std::atomic_bool>(false);

cortex-cpp/controllers/server.h

@@ -66,6 +66,8 @@ class server : public drogon::HttpController<server>,
   // ADD_METHOD_TO(server::handlePrelight, "/v1/embeddings", Options);
 
   // PATH_ADD("/llama/chat_completion", Post);
+  METHOD_ADD(server::UnloadEngine, "unloadengine", Post);
+
   METHOD_LIST_END
   void ChatCompletion(
       const HttpRequestPtr& req,
@@ -91,6 +93,8 @@ class server : public drogon::HttpController<server>,
   void FineTuning(
       const HttpRequestPtr& req,
       std::function<void(const HttpResponsePtr&)>&& callback) override;
+  void UnloadEngine(const HttpRequestPtr& req,
+                    std::function<void(const HttpResponsePtr&)>&& callback);
 
  private:
   void ProcessStreamRes(std::function<void(const HttpResponsePtr&)> cb,

cortex-cpp/main.cc

@@ -44,7 +44,6 @@ int main(int argc, char* argv[]) {
   int thread_num = 1;
   std::string host = "127.0.0.1";
   int port = 3928;
-  std::string uploads_folder_path;
 
   // Number of cortex-cpp threads
   if (argc > 1) {
@@ -61,11 +60,6 @@ int main(int argc, char* argv[]) {
     port = std::atoi(argv[3]);  // Convert string argument to int
   }
 
-  // Uploads folder path
-  if (argc > 4) {
-    uploads_folder_path = argv[4];
-  }
-
   int logical_cores = std::thread::hardware_concurrency();
   int drogon_thread_num = std::max(thread_num, logical_cores);
   // cortex_utils::nitro_logo();
@@ -79,10 +73,6 @@ int main(int argc, char* argv[]) {
   LOG_INFO << "Please load your model";
   drogon::app().addListener(host, port);
   drogon::app().setThreadNum(drogon_thread_num);
-  if (!uploads_folder_path.empty()) {
-    LOG_INFO << "Drogon uploads folder is at: " << uploads_folder_path;
-    drogon::app().setUploadPath(uploads_folder_path);
-  }
   LOG_INFO << "Number of thread is:" << drogon::app().getThreadNum();
 
   drogon::app().run();

cortex-cpp/package.json

@@ -9,10 +9,11 @@
     "url": "git+https://github.com/janhq/cortex.git"
   },
   "scripts": {
-    "install": "prebuild-install --runtime napi --backend cmake-js --config Release || cmake-js rebuild --config Release",
-    "build": "cmake-js configure --config Release && cmake-js build --config Release",
-    "rebuild": "cmake-js rebuild --config Release",
-    "prebuild": "prebuild --runtime napi --backend cmake-js --all --strip --verbose --config Release --include-regex \"\\.(node|exp|lib|so)$\"",
+    "install": "prebuild-install --runtime napi --backend cmake-js --config Release || yarn rebuild",
+    "build-deps": "cmake-js configure --directory ./cortex-cpp-deps --out ./build-deps/cortex-cpp-deps --config Release && cmake-js build --directory ./cortex-cpp-deps --out ./build-deps/cortex-cpp-deps --config Release",
+    "build": "yarn build-deps && cmake-js configure --config Release && cmake-js build --config Release",
+    "rebuild": "yarn build-deps && cmake-js rebuild --config Release",
+    "prebuild": "yarn build-deps && prebuild --runtime napi --backend cmake-js --all --strip --verbose --config Release --include-regex \"\\.(node|exp|lib|so)$\"",
     "upload": "prebuild --runtime napi --backend cmake-js --upload ${GITHUB_TOKEN}"
   },
   "author": "Jan <[email protected]>",
@@ -35,7 +36,6 @@
     ]
   },
   "files": [
-    "binding/*.js",
-    "binding/*.d.ts"
+    "**"
   ]
 }

cortex-js/Makefile

@@ -5,15 +5,15 @@ AZURE_TENANT_ID ?= xxxx
 AZURE_CLIENT_SECRET ?= xxxx
 AZURE_CERT_NAME ?= xxxx
 DEVELOPER_ID ?= xxxx
-CORTEX_EXE_IN ?= "dist/cortexso-win.exe"
+CORTEX_EXE_IN ?= "dist/cortexso.exe"
 CORTEX_EXE_OUT ?= "cortex.exe"
 CORTEX_VERSION ?= "0.0.0.1"
 
 update-app-info:
 ifeq ($(OS),Windows_NT)
 	@powershell -Command 'npx resedit --in $(CORTEX_EXE_IN) --out $(CORTEX_EXE_OUT) --icon "1,cortex.ico" --no-grow --company-name "Homebrew Computer Pte Ltd" --file-description "cortex cli" --file-version "$(CORTEX_VERSION)" --internal-name "cortex" --product-name "cortex" --product-version "$(CORTEX_VERSION)"'
 else ifeq ($(shell uname -s),Linux)
-	@cp ./dist/cortexso-linux ./cortex
+	@cp ./dist/cortexso ./cortex
 endif
 
 codesign-binary:

cortex-js/installer.iss

@@ -37,3 +37,50 @@ Name: "quicklaunchicon"; Description: "Create a &Quick Launch icon"; GroupDescri
 [Icons]
 Name: "{commondesktop}\Cortexso"; Filename: "{app}\cortex.exe"; Tasks: desktopicon
 Name: "{userappdata}\Microsoft\Internet Explorer\Quick Launch\Cortexso"; Filename: "{app}\cortex.exe"; Tasks: quicklaunchicon
+
+; Define the uninstall run section to execute commands before uninstallation
+[UninstallRun]
+Filename: "{app}\cortex.exe"; Parameters: "stop"; StatusMsg: "Stopping Cortexso service..."; Flags: runhidden
+
+; Use Pascal scripting to delete the directory for all users
+[Code]
+function GetUsersFolder: String;
+var
+  WinDir: String;
+begin
+  WinDir := ExpandConstant('{win}');
+  Result := Copy(WinDir, 1, Pos('\Windows', WinDir) - 1) + '\Users';
+end;
+
+procedure DeleteUserCortexFolder;
+var
+  UsersFolder: String;
+  FindRec: TFindRec;
+begin
+  UsersFolder := GetUsersFolder;
+  if FindFirst(UsersFolder + '\*', FindRec) then
+  begin
+    try
+      repeat
+        if (FindRec.Attributes and FILE_ATTRIBUTE_DIRECTORY <> 0) and
+           (FindRec.Name <> '.') and (FindRec.Name <> '..') then
+        begin
+          if DirExists(UsersFolder + '\' + FindRec.Name + '\cortex') then
+          begin
+            DelTree(UsersFolder + '\' + FindRec.Name + '\cortex', True, True, True);
+          end;
+        end;
+      until not FindNext(FindRec);
+    finally
+      FindClose(FindRec);
+    end;
+  end;
+end;
+
+procedure CurUninstallStepChanged(CurUninstallStep: TUninstallStep);
+begin
+  if CurUninstallStep = usPostUninstall then
+  begin
+    DeleteUserCortexFolder;
+  end;
+end;