feat: e2e test ci (#1238)

* feat: download pre-release * chore: add e2e test * feat: e2e test CI * fix: timeout * fix: exit code * fix: timeout None * fix: test * fix: clone exe * fix: cd * fix: temp comment * fix: windows and macos * fix: test * fix: windows * fix: readable * ci: e2e testing use GITHUB_TOKEN to increase github api rate limit * ci: enable e2e testing in quality gate new PR --------- Co-authored-by: Hien To <[email protected]>
janhq · Oct 2, 2024 · 23746ee · 23746ee
1 parent d67029b
commit 23746ee
Show file tree

Hide file tree

Showing 8 changed files with 117 additions and 11 deletions.
diff --git a/.github/workflows/cortex-cpp-quality-gate.yml b/.github/workflows/cortex-cpp-quality-gate.yml
@@ -2,7 +2,7 @@ name: CI Quality Gate Cortex CPP
 
 on:
   pull_request:
-    types: [opened, synchronize, reopened]
+    types: [opened, synchronize, reopened, ready_for_review]
     paths:
       [
         "engine/**",
@@ -53,11 +53,10 @@ jobs:
         with:
           submodules: recursive
 
-      - name: use python for linux
-        if: runner.os == 'Linux'
-        uses: actions/setup-python@v4
+      - name: use python
+        uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.10'
 
       - name: Install tools on Linux
         if: runner.os == 'Linux'
@@ -106,7 +105,38 @@ jobs:
         run: |
           cd engine
           make run-unit-tests
-  
+
+      - name: Run e2e tests
+        if: runner.os != 'Windows' && github.event.pull_request.draft == false
+        run: |
+          cd engine
+          cp build/cortex build/cortex-nightly
+          cp build/cortex build/cortex-beta
+          python -m pip install --upgrade pip
+          python -m pip install pytest
+          python -m pip install requests          
+          python e2e-test/main.py
+          rm build/cortex-nightly
+          rm build/cortex-beta
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+
+      - name: Run e2e tests
+        if: runner.os == 'Windows' && github.event.pull_request.draft == false
+        run: |
+          cd engine
+          cp build/cortex.exe build/cortex-nightly.exe
+          cp build/cortex.exe build/cortex-beta.exe
+          python -m pip install --upgrade pip
+          python -m pip install pytest
+          python -m pip install requests          
+          python e2e-test/main.py
+          rm build/cortex-nightly.exe
+          rm build/cortex-beta.exe
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Pre-package
         run: |
           cd engine

diff --git a/.github/workflows/template-build-linux-x64.yml b/.github/workflows/template-build-linux-x64.yml
@@ -127,6 +127,25 @@ jobs:
         run: |
           cd engine
           make build CMAKE_EXTRA_FLAGS="${{ inputs.cmake-flags }}" BUILD_DEPS_CMAKE_EXTRA_FLAGS="${{ inputs.build-deps-cmake-flags }}"
+          
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10' 
+
+      - name: Run e2e tests
+        run: |
+          cd engine
+          cp build/cortex build/cortex-nightly
+          cp build/cortex build/cortex-beta
+          python -m pip install --upgrade pip
+          python -m pip install pytest
+          python -m pip install requests          
+          python e2e-test/main.py
+          rm build/cortex-nightly
+          rm build/cortex-beta
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Pre-package
         run: |

diff --git a/.github/workflows/template-build-macos.yml b/.github/workflows/template-build-macos.yml
@@ -152,7 +152,26 @@ jobs:
         run: |
           cd engine
           make build CMAKE_EXTRA_FLAGS="${{ inputs.cmake-flags }}" BUILD_DEPS_CMAKE_EXTRA_FLAGS="${{ inputs.build-deps-cmake-flags }}"
-
+      
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10' 
+
+      - name: Run e2e tests
+        run: |
+          cd engine
+          cp build/cortex build/cortex-nightly
+          cp build/cortex build/cortex-beta
+          python -m pip install --upgrade pip
+          python -m pip install pytest
+          python -m pip install requests          
+          python e2e-test/main.py
+          rm build/cortex-nightly
+          rm build/cortex-beta
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Pre-package
         run: |
           cd engine

diff --git a/.github/workflows/template-build-windows-x64.yml b/.github/workflows/template-build-windows-x64.yml
@@ -164,6 +164,25 @@ jobs:
           cd engine
           make build CMAKE_EXTRA_FLAGS="${{ inputs.cmake-flags }}" BUILD_DEPS_CMAKE_EXTRA_FLAGS="${{ inputs.build-deps-cmake-flags }}"
 
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10' 
+
+      - name: Run e2e tests
+        run: |
+          cd engine
+          cp build/cortex.exe build/cortex-nightly.exe
+          cp build/cortex.exe build/cortex-beta.exe
+          python -m pip install --upgrade pip
+          python -m pip install pytest
+          python -m pip install requests          
+          python e2e-test/main.py
+          rm build/cortex-nightly.exe
+          rm build/cortex-beta.exe
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Pre-package
         run: |
           cd engine

diff --git a/engine/e2e-test/test_cli_engine_install.py b/engine/e2e-test/test_cli_engine_install.py
@@ -9,7 +9,7 @@ class TestCliEngineInstall:
 
     def test_engines_install_llamacpp_should_be_successfully(self):
         exit_code, output, error = run(
-            "Install Engine", ["engines", "install", "cortex.llamacpp"], timeout=600
+            "Install Engine", ["engines", "install", "cortex.llamacpp"], timeout=None
         )
         assert "Start downloading" in output, "Should display downloading message"
         assert exit_code == 0, f"Install engine failed with error: {error}"

diff --git a/engine/e2e-test/test_create_log_folder.py b/engine/e2e-test/test_create_log_folder.py
@@ -24,4 +24,8 @@ def setup_and_teardown(self):
 
     def test_create_log_folder_run_successfully(self):
         root = Path.home()
-        assert os.path.exists(root / "cortexcpp" / "logs")
+        assert (
+            os.path.exists(root / "cortexcpp" / "logs")
+            or os.path.exists(root / "cortexcpp-beta" / "logs")
+            or os.path.exists(root / "cortexcpp-nightly" / "logs")
+        )
diff --git a/engine/e2e-test/test_runner.py b/engine/e2e-test/test_runner.py
@@ -7,7 +7,7 @@
 from typing import List
 
 # You might want to change the path of the executable based on your build directory
-executable_windows_path = "build\\Debug\\cortex.exe"
+executable_windows_path = "build\\cortex.exe"
 executable_unix_path = "build/cortex"
 
 # Timeout

diff --git a/engine/services/engine_service.cc b/engine/services/engine_service.cc
@@ -217,6 +217,10 @@ cpp::result<bool, std::string> EngineService::UninstallEngine(
 
 cpp::result<bool, std::string> EngineService::DownloadEngine(
     const std::string& engine, const std::string& version) {
+
+  // Check if GITHUB_TOKEN env exist
+  const char* github_token = std::getenv("GITHUB_TOKEN");
+
   auto get_params = [&engine, &version]() -> std::vector<std::string> {
     if (version == "latest") {
       return {"repos", "janhq", engine, "releases", version};
@@ -232,7 +236,18 @@ cpp::result<bool, std::string> EngineService::DownloadEngine(
   };
 
   httplib::Client cli(url_obj.GetProtocolAndHost());
-  if (auto res = cli.Get(url_obj.GetPathAndQuery());
+
+  httplib::Headers headers;
+
+  if (github_token) {
+    std::string auth_header = "token " + std::string(github_token);
+    headers.insert({"Authorization", auth_header});
+    CTL_INF("Using authentication with GitHub token.");
+  } else {
+    CTL_INF("No GitHub token found. Sending request without authentication.");
+  }
+
+  if (auto res = cli.Get(url_obj.GetPathAndQuery(), headers);
       res->status == httplib::StatusCode::OK_200) {
     auto body = json::parse(res->body);
     auto get_data =