test: add e2e test to chat with tinyllama

.github/workflows/jan-electron-linter-and-test.yml

@@ -84,6 +84,18 @@ jobs:
           rm -rf ~/jan
           make clean
 
+      - uses: suisei-cn/actions-download-file@818d6b7dc8fe73f2f924b6241f2b1134ca1377d9 # 1.6.0
+        id: downloadfile
+        name: Download models
+        with:
+          url: "http://minio.jan.ai:9000/models/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf"
+          target: electron/test-data/models/tinyllama-1.1b/
+
+      - name: 'Prepare models.json'
+        continue-on-error: true
+        run: |
+          cp extensions/inference-nitro-extension/resources/models/tinyllama-1.1b/model.json electron/test-data/models/tinyllama-1.1b/
+
       - name: Get Commit Message for PR
         if: github.event_name == 'pull_request'
         run: |
@@ -129,6 +141,18 @@ jobs:
           rm -rf ~/jan
           make clean
 
+      - uses: suisei-cn/actions-download-file@818d6b7dc8fe73f2f924b6241f2b1134ca1377d9 # 1.6.0
+        id: downloadfile
+        name: Download models
+        with:
+          url: "http://minio.jan.ai:9000/models/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf"
+          target: electron/test-data/models/tinyllama-1.1b/
+
+      - name: 'Prepare models.json'
+        continue-on-error: true
+        run: |
+          cp extensions/inference-nitro-extension/resources/models/tinyllama-1.1b/model.json electron/test-data/models/tinyllama-1.1b/
+
       - name: Linter and test
         run: |
           npm config set registry https://registry.npmjs.org --global
@@ -168,6 +192,19 @@ jobs:
           }
           make clean
 
+      - uses: suisei-cn/actions-download-file@818d6b7dc8fe73f2f924b6241f2b1134ca1377d9 # 1.6.0
+        id: downloadfile
+        name: Download models
+        with:
+          url: "http://minio.jan.ai:9000/models/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf"
+          target: electron/test-data/models/tinyllama-1.1b/
+
+      - name: 'Prepare models.json'
+        shell: bash
+        continue-on-error: true
+        run: |
+          cp extensions/inference-nitro-extension/resources/models/tinyllama-1.1b/model.json electron/test-data/models/tinyllama-1.1b/
+
       - name: Get Commit Message for push event
         if: github.event_name == 'push'
         shell: bash
@@ -216,6 +253,19 @@ jobs:
           }
           make clean
 
+      - uses: suisei-cn/actions-download-file@818d6b7dc8fe73f2f924b6241f2b1134ca1377d9 # 1.6.0
+        id: downloadfile
+        name: Download models
+        with:
+          url: "http://minio.jan.ai:9000/models/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf"
+          target: electron/test-data/models/tinyllama-1.1b/
+
+      - name: 'Prepare models.json'
+        shell: bash
+        continue-on-error: true
+        run: |
+          cp extensions/inference-nitro-extension/resources/models/tinyllama-1.1b/model.json electron/test-data/models/tinyllama-1.1b/
+
       - name: Get Commit Message for PR
         if: github.event_name == 'pull_request'
         shell: bash
@@ -265,6 +315,19 @@ jobs:
           }
           make clean
 
+      - uses: suisei-cn/actions-download-file@818d6b7dc8fe73f2f924b6241f2b1134ca1377d9 # 1.6.0
+        id: downloadfile
+        name: Download models
+        with:
+          url: "http://minio.jan.ai:9000/models/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf"
+          target: electron/test-data/models/tinyllama-1.1b/
+
+      - name: 'Prepare models.json'
+        shell: bash
+        continue-on-error: true
+        run: |
+          cp extensions/inference-nitro-extension/resources/models/tinyllama-1.1b/model.json electron/test-data/models/tinyllama-1.1b/
+
       - name: Linter and test
         shell: powershell
         run: |
@@ -292,6 +355,18 @@ jobs:
           rm -rf ~/jan
           make clean
 
+      - uses: suisei-cn/actions-download-file@818d6b7dc8fe73f2f924b6241f2b1134ca1377d9 # 1.6.0
+        id: downloadfile
+        name: Download models
+        with:
+          url: "http://minio.jan.ai:9000/models/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf"
+          target: electron/test-data/models/tinyllama-1.1b/
+
+      - name: 'Prepare models.json'
+        continue-on-error: true
+        run: |
+          cp extensions/inference-nitro-extension/resources/models/tinyllama-1.1b/model.json electron/test-data/models/tinyllama-1.1b/
+
       - name: Get Commit Message for PR
         if: github.event_name == 'pull_request'
         run: |
@@ -389,6 +464,18 @@ jobs:
           rm -rf ~/jan
           make clean
 
+      - uses: suisei-cn/actions-download-file@818d6b7dc8fe73f2f924b6241f2b1134ca1377d9 # 1.6.0
+        id: downloadfile
+        name: Download models
+        with:
+          url: "http://minio.jan.ai:9000/models/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf"
+          target: electron/test-data/models/tinyllama-1.1b/
+
+      - name: 'Prepare models.json'
+        continue-on-error: true
+        run: |
+          cp extensions/inference-nitro-extension/resources/models/tinyllama-1.1b/model.json electron/test-data/models/tinyllama-1.1b/
+
       - name: Linter and test
         run: |
           export DISPLAY=$(w -h | awk 'NR==1 {print $2}')

electron/tests/config/constants.ts

@@ -1,4 +1,5 @@
 export const Constants = {
   VIDEO_DIR: './playwright-video',
+  TRACE_DIR: './playwright-trace',
   TIMEOUT: '300000',
 }

electron/tests/config/fixtures.ts

@@ -19,7 +19,9 @@ import { CommonActions } from '../pages/commonActions'
 export let electronApp: ElectronApplication
 export let page: Page
 export let appInfo: ElectronAppInfo
+let context: BrowserContext
 export const TIMEOUT = parseInt(process.env.TEST_TIMEOUT || Constants.TIMEOUT)
+const TRACE_PATH = Constants.TRACE_DIR + 'trace.zip'
 
 export async function setupElectron() {
   console.log(`TEST TIMEOUT: ${TIMEOUT}`)
@@ -99,13 +101,15 @@ export const test = base.extend<
       // After the test, we can check whether the test passed or failed.
       if (testInfo.status !== testInfo.expectedStatus) {
         await commonActions.takeScreenshot('')
+        await context.tracing.stopChunk({ path: TRACE_PATH })
+        await testInfo.attach('trace', { path: TRACE_PATH })
       }
     },
     { auto: true },
   ],
-
 })
 
+
 test.beforeAll(async () => {
   test.setTimeout(TIMEOUT)
   await setupElectron()
@@ -115,6 +119,7 @@ test.beforeAll(async () => {
   })
 })
 
+
 test.afterAll(async () => {
   // temporally disabling this due to the config for parallel testing WIP
   // teardownElectron()

electron/tests/e2e/thread.e2e.spec.ts

@@ -1,7 +1,78 @@
 import { expect } from '@playwright/test'
 import { page, test, TIMEOUT } from '../config/fixtures'
 
-test('Select GPT model from Hub and Chat with Invalid API Key', async ({ hubPage }) => {
+test.describe('Interact with local model', () => {
+  test('Select USE local model from Hub and Chat', async ({ hubPage }) => {
+    await hubPage.navigateByMenu()
+    await hubPage.verifyContainerVisible()
+
+    // Select the first GPT model
+    await page
+      .locator('[data-testid^="use-model-btn"][data-testid*="tinyllama"]')
+      .first().click()
+
+    await page
+      .getByTestId('txt-input-chat')
+      .fill('How many r\'s in strawberry?')
+
+    // send chat
+    await page
+      .getByTestId('btn-send-chat')
+      .click()
+
+    await expect(page.locator('[data-testid^="toaster-"]')).toBeVisible({
+      timeout: TIMEOUT,
+    })
+
+    await hubPage.waitLoadersCompleted()
+
+    await expect(page.getByTestId('btn-stop-chat')).not.toBeVisible({
+      timeout: TIMEOUT,
+    })
+
+    await expect(page.getByTestId('error-message')).not.toBeVisible({
+      timeout: TIMEOUT,
+    })
+
+    await expect(page.getByTestId('btn-regenerate-msg')).toBeVisible({ timeout: TIMEOUT })
+
+    const text = await page.getByTestId('lbl-token-speed').textContent()
+    const tokenSpeed = parseFloat(text.match(/\d+(\.\d+)?/)[0])
+    // Assertion to check if token speed is higher than 3 t/s
+    expect(tokenSpeed).toBeGreaterThan(3)
+  })
+
+  test('Regenerate msg and verify API request ', async ({ hubPage }) => {
+    let apiRequest
+    await page.route('**/inferences/server/chat_completion', route => {
+      apiRequest = route.request()
+      route.continue()
+    })
+
+    await page.getByTestId('btn-regenerate-msg').click()
+
+    await page.waitForResponse('**/inferences/server/chat_completion')
+
+    await hubPage.waitLoadersCompleted()
+
+    await expect(page.getByTestId('error-message')).not.toBeVisible({
+      timeout: TIMEOUT,
+    })
+
+    // Verify request contains correct param
+    const requestBody = JSON.parse(await apiRequest.postData())
+    expect(requestBody).toMatchObject({
+      'engine': 'cortex.llamacpp',
+      'frequency_penalty': 0,
+      'max_tokens': 2048,
+      'presence_penalty': 0,
+      'temperature': 0.6,
+      'top_p': 0.95,
+    })
+  })
+})
+
+test('Select USE GPT model from Hub and Chat with Invalid API Key', async ({ hubPage }) => {
   await hubPage.navigateByMenu()
   await hubPage.verifyContainerVisible()
 
@@ -23,13 +94,19 @@ test('Select GPT model from Hub and Chat with Invalid API Key', async ({ hubPage
     .getByTestId('btn-send-chat')
     .click()
 
-  await page.waitForFunction(() => {
-    const loaders = document.querySelectorAll('[data-testid$="loader"]');
-    return !loaders.length;
-  }, { timeout: TIMEOUT });
+  await hubPage.waitLoadersCompleted()
 
   const APIKeyError = page.getByTestId('invalid-API-key-error')
   await expect(APIKeyError).toBeVisible({
     timeout: TIMEOUT,
   })
 })
+
+test('Thread dropdown option should be visible', async () => {
+
+  await page.getByTestId('thread-menu').first().hover({ force: true })
+
+  await expect(page.getByTestId('btn-edit-title').first()).toBeVisible()
+  await expect(page.getByTestId('btn-clean-thread').first()).toBeVisible()
+  await expect(page.getByTestId('btn-delete-thread').first()).toBeVisible()
+})

electron/tests/pages/basePage.ts

@@ -1,6 +1,6 @@
 import { Page, expect } from '@playwright/test'
 import { CommonActions } from './commonActions'
-import { TIMEOUT } from '../config/fixtures'
+import { page, TIMEOUT } from '../config/fixtures'
 
 export class BasePage {
   menuId: string
@@ -51,4 +51,11 @@ export class BasePage {
       })
     return isVisible
   }
+
+  async waitLoadersCompleted() {
+    await page.waitForFunction(() => {
+      const loaders = document.querySelectorAll('[data-testid$="loader"]')
+      return !loaders.length
+    }, { timeout: TIMEOUT })
+  }
 }

electron/tests/pages/commonActions.ts

@@ -1,5 +1,9 @@
+import { promisify } from 'util';
 import { Page, TestInfo } from '@playwright/test'
 import { page } from '../config/fixtures'
+import { exec } from 'child_process';
+
+const execPromise = promisify(exec);
 
 export class CommonActions {
   private testData = new Map<string, string>()
@@ -31,4 +35,22 @@ export class CommonActions {
   getValue(key: string) {
     return this.testData.get(key)
   }
+
+  async downloadFileWithCurl(url: string, savePath: string, timeout: number): Promise<void> {
+    try {
+      const curlCommand = `curl -o "${savePath}" "${url}"`;
+
+      const { stdout, stderr } = await execPromise(curlCommand);
+
+      if (stderr) {
+        console.error(`Error downloading the file: ${stderr}`);
+        throw new Error(stderr);
+      }
+
+      console.log(`File downloaded successfully to ${savePath}`);
+    } catch (error) {
+      console.error(`Failed to download the file: ${error.message}`);
+      throw error;
+    }
+  }
 }

extensions/inference-nitro-extension/download.bat

@@ -1,3 +1,3 @@
 @echo off
 set /p CORTEX_VERSION=<./bin/version.txt
-.\node_modules\.bin\download https://github.com/janhq/cortex/releases/download/v%CORTEX_VERSION%/cortex-cpp-%CORTEX_VERSION%-windows-amd64.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/cortex/releases/download/v%CORTEX_VERSION%/cortex-cpp-%CORTEX_VERSION%-windows-amd64.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7 && .\node_modules\.bin\download https://github.com/janhq/cortex/releases/download/v%CORTEX_VERSION%/cortex-cpp-%CORTEX_VERSION%-windows-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu && .\node_modules\.bin\download https://github.com/janhq/cortex/releases/download/v%CORTEX_VERSION%/cortex-cpp-%CORTEX_VERSION%-windows-amd64.tar.gz -e --strip 1 -o ./bin/win-vulkan && .\node_modules\.bin\download https://github.com/janhq/cortex.llamacpp/releases/download/v0.1.25/cortex.llamacpp-0.1.25-windows-amd64-noavx-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0/engines/cortex.llamacpp && .\node_modules\.bin\download https://github.com/janhq/cortex.llamacpp/releases/download/v0.1.25/cortex.llamacpp-0.1.25-windows-amd64-noavx-cuda-11-7.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7/engines/cortex.llamacpp && .\node_modules\.bin\download https://github.com/janhq/cortex.llamacpp/releases/download/v0.1.25/cortex.llamacpp-0.1.25-windows-amd64-noavx.tar.gz -e --strip 1 -o ./bin/win-cpu/engines/cortex.llamacpp && .\node_modules\.bin\download https://github.com/janhq/cortex.llamacpp/releases/download/v0.1.25/cortex.llamacpp-0.1.25-windows-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/win-vulkan/engines/cortex.llamacpp
+.\node_modules\.bin\download https://github.com/janhq/cortex/releases/download/v%CORTEX_VERSION%/cortex-cpp-%CORTEX_VERSION%-windows-amd64.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/cortex/releases/download/v%CORTEX_VERSION%/cortex-cpp-%CORTEX_VERSION%-windows-amd64.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7 && .\node_modules\.bin\download https://github.com/janhq/cortex/releases/download/v%CORTEX_VERSION%/cortex-cpp-%CORTEX_VERSION%-windows-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu && .\node_modules\.bin\download https://github.com/janhq/cortex/releases/download/v%CORTEX_VERSION%/cortex-cpp-%CORTEX_VERSION%-windows-amd64.tar.gz -e --strip 1 -o ./bin/win-vulkan && .\node_modules\.bin\download https://github.com/janhq/cortex.llamacpp/releases/download/v0.1.28/cortex.llamacpp-0.1.28-windows-amd64-noavx-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0/engines/cortex.llamacpp && .\node_modules\.bin\download https://github.com/janhq/cortex.llamacpp/releases/download/v0.1.28/cortex.llamacpp-0.1.28-windows-amd64-noavx-cuda-11-7.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7/engines/cortex.llamacpp && .\node_modules\.bin\download https://github.com/janhq/cortex.llamacpp/releases/download/v0.1.28/cortex.llamacpp-0.1.28-windows-amd64-noavx.tar.gz -e --strip 1 -o ./bin/win-cpu/engines/cortex.llamacpp && .\node_modules\.bin\download https://github.com/janhq/cortex.llamacpp/releases/download/v0.1.28/cortex.llamacpp-0.1.28-windows-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/win-vulkan/engines/cortex.llamacpp