fix: move tensorrt executable to engine (#2400)

* fix: move tensorrt executable to engine Signed-off-by: James <[email protected]> * some update Signed-off-by: hiro <[email protected]> * chore: bump tensorrt version * fix: wrong destroy path * fix: install extensions in parallel * chore: update path for tensorrt engine (#2404) Signed-off-by: James <[email protected]> Co-authored-by: James <[email protected]> --------- Signed-off-by: James <[email protected]> Signed-off-by: hiro <[email protected]> Co-authored-by: James <[email protected]> Co-authored-by: hiro <[email protected]> Co-authored-by: Louis <[email protected]>
janhq · Mar 18, 2024 · c81a33f · c81a33f
1 parent 60cf8de
commit c81a33f
Show file tree

Hide file tree

Showing 17 changed files with 283 additions and 84 deletions.
diff --git a/core/src/api/index.ts b/core/src/api/index.ts
@@ -33,7 +33,7 @@ export enum AppRoute {
   stopServer = 'stopServer',
   log = 'log',
   logServer = 'logServer',
-  systemInformations = 'systemInformations',
+  systemInformation = 'systemInformation',
   showToast = 'showToast',
 }
 
@@ -95,6 +95,7 @@ export enum FileManagerRoute {
   getUserHomePath = 'getUserHomePath',
   fileStat = 'fileStat',
   writeBlob = 'writeBlob',
+  mkdir = 'mkdir',
 }
 
 export type ApiFunction = (...args: any[]) => any

diff --git a/core/src/core.ts b/core/src/core.ts
@@ -1,4 +1,4 @@
-import { DownloadRequest, FileStat, NetworkConfig } from './types'
+import { DownloadRequest, FileStat, NetworkConfig, SystemInformation } from './types'
 
 /**
  * Execute a extension module function in main process
@@ -110,7 +110,8 @@ const isSubdirectory: (from: string, to: string) => Promise<boolean> = (from: st
  * Get system information
  * @returns {Promise<any>} - A promise that resolves with the system information.
  */
-const systemInformations: () => Promise<any> = () => global.core.api?.systemInformations()
+const systemInformation: () => Promise<SystemInformation> = () =>
+  global.core.api?.systemInformation()
 
 /**
  * Show toast message from browser processes.
@@ -146,7 +147,7 @@ export {
   log,
   isSubdirectory,
   getUserHomePath,
-  systemInformations,
+  systemInformation,
   showToast,
   FileStat,
 }
diff --git a/core/src/extensions/ai-engines/LocalOAIEngine.ts b/core/src/extensions/ai-engines/LocalOAIEngine.ts
@@ -1,4 +1,4 @@
-import { executeOnMain, getJanDataFolderPath, joinPath } from '../../core'
+import { executeOnMain, getJanDataFolderPath, joinPath, systemInformation } from '../../core'
 import { events } from '../../events'
 import { Model, ModelEvent } from '../../types'
 import { OAIEngine } from './OAIEngine'
@@ -30,11 +30,11 @@ export abstract class LocalOAIEngine extends OAIEngine {
     if (model.engine.toString() !== this.provider) return
 
     const modelFolder = await joinPath([await getJanDataFolderPath(), this.modelFolder, model.id])
-
+    const systemInfo = await systemInformation()
     const res = await executeOnMain(this.nodeModule, this.loadModelFunctionName, {
       modelFolder,
       model,
-    })
+    }, systemInfo)
 
     if (res?.error) {
       events.emit(ModelEvent.OnModelFail, {

diff --git a/core/src/extensions/monitoring.ts b/core/src/extensions/monitoring.ts
@@ -1,5 +1,5 @@
 import { BaseExtension, ExtensionTypeEnum } from '../extension'
-import { GpuSetting, MonitoringInterface } from '../index'
+import { GpuSetting, MonitoringInterface, OperatingSystemInfo } from '../index'
 
 /**
  * Monitoring extension for system monitoring.
@@ -16,4 +16,5 @@ export abstract class MonitoringExtension extends BaseExtension implements Monit
   abstract getGpuSetting(): Promise<GpuSetting>
   abstract getResourcesInfo(): Promise<any>
   abstract getCurrentLoad(): Promise<any>
+  abstract getOsInfo(): Promise<OperatingSystemInfo>
 }
diff --git a/core/src/fs.ts b/core/src/fs.ts
@@ -37,6 +37,8 @@ const readdirSync = (...args: any[]) => global.core.api?.readdirSync(...args)
  */
 const mkdirSync = (...args: any[]) => global.core.api?.mkdirSync(...args)
 
+const mkdir = (...args: any[]) => global.core.api?.mkdir(...args)
+
 /**
  * Removes a directory at the specified path.
  * @returns {Promise<any>} A Promise that resolves when the directory is removed successfully.
@@ -92,6 +94,7 @@ export const fs = {
   existsSync,
   readdirSync,
   mkdirSync,
+  mkdir,
   rmdirSync,
   unlinkSync,
   appendFileSync,

diff --git a/core/src/node/api/processors/fsExt.ts b/core/src/node/api/processors/fsExt.ts
@@ -88,4 +88,16 @@ export class FSExt implements Processor {
       })
     })
   }
+
+  mkdir(path: string): Promise<void> {
+    return new Promise((resolve, reject) => {
+      fs.mkdir(path, { recursive: true }, (err) => {
+        if (err) {
+          reject(err)
+        } else {
+          resolve()
+        }
+      })
+    })
+  }
 }
diff --git a/core/src/node/extension/store.ts b/core/src/node/extension/store.ts
@@ -93,24 +93,25 @@ export function persistExtensions() {
  */
 export async function installExtensions(extensions: any) {
   const installed: Extension[] = []
-  for (const ext of extensions) {
-    // Set install options and activation based on input type
+  const installations = extensions.map((ext: any): Promise<void> => {
     const isObject = typeof ext === 'object'
     const spec = isObject ? [ext.specifier, ext] : [ext]
     const activate = isObject ? ext.activate !== false : true
 
     // Install and possibly activate extension
     const extension = new Extension(...spec)
     if (!extension.origin) {
-      continue
+      return Promise.resolve()
     }
-    await extension._install()
-    if (activate) extension.setActive(true)
+    return extension._install().then(() => {
+      if (activate) extension.setActive(true)
+      // Add extension to store if needed
+      addExtension(extension)
+      installed.push(extension)
+    })
+  })
 
-    // Add extension to store if needed
-    addExtension(extension)
-    installed.push(extension)
-  }
+  await Promise.all(installations)
 
   // Return list of all installed extensions
   return installed

diff --git a/core/src/types/miscellaneous/systemResourceInfo.ts b/core/src/types/miscellaneous/systemResourceInfo.ts
@@ -30,3 +30,27 @@ export type GpuSettingInfo = {
   name: string
   arch?: string
 }
+
+export type SystemInformation = {
+  gpuSetting: GpuSetting
+  osInfo?: OperatingSystemInfo
+}
+
+export const SupportedPlatforms = ['win32', 'linux', 'darwin'] as const
+export type SupportedPlatformTuple = typeof SupportedPlatforms
+export type SupportedPlatform = SupportedPlatformTuple[number]
+
+export type OperatingSystemInfo = {
+  platform: SupportedPlatform | 'unknown'
+  arch: string
+  release: string
+  machine: string
+  version: string
+  totalMem: number
+  freeMem: number
+}
+
+export type CpuCoreInfo = {
+  model: string
+  speed: number
+}
diff --git a/extensions/monitoring-extension/src/index.ts b/extensions/monitoring-extension/src/index.ts
@@ -1,4 +1,9 @@
-import { GpuSetting, MonitoringExtension, executeOnMain } from '@janhq/core'
+import {
+  GpuSetting,
+  MonitoringExtension,
+  OperatingSystemInfo,
+  executeOnMain,
+} from '@janhq/core'
 
 /**
  * JanMonitoringExtension is a extension that provides system monitoring functionality.
@@ -41,4 +46,8 @@ export default class JanMonitoringExtension extends MonitoringExtension {
   getCurrentLoad(): Promise<any> {
     return executeOnMain(NODE, 'getCurrentLoad')
   }
+
+  getOsInfo(): Promise<OperatingSystemInfo> {
+    return executeOnMain(NODE, 'getOsInfo')
+  }
 }
diff --git a/extensions/monitoring-extension/src/node/index.ts b/extensions/monitoring-extension/src/node/index.ts
@@ -1,9 +1,16 @@
-import { GpuSetting, GpuSettingInfo, ResourceInfo } from '@janhq/core'
+import {
+  GpuSetting,
+  GpuSettingInfo,
+  OperatingSystemInfo,
+  ResourceInfo,
+  SupportedPlatforms,
+} from '@janhq/core'
 import { getJanDataFolderPath, log } from '@janhq/core/node'
 import { mem, cpu } from 'node-os-utils'
 import { exec } from 'child_process'
 import { writeFileSync, existsSync, readFileSync, mkdirSync } from 'fs'
 import path from 'path'
+import os from 'os'
 
 /**
  * Path to the settings directory
@@ -320,3 +327,20 @@ const updateCudaExistence = (
   data.is_initial = false
   return data
 }
+
+export const getOsInfo = (): OperatingSystemInfo => {
+  const platform =
+    SupportedPlatforms.find((p) => p === process.platform) || 'unknown'
+
+  const osInfo: OperatingSystemInfo = {
+    platform: platform,
+    arch: process.arch,
+    release: os.release(),
+    machine: os.machine(),
+    version: os.version(),
+    totalMem: os.totalmem(),
+    freeMem: os.freemem(),
+  }
+
+  return osInfo
+}
diff --git a/extensions/tensorrt-llm-extension/package.json b/extensions/tensorrt-llm-extension/package.json
@@ -18,6 +18,8 @@
       "0.1.0"
     ]
   },
+  "tensorrtVersion": "0.1.6",
+  "provider": "nitro-tensorrt-llm",
   "scripts": {
     "build": "tsc --module commonjs && rollup -c rollup.config.ts",
     "build:publish:win32": "rimraf *.tgz --glob && npm run build && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install",

diff --git a/extensions/tensorrt-llm-extension/rollup.config.ts b/extensions/tensorrt-llm-extension/rollup.config.ts
@@ -16,9 +16,10 @@ export default [
     plugins: [
       replace({
         EXTENSION_NAME: JSON.stringify(packageJson.name),
-        TENSORRT_VERSION: JSON.stringify('0.1.5'),
+        TENSORRT_VERSION: JSON.stringify(packageJson.tensorrtVersion),
+        PROVIDER: JSON.stringify(packageJson.provider),
         DOWNLOAD_RUNNER_URL:
-          process.platform === 'darwin' || process.platform === 'win32'
+          process.platform === 'win32'
             ? JSON.stringify(
                 'https://github.com/janhq/nitro-tensorrt-llm/releases/download/windows-v<version>/nitro-windows-v<version>-amd64-tensorrt-llm-<gpuarch>.tar.gz'
               )
@@ -52,11 +53,14 @@ export default [
     },
     plugins: [
       replace({
+        EXTENSION_NAME: JSON.stringify(packageJson.name),
+        TENSORRT_VERSION: JSON.stringify(packageJson.tensorrtVersion),
+        PROVIDER: JSON.stringify(packageJson.provider),
         LOAD_MODEL_URL: JSON.stringify(
           `${packageJson.config?.protocol ?? 'http'}://${packageJson.config?.host}:${packageJson.config?.port}/inferences/tensorrtllm/loadmodel`
         ),
         TERMINATE_ENGINE_URL: JSON.stringify(
-          `${packageJson.config?.protocol ?? 'http'}://${packageJson.config?.host}:${packageJson.config?.port}/inferences/processmanager/destroy`
+          `${packageJson.config?.protocol ?? 'http'}://${packageJson.config?.host}:${packageJson.config?.port}/processmanager/destroy`
         ),
         ENGINE_HOST: JSON.stringify(packageJson.config?.host ?? '127.0.0.1'),
         ENGINE_PORT: JSON.stringify(packageJson.config?.port ?? '3928'),

diff --git a/extensions/tensorrt-llm-extension/src/@types/global.d.ts b/extensions/tensorrt-llm-extension/src/@types/global.d.ts
@@ -8,3 +8,4 @@ declare const DOWNLOAD_RUNNER_URL: string
 declare const TENSORRT_VERSION: string
 declare const COMPATIBILITY: object
 declare const EXTENSION_NAME: string
+declare const PROVIDER: string