diff --git a/cortex-js/package.json b/cortex-js/package.json index 7aea006a7..f22c50b8e 100644 --- a/cortex-js/package.json +++ b/cortex-js/package.json @@ -53,16 +53,18 @@ "@nestjs/sequelize": "^10.0.1", "@nestjs/swagger": "^7.3.1", "@terascope/fetch-github-release": "^0.8.8", + "@types/node-os-utils": "^1.3.4", "axios": "^1.6.8", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", "cli-progress": "^3.12.0", - "cortex-cpp": "0.5.0-36", + "cortex-cpp": "0.5.0-40", "cpu-instructions": "^0.0.11", "decompress": "^4.2.1", "hyllama": "^0.2.2", "js-yaml": "^4.1.0", "nest-commander": "^3.13.0", + "node-os-utils": "^1.3.7", "ora": "5.4.1", "readline": "^1.3.0", "reflect-metadata": "^0.2.0", @@ -70,7 +72,7 @@ "sequelize": "^6.37.3", "sequelize-typescript": "^2.1.6", "sqlite3": "^5.1.7", - "systeminformation": "^5.22.11", + "systeminformation": "^5.23.4", "ulid": "^2.3.0", "uuid": "^9.0.1", "whatwg-url": "^14.0.0", diff --git a/cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts b/cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts index c65075e5d..28d15a7f1 100644 --- a/cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts +++ b/cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts @@ -41,12 +41,13 @@ export class EnginesInitCommand extends BaseCommand { const configs = await fileManagerService.getConfig(); const host = configs.cortexCppHost; const port = configs.cortexCppPort; - // Should stop cortex before installing engine - const stopCortexSpinner = ora('Stopping cortex...').start(); + // Should unload engine before installing engine + const unloadCortexEngineSpinner = ora('Unloading cortex...').start(); if (await this.cortexUsecases.healthCheck(host, port)) { - await this.cortexUsecases.stopCortex(); + await this.cortexUsecases.unloadCortexEngine(engine as Engines); } - stopCortexSpinner.succeed('Cortex stopped'); + + unloadCortexEngineSpinner.succeed('Cortex unloaded'); console.log(`Installing engine ${engine}...`); await this.cortex.engines.init(engine, params); diff --git a/cortex-js/src/infrastructure/commanders/ps.command.ts b/cortex-js/src/infrastructure/commanders/ps.command.ts index b4c2d30b0..f2268cdd6 100644 --- a/cortex-js/src/infrastructure/commanders/ps.command.ts +++ b/cortex-js/src/infrastructure/commanders/ps.command.ts @@ -12,6 +12,7 @@ import { firstValueFrom } from 'rxjs'; import { HttpService } from '@nestjs/axios'; import { CORTEX_CPP_MODELS_URL } from '../constants/cortex'; import { fileManagerService } from '../services/file-manager/file-manager.service'; +import { getMemoryInformation } from '@/utils/system-resource'; @SubCommand({ name: 'ps', @@ -45,9 +46,9 @@ export class PSCommand extends BaseCommand { totalVram, }); } - const memoryData = await systeminformation.mem(); + const { total, used } = await getMemoryInformation() const memoryUsage = ( - (memoryData.active / memoryData.total) * + (used / total) * 100 ).toFixed(2); const consumedTable = { diff --git a/cortex-js/src/infrastructure/constants/cortex.ts b/cortex-js/src/infrastructure/constants/cortex.ts index 04d80634c..9e07868f8 100644 --- a/cortex-js/src/infrastructure/constants/cortex.ts +++ b/cortex-js/src/infrastructure/constants/cortex.ts @@ -26,6 +26,11 @@ export const CORTEX_CPP_PROCESS_DESTROY_URL = ( port: number = defaultCortexCppPort, ) => `http://${host}:${port}/processmanager/destroy`; +export const CORTEX_CPP_UNLOAD_ENGINE_URL = ( + host: string = defaultCortexCppHost, + port: number = defaultCortexCppPort, +) => `http://${host}:${port}/inferences/server/unloadengine`; + export const CORTEX_CPP_HEALTH_Z_URL = ( host: string = defaultCortexCppHost, port: number = defaultCortexCppPort, diff --git a/cortex-js/src/infrastructure/services/resources-manager/resources-manager.service.ts b/cortex-js/src/infrastructure/services/resources-manager/resources-manager.service.ts index da7a0c402..8b8752af6 100644 --- a/cortex-js/src/infrastructure/services/resources-manager/resources-manager.service.ts +++ b/cortex-js/src/infrastructure/services/resources-manager/resources-manager.service.ts @@ -1,18 +1,20 @@ +import osUtils from 'node-os-utils' import { ResourceStatus, UsedMemInfo, } from '@/domain/models/resource.interface'; +import { getMemoryInformation, MemoryInformation } from '@/utils/system-resource'; import { Injectable } from '@nestjs/common'; import systemInformation, { Systeminformation } from 'systeminformation'; @Injectable() export class ResourcesManagerService { async getResourceStatuses(): Promise { - const promises = [systemInformation.currentLoad(), systemInformation.mem()]; + const promises = [systemInformation.currentLoad(), getMemoryInformation()]; const results = await Promise.all(promises); const cpuUsage = results[0] as Systeminformation.CurrentLoadData; - const memory = results[1] as Systeminformation.MemData; + const memory = results[1] as MemoryInformation const memInfo: UsedMemInfo = { total: memory.total, used: memory.used, diff --git a/cortex-js/src/usecases/cortex/cortex.usecases.ts b/cortex-js/src/usecases/cortex/cortex.usecases.ts index 573a9db00..74fbe942e 100644 --- a/cortex-js/src/usecases/cortex/cortex.usecases.ts +++ b/cortex-js/src/usecases/cortex/cortex.usecases.ts @@ -13,11 +13,13 @@ import { fileManagerService } from '@/infrastructure/services/file-manager/file- import { CORTEX_CPP_HEALTH_Z_URL, CORTEX_CPP_PROCESS_DESTROY_URL, + CORTEX_CPP_UNLOAD_ENGINE_URL, CORTEX_JS_SYSTEM_URL, defaultCortexJsHost, defaultCortexJsPort, } from '@/infrastructure/constants/cortex'; import { openSync } from 'fs'; +import { Engines } from '@/infrastructure/commanders/types/engine.interface'; @Injectable() export class CortexUsecases implements BeforeApplicationShutdown { @@ -123,6 +125,29 @@ export class CortexUsecases implements BeforeApplicationShutdown { } } + /** + * Unload the engine + */ + async unloadCortexEngine(engine: Engines): Promise { + const configs = await fileManagerService.getConfig(); + try { + await firstValueFrom( + this.httpService.post( + CORTEX_CPP_UNLOAD_ENGINE_URL( + configs.cortexCppHost, + configs.cortexCppPort, + ), + ), + ); + } finally { + return { + message: `${engine} unloaded successfully`, + status: 'success', + }; + } + } + + /** * Check whether the Cortex CPP is healthy * @param host diff --git a/cortex-js/src/utils/system-resource.ts b/cortex-js/src/utils/system-resource.ts new file mode 100644 index 000000000..527884a9c --- /dev/null +++ b/cortex-js/src/utils/system-resource.ts @@ -0,0 +1,16 @@ +import osUtils from 'node-os-utils' + +export type MemoryInformation = { + total: osUtils.MemUsedInfo['totalMemMb']; + used: osUtils.MemUsedInfo['usedMemMb']; + free: osUtils.MemFreeInfo['freeMemMb'] +} + +export const getMemoryInformation = async (): Promise => { + const [usedMemoryInfo, freeMemoryInfo] = await Promise.all([osUtils.mem.used(), osUtils.mem.free()]) + return { + total: usedMemoryInfo.totalMemMb, + used: usedMemoryInfo.usedMemMb, + free: freeMemoryInfo.freeMemMb + } +} \ No newline at end of file