From dc005c7b92c5c59d5c5cb9068eac702749464363 Mon Sep 17 00:00:00 2001 From: marknguyen1302 Date: Wed, 17 Jul 2024 16:37:49 +0700 Subject: [PATCH 1/3] fix wrong engine list --- .../infrastructure/commanders/engines/engines-get.command.ts | 2 +- .../infrastructure/commanders/engines/engines-list.command.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cortex-js/src/infrastructure/commanders/engines/engines-get.command.ts b/cortex-js/src/infrastructure/commanders/engines/engines-get.command.ts index d72766dbe..f002a84a4 100644 --- a/cortex-js/src/infrastructure/commanders/engines/engines-get.command.ts +++ b/cortex-js/src/infrastructure/commanders/engines/engines-get.command.ts @@ -27,7 +27,7 @@ export class EnginesGetCommand extends CommandRunner { } else { console.table({ ...engine, - name: EngineNamesMap[engine.name as Engines], + name: EngineNamesMap[engine.name as Engines] || engine.name, }); } }); diff --git a/cortex-js/src/infrastructure/commanders/engines/engines-list.command.ts b/cortex-js/src/infrastructure/commanders/engines/engines-list.command.ts index fa49aa302..e19d4aedd 100644 --- a/cortex-js/src/infrastructure/commanders/engines/engines-list.command.ts +++ b/cortex-js/src/infrastructure/commanders/engines/engines-list.command.ts @@ -2,7 +2,7 @@ import { CommandRunner, SubCommand } from 'nest-commander'; import { SetCommandContext } from '../decorators/CommandContext'; import { ContextService } from '@/infrastructure/services/context/context.service'; import { EnginesUsecases } from '@/usecases/engines/engines.usecase'; -import { EngineNamesMap, Engines } from '../types/engine.interface'; +import { EngineNamesMap } from '../types/engine.interface'; @SubCommand({ name: 'list', @@ -21,7 +21,7 @@ export class EnginesListCommand extends CommandRunner { return this.enginesUsecases.getEngines().then((engines) => { const enginesTable = engines.map((engine) => ({ ...engine, - name: EngineNamesMap[engine.name as Engines], + name: EngineNamesMap[engine.name as string] || engine.name, })); console.table(enginesTable); }); From 810954c6a4565aadefebfccc5ca1e5634f07b29b Mon Sep 17 00:00:00 2001 From: marknguyen1302 Date: Wed, 17 Jul 2024 17:29:36 +0700 Subject: [PATCH 2/3] change product name, description for engines --- .../infrastructure/providers/cortex/cortex.module.ts | 9 +++++++++ .../providers/cortex/llamacpp.provider.ts | 9 +++++++++ .../infrastructure/providers/cortex/onnx.provider.ts | 9 +++++++++ .../providers/cortex/tensorrtllm.provider.ts | 9 +++++++++ .../repositories/extensions/extension.repository.ts | 10 ++++++---- 5 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 cortex-js/src/infrastructure/providers/cortex/llamacpp.provider.ts create mode 100644 cortex-js/src/infrastructure/providers/cortex/onnx.provider.ts create mode 100644 cortex-js/src/infrastructure/providers/cortex/tensorrtllm.provider.ts diff --git a/cortex-js/src/infrastructure/providers/cortex/cortex.module.ts b/cortex-js/src/infrastructure/providers/cortex/cortex.module.ts index 4cd9147e0..42d80effd 100644 --- a/cortex-js/src/infrastructure/providers/cortex/cortex.module.ts +++ b/cortex-js/src/infrastructure/providers/cortex/cortex.module.ts @@ -2,6 +2,9 @@ import { Module } from '@nestjs/common'; import CortexProvider from './cortex.provider'; import { HttpModule } from '@nestjs/axios'; import { FileManagerModule } from '@/infrastructure/services/file-manager/file-manager.module'; +import Onnxprovider from './onnx.provider'; +import LlamaCPPProvider from './llamacpp.provider'; +import TensorrtLLMProvider from './tensorrtllm.provider'; @Module({ imports: [HttpModule, FileManagerModule], @@ -10,12 +13,18 @@ import { FileManagerModule } from '@/infrastructure/services/file-manager/file-m provide: 'CORTEX_PROVIDER', useClass: CortexProvider, }, + Onnxprovider, + LlamaCPPProvider, + TensorrtLLMProvider, ], exports: [ { provide: 'CORTEX_PROVIDER', useClass: CortexProvider, }, + Onnxprovider, + LlamaCPPProvider, + TensorrtLLMProvider, ], }) export class CortexProviderModule {} diff --git a/cortex-js/src/infrastructure/providers/cortex/llamacpp.provider.ts b/cortex-js/src/infrastructure/providers/cortex/llamacpp.provider.ts new file mode 100644 index 000000000..b39feafca --- /dev/null +++ b/cortex-js/src/infrastructure/providers/cortex/llamacpp.provider.ts @@ -0,0 +1,9 @@ +import { Injectable } from '@nestjs/common'; +import CortexProvider from './cortex.provider'; + +@Injectable() +export default class LlamaCPPProvider extends CortexProvider { + productName = 'LlamaCPP Inference Engine'; + description = + 'This extension enables chat completion API calls using the LlamaCPP engine'; +} diff --git a/cortex-js/src/infrastructure/providers/cortex/onnx.provider.ts b/cortex-js/src/infrastructure/providers/cortex/onnx.provider.ts new file mode 100644 index 000000000..bf2652a7c --- /dev/null +++ b/cortex-js/src/infrastructure/providers/cortex/onnx.provider.ts @@ -0,0 +1,9 @@ +import { Injectable } from '@nestjs/common'; +import CortexProvider from './cortex.provider'; + +@Injectable() +export default class Onnxprovider extends CortexProvider { + productName = 'Onnx Inference Engine'; + description = + 'This extension enables chat completion API calls using the Onnx engine'; +} diff --git a/cortex-js/src/infrastructure/providers/cortex/tensorrtllm.provider.ts b/cortex-js/src/infrastructure/providers/cortex/tensorrtllm.provider.ts new file mode 100644 index 000000000..c870b1ddf --- /dev/null +++ b/cortex-js/src/infrastructure/providers/cortex/tensorrtllm.provider.ts @@ -0,0 +1,9 @@ +import { Injectable } from '@nestjs/common'; +import CortexProvider from './cortex.provider'; + +@Injectable() +export default class TensorrtLLMProvider extends CortexProvider { + productName = 'TensorrtLLM Inference Engine'; + description = + 'This extension enables chat completion API calls using the TensorrtLLM engine'; +} diff --git a/cortex-js/src/infrastructure/repositories/extensions/extension.repository.ts b/cortex-js/src/infrastructure/repositories/extensions/extension.repository.ts index d8ad85994..62c8b1482 100644 --- a/cortex-js/src/infrastructure/repositories/extensions/extension.repository.ts +++ b/cortex-js/src/infrastructure/repositories/extensions/extension.repository.ts @@ -7,8 +7,10 @@ import { FileManagerService } from '@/infrastructure/services/file-manager/file- import { existsSync } from 'fs'; import { Engines } from '@/infrastructure/commanders/types/engine.interface'; import { OAIEngineExtension } from '@/domain/abstracts/oai.abstract'; -import CortexProvider from '@/infrastructure/providers/cortex/cortex.provider'; import { HttpService } from '@nestjs/axios'; +import LlamaCPPProvider from '@/infrastructure/providers/cortex/llamacpp.provider'; +import Onnxprovider from '@/infrastructure/providers/cortex/onnx.provider'; +import TensorrtLLMProvider from '@/infrastructure/providers/cortex/tensorrtllm.provider'; @Injectable() export class ExtensionRepositoryImpl implements ExtensionRepository { @@ -44,7 +46,7 @@ export class ExtensionRepositoryImpl implements ExtensionRepository { } private async loadCoreExtensions() { - const llamaCPPEngine = new CortexProvider( + const llamaCPPEngine = new LlamaCPPProvider( this.httpService, this.fileManagerService, ); @@ -56,7 +58,7 @@ export class ExtensionRepositoryImpl implements ExtensionRepository { ), ); - const onnxEngine = new CortexProvider( + const onnxEngine = new Onnxprovider( this.httpService, this.fileManagerService, ); @@ -68,7 +70,7 @@ export class ExtensionRepositoryImpl implements ExtensionRepository { ), ); - const tensorrtLLMEngine = new CortexProvider( + const tensorrtLLMEngine = new TensorrtLLMProvider( this.httpService, this.fileManagerService, ); From 12d165f6422a0da4daa0f4fdd63364302a0ea56a Mon Sep 17 00:00:00 2001 From: marknguyen1302 Date: Wed, 17 Jul 2024 17:43:50 +0700 Subject: [PATCH 3/3] refactor --- .../src/infrastructure/providers/cortex/llamacpp.provider.ts | 2 ++ cortex-js/src/infrastructure/providers/cortex/onnx.provider.ts | 2 ++ .../infrastructure/providers/cortex/tensorrtllm.provider.ts | 2 ++ .../repositories/extensions/extension.repository.ts | 3 --- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cortex-js/src/infrastructure/providers/cortex/llamacpp.provider.ts b/cortex-js/src/infrastructure/providers/cortex/llamacpp.provider.ts index b39feafca..85a76c55b 100644 --- a/cortex-js/src/infrastructure/providers/cortex/llamacpp.provider.ts +++ b/cortex-js/src/infrastructure/providers/cortex/llamacpp.provider.ts @@ -1,8 +1,10 @@ import { Injectable } from '@nestjs/common'; import CortexProvider from './cortex.provider'; +import { Engines } from '@/infrastructure/commanders/types/engine.interface'; @Injectable() export default class LlamaCPPProvider extends CortexProvider { + name = Engines.llamaCPP; productName = 'LlamaCPP Inference Engine'; description = 'This extension enables chat completion API calls using the LlamaCPP engine'; diff --git a/cortex-js/src/infrastructure/providers/cortex/onnx.provider.ts b/cortex-js/src/infrastructure/providers/cortex/onnx.provider.ts index bf2652a7c..4a3ef2fa9 100644 --- a/cortex-js/src/infrastructure/providers/cortex/onnx.provider.ts +++ b/cortex-js/src/infrastructure/providers/cortex/onnx.provider.ts @@ -1,8 +1,10 @@ import { Injectable } from '@nestjs/common'; import CortexProvider from './cortex.provider'; +import { Engines } from '@/infrastructure/commanders/types/engine.interface'; @Injectable() export default class Onnxprovider extends CortexProvider { + name = Engines.onnx; productName = 'Onnx Inference Engine'; description = 'This extension enables chat completion API calls using the Onnx engine'; diff --git a/cortex-js/src/infrastructure/providers/cortex/tensorrtllm.provider.ts b/cortex-js/src/infrastructure/providers/cortex/tensorrtllm.provider.ts index c870b1ddf..65c63b489 100644 --- a/cortex-js/src/infrastructure/providers/cortex/tensorrtllm.provider.ts +++ b/cortex-js/src/infrastructure/providers/cortex/tensorrtllm.provider.ts @@ -1,8 +1,10 @@ import { Injectable } from '@nestjs/common'; import CortexProvider from './cortex.provider'; +import { Engines } from '@/infrastructure/commanders/types/engine.interface'; @Injectable() export default class TensorrtLLMProvider extends CortexProvider { + name = Engines.tensorrtLLM; productName = 'TensorrtLLM Inference Engine'; description = 'This extension enables chat completion API calls using the TensorrtLLM engine'; diff --git a/cortex-js/src/infrastructure/repositories/extensions/extension.repository.ts b/cortex-js/src/infrastructure/repositories/extensions/extension.repository.ts index 62c8b1482..3ebd58e60 100644 --- a/cortex-js/src/infrastructure/repositories/extensions/extension.repository.ts +++ b/cortex-js/src/infrastructure/repositories/extensions/extension.repository.ts @@ -50,7 +50,6 @@ export class ExtensionRepositoryImpl implements ExtensionRepository { this.httpService, this.fileManagerService, ); - llamaCPPEngine.name = Engines.llamaCPP; llamaCPPEngine.initalized = existsSync( join( await this.fileManagerService.getCortexCppEnginePath(), @@ -62,7 +61,6 @@ export class ExtensionRepositoryImpl implements ExtensionRepository { this.httpService, this.fileManagerService, ); - onnxEngine.name = Engines.onnx; onnxEngine.initalized = existsSync( join( await this.fileManagerService.getCortexCppEnginePath(), @@ -74,7 +72,6 @@ export class ExtensionRepositoryImpl implements ExtensionRepository { this.httpService, this.fileManagerService, ); - tensorrtLLMEngine.name = Engines.tensorrtLLM; tensorrtLLMEngine.initalized = existsSync( join( await this.fileManagerService.getCortexCppEnginePath(),