diff --git a/cortex-js/src/infrastructure/commanders/engines/engines-get.command.ts b/cortex-js/src/infrastructure/commanders/engines/engines-get.command.ts index d72766dbe..f002a84a4 100644 --- a/cortex-js/src/infrastructure/commanders/engines/engines-get.command.ts +++ b/cortex-js/src/infrastructure/commanders/engines/engines-get.command.ts @@ -27,7 +27,7 @@ export class EnginesGetCommand extends CommandRunner { } else { console.table({ ...engine, - name: EngineNamesMap[engine.name as Engines], + name: EngineNamesMap[engine.name as Engines] || engine.name, }); } }); diff --git a/cortex-js/src/infrastructure/commanders/engines/engines-list.command.ts b/cortex-js/src/infrastructure/commanders/engines/engines-list.command.ts index fa49aa302..e19d4aedd 100644 --- a/cortex-js/src/infrastructure/commanders/engines/engines-list.command.ts +++ b/cortex-js/src/infrastructure/commanders/engines/engines-list.command.ts @@ -2,7 +2,7 @@ import { CommandRunner, SubCommand } from 'nest-commander'; import { SetCommandContext } from '../decorators/CommandContext'; import { ContextService } from '@/infrastructure/services/context/context.service'; import { EnginesUsecases } from '@/usecases/engines/engines.usecase'; -import { EngineNamesMap, Engines } from '../types/engine.interface'; +import { EngineNamesMap } from '../types/engine.interface'; @SubCommand({ name: 'list', @@ -21,7 +21,7 @@ export class EnginesListCommand extends CommandRunner { return this.enginesUsecases.getEngines().then((engines) => { const enginesTable = engines.map((engine) => ({ ...engine, - name: EngineNamesMap[engine.name as Engines], + name: EngineNamesMap[engine.name as string] || engine.name, })); console.table(enginesTable); }); diff --git a/cortex-js/src/infrastructure/providers/cortex/cortex.module.ts b/cortex-js/src/infrastructure/providers/cortex/cortex.module.ts index 4cd9147e0..42d80effd 100644 --- a/cortex-js/src/infrastructure/providers/cortex/cortex.module.ts +++ b/cortex-js/src/infrastructure/providers/cortex/cortex.module.ts @@ -2,6 +2,9 @@ import { Module } from '@nestjs/common'; import CortexProvider from './cortex.provider'; import { HttpModule } from '@nestjs/axios'; import { FileManagerModule } from '@/infrastructure/services/file-manager/file-manager.module'; +import Onnxprovider from './onnx.provider'; +import LlamaCPPProvider from './llamacpp.provider'; +import TensorrtLLMProvider from './tensorrtllm.provider'; @Module({ imports: [HttpModule, FileManagerModule], @@ -10,12 +13,18 @@ import { FileManagerModule } from '@/infrastructure/services/file-manager/file-m provide: 'CORTEX_PROVIDER', useClass: CortexProvider, }, + Onnxprovider, + LlamaCPPProvider, + TensorrtLLMProvider, ], exports: [ { provide: 'CORTEX_PROVIDER', useClass: CortexProvider, }, + Onnxprovider, + LlamaCPPProvider, + TensorrtLLMProvider, ], }) export class CortexProviderModule {} diff --git a/cortex-js/src/infrastructure/providers/cortex/llamacpp.provider.ts b/cortex-js/src/infrastructure/providers/cortex/llamacpp.provider.ts new file mode 100644 index 000000000..85a76c55b --- /dev/null +++ b/cortex-js/src/infrastructure/providers/cortex/llamacpp.provider.ts @@ -0,0 +1,11 @@ +import { Injectable } from '@nestjs/common'; +import CortexProvider from './cortex.provider'; +import { Engines } from '@/infrastructure/commanders/types/engine.interface'; + +@Injectable() +export default class LlamaCPPProvider extends CortexProvider { + name = Engines.llamaCPP; + productName = 'LlamaCPP Inference Engine'; + description = + 'This extension enables chat completion API calls using the LlamaCPP engine'; +} diff --git a/cortex-js/src/infrastructure/providers/cortex/onnx.provider.ts b/cortex-js/src/infrastructure/providers/cortex/onnx.provider.ts new file mode 100644 index 000000000..4a3ef2fa9 --- /dev/null +++ b/cortex-js/src/infrastructure/providers/cortex/onnx.provider.ts @@ -0,0 +1,11 @@ +import { Injectable } from '@nestjs/common'; +import CortexProvider from './cortex.provider'; +import { Engines } from '@/infrastructure/commanders/types/engine.interface'; + +@Injectable() +export default class Onnxprovider extends CortexProvider { + name = Engines.onnx; + productName = 'Onnx Inference Engine'; + description = + 'This extension enables chat completion API calls using the Onnx engine'; +} diff --git a/cortex-js/src/infrastructure/providers/cortex/tensorrtllm.provider.ts b/cortex-js/src/infrastructure/providers/cortex/tensorrtllm.provider.ts new file mode 100644 index 000000000..65c63b489 --- /dev/null +++ b/cortex-js/src/infrastructure/providers/cortex/tensorrtllm.provider.ts @@ -0,0 +1,11 @@ +import { Injectable } from '@nestjs/common'; +import CortexProvider from './cortex.provider'; +import { Engines } from '@/infrastructure/commanders/types/engine.interface'; + +@Injectable() +export default class TensorrtLLMProvider extends CortexProvider { + name = Engines.tensorrtLLM; + productName = 'TensorrtLLM Inference Engine'; + description = + 'This extension enables chat completion API calls using the TensorrtLLM engine'; +} diff --git a/cortex-js/src/infrastructure/repositories/extensions/extension.repository.ts b/cortex-js/src/infrastructure/repositories/extensions/extension.repository.ts index d8ad85994..3ebd58e60 100644 --- a/cortex-js/src/infrastructure/repositories/extensions/extension.repository.ts +++ b/cortex-js/src/infrastructure/repositories/extensions/extension.repository.ts @@ -7,8 +7,10 @@ import { FileManagerService } from '@/infrastructure/services/file-manager/file- import { existsSync } from 'fs'; import { Engines } from '@/infrastructure/commanders/types/engine.interface'; import { OAIEngineExtension } from '@/domain/abstracts/oai.abstract'; -import CortexProvider from '@/infrastructure/providers/cortex/cortex.provider'; import { HttpService } from '@nestjs/axios'; +import LlamaCPPProvider from '@/infrastructure/providers/cortex/llamacpp.provider'; +import Onnxprovider from '@/infrastructure/providers/cortex/onnx.provider'; +import TensorrtLLMProvider from '@/infrastructure/providers/cortex/tensorrtllm.provider'; @Injectable() export class ExtensionRepositoryImpl implements ExtensionRepository { @@ -44,11 +46,10 @@ export class ExtensionRepositoryImpl implements ExtensionRepository { } private async loadCoreExtensions() { - const llamaCPPEngine = new CortexProvider( + const llamaCPPEngine = new LlamaCPPProvider( this.httpService, this.fileManagerService, ); - llamaCPPEngine.name = Engines.llamaCPP; llamaCPPEngine.initalized = existsSync( join( await this.fileManagerService.getCortexCppEnginePath(), @@ -56,11 +57,10 @@ export class ExtensionRepositoryImpl implements ExtensionRepository { ), ); - const onnxEngine = new CortexProvider( + const onnxEngine = new Onnxprovider( this.httpService, this.fileManagerService, ); - onnxEngine.name = Engines.onnx; onnxEngine.initalized = existsSync( join( await this.fileManagerService.getCortexCppEnginePath(), @@ -68,11 +68,10 @@ export class ExtensionRepositoryImpl implements ExtensionRepository { ), ); - const tensorrtLLMEngine = new CortexProvider( + const tensorrtLLMEngine = new TensorrtLLMProvider( this.httpService, this.fileManagerService, ); - tensorrtLLMEngine.name = Engines.tensorrtLLM; tensorrtLLMEngine.initalized = existsSync( join( await this.fileManagerService.getCortexCppEnginePath(),