diff --git a/cortex-js/src/infrastructure/commanders/engines.command.ts b/cortex-js/src/infrastructure/commanders/engines.command.ts index 5c4e39cf0..9f3e84b82 100644 --- a/cortex-js/src/infrastructure/commanders/engines.command.ts +++ b/cortex-js/src/infrastructure/commanders/engines.command.ts @@ -75,4 +75,12 @@ export class EnginesCommand extends BaseCommand { parseVulkan() { return true; } + + @Option({ + flags: '-v, --version ', + description: 'Select version to install', + }) + parseVersion(value: string) { + return value; + } } diff --git a/cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts b/cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts index d5251cea3..32c9047d4 100644 --- a/cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts +++ b/cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts @@ -1,5 +1,4 @@ -import { exit, stdin, stdout } from 'node:process'; -import { Option, SubCommand } from 'nest-commander'; +import { SubCommand } from 'nest-commander'; import { SetCommandContext } from '../decorators/CommandContext'; import { ContextService } from '@/infrastructure/services/context/context.service'; import { Engines } from '../types/engine.interface'; @@ -10,6 +9,7 @@ import { defaultInstallationOptions } from '@/utils/init'; import { Presets, SingleBar } from 'cli-progress'; import { CortexClient } from '../services/cortex.client'; import ora from 'ora'; +import { InitEngineDto } from '@/infrastructure/dtos/engines/engines.dto'; @SubCommand({ name: ' init', @@ -31,7 +31,7 @@ export class EnginesInitCommand extends BaseCommand { async runCommand( passedParams: string[], - options: { vulkan: boolean }, + options: InitEngineDto, ): Promise { const engine = passedParams[0]; const params = passedParams.includes(Engines.llamaCPP) @@ -51,6 +51,7 @@ export class EnginesInitCommand extends BaseCommand { } stopCortexSpinner.succeed('Cortex stopped'); console.log(`Installing engine ${engine}...`); + await this.cortex.engines.init(engine, params); const response = await this.cortex.events.downloadEvent(); @@ -77,13 +78,4 @@ export class EnginesInitCommand extends BaseCommand { console.log('Engine installed successfully'); process.exit(0); } - - @Option({ - flags: '-vk, --vulkan', - description: 'Install Vulkan engine', - defaultValue: false, - }) - parseVulkan() { - return true; - } } diff --git a/cortex-js/src/infrastructure/commanders/types/init-options.interface.ts b/cortex-js/src/infrastructure/commanders/types/init-options.interface.ts index 55444f04e..c8586a78c 100644 --- a/cortex-js/src/infrastructure/commanders/types/init-options.interface.ts +++ b/cortex-js/src/infrastructure/commanders/types/init-options.interface.ts @@ -5,4 +5,5 @@ export interface InitOptions { cudaVersion?: '11' | '12'; silent?: boolean; vulkan?: boolean; + version?: string; } diff --git a/cortex-js/src/infrastructure/controllers/engines.controller.ts b/cortex-js/src/infrastructure/controllers/engines.controller.ts index f96d896a0..2aa770aab 100644 --- a/cortex-js/src/infrastructure/controllers/engines.controller.ts +++ b/cortex-js/src/infrastructure/controllers/engines.controller.ts @@ -82,7 +82,7 @@ export class EnginesController { @Post(':name(*)/init') initialize(@Param('name') name: string, @Body() body: InitEngineDto | undefined, @Res() res: Response) { try{ - this.initUsescases.installEngine(body, 'latest', name, true); + this.initUsescases.installEngine(body, name, true); res.json({ message: 'Engine initialization started successfully.', }) diff --git a/cortex-js/src/infrastructure/dtos/engines/engines.dto.ts b/cortex-js/src/infrastructure/dtos/engines/engines.dto.ts index ec323560e..5180f7335 100644 --- a/cortex-js/src/infrastructure/dtos/engines/engines.dto.ts +++ b/cortex-js/src/infrastructure/dtos/engines/engines.dto.ts @@ -55,6 +55,7 @@ export class InitEngineDto { }) @IsString() runMode?: 'CPU' | 'GPU'; + @ApiProperty({ type: String, example: 'Nvidia', @@ -62,6 +63,7 @@ export class InitEngineDto { }) @IsString() gpuType?: 'Nvidia' | 'Others (Vulkan)'; + @ApiProperty({ type: String, example: 'AVX', @@ -69,6 +71,7 @@ export class InitEngineDto { }) @IsString() instructions?: 'AVX' | 'AVX2' | 'AVX512' | undefined; + @ApiProperty({ type: String, example: '11', @@ -76,6 +79,7 @@ export class InitEngineDto { }) @IsString() cudaVersion?: '11' | '12'; + @ApiProperty({ type: Boolean, example: true, @@ -83,6 +87,7 @@ export class InitEngineDto { }) @IsBoolean() silent?: boolean; + @ApiProperty({ type: Boolean, example: true, @@ -90,4 +95,12 @@ export class InitEngineDto { }) @IsBoolean() vulkan?: boolean; + + @ApiProperty({ + type: String, + example: true, + description: 'Engine version.', + }) + @IsBoolean() + version?: string; } diff --git a/cortex-js/src/usecases/engines/engines.usecase.ts b/cortex-js/src/usecases/engines/engines.usecase.ts index 79e8c46a9..e8f421a8d 100644 --- a/cortex-js/src/usecases/engines/engines.usecase.ts +++ b/cortex-js/src/usecases/engines/engines.usecase.ts @@ -74,7 +74,6 @@ export class EnginesUsecases { */ installEngine = async ( options?: InitOptions, - version: string = 'latest', engine: string = 'default', force: boolean = false, ): Promise => { @@ -95,30 +94,34 @@ export class EnginesUsecases { engine === Engines.llamaCPP && (options?.vulkan || (options?.runMode === 'GPU' && options?.gpuType !== 'Nvidia')); - installPackages.push(this.installAcceleratedEngine(version, engine, [ - process.platform === 'win32' - ? '-windows' - : process.platform === 'darwin' - ? '-mac' - : '-linux', - // CPU Instructions - CPU | GPU Non-Vulkan - options?.instructions && !isVulkan - ? `-${options?.instructions?.toLowerCase()}` - : '', - // Cuda - options?.runMode === 'GPU' && options?.gpuType === 'Nvidia' && !isVulkan - ? `cuda-${options.cudaVersion ?? '12'}` - : '', - // Vulkan - isVulkan ? '-vulkan' : '', + installPackages.push( + this.installAcceleratedEngine(options?.version ?? 'latest', engine, [ + process.platform === 'win32' + ? '-windows' + : process.platform === 'darwin' + ? '-mac' + : '-linux', + // CPU Instructions - CPU | GPU Non-Vulkan + options?.instructions && !isVulkan + ? `-${options?.instructions?.toLowerCase()}` + : '', + // Cuda + options?.runMode === 'GPU' && + options?.gpuType === 'Nvidia' && + !isVulkan + ? `cuda-${options.cudaVersion ?? '12'}` + : '', + // Vulkan + isVulkan ? '-vulkan' : '', - // Arch - engine !== Engines.tensorrtLLM - ? process.arch === 'arm64' - ? '-arm64' - : '-amd64' - : '', - ])); + // Arch + engine !== Engines.tensorrtLLM + ? process.arch === 'arm64' + ? '-arm64' + : '-amd64' + : '', + ]), + ); } if ( @@ -128,11 +131,13 @@ export class EnginesUsecases { options?.gpuType === 'Nvidia' && !options?.vulkan) ) - installPackages.push(this.installCudaToolkitDependency( - engine === Engines.tensorrtLLM - ? MIN_CUDA_VERSION - : options?.cudaVersion, - )); + installPackages.push( + this.installCudaToolkitDependency( + engine === Engines.tensorrtLLM + ? MIN_CUDA_VERSION + : options?.cudaVersion, + ), + ); await Promise.all(installPackages); // Update states await this.extensionRepository.findOne(engine).then((e) => { @@ -252,6 +257,8 @@ export class EnginesUsecases { matchers.every((matcher) => asset.name.includes(matcher)), ); + console.log('Downloading ', toDownloadAsset.name); + if (!toDownloadAsset) { console.log( `Could not find engine file for platform ${process.platform}`,