diff --git a/README.md b/README.md index f81acab38..c42c037fb 100644 --- a/README.md +++ b/README.md @@ -68,12 +68,7 @@ To install Cortex CLI, follow the steps below: npm i -g @janhq/cortex ``` -2. Initialize a compatible engine: -``` bash -cortex init -``` - -3. Download a GGUF model from Hugging Face: +2. Download a GGUF model from Hugging Face: ``` bash # Pull a model most compatible with your hardware cortex pull llama3 @@ -84,12 +79,12 @@ cortex pull llama3:7b # Pull a model with the HuggingFace `model_id` cortex pull microsoft/Phi-3-mini-4k-instruct-gguf ``` -4. Load the model: +3. Load the model: ``` bash cortex models start llama3:7b ``` -5. Start chatting with the model: +4. Start chatting with the model: ``` bash cortex chat tell me a joke ``` diff --git a/cortex-js/README.md b/cortex-js/README.md index 806c8c873..a0af544df 100644 --- a/cortex-js/README.md +++ b/cortex-js/README.md @@ -68,12 +68,7 @@ To install Cortex CLI, follow the steps below: npm i -g @janhq/cortex ``` -2. Initialize a compatible engine: -``` bash -cortex init -``` - -3. Download a GGUF model from Hugging Face: +2. Download a GGUF model from Hugging Face: ``` bash # Pull a model most compatible with your hardware cortex pull llama3 @@ -84,12 +79,12 @@ cortex pull llama3:7b # Pull a model with the HuggingFace `model_id` cortex pull microsoft/Phi-3-mini-4k-instruct-gguf ``` -4. Load the model: +3. Load the model: ``` bash cortex models start llama3:7b ``` -5. Start chatting with the model: +4. Start chatting with the model: ``` bash cortex chat tell me a joke ``` diff --git a/cortex-js/package.json b/cortex-js/package.json index ed40df8a8..557a6d8a9 100644 --- a/cortex-js/package.json +++ b/cortex-js/package.json @@ -26,7 +26,8 @@ "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", "test:e2e": "jest --config ./test/jest-e2e.json", "typeorm": "typeorm-ts-node-esm", - "build:dev": "npx nest build && chmod +x ./dist/src/command.js && npm link" + "build:dev": "npx nest build && chmod +x ./dist/src/command.js && npm link", + "postinstall": "cortex init -s" }, "dependencies": { "@huggingface/gguf": "^0.1.5", diff --git a/cortex-js/src/infrastructure/commanders/init.command.ts b/cortex-js/src/infrastructure/commanders/init.command.ts index adf8eba4b..033245fcc 100644 --- a/cortex-js/src/infrastructure/commanders/init.command.ts +++ b/cortex-js/src/infrastructure/commanders/init.command.ts @@ -1,4 +1,9 @@ -import { CommandRunner, InquirerService, SubCommand } from 'nest-commander'; +import { + CommandRunner, + InquirerService, + SubCommand, + Option, +} from 'nest-commander'; import { InitCliUsecases } from './usecases/init.cli.usecases'; import { InitOptions } from './types/init-options.interface'; @@ -16,6 +21,38 @@ export class InitCommand extends CommandRunner { } async run(input: string[], options?: InitOptions): Promise { + if (options?.silent) { + return this.initSilently(input); + } else { + return this.initPrompts(input, options); + } + } + + private initSilently = async (input: string[], options: InitOptions = {}) => { + const version = input[0] ?? 'latest'; + if (process.platform === 'darwin') { + const engineFileName = this.initUsecases.parseEngineFileName(options); + return this.initUsecases.installEngine(engineFileName, version); + } + // If Nvidia Driver is installed -> GPU + options.runMode = (await this.initUsecases.checkNvidiaGPUExist()) + ? 'GPU' + : 'CPU'; + // CPU Instructions detection + options.gpuType = 'Nvidia'; + options.installCuda = 'Yes'; + options.instructions = await this.initUsecases.detectInstructions(); + const engineFileName = this.initUsecases.parseEngineFileName(options); + return this.initUsecases.installEngine(engineFileName, version); + }; + + /** + * Manual initalization + * To setup cortex's dependencies + * @param input + * @param options GPU | CPU / Nvidia | Others (Vulkan) / AVX | AVX2 | AVX512 + */ + private initPrompts = async (input: string[], options?: InitOptions) => { options = await this.inquirerService.ask( 'init-run-mode-questions', options, @@ -33,5 +70,14 @@ export class InitCommand extends CommandRunner { if (options.installCuda === 'Yes') { await this.initUsecases.installCudaToolkitDependency(options); } + }; + + @Option({ + flags: '-s, --silent', + description: 'Init without asking questions', + defaultValue: false, + }) + parseSilent() { + return true; } } diff --git a/cortex-js/src/infrastructure/commanders/types/init-options.interface.ts b/cortex-js/src/infrastructure/commanders/types/init-options.interface.ts index 24d460bbb..05ff929eb 100644 --- a/cortex-js/src/infrastructure/commanders/types/init-options.interface.ts +++ b/cortex-js/src/infrastructure/commanders/types/init-options.interface.ts @@ -4,4 +4,5 @@ export interface InitOptions { instructions?: 'AVX' | 'AVX2' | 'AVX512' | undefined; cudaVersion?: '11' | '12'; installCuda?: 'Yes' | string; + silent?: boolean; } diff --git a/cortex-js/src/infrastructure/commanders/usecases/init.cli.usecases.ts b/cortex-js/src/infrastructure/commanders/usecases/init.cli.usecases.ts index 34527d857..300356359 100644 --- a/cortex-js/src/infrastructure/commanders/usecases/init.cli.usecases.ts +++ b/cortex-js/src/infrastructure/commanders/usecases/init.cli.usecases.ts @@ -9,11 +9,13 @@ import { Injectable } from '@nestjs/common'; import { firstValueFrom } from 'rxjs'; import { FileManagerService } from '@/file-manager/file-manager.service'; import { rm } from 'fs/promises'; +import { exec } from 'child_process'; @Injectable() export class InitCliUsecases { - CORTEX_RELEASES_URL = 'https://api.github.com/repos/janhq/cortex/releases'; - CUDA_DOWNLOAD_URL = + private readonly CORTEX_RELEASES_URL = + 'https://api.github.com/repos/janhq/cortex/releases'; + private readonly CUDA_DOWNLOAD_URL = 'https://catalog.jan.ai/dist/cuda-dependencies///cuda.tar.gz'; constructor( @@ -109,7 +111,7 @@ export class InitCliUsecases { await rm(destination, { force: true }); }; - parseEngineFileName = (options: InitOptions) => { + parseEngineFileName = (options?: InitOptions) => { const platform = process.platform === 'win32' ? 'windows' @@ -118,12 +120,14 @@ export class InitCliUsecases { : process.platform; const arch = process.arch === 'arm64' ? process.arch : 'amd64'; const cudaVersion = - options.runMode === 'GPU' + options?.runMode === 'GPU' ? options.gpuType === 'Nvidia' ? '-cuda-' + (options.cudaVersion === '11' ? '11-7' : '12-0') : '-vulkan' : ''; - const instructions = options.instructions ? `-${options.instructions}` : ''; + const instructions = options?.instructions + ? `-${options.instructions}` + : ''; const engineName = `${platform}-${arch}${instructions.toLowerCase()}${cudaVersion}`; return `${engineName}.tar.gz`; }; @@ -173,10 +177,6 @@ export class InitCliUsecases { return undefined; // No CUDA Toolkit found }; - checkFileExistenceInPaths = (file: string, paths: string[]): boolean => { - return paths.some((p) => existsSync(join(p, file))); - }; - installCudaToolkitDependency = async (options: InitOptions) => { const platform = process.platform === 'win32' ? 'windows' : 'linux'; @@ -232,4 +232,61 @@ export class InitCliUsecases { } await rm(destination, { force: true }); }; + + // Function to check for NVIDIA GPU + checkNvidiaGPUExist = (): Promise => { + return new Promise((resolve) => { + // Execute the nvidia-smi command + exec('nvidia-smi', (error) => { + if (error) { + // If there's an error, it means nvidia-smi is not installed or there's no NVIDIA GPU + console.log('NVIDIA GPU not detected or nvidia-smi not installed.'); + resolve(false); + } else { + // If the command executes successfully, NVIDIA GPU is present + console.log('NVIDIA GPU detected.'); + resolve(true); + } + }); + }); + }; + + detectInstructions = (): Promise<'AVX' | 'AVX2' | 'AVX512' | undefined> => { + return new Promise<'AVX' | 'AVX2' | 'AVX512' | undefined>((res) => { + // Execute the cpuinfo command + + exec( + join( + __dirname, + `../../../../bin/cpuinfo${process.platform !== 'linux' ? '.exe' : ''}`, + ), + (error, stdout) => { + if (error) { + // If there's an error, it means lscpu is not installed + console.log('CPUInfo is not installed.'); + res('AVX'); + } else { + // If the command executes successfully, parse the output to detect CPU instructions + if (stdout.includes('"AVX512": "true"')) { + console.log('AVX-512 instructions detected.'); + res('AVX512'); + } else if ('"AVX2": "true"') { + console.log('AVX2 instructions detected.'); + res('AVX2'); + } else { + console.log('AVXs instructions detected.'); + res('AVX'); + } + } + }, + ); + }); + }; + + private checkFileExistenceInPaths = ( + file: string, + paths: string[], + ): boolean => { + return paths.some((p) => existsSync(join(p, file))); + }; }