diff --git a/cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts b/cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts index 8e7394eac..1763d8757 100644 --- a/cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts +++ b/cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts @@ -3,6 +3,8 @@ import { SetCommandContext } from '../decorators/CommandContext'; import { ContextService } from '@/infrastructure/services/context/context.service'; import { InitCliUsecases } from '../usecases/init.cli.usecases'; import { Engines } from '../types/engine.interface'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; +import { FileManagerService } from '@/infrastructure/services/file-manager/file-manager.service'; @SubCommand({ name: 'init', @@ -16,6 +18,8 @@ import { Engines } from '../types/engine.interface'; export class EnginesInitCommand extends CommandRunner { constructor( private readonly initUsecases: InitCliUsecases, + private readonly cortexUsecases: CortexUsecases, + private readonly fileManagerService: FileManagerService, readonly contextService: ContextService, ) { super(); @@ -26,14 +30,24 @@ export class EnginesInitCommand extends CommandRunner { const options = passedParams.includes(Engines.llamaCPP) ? await this.initUsecases.defaultInstallationOptions() : {}; + + const configs = await this.fileManagerService.getConfig(); + const host = configs.cortexCppHost; + const port = configs.cortexCppPort; + // Should stop cortex before installing engine + if (await this.cortexUsecases.healthCheck(host, port)) { + await this.cortexUsecases.stopCortex(); + } return this.initUsecases .installEngine( options, engine.includes('@') ? engine.split('@')[1] : 'latest', engine, - true + true, ) .then(() => console.log('Engine installed successfully!')) - .catch(() => console.error('Engine not found or installation failed!')); + .catch((e) => + console.error('Install engine failed with reason: %s', e.message ?? e), + ); } } diff --git a/cortex-js/src/infrastructure/commanders/models.command.ts b/cortex-js/src/infrastructure/commanders/models.command.ts index aafe50fc9..17af50c0f 100644 --- a/cortex-js/src/infrastructure/commanders/models.command.ts +++ b/cortex-js/src/infrastructure/commanders/models.command.ts @@ -6,6 +6,7 @@ import { ModelStopCommand } from './models/model-stop.command'; import { ModelPullCommand } from './models/model-pull.command'; import { ModelRemoveCommand } from './models/model-remove.command'; import { ModelUpdateCommand } from './models/model-update.command'; +import { RunCommand } from './shortcuts/run.command'; @SubCommand({ name: 'models', @@ -17,6 +18,7 @@ import { ModelUpdateCommand } from './models/model-update.command'; ModelGetCommand, ModelRemoveCommand, ModelUpdateCommand, + RunCommand, ], description: 'Subcommands for managing models', }) diff --git a/cortex-js/src/infrastructure/commanders/models/model-start.command.ts b/cortex-js/src/infrastructure/commanders/models/model-start.command.ts index 7eee1f990..d2078cc9f 100644 --- a/cortex-js/src/infrastructure/commanders/models/model-start.command.ts +++ b/cortex-js/src/infrastructure/commanders/models/model-start.command.ts @@ -60,7 +60,7 @@ export class ModelStartCommand extends CommandRunner { /^(http|https):\/\/[^/]+\/.*/.test(existingModel.files[0]) ) { console.error( - `${modelId} not found on filesystem. Please try 'cortex pull ${modelId}' first.`, + `${modelId} not found on filesystem.\nPlease try 'cortex pull ${modelId}' first.`, ); process.exit(1); } diff --git a/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts b/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts index beee3ec45..0ebbca61b 100644 --- a/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts +++ b/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts @@ -52,13 +52,9 @@ export class RunCommand extends CommandRunner { exit(1); } } - // If not exist // Try Pull if (!(await this.modelsCliUsecases.getModel(modelId))) { - console.log( - `${modelId} not found on filesystem. Downloading from remote: https://huggingface.co/cortexso if possible.`, - ); await this.modelsCliUsecases.pullModel(modelId).catch((e: Error) => { if (e instanceof ModelNotFoundException) console.error('Model does not exist.'); @@ -78,6 +74,7 @@ export class RunCommand extends CommandRunner { process.exit(1); } + // Check model compatibility on this machine checkModelCompatibility(modelId); const engine = existingModel.engine || Engines.llamaCPP; diff --git a/cortex-js/src/infrastructure/commanders/usecases/chat.cli.usecases.ts b/cortex-js/src/infrastructure/commanders/usecases/chat.cli.usecases.ts index 710a13902..ba78119ef 100644 --- a/cortex-js/src/infrastructure/commanders/usecases/chat.cli.usecases.ts +++ b/cortex-js/src/infrastructure/commanders/usecases/chat.cli.usecases.ts @@ -37,7 +37,7 @@ export class ChatCliUsecases { attach: boolean = true, stopModel: boolean = true, ): Promise { - if (attach) console.log(`Inorder to exit, type '${this.exitClause}'.`); + if (attach) console.log(`In order to exit, type '${this.exitClause}'.`); const thread = await this.getOrCreateNewThread(modelId, threadId); const messages: ChatCompletionMessage[] = ( await this.messagesUsecases.getLastMessagesByThread(thread.id, 10) diff --git a/cortex-js/src/infrastructure/commanders/usecases/models.cli.usecases.ts b/cortex-js/src/infrastructure/commanders/usecases/models.cli.usecases.ts index cb140559e..cb9bc4e15 100644 --- a/cortex-js/src/infrastructure/commanders/usecases/models.cli.usecases.ts +++ b/cortex-js/src/infrastructure/commanders/usecases/models.cli.usecases.ts @@ -3,9 +3,7 @@ import { ModelsUsecases } from '@/usecases/models/models.usecases'; import { Model } from '@/domain/models/model.interface'; import { InquirerService } from 'nest-commander'; import { Inject, Injectable } from '@nestjs/common'; -import { Presets, SingleBar } from 'cli-progress'; -import { HttpService } from '@nestjs/axios'; import { StartModelSuccessDto } from '@/infrastructure/dtos/models/start-model-success.dto'; import { UpdateModelDto } from '@/infrastructure/dtos/models/update-model.dto'; import { FileManagerService } from '@/infrastructure/services/file-manager/file-manager.service'; @@ -14,6 +12,7 @@ import { load } from 'js-yaml'; import { existsSync, readdirSync, readFileSync } from 'fs'; import { isLocalModel } from '@/utils/normalize-model-id'; import { HuggingFaceRepoSibling } from '@/domain/models/huggingface.interface'; +import { printLastErrorLines } from '@/utils/logs'; @Injectable() export class ModelsCliUsecases { @@ -21,7 +20,6 @@ export class ModelsCliUsecases { private readonly modelsUsecases: ModelsUsecases, @Inject(InquirerService) private readonly inquirerService: InquirerService, - private readonly httpService: HttpService, private readonly fileService: FileManagerService, ) {} @@ -40,11 +38,16 @@ export class ModelsCliUsecases { ...parsedPreset, })) .then((settings) => this.modelsUsecases.startModel(modelId, settings)) - .catch(() => { - return { - modelId: modelId, - message: 'Model not found', - }; + .catch(async (e) => { + console.error('Model start failed with reason:', e.message); + + printLastErrorLines(await this.fileService.getDataFolderPath(), 5); + + console.log( + 'For more information, please check the logs at: %s', + join(await this.fileService.getDataFolderPath(), 'cortex.log'), + ); + process.exit(1); }); } @@ -114,6 +117,11 @@ export class ModelsCliUsecases { console.error('Model already exists'); process.exit(1); } + // Checking dependencies + + console.log( + `${modelId} not found on filesystem.\nDownloading from remote: https://huggingface.co/${modelId.includes('/') ? modelId : 'cortexso'} ...`, + ); await this.modelsUsecases.pullModel(modelId, true, (files) => { return new Promise(async (resolve) => { const listChoices = files diff --git a/cortex-js/src/usecases/cortex/cortex.usecases.ts b/cortex-js/src/usecases/cortex/cortex.usecases.ts index 2410b34bf..cb8aef592 100644 --- a/cortex-js/src/usecases/cortex/cortex.usecases.ts +++ b/cortex-js/src/usecases/cortex/cortex.usecases.ts @@ -12,6 +12,7 @@ import { CORTEX_CPP_PROCESS_DESTROY_URL, CORTEX_JS_STOP_API_SERVER_URL, } from '@/infrastructure/constants/cortex'; +import { createWriteStream, openSync } from 'fs'; @Injectable() export class CortexUsecases { @@ -25,8 +26,8 @@ export class CortexUsecases { /** * Start the Cortex CPP process - * @param attach - * @returns + * @param attach + * @returns */ async startCortex( attach: boolean = false, @@ -55,11 +56,16 @@ export class CortexUsecases { 'cortex-cpp', ); + const writer = openSync( + join(await this.fileManagerService.getDataFolderPath(), 'cortex.log'), + 'a+', + ); + // go up one level to get the binary folder, have to also work on windows this.cortexProcess = spawn(cortexCppPath, args, { detached: !attach, cwd: cortexCppFolderPath, - stdio: attach ? 'inherit' : undefined, + stdio: [0, writer, writer], env: { ...process.env, CUDA_VISIBLE_DEVICES: '0', @@ -135,7 +141,13 @@ export class CortexUsecases { .catch(() => {}); } - private healthCheck(host: string, port: number): Promise { + /** + * Check whether the Cortex CPP is healthy + * @param host + * @param port + * @returns + */ + healthCheck(host: string, port: number): Promise { return fetch(CORTEX_CPP_HEALTH_Z_URL(host, port)) .then((res) => { if (res.ok) { diff --git a/cortex-js/src/usecases/models/models.usecases.ts b/cortex-js/src/usecases/models/models.usecases.ts index 668514764..b6b08a3d0 100644 --- a/cortex-js/src/usecases/models/models.usecases.ts +++ b/cortex-js/src/usecases/models/models.usecases.ts @@ -163,7 +163,7 @@ export class ModelsUsecases { modelId, }; } - + console.log('Loading model...'); // update states and emitting event this.activeModelStatuses[modelId] = { model: modelId, @@ -233,7 +233,7 @@ export class ModelsUsecases { e, TelemetrySource.CORTEX_CPP, ); - return { + throw { message: e.message, modelId, }; @@ -365,7 +365,7 @@ export class ModelsUsecases { const model: CreateModelDto = load( readFileSync(join(modelFolder, 'model.yml'), 'utf-8'), ) as CreateModelDto; - if (model.engine === Engines.llamaCPP) { + if (model.engine === Engines.llamaCPP && model.files) { const fileUrl = join( await this.fileManagerService.getModelsPath(), normalizeModelId(modelId), @@ -373,6 +373,17 @@ export class ModelsUsecases { ); model.files = [fileUrl]; model.name = modelId.replace(':default', ''); + } else if (model.engine === Engines.llamaCPP) { + model.files = [ + join( + await this.fileManagerService.getModelsPath(), + normalizeModelId(modelId), + basename( + files.find((e) => e.rfilename.endsWith('.gguf'))?.rfilename ?? + files[0].rfilename, + ), + ), + ]; } else { model.files = [modelFolder]; } @@ -387,7 +398,10 @@ export class ModelsUsecases { const fileUrl = join( await this.fileManagerService.getModelsPath(), normalizeModelId(modelId), - basename(files[0].rfilename), + basename( + files.find((e) => e.rfilename.endsWith('.gguf'))?.rfilename ?? + files[0].rfilename, + ), ); await this.update(modelId, { files: [fileUrl], diff --git a/cortex-js/src/utils/logs.ts b/cortex-js/src/utils/logs.ts new file mode 100644 index 000000000..085be60a9 --- /dev/null +++ b/cortex-js/src/utils/logs.ts @@ -0,0 +1,32 @@ +import { createReadStream } from 'fs'; +import { join } from 'path'; +import { createInterface } from 'readline'; + +/** + * Print the last N lines of a file that contain the word 'ERROR' + * @param filename + * @param numLines + */ +export async function printLastErrorLines( + dataFolderPath: string, + numLines: number = 5, +): Promise { + const errorLines: string[] = []; + + const fileStream = createReadStream(join(dataFolderPath, 'cortex.log')); + const rl = createInterface({ + input: fileStream, + crlfDelay: Infinity, + }); + + for await (const line of rl) { + errorLines.push(line); + if (errorLines.length > numLines) { + errorLines.shift(); + } + } + + console.log(`Last errors:`); + errorLines.forEach((line) => console.log(line)); + console.log('...'); +}