diff --git a/cortex-js/src/infrastructure/commanders/models.command.ts b/cortex-js/src/infrastructure/commanders/models.command.ts index aafe50fc9..17af50c0f 100644 --- a/cortex-js/src/infrastructure/commanders/models.command.ts +++ b/cortex-js/src/infrastructure/commanders/models.command.ts @@ -6,6 +6,7 @@ import { ModelStopCommand } from './models/model-stop.command'; import { ModelPullCommand } from './models/model-pull.command'; import { ModelRemoveCommand } from './models/model-remove.command'; import { ModelUpdateCommand } from './models/model-update.command'; +import { RunCommand } from './shortcuts/run.command'; @SubCommand({ name: 'models', @@ -17,6 +18,7 @@ import { ModelUpdateCommand } from './models/model-update.command'; ModelGetCommand, ModelRemoveCommand, ModelUpdateCommand, + RunCommand, ], description: 'Subcommands for managing models', }) diff --git a/cortex-js/src/infrastructure/commanders/models/model-start.command.ts b/cortex-js/src/infrastructure/commanders/models/model-start.command.ts index 7eee1f990..02e29b065 100644 --- a/cortex-js/src/infrastructure/commanders/models/model-start.command.ts +++ b/cortex-js/src/infrastructure/commanders/models/model-start.command.ts @@ -60,7 +60,7 @@ export class ModelStartCommand extends CommandRunner { /^(http|https):\/\/[^/]+\/.*/.test(existingModel.files[0]) ) { console.error( - `${modelId} not found on filesystem. Please try 'cortex pull ${modelId}' first.`, + `The model ${modelId} not found on filesystem.\nPlease try 'cortex pull ${modelId}' first.`, ); process.exit(1); } diff --git a/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts b/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts index beee3ec45..0ebbca61b 100644 --- a/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts +++ b/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts @@ -52,13 +52,9 @@ export class RunCommand extends CommandRunner { exit(1); } } - // If not exist // Try Pull if (!(await this.modelsCliUsecases.getModel(modelId))) { - console.log( - `${modelId} not found on filesystem. Downloading from remote: https://huggingface.co/cortexso if possible.`, - ); await this.modelsCliUsecases.pullModel(modelId).catch((e: Error) => { if (e instanceof ModelNotFoundException) console.error('Model does not exist.'); @@ -78,6 +74,7 @@ export class RunCommand extends CommandRunner { process.exit(1); } + // Check model compatibility on this machine checkModelCompatibility(modelId); const engine = existingModel.engine || Engines.llamaCPP; diff --git a/cortex-js/src/infrastructure/commanders/usecases/chat.cli.usecases.ts b/cortex-js/src/infrastructure/commanders/usecases/chat.cli.usecases.ts index 710a13902..ba78119ef 100644 --- a/cortex-js/src/infrastructure/commanders/usecases/chat.cli.usecases.ts +++ b/cortex-js/src/infrastructure/commanders/usecases/chat.cli.usecases.ts @@ -37,7 +37,7 @@ export class ChatCliUsecases { attach: boolean = true, stopModel: boolean = true, ): Promise { - if (attach) console.log(`Inorder to exit, type '${this.exitClause}'.`); + if (attach) console.log(`In order to exit, type '${this.exitClause}'.`); const thread = await this.getOrCreateNewThread(modelId, threadId); const messages: ChatCompletionMessage[] = ( await this.messagesUsecases.getLastMessagesByThread(thread.id, 10) diff --git a/cortex-js/src/infrastructure/commanders/usecases/models.cli.usecases.ts b/cortex-js/src/infrastructure/commanders/usecases/models.cli.usecases.ts index cb140559e..593f28ecb 100644 --- a/cortex-js/src/infrastructure/commanders/usecases/models.cli.usecases.ts +++ b/cortex-js/src/infrastructure/commanders/usecases/models.cli.usecases.ts @@ -3,9 +3,7 @@ import { ModelsUsecases } from '@/usecases/models/models.usecases'; import { Model } from '@/domain/models/model.interface'; import { InquirerService } from 'nest-commander'; import { Inject, Injectable } from '@nestjs/common'; -import { Presets, SingleBar } from 'cli-progress'; -import { HttpService } from '@nestjs/axios'; import { StartModelSuccessDto } from '@/infrastructure/dtos/models/start-model-success.dto'; import { UpdateModelDto } from '@/infrastructure/dtos/models/update-model.dto'; import { FileManagerService } from '@/infrastructure/services/file-manager/file-manager.service'; @@ -14,6 +12,7 @@ import { load } from 'js-yaml'; import { existsSync, readdirSync, readFileSync } from 'fs'; import { isLocalModel } from '@/utils/normalize-model-id'; import { HuggingFaceRepoSibling } from '@/domain/models/huggingface.interface'; +import { printLastErrorLines } from '@/utils/logs'; @Injectable() export class ModelsCliUsecases { @@ -21,7 +20,6 @@ export class ModelsCliUsecases { private readonly modelsUsecases: ModelsUsecases, @Inject(InquirerService) private readonly inquirerService: InquirerService, - private readonly httpService: HttpService, private readonly fileService: FileManagerService, ) {} @@ -40,11 +38,16 @@ export class ModelsCliUsecases { ...parsedPreset, })) .then((settings) => this.modelsUsecases.startModel(modelId, settings)) - .catch(() => { - return { - modelId: modelId, - message: 'Model not found', - }; + .catch(async (e) => { + console.error('Model start failed with reason:', e.message); + + printLastErrorLines(await this.fileService.getDataFolderPath(), 5) + + console.log( + 'For more information, please check the logs at: %s', + join(await this.fileService.getDataFolderPath(), 'cortex.log'), + ); + process.exit(1); }); } @@ -114,6 +117,11 @@ export class ModelsCliUsecases { console.error('Model already exists'); process.exit(1); } + // Checking dependencies + + console.log( + `The model ${modelId} not found on filesystem.\nDownloading from remote: https://huggingface.co/cortexso...`, + ); await this.modelsUsecases.pullModel(modelId, true, (files) => { return new Promise(async (resolve) => { const listChoices = files diff --git a/cortex-js/src/usecases/cortex/cortex.usecases.ts b/cortex-js/src/usecases/cortex/cortex.usecases.ts index 2410b34bf..50e7dc969 100644 --- a/cortex-js/src/usecases/cortex/cortex.usecases.ts +++ b/cortex-js/src/usecases/cortex/cortex.usecases.ts @@ -12,6 +12,7 @@ import { CORTEX_CPP_PROCESS_DESTROY_URL, CORTEX_JS_STOP_API_SERVER_URL, } from '@/infrastructure/constants/cortex'; +import { createWriteStream, openSync } from 'fs'; @Injectable() export class CortexUsecases { @@ -25,8 +26,8 @@ export class CortexUsecases { /** * Start the Cortex CPP process - * @param attach - * @returns + * @param attach + * @returns */ async startCortex( attach: boolean = false, @@ -55,11 +56,16 @@ export class CortexUsecases { 'cortex-cpp', ); + const writer = openSync( + join(await this.fileManagerService.getDataFolderPath(), 'cortex.log'), + 'a+', + ); + // go up one level to get the binary folder, have to also work on windows this.cortexProcess = spawn(cortexCppPath, args, { detached: !attach, cwd: cortexCppFolderPath, - stdio: attach ? 'inherit' : undefined, + stdio: [0, writer, writer], env: { ...process.env, CUDA_VISIBLE_DEVICES: '0', diff --git a/cortex-js/src/usecases/models/models.usecases.ts b/cortex-js/src/usecases/models/models.usecases.ts index 668514764..e332eb497 100644 --- a/cortex-js/src/usecases/models/models.usecases.ts +++ b/cortex-js/src/usecases/models/models.usecases.ts @@ -163,7 +163,7 @@ export class ModelsUsecases { modelId, }; } - + console.log('Loading model...'); // update states and emitting event this.activeModelStatuses[modelId] = { model: modelId, @@ -233,7 +233,7 @@ export class ModelsUsecases { e, TelemetrySource.CORTEX_CPP, ); - return { + throw { message: e.message, modelId, }; diff --git a/cortex-js/src/utils/logs.ts b/cortex-js/src/utils/logs.ts new file mode 100644 index 000000000..085be60a9 --- /dev/null +++ b/cortex-js/src/utils/logs.ts @@ -0,0 +1,32 @@ +import { createReadStream } from 'fs'; +import { join } from 'path'; +import { createInterface } from 'readline'; + +/** + * Print the last N lines of a file that contain the word 'ERROR' + * @param filename + * @param numLines + */ +export async function printLastErrorLines( + dataFolderPath: string, + numLines: number = 5, +): Promise { + const errorLines: string[] = []; + + const fileStream = createReadStream(join(dataFolderPath, 'cortex.log')); + const rl = createInterface({ + input: fileStream, + crlfDelay: Infinity, + }); + + for await (const line of rl) { + errorLines.push(line); + if (errorLines.length > numLines) { + errorLines.shift(); + } + } + + console.log(`Last errors:`); + errorLines.forEach((line) => console.log(line)); + console.log('...'); +}