diff --git a/cortex-js/package.json b/cortex-js/package.json index 23383a1c6..a1fc603e3 100644 --- a/cortex-js/package.json +++ b/cortex-js/package.json @@ -52,10 +52,11 @@ "class-validator": "^0.14.1", "cli-progress": "^3.12.0", "cortexso-node": "^0.0.4", - "cpu-instructions": "^0.0.10", + "cpu-instructions": "^0.0.11", "decompress": "^4.2.1", "js-yaml": "^4.1.0", "nest-commander": "^3.13.0", + "ora": "5.4.1", "readline": "^1.3.0", "reflect-metadata": "^0.2.0", "rxjs": "^7.8.1", diff --git a/cortex-js/src/command.ts b/cortex-js/src/command.ts index 274e401b3..1573a2a38 100644 --- a/cortex-js/src/command.ts +++ b/cortex-js/src/command.ts @@ -1,12 +1,14 @@ -#!/usr/bin/env node --no-warnings +#!/usr/bin/env node +import ora from 'ora'; +const dependenciesSpinner = ora('Loading dependencies...').start(); +const time = Date.now(); import { CommandFactory } from 'nest-commander'; import { CommandModule } from './command.module'; import { TelemetryUsecases } from './usecases/telemetry/telemetry.usecases'; import { TelemetrySource } from './domain/telemetry/telemetry.interface'; -import { AsyncLocalStorage } from 'async_hooks'; import { ContextService } from '@/infrastructure/services/context/context.service'; -export const asyncLocalStorage = new AsyncLocalStorage(); +dependenciesSpinner.succeed('Dependencies loaded in ' + (Date.now() - time) + 'ms'); async function bootstrap() { let telemetryUseCase: TelemetryUsecases | null = null; diff --git a/cortex-js/src/domain/telemetry/telemetry.interface.ts b/cortex-js/src/domain/telemetry/telemetry.interface.ts index c897a436e..15e330586 100644 --- a/cortex-js/src/domain/telemetry/telemetry.interface.ts +++ b/cortex-js/src/domain/telemetry/telemetry.interface.ts @@ -103,6 +103,7 @@ export interface TelemetryAnonymized { sessionId: string | null; lastActiveAt?: string | null; } + export interface BenchmarkHardware { gpu: any[]; cpu: any; diff --git a/cortex-js/src/infrastructure/commanders/chat.command.ts b/cortex-js/src/infrastructure/commanders/chat.command.ts index 8b72c7b37..fb4ba0427 100644 --- a/cortex-js/src/infrastructure/commanders/chat.command.ts +++ b/cortex-js/src/infrastructure/commanders/chat.command.ts @@ -4,6 +4,7 @@ import { Option, InquirerService, } from 'nest-commander'; +import ora from 'ora'; import { ChatCliUsecases } from './usecases/chat.cli.usecases'; import { exit } from 'node:process'; import { PSCliUsecases } from './usecases/ps.cli.usecases'; @@ -48,6 +49,7 @@ export class ChatCommand extends CommandRunner { async run(passedParams: string[], options: ChatOptions): Promise { let modelId = passedParams[0]; + const checkingSpinner = ora('Checking model...').start(); // First attempt to get message from input or options // Extract input from 1 to end of array let message = options.message ?? passedParams.slice(1).join(' '); @@ -66,10 +68,11 @@ export class ChatCommand extends CommandRunner { } else if (models.length > 0) { modelId = await this.modelInquiry(models); } else { - console.error('Model ID is required'); + checkingSpinner.fail('Model ID is required'); exit(1); } } + checkingSpinner.succeed(`Model found`); if (!message) options.attach = true; const result = await this.chatCliUsecases.chat( diff --git a/cortex-js/src/infrastructure/commanders/embeddings.command.ts b/cortex-js/src/infrastructure/commanders/embeddings.command.ts index 473c84254..a04bd9839 100644 --- a/cortex-js/src/infrastructure/commanders/embeddings.command.ts +++ b/cortex-js/src/infrastructure/commanders/embeddings.command.ts @@ -4,6 +4,7 @@ import { Option, SubCommand, } from 'nest-commander'; +import ora from 'ora'; import { ModelsUsecases } from '@/usecases/models/models.usecases'; import { PSCliUsecases } from './usecases/ps.cli.usecases'; import { ChatCliUsecases } from './usecases/chat.cli.usecases'; @@ -39,6 +40,7 @@ export class EmbeddingCommand extends CommandRunner { options: EmbeddingCommandOptions, ): Promise { let modelId = passedParams[0]; + const checkingSpinner = ora('Checking model...').start(); // First attempt to get message from input or options let input: string | string[] = options.input ?? passedParams.splice(1); @@ -54,11 +56,11 @@ export class EmbeddingCommand extends CommandRunner { } else if (models.length > 0) { modelId = await this.modelInquiry(models); } else { - console.error('Model ID is required'); + checkingSpinner.fail('Model ID is required'); process.exit(1); } } - + checkingSpinner.succeed(`Model found`); return this.chatCliUsecases .embeddings(modelId, input) .then((res) => diff --git a/cortex-js/src/infrastructure/commanders/models/model-start.command.ts b/cortex-js/src/infrastructure/commanders/models/model-start.command.ts index d2078cc9f..5cec03037 100644 --- a/cortex-js/src/infrastructure/commanders/models/model-start.command.ts +++ b/cortex-js/src/infrastructure/commanders/models/model-start.command.ts @@ -4,6 +4,7 @@ import { Option, InquirerService, } from 'nest-commander'; +import ora from 'ora'; import { exit } from 'node:process'; import { ModelsCliUsecases } from '@commanders/usecases/models.cli.usecases'; import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; @@ -44,11 +45,12 @@ export class ModelStartCommand extends CommandRunner { async run(passedParams: string[], options: ModelStartOptions): Promise { let modelId = passedParams[0]; + const checkingSpinner = ora('Checking model...').start(); if (!modelId) { try { modelId = await this.modelInquiry(); } catch { - console.error('Model ID is required'); + checkingSpinner.fail('Model ID is required'); exit(1); } } @@ -59,26 +61,25 @@ export class ModelStartCommand extends CommandRunner { !Array.isArray(existingModel.files) || /^(http|https):\/\/[^/]+\/.*/.test(existingModel.files[0]) ) { - console.error( - `${modelId} not found on filesystem.\nPlease try 'cortex pull ${modelId}' first.`, - ); + checkingSpinner.fail(`Model ${modelId} not found on filesystem.\nPlease try 'cortex pull ${modelId}' first.`); process.exit(1); } checkModelCompatibility(modelId); - + checkingSpinner.succeed('Model found'); const engine = existingModel.engine || Engines.llamaCPP; // Pull engine if not exist if ( !existsSync(join(await this.fileService.getCortexCppEnginePath(), engine)) ) { + const engineSpinner = ora('Installing engine...').start(); await this.initUsecases.installEngine( await this.initUsecases.defaultInstallationOptions(), 'latest', engine, ); + engineSpinner.succeed(); } - await this.cortexUsecases .startCortex(options.attach) .then(() => this.modelsCliUsecases.startModel(modelId, options.preset)) diff --git a/cortex-js/src/infrastructure/commanders/ps.command.ts b/cortex-js/src/infrastructure/commanders/ps.command.ts index c00bbed2c..5d900c328 100644 --- a/cortex-js/src/infrastructure/commanders/ps.command.ts +++ b/cortex-js/src/infrastructure/commanders/ps.command.ts @@ -1,7 +1,9 @@ +import ora from 'ora'; import { CommandRunner, SubCommand } from 'nest-commander'; import { PSCliUsecases } from './usecases/ps.cli.usecases'; import { SetCommandContext } from './decorators/CommandContext'; import { ContextService } from '../services/context/context.service'; +import { ModelStat } from './types/model-stat.interface'; @SubCommand({ name: 'ps', @@ -16,12 +18,20 @@ export class PSCommand extends CommandRunner { super(); } async run(): Promise { + const runningSpinner = ora('Running PS command...').start(); + let checkingSpinner: ora.Ora return this.usecases .getModels() - .then(console.table) - .then(() => this.usecases.isAPIServerOnline()) + .then((models: ModelStat[]) => { + runningSpinner.succeed(); + console.table(models); + }) + .then(() => { + checkingSpinner = ora('Checking API server...').start(); + return this.usecases.isAPIServerOnline(); + }) .then((isOnline) => { - if (isOnline) console.log('API server is online'); + checkingSpinner.succeed(isOnline ? 'API server is online' : 'API server is offline'); }); } -} +} \ No newline at end of file diff --git a/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts b/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts index 0ebbca61b..5c6e96ffe 100644 --- a/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts +++ b/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts @@ -6,6 +6,7 @@ import { InquirerService, } from 'nest-commander'; import { exit } from 'node:process'; +import ora from 'ora'; import { ChatCliUsecases } from '@commanders/usecases/chat.cli.usecases'; import { ModelsCliUsecases } from '@commanders/usecases/models.cli.usecases'; import { ModelNotFoundException } from '@/infrastructure/exception/model-not-found.exception'; @@ -44,21 +45,23 @@ export class RunCommand extends CommandRunner { async run(passedParams: string[], options: RunOptions): Promise { let modelId = passedParams[0]; + const checkingSpinner = ora('Checking model...').start(); if (!modelId) { try { modelId = await this.modelInquiry(); } catch { - console.error('Model ID is required'); + checkingSpinner.fail('Model ID is required'); exit(1); } } // If not exist // Try Pull if (!(await this.modelsCliUsecases.getModel(modelId))) { + checkingSpinner.succeed('Model not found. Attempting to pull...'); await this.modelsCliUsecases.pullModel(modelId).catch((e: Error) => { if (e instanceof ModelNotFoundException) - console.error('Model does not exist.'); - else console.error(e.message ?? e); + checkingSpinner.fail('Model does not exist.'); + else checkingSpinner.fail(e.message ?? e); exit(1); }); } @@ -70,23 +73,27 @@ export class RunCommand extends CommandRunner { !Array.isArray(existingModel.files) || /^(http|https):\/\/[^/]+\/.*/.test(existingModel.files[0]) ) { - console.error('Model is not available.'); + checkingSpinner.fail( + `Model is not available` + ); process.exit(1); } + checkingSpinner.succeed('Model found'); // Check model compatibility on this machine checkModelCompatibility(modelId); - const engine = existingModel.engine || Engines.llamaCPP; // Pull engine if not exist if ( !existsSync(join(await this.fileService.getCortexCppEnginePath(), engine)) ) { + const engineSpinner = ora('Installing engine...').start(); await this.initUsecases.installEngine( await this.initUsecases.defaultInstallationOptions(), 'latest', engine, ); + engineSpinner.succeed('Engine installed'); } return this.cortexUsecases diff --git a/cortex-js/src/infrastructure/commanders/usecases/ps.cli.usecases.ts b/cortex-js/src/infrastructure/commanders/usecases/ps.cli.usecases.ts index 6fad12b6b..20d68fc01 100644 --- a/cortex-js/src/infrastructure/commanders/usecases/ps.cli.usecases.ts +++ b/cortex-js/src/infrastructure/commanders/usecases/ps.cli.usecases.ts @@ -1,4 +1,5 @@ import { HttpStatus, Injectable } from '@nestjs/common'; +import ora from 'ora'; import { CORTEX_CPP_MODELS_URL, CORTEX_JS_HEALTH_URL, @@ -26,6 +27,7 @@ export class PSCliUsecases { */ async getModels(): Promise { const configs = await this.fileService.getConfig(); + const runningSpinner = ora('Getting models...').start(); return new Promise((resolve, reject) => firstValueFrom( this.httpService.get( @@ -40,6 +42,7 @@ export class PSCliUsecases { Array.isArray(data.data) && data.data.length > 0 ) { + runningSpinner.succeed(); resolve( data.data.map((e) => { const startTime = e.start_time ?? new Date(); @@ -59,7 +62,10 @@ export class PSCliUsecases { } else reject(); }) .catch(reject), - ).catch(() => []); + ).catch(() => { + runningSpinner.succeed(''); + return []; + }); } /** diff --git a/cortex-js/src/usecases/models/models.usecases.ts b/cortex-js/src/usecases/models/models.usecases.ts index b6b08a3d0..6d714799d 100644 --- a/cortex-js/src/usecases/models/models.usecases.ts +++ b/cortex-js/src/usecases/models/models.usecases.ts @@ -1,10 +1,11 @@ +import ora from 'ora'; import { CreateModelDto } from '@/infrastructure/dtos/models/create-model.dto'; import { UpdateModelDto } from '@/infrastructure/dtos/models/update-model.dto'; import { BadRequestException, Injectable } from '@nestjs/common'; import { Model, ModelSettingParams } from '@/domain/models/model.interface'; import { ModelNotFoundException } from '@/infrastructure/exception/model-not-found.exception'; import { basename, join } from 'path'; -import { promises, existsSync, mkdirSync, rmdirSync, readFileSync } from 'fs'; +import { promises, existsSync, mkdirSync, readFileSync, rmSync } from 'fs'; import { StartModelSuccessDto } from '@/infrastructure/dtos/models/start-model-success.dto'; import { ExtensionRepository } from '@/domain/repositories/extension.interface'; import { EngineExtension } from '@/domain/abstracts/engine.abstract'; @@ -123,8 +124,7 @@ export class ModelsUsecases { .remove(id) .then( () => - existsSync(modelFolder) && - rmdirSync(modelFolder, { recursive: true }), + existsSync(modelFolder) && rmSync(modelFolder, { recursive: true }), ) .then(() => { const modelEvent: ModelEvent = { @@ -163,7 +163,7 @@ export class ModelsUsecases { modelId, }; } - console.log('Loading model...'); + const loadingModelSpinner = ora('Loading model...').start(); // update states and emitting event this.activeModelStatuses[modelId] = { model: modelId, @@ -210,10 +210,13 @@ export class ModelsUsecases { }; this.eventEmitter.emit('model.event', modelEvent); }) - .then(() => ({ - message: 'Model loaded successfully', - modelId, - })) + .then(() => { + loadingModelSpinner.succeed('Model loaded'); + return { + message: 'Model loaded successfully', + modelId, + }; + }) .catch(async (e) => { // remove the model from this.activeModelStatus. delete this.activeModelStatuses[modelId]; @@ -229,6 +232,7 @@ export class ModelsUsecases { modelId, }; } + loadingModelSpinner.fail('Model loading failed'); await this.telemetryUseCases.createCrashReport( e, TelemetrySource.CORTEX_CPP, @@ -359,7 +363,9 @@ export class ModelsUsecases { toDownloads, // Post processing async () => { - console.log('Update model metadata...'); + const uploadModelMetadataSpiner = ora( + 'Updating model metadata...', + ).start(); // Post processing after download if (existsSync(join(modelFolder, 'model.yml'))) { const model: CreateModelDto = load( @@ -409,6 +415,7 @@ export class ModelsUsecases { }); } } + uploadModelMetadataSpiner.succeed('Model metadata updated'); const modelEvent: ModelEvent = { model: modelId, event: 'model-downloaded',