diff --git a/cortex-js/src/command.module.ts b/cortex-js/src/command.module.ts index 91935de69..7930b362a 100644 --- a/cortex-js/src/command.module.ts +++ b/cortex-js/src/command.module.ts @@ -3,7 +3,6 @@ import { ModelsModule } from './usecases/models/models.module'; import { DatabaseModule } from './infrastructure/database/database.module'; import { ConfigModule } from '@nestjs/config'; import { CortexModule } from './usecases/cortex/cortex.module'; -import { ServeCommand } from './infrastructure/commanders/serve.command'; import { ModelsCommand } from './infrastructure/commanders/models.command'; import { ExtensionModule } from './infrastructure/repositories/extensions/extension.module'; import { HttpModule } from '@nestjs/axios'; @@ -45,7 +44,6 @@ import { EnginesListCommand } from './infrastructure/commanders/engines/engines- import { EnginesGetCommand } from './infrastructure/commanders/engines/engines-get.command'; import { EnginesInitCommand } from './infrastructure/commanders/engines/engines-init.command'; - @Module({ imports: [ ConfigModule.forRoot({ @@ -73,7 +71,6 @@ import { EnginesInitCommand } from './infrastructure/commanders/engines/engines- providers: [ CortexCommand, ModelsCommand, - ServeCommand, ChatCommand, PSCommand, KillCommand, diff --git a/cortex-js/src/domain/abstracts/oai.abstract.ts b/cortex-js/src/domain/abstracts/oai.abstract.ts index 3b75edfbf..ae1dc95e4 100644 --- a/cortex-js/src/domain/abstracts/oai.abstract.ts +++ b/cortex-js/src/domain/abstracts/oai.abstract.ts @@ -25,7 +25,7 @@ export abstract class OAIEngineExtension extends EngineExtension { const additionalHeaders = _.omit(headers, [ 'content-type', 'authorization', - 'content-length' + 'content-length', ]); const response = await firstValueFrom( this.httpService.post(this.apiUrl, payload, { diff --git a/cortex-js/src/domain/config/config.interface.ts b/cortex-js/src/domain/config/config.interface.ts index f7378dd35..b99d76600 100644 --- a/cortex-js/src/domain/config/config.interface.ts +++ b/cortex-js/src/domain/config/config.interface.ts @@ -2,4 +2,7 @@ export interface Config { dataFolderPath: string; cortexCppHost: string; cortexCppPort: number; + // todo: will remove optional when all command request api server + apiServerPort?: number; + apiServerHost?: string; } diff --git a/cortex-js/src/infrastructure/commanders/base.command.ts b/cortex-js/src/infrastructure/commanders/base.command.ts new file mode 100644 index 000000000..25b508236 --- /dev/null +++ b/cortex-js/src/infrastructure/commanders/base.command.ts @@ -0,0 +1,29 @@ +import { CommandRunner } from 'nest-commander'; +import { Injectable } from '@nestjs/common'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; +import ora from 'ora'; + +@Injectable() +export abstract class BaseCommand extends CommandRunner { + constructor(readonly cortexUseCases: CortexUsecases) { + super(); + } + protected abstract runCommand( + passedParam: string[], + options?: Record, + ): Promise; + + async run( + passedParam: string[], + options?: Record, + ): Promise { + const checkingSpinner = ora('Checking API server online...').start(); + const result = await this.cortexUseCases.isAPIServerOnline(); + if (!result) { + checkingSpinner.fail('API server is offline'); + process.exit(1); + } + checkingSpinner.succeed('API server is online'); + await this.runCommand(passedParam, options); + } +} diff --git a/cortex-js/src/infrastructure/commanders/benchmark.command.ts b/cortex-js/src/infrastructure/commanders/benchmark.command.ts index ae23f13fc..5e36f51d4 100644 --- a/cortex-js/src/infrastructure/commanders/benchmark.command.ts +++ b/cortex-js/src/infrastructure/commanders/benchmark.command.ts @@ -4,6 +4,8 @@ import { BenchmarkConfig, ParametersConfig, } from './types/benchmark-config.interface'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; +import { BaseCommand } from './base.command'; @SubCommand({ name: 'benchmark', @@ -14,12 +16,15 @@ import { description: 'Benchmark and analyze the performance of a specific AI model using a variety of system resources', }) -export class BenchmarkCommand extends CommandRunner { - constructor(private readonly benchmarkUsecases: BenchmarkCliUsecases) { - super(); +export class BenchmarkCommand extends BaseCommand { + constructor( + private readonly benchmarkUsecases: BenchmarkCliUsecases, + readonly cortexUsecases: CortexUsecases, + ) { + super(cortexUsecases); } - async run( + async runCommand( passedParams: string[], options?: Partial, ): Promise { diff --git a/cortex-js/src/infrastructure/commanders/chat.command.ts b/cortex-js/src/infrastructure/commanders/chat.command.ts index 9c39287a5..027ab9c80 100644 --- a/cortex-js/src/infrastructure/commanders/chat.command.ts +++ b/cortex-js/src/infrastructure/commanders/chat.command.ts @@ -1,10 +1,5 @@ -import { - CommandRunner, - SubCommand, - Option, - InquirerService, -} from 'nest-commander'; -import ora from 'ora'; +import { existsSync } from 'fs'; +import { SubCommand, Option, InquirerService } from 'nest-commander'; import { ChatCliUsecases } from './usecases/chat.cli.usecases'; import { exit } from 'node:process'; import { PSCliUsecases } from './usecases/ps.cli.usecases'; @@ -17,6 +12,14 @@ import { TelemetrySource, } from '@/domain/telemetry/telemetry.interface'; import { ContextService } from '../services/context/context.service'; +import { BaseCommand } from './base.command'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; +import { ModelsCliUsecases } from './usecases/models.cli.usecases'; +import { Engines } from './types/engine.interface'; +import { join } from 'path'; +import { EnginesUsecases } from '@/usecases/engines/engines.usecase'; +import { FileManagerService } from '../services/file-manager/file-manager.service'; +import { isLocalModel } from '@/utils/normalize-model-id'; type ChatOptions = { threadId?: string; @@ -35,7 +38,7 @@ type ChatOptions = { }, }) @SetCommandContext() -export class ChatCommand extends CommandRunner { +export class ChatCommand extends BaseCommand { constructor( private readonly inquirerService: InquirerService, private readonly chatCliUsecases: ChatCliUsecases, @@ -43,11 +46,18 @@ export class ChatCommand extends CommandRunner { private readonly psCliUsecases: PSCliUsecases, readonly contextService: ContextService, private readonly telemetryUsecases: TelemetryUsecases, + readonly cortexUsecases: CortexUsecases, + readonly modelsCliUsecases: ModelsCliUsecases, + private readonly fileService: FileManagerService, + private readonly initUsecases: EnginesUsecases, ) { - super(); + super(cortexUsecases); } - async run(passedParams: string[], options: ChatOptions): Promise { + async runCommand( + passedParams: string[], + options: ChatOptions, + ): Promise { let modelId = passedParams[0]; // First attempt to get message from input or options // Extract input from 1 to end of array @@ -59,7 +69,7 @@ export class ChatCommand extends CommandRunner { // first input might be message input message = passedParams.length ? passedParams.join(' ') - : options.message ?? ''; + : (options.message ?? ''); // If model ID is not provided, prompt user to select from running models const models = await this.psCliUsecases.getModels(); if (models.length === 1) { @@ -71,14 +81,20 @@ export class ChatCommand extends CommandRunner { } } + const existingModel = await this.modelsCliUsecases.getModel(modelId); + if (!existingModel || !isLocalModel(existingModel.files)) { + process.exit(1); + } + + const engine = existingModel.engine || Engines.llamaCPP; + // Pull engine if not exist + if ( + !existsSync(join(await this.fileService.getCortexCppEnginePath(), engine)) + ) { + await this.initUsecases.installEngine(undefined, 'latest', engine); + } + if (!message) options.attach = true; - const result = await this.chatCliUsecases.chat( - modelId, - options.threadId, - message, // Accept both message from inputs or arguments - options.attach, - false, // Do not stop cortex session or loaded model - ); this.telemetryUsecases.sendEvent( [ { @@ -88,7 +104,18 @@ export class ChatCommand extends CommandRunner { ], TelemetrySource.CLI, ); - return result; + return this.cortexUsecases + .startCortex() + .then(() => this.modelsCliUsecases.startModel(modelId)) + .then(() => + this.chatCliUsecases.chat( + modelId, + options.threadId, + message, // Accept both message from inputs or arguments + options.attach, + false, // Do not stop cortex session or loaded model + ), + ); } modelInquiry = async (models: ModelStat[]) => { diff --git a/cortex-js/src/infrastructure/commanders/configs.command.ts b/cortex-js/src/infrastructure/commanders/configs.command.ts index c7e841ac2..5801891ca 100644 --- a/cortex-js/src/infrastructure/commanders/configs.command.ts +++ b/cortex-js/src/infrastructure/commanders/configs.command.ts @@ -1,9 +1,11 @@ -import { CommandRunner, SubCommand } from 'nest-commander'; +import { SubCommand } from 'nest-commander'; import { SetCommandContext } from './decorators/CommandContext'; import { ContextService } from '@/infrastructure/services/context/context.service'; import { ConfigsGetCommand } from './configs/configs-get.command'; import { ConfigsListCommand } from './configs/configs-list.command'; import { ConfigsSetCommand } from './configs/configs-set.command'; +import { BaseCommand } from './base.command'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; @SubCommand({ name: 'configs', @@ -11,12 +13,15 @@ import { ConfigsSetCommand } from './configs/configs-set.command'; subCommands: [ConfigsGetCommand, ConfigsListCommand, ConfigsSetCommand], }) @SetCommandContext() -export class ConfigsCommand extends CommandRunner { - constructor(readonly contextService: ContextService) { - super(); +export class ConfigsCommand extends BaseCommand { + constructor( + readonly contextService: ContextService, + readonly cortexUseCases: CortexUsecases, + ) { + super(cortexUseCases); } - async run(): Promise { + async runCommand(): Promise { this.command?.help(); } } diff --git a/cortex-js/src/infrastructure/commanders/configs/configs-get.command.ts b/cortex-js/src/infrastructure/commanders/configs/configs-get.command.ts index 8ad60545b..4e815ebd8 100644 --- a/cortex-js/src/infrastructure/commanders/configs/configs-get.command.ts +++ b/cortex-js/src/infrastructure/commanders/configs/configs-get.command.ts @@ -2,6 +2,8 @@ import { CommandRunner, SubCommand } from 'nest-commander'; import { SetCommandContext } from '../decorators/CommandContext'; import { ContextService } from '@/infrastructure/services/context/context.service'; import { ConfigsUsecases } from '@/usecases/configs/configs.usecase'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; +import { BaseCommand } from '../base.command'; @SubCommand({ name: 'get', @@ -12,15 +14,16 @@ import { ConfigsUsecases } from '@/usecases/configs/configs.usecase'; }, }) @SetCommandContext() -export class ConfigsGetCommand extends CommandRunner { +export class ConfigsGetCommand extends BaseCommand { constructor( private readonly configsUsecases: ConfigsUsecases, readonly contextService: ContextService, + readonly cortexUsecases: CortexUsecases, ) { - super(); + super(cortexUsecases); } - async run(passedParams: string[]): Promise { + async runCommand(passedParams: string[]): Promise { return this.configsUsecases .getGroupConfigs(passedParams[0]) .then(console.table); diff --git a/cortex-js/src/infrastructure/commanders/configs/configs-list.command.ts b/cortex-js/src/infrastructure/commanders/configs/configs-list.command.ts index 3980412a2..500ef0d35 100644 --- a/cortex-js/src/infrastructure/commanders/configs/configs-list.command.ts +++ b/cortex-js/src/infrastructure/commanders/configs/configs-list.command.ts @@ -1,22 +1,25 @@ -import { CommandRunner, SubCommand } from 'nest-commander'; +import { SubCommand } from 'nest-commander'; import { SetCommandContext } from '../decorators/CommandContext'; import { ContextService } from '@/infrastructure/services/context/context.service'; import { ConfigsUsecases } from '@/usecases/configs/configs.usecase'; +import { BaseCommand } from '../base.command'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; @SubCommand({ name: 'list', description: 'Get all cortex configurations', }) @SetCommandContext() -export class ConfigsListCommand extends CommandRunner { +export class ConfigsListCommand extends BaseCommand { constructor( private readonly configsUsecases: ConfigsUsecases, readonly contextService: ContextService, + readonly cortexUsecases: CortexUsecases, ) { - super(); + super(cortexUsecases); } - async run(): Promise { + async runCommand(): Promise { return this.configsUsecases.getConfigs().then(console.table); } } diff --git a/cortex-js/src/infrastructure/commanders/configs/configs-set.command.ts b/cortex-js/src/infrastructure/commanders/configs/configs-set.command.ts index 1e5d46327..4c12785cc 100644 --- a/cortex-js/src/infrastructure/commanders/configs/configs-set.command.ts +++ b/cortex-js/src/infrastructure/commanders/configs/configs-set.command.ts @@ -2,6 +2,8 @@ import { CommandRunner, SubCommand, Option } from 'nest-commander'; import { SetCommandContext } from '../decorators/CommandContext'; import { ContextService } from '@/infrastructure/services/context/context.service'; import { ConfigsUsecases } from '@/usecases/configs/configs.usecase'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; +import { BaseCommand } from '../base.command'; interface ConfigsSetOption { key: string; @@ -14,15 +16,19 @@ interface ConfigsSetOption { description: 'Set a cortex configuration', }) @SetCommandContext() -export class ConfigsSetCommand extends CommandRunner { +export class ConfigsSetCommand extends BaseCommand { constructor( private readonly configsUsecases: ConfigsUsecases, readonly contextService: ContextService, + readonly cortexUsecases: CortexUsecases, ) { - super(); + super(cortexUsecases); } - async run(passedParams: string[], options: ConfigsSetOption): Promise { + async runCommand( + passedParams: string[], + options: ConfigsSetOption, + ): Promise { return this.configsUsecases .saveConfig(options.key, options.value, options.group) .then(() => console.log('Set configuration successfully')); diff --git a/cortex-js/src/infrastructure/commanders/cortex-command.commander.ts b/cortex-js/src/infrastructure/commanders/cortex-command.commander.ts index 8b5663b30..086fbf36d 100644 --- a/cortex-js/src/infrastructure/commanders/cortex-command.commander.ts +++ b/cortex-js/src/infrastructure/commanders/cortex-command.commander.ts @@ -1,27 +1,36 @@ -import { RootCommand, CommandRunner } from 'nest-commander'; -import { ServeCommand } from './serve.command'; +import { RootCommand, CommandRunner, Option } from 'nest-commander'; import { ChatCommand } from './chat.command'; import { ModelsCommand } from './models.command'; import { RunCommand } from './shortcuts/run.command'; import { ModelPullCommand } from './models/model-pull.command'; import { PSCommand } from './ps.command'; import { KillCommand } from './kill.command'; -import pkg from '@/../package.json'; import { PresetCommand } from './presets.command'; import { TelemetryCommand } from './telemetry.command'; import { SetCommandContext } from './decorators/CommandContext'; import { EmbeddingCommand } from './embeddings.command'; import { BenchmarkCommand } from './benchmark.command'; import chalk from 'chalk'; -import { printSlogan } from '@/utils/logo'; import { ContextService } from '../services/context/context.service'; import { EnginesCommand } from './engines.command'; import { ConfigsCommand } from './configs.command'; +import { defaultCortexJsHost, defaultCortexJsPort } from '../constants/cortex'; +import { getApp } from '@/app'; +import { FileManagerService } from '../services/file-manager/file-manager.service'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; +import { ServeStopCommand } from './sub-commands/serve-stop.command'; +import ora from 'ora'; + +type ServeOptions = { + address?: string; + port?: number; + logs?: boolean; + dataFolder?: string; +}; @RootCommand({ subCommands: [ ModelsCommand, - ServeCommand, ChatCommand, RunCommand, ModelPullCommand, @@ -33,20 +42,119 @@ import { ConfigsCommand } from './configs.command'; BenchmarkCommand, EnginesCommand, ConfigsCommand, + ServeStopCommand, ], description: 'Cortex CLI', }) @SetCommandContext() export class CortexCommand extends CommandRunner { - constructor(readonly contextService: ContextService) { + constructor( + readonly contextService: ContextService, + readonly fileManagerService: FileManagerService, + readonly cortexUseCases: CortexUsecases, + ) { super(); } - async run(): Promise { - printSlogan(); - console.log('\n'); - console.log(`Cortex CLI - v${pkg.version}`); - console.log(chalk.blue(`Github: ${pkg.homepage}`)); - console.log('\n'); - this.command?.help(); + + async run(passedParams: string[], options?: ServeOptions): Promise { + const host = options?.address || defaultCortexJsHost; + const port = options?.port || defaultCortexJsPort; + const showLogs = options?.logs || false; + const dataFolderPath = options?.dataFolder; + + return this.startServer(host, port, showLogs, dataFolderPath); + } + + private async startServer( + host: string, + port: number, + attach: boolean, + dataFolderPath?: string, + ) { + const config = await this.fileManagerService.getConfig(); + try { + const startEngineSpinner = ora('Starting Cortex...'); + await this.cortexUseCases.startCortex().catch((e) => { + startEngineSpinner.fail('Failed to start Cortex'); + throw e; + }); + startEngineSpinner.succeed('Cortex started successfully'); + const isServerOnline = await this.cortexUseCases.isAPIServerOnline(); + if (isServerOnline) { + const { + apiServerHost: configApiServerHost, + apiServerPort: configApiServerPort, + } = await this.fileManagerService.getConfig(); + console.log( + chalk.blue( + `Server is already running at http://${configApiServerHost}:${configApiServerPort}. Please use 'cortex stop' to stop the server.`, + ), + ); + process.exit(0); + } + if (dataFolderPath) { + await this.fileManagerService.writeConfigFile({ + ...config, + dataFolderPath, + }); + // load config again to create the data folder + await this.fileManagerService.getConfig(dataFolderPath); + } + if (attach) { + const app = await getApp(); + await app.listen(port, host); + } else { + await this.cortexUseCases.startServerDetached(host, port); + } + console.log(chalk.blue(`Started server at http://${host}:${port}`)); + console.log( + chalk.blue(`API Playground available at http://${host}:${port}/api`), + ); + await this.fileManagerService.writeConfigFile({ + ...config, + apiServerHost: host, + apiServerPort: port, + dataFolderPath: dataFolderPath || config.dataFolderPath, + }); + } catch (e) { + console.error(e); + // revert the data folder path if it was set + await this.fileManagerService.writeConfigFile({ + ...config, + }); + console.error(`Failed to start server. Is port ${port} in use?`); + } + } + + @Option({ + flags: '-a, --address
', + description: 'Address to use', + }) + parseHost(value: string) { + return value; + } + + @Option({ + flags: '-p, --port ', + description: 'Port to serve the application', + }) + parsePort(value: string) { + return parseInt(value, 10); + } + + @Option({ + flags: '-l, --logs', + description: 'Show logs', + }) + parseLogs() { + return true; + } + + @Option({ + flags: '--dataFolder ', + description: 'Set the data folder directory', + }) + parseDataFolder(value: string) { + return value; } } diff --git a/cortex-js/src/infrastructure/commanders/embeddings.command.ts b/cortex-js/src/infrastructure/commanders/embeddings.command.ts index a04bd9839..be90b0dd4 100644 --- a/cortex-js/src/infrastructure/commanders/embeddings.command.ts +++ b/cortex-js/src/infrastructure/commanders/embeddings.command.ts @@ -10,6 +10,8 @@ import { PSCliUsecases } from './usecases/ps.cli.usecases'; import { ChatCliUsecases } from './usecases/chat.cli.usecases'; import { inspect } from 'util'; import { ModelStat } from './types/model-stat.interface'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; +import { BaseCommand } from './base.command'; interface EmbeddingCommandOptions { encoding_format?: string; @@ -26,16 +28,17 @@ interface EmbeddingCommandOptions { 'Model to use for embedding. If not provided, it will prompt to select from running models.', }, }) -export class EmbeddingCommand extends CommandRunner { +export class EmbeddingCommand extends BaseCommand { constructor( private readonly chatCliUsecases: ChatCliUsecases, private readonly modelsUsecases: ModelsUsecases, private readonly psCliUsecases: PSCliUsecases, private readonly inquirerService: InquirerService, + readonly cortexUsecases: CortexUsecases, ) { - super(); + super(cortexUsecases); } - async run( + async runCommand( passedParams: string[], options: EmbeddingCommandOptions, ): Promise { diff --git a/cortex-js/src/infrastructure/commanders/engines.command.ts b/cortex-js/src/infrastructure/commanders/engines.command.ts index 9b0744562..a046be028 100644 --- a/cortex-js/src/infrastructure/commanders/engines.command.ts +++ b/cortex-js/src/infrastructure/commanders/engines.command.ts @@ -1,5 +1,5 @@ import { invert } from 'lodash'; -import { CommandRunner, SubCommand } from 'nest-commander'; +import { Option, SubCommand } from 'nest-commander'; import { SetCommandContext } from './decorators/CommandContext'; import { ContextService } from '@/infrastructure/services/context/context.service'; import { EnginesListCommand } from './engines/engines-list.command'; @@ -7,6 +7,8 @@ import { EnginesGetCommand } from './engines/engines-get.command'; import { EnginesInitCommand } from './engines/engines-init.command'; import { ModuleRef } from '@nestjs/core'; import { EngineNamesMap } from './types/engine.interface'; +import { BaseCommand } from './base.command'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; @SubCommand({ name: 'engines', @@ -15,7 +17,7 @@ import { EngineNamesMap } from './types/engine.interface'; arguments: ' [subcommand]', }) @SetCommandContext() -export class EnginesCommand extends CommandRunner { +export class EnginesCommand extends BaseCommand { commandMap: { [key: string]: any } = { list: EnginesListCommand, get: EnginesGetCommand, @@ -25,10 +27,11 @@ export class EnginesCommand extends CommandRunner { constructor( readonly contextService: ContextService, private readonly moduleRef: ModuleRef, + readonly cortexUsecases: CortexUsecases, ) { - super(); + super(cortexUsecases); } - async run(passedParam: string[]): Promise { + async runCommand(passedParam: string[], options: { vulkan: boolean }) { const [parameter, command] = passedParam; if (command !== 'list' && !parameter) { console.error('Engine name is required.'); @@ -42,15 +45,28 @@ export class EnginesCommand extends CommandRunner { return; } const engine = invert(EngineNamesMap)[parameter] || parameter; - await this.runCommand(commandClass, [engine]); + await this.runEngineCommand(commandClass, [engine], options); } - private async runCommand(commandClass: any, params: string[] = []) { + private async runEngineCommand( + commandClass: any, + params: string[] = [], + options?: { vulkan: boolean }, + ) { const commandInstance = this.moduleRef.get(commandClass, { strict: false }); if (commandInstance) { - await commandInstance.run(params); + await commandInstance.run(params, options); } else { console.error('Command not found.'); } } + + @Option({ + flags: '-vk, --vulkan', + description: 'Install Vulkan engine', + defaultValue: false, + }) + parseVulkan() { + return true; + } } diff --git a/cortex-js/src/infrastructure/commanders/engines/engines-get.command.ts b/cortex-js/src/infrastructure/commanders/engines/engines-get.command.ts index f002a84a4..ccb9e43e0 100644 --- a/cortex-js/src/infrastructure/commanders/engines/engines-get.command.ts +++ b/cortex-js/src/infrastructure/commanders/engines/engines-get.command.ts @@ -1,8 +1,10 @@ -import { CommandRunner, SubCommand } from 'nest-commander'; +import { SubCommand } from 'nest-commander'; import { SetCommandContext } from '../decorators/CommandContext'; import { ContextService } from '@/infrastructure/services/context/context.service'; import { EnginesUsecases } from '@/usecases/engines/engines.usecase'; import { EngineNamesMap, Engines } from '../types/engine.interface'; +import { BaseCommand } from '../base.command'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; @SubCommand({ name: ' get', @@ -12,15 +14,16 @@ import { EngineNamesMap, Engines } from '../types/engine.interface'; }, }) @SetCommandContext() -export class EnginesGetCommand extends CommandRunner { +export class EnginesGetCommand extends BaseCommand { constructor( private readonly engineUsecases: EnginesUsecases, readonly contextService: ContextService, + readonly cortexUsecases: CortexUsecases, ) { - super(); + super(cortexUsecases); } - async run(passedParams: string[]): Promise { + async runCommand(passedParams: string[]): Promise { return this.engineUsecases.getEngine(passedParams[0]).then((engine) => { if (!engine) { console.error('Engine not found.'); diff --git a/cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts b/cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts index 65b758075..4e7c6e8f7 100644 --- a/cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts +++ b/cortex-js/src/infrastructure/commanders/engines/engines-init.command.ts @@ -1,10 +1,11 @@ -import { CommandRunner, Option, SubCommand } from 'nest-commander'; +import { Option, SubCommand } from 'nest-commander'; import { SetCommandContext } from '../decorators/CommandContext'; import { ContextService } from '@/infrastructure/services/context/context.service'; import { Engines } from '../types/engine.interface'; import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; import { FileManagerService } from '@/infrastructure/services/file-manager/file-manager.service'; import { EnginesUsecases } from '@/usecases/engines/engines.usecase'; +import { BaseCommand } from '../base.command'; @SubCommand({ name: ' init', @@ -14,17 +15,17 @@ import { EnginesUsecases } from '@/usecases/engines/engines.usecase'; }, }) @SetCommandContext() -export class EnginesInitCommand extends CommandRunner { +export class EnginesInitCommand extends BaseCommand { constructor( private readonly engineUsecases: EnginesUsecases, private readonly cortexUsecases: CortexUsecases, private readonly fileManagerService: FileManagerService, readonly contextService: ContextService, ) { - super(); + super(cortexUsecases); } - async run( + async runCommand( passedParams: string[], options: { vulkan: boolean }, ): Promise { diff --git a/cortex-js/src/infrastructure/commanders/engines/engines-list.command.ts b/cortex-js/src/infrastructure/commanders/engines/engines-list.command.ts index e19d4aedd..4de21c0eb 100644 --- a/cortex-js/src/infrastructure/commanders/engines/engines-list.command.ts +++ b/cortex-js/src/infrastructure/commanders/engines/engines-list.command.ts @@ -1,23 +1,26 @@ -import { CommandRunner, SubCommand } from 'nest-commander'; +import { SubCommand } from 'nest-commander'; import { SetCommandContext } from '../decorators/CommandContext'; import { ContextService } from '@/infrastructure/services/context/context.service'; import { EnginesUsecases } from '@/usecases/engines/engines.usecase'; import { EngineNamesMap } from '../types/engine.interface'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; +import { BaseCommand } from '../base.command'; @SubCommand({ name: 'list', description: 'Get all cortex engines', }) @SetCommandContext() -export class EnginesListCommand extends CommandRunner { +export class EnginesListCommand extends BaseCommand { constructor( private readonly enginesUsecases: EnginesUsecases, readonly contextService: ContextService, + readonly cortexUseCases: CortexUsecases, ) { - super(); + super(cortexUseCases); } - async run(): Promise { + async runCommand(): Promise { return this.enginesUsecases.getEngines().then((engines) => { const enginesTable = engines.map((engine) => ({ ...engine, diff --git a/cortex-js/src/infrastructure/commanders/kill.command.ts b/cortex-js/src/infrastructure/commanders/kill.command.ts index e6f40742a..922b74171 100644 --- a/cortex-js/src/infrastructure/commanders/kill.command.ts +++ b/cortex-js/src/infrastructure/commanders/kill.command.ts @@ -1,21 +1,22 @@ -import { CommandRunner, SubCommand } from 'nest-commander'; +import { SubCommand } from 'nest-commander'; import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; import { SetCommandContext } from './decorators/CommandContext'; import { ContextService } from '../services/context/context.service'; +import { BaseCommand } from './base.command'; @SubCommand({ name: 'kill', description: 'Kill running cortex processes', }) @SetCommandContext() -export class KillCommand extends CommandRunner { +export class KillCommand extends BaseCommand { constructor( private readonly cortexUsecases: CortexUsecases, readonly contextService: ContextService, ) { - super(); + super(cortexUsecases); } - async run(): Promise { + async runCommand(): Promise { return this.cortexUsecases .stopCortex() .then(this.cortexUsecases.stopServe) diff --git a/cortex-js/src/infrastructure/commanders/models.command.ts b/cortex-js/src/infrastructure/commanders/models.command.ts index d59f387a1..71aa66171 100644 --- a/cortex-js/src/infrastructure/commanders/models.command.ts +++ b/cortex-js/src/infrastructure/commanders/models.command.ts @@ -7,6 +7,7 @@ import { ModelPullCommand } from './models/model-pull.command'; import { ModelRemoveCommand } from './models/model-remove.command'; import { ModelUpdateCommand } from './models/model-update.command'; import { RunCommand } from './shortcuts/run.command'; +import { BaseCommand } from './base.command'; @SubCommand({ name: 'models', @@ -22,8 +23,8 @@ import { RunCommand } from './shortcuts/run.command'; ], description: 'Subcommands for managing models', }) -export class ModelsCommand extends CommandRunner { - async run(): Promise { +export class ModelsCommand extends BaseCommand { + async runCommand(): Promise { this.command?.help(); } } diff --git a/cortex-js/src/infrastructure/commanders/models/model-get.command.ts b/cortex-js/src/infrastructure/commanders/models/model-get.command.ts index 16c26ae66..c519f71e7 100644 --- a/cortex-js/src/infrastructure/commanders/models/model-get.command.ts +++ b/cortex-js/src/infrastructure/commanders/models/model-get.command.ts @@ -1,8 +1,10 @@ -import { CommandRunner, SubCommand } from 'nest-commander'; +import { SubCommand } from 'nest-commander'; import { ModelsCliUsecases } from '@commanders/usecases/models.cli.usecases'; import { exit } from 'node:process'; import { SetCommandContext } from '../decorators/CommandContext'; import { ContextService } from '@/infrastructure/services/context/context.service'; +import { BaseCommand } from '../base.command'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; @SubCommand({ name: 'get', @@ -13,15 +15,16 @@ import { ContextService } from '@/infrastructure/services/context/context.servic }, }) @SetCommandContext() -export class ModelGetCommand extends CommandRunner { +export class ModelGetCommand extends BaseCommand { constructor( private readonly modelsCliUsecases: ModelsCliUsecases, readonly contextService: ContextService, + readonly cortexUseCases: CortexUsecases, ) { - super(); + super(cortexUseCases); } - async run(passedParams: string[]): Promise { + async runCommand(passedParams: string[]): Promise { if (passedParams.length === 0) { console.error('Model ID is required'); exit(1); diff --git a/cortex-js/src/infrastructure/commanders/models/model-list.command.ts b/cortex-js/src/infrastructure/commanders/models/model-list.command.ts index 369ed496d..dac316f57 100644 --- a/cortex-js/src/infrastructure/commanders/models/model-list.command.ts +++ b/cortex-js/src/infrastructure/commanders/models/model-list.command.ts @@ -1,22 +1,28 @@ -import { CommandRunner, SubCommand, Option } from 'nest-commander'; +import { SubCommand, Option } from 'nest-commander'; import { ModelsCliUsecases } from '../usecases/models.cli.usecases'; import { SetCommandContext } from '../decorators/CommandContext'; import { ContextService } from '@/infrastructure/services/context/context.service'; +import { BaseCommand } from '../base.command'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; interface ModelListOptions { format: 'table' | 'json'; } @SubCommand({ name: 'list', description: 'List all models locally.' }) @SetCommandContext() -export class ModelListCommand extends CommandRunner { +export class ModelListCommand extends BaseCommand { constructor( private readonly modelsCliUsecases: ModelsCliUsecases, readonly contextService: ContextService, + readonly cortexUseCases: CortexUsecases, ) { - super(); + super(cortexUseCases); } - async run(passedParams: string[], option: ModelListOptions): Promise { + async runCommand( + passedParams: string[], + option: ModelListOptions, + ): Promise { const models = await this.modelsCliUsecases.listAllModels(); option.format === 'table' ? console.table( diff --git a/cortex-js/src/infrastructure/commanders/models/model-pull.command.ts b/cortex-js/src/infrastructure/commanders/models/model-pull.command.ts index ad9021b08..c9c414e4c 100644 --- a/cortex-js/src/infrastructure/commanders/models/model-pull.command.ts +++ b/cortex-js/src/infrastructure/commanders/models/model-pull.command.ts @@ -15,6 +15,8 @@ import { FileManagerService } from '@/infrastructure/services/file-manager/file- import { checkModelCompatibility } from '@/utils/model-check'; import { Engines } from '../types/engine.interface'; import { EnginesUsecases } from '@/usecases/engines/engines.usecase'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; +import { BaseCommand } from '../base.command'; @SubCommand({ name: 'pull', @@ -25,18 +27,19 @@ import { EnginesUsecases } from '@/usecases/engines/engines.usecase'; 'Download a model from a registry. Working with HuggingFace repositories. For available models, please visit https://huggingface.co/cortexso', }) @SetCommandContext() -export class ModelPullCommand extends CommandRunner { +export class ModelPullCommand extends BaseCommand { constructor( private readonly modelsCliUsecases: ModelsCliUsecases, private readonly engineUsecases: EnginesUsecases, private readonly fileService: FileManagerService, readonly contextService: ContextService, private readonly telemetryUsecases: TelemetryUsecases, + readonly cortexUsecases: CortexUsecases, ) { - super(); + super(cortexUsecases); } - async run(passedParams: string[]) { + async runCommand(passedParams: string[]) { if (passedParams.length < 1) { console.error('Model Id is required'); exit(1); diff --git a/cortex-js/src/infrastructure/commanders/models/model-remove.command.ts b/cortex-js/src/infrastructure/commanders/models/model-remove.command.ts index c04e36128..18b21144a 100644 --- a/cortex-js/src/infrastructure/commanders/models/model-remove.command.ts +++ b/cortex-js/src/infrastructure/commanders/models/model-remove.command.ts @@ -1,8 +1,10 @@ -import { CommandRunner, SubCommand } from 'nest-commander'; +import { SubCommand } from 'nest-commander'; import { ModelsCliUsecases } from '@commanders/usecases/models.cli.usecases'; import { exit } from 'node:process'; import { SetCommandContext } from '../decorators/CommandContext'; import { ContextService } from '@/infrastructure/services/context/context.service'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; +import { BaseCommand } from '../base.command'; @SubCommand({ name: 'remove', @@ -13,15 +15,16 @@ import { ContextService } from '@/infrastructure/services/context/context.servic }, }) @SetCommandContext() -export class ModelRemoveCommand extends CommandRunner { +export class ModelRemoveCommand extends BaseCommand { constructor( private readonly modelsCliUsecases: ModelsCliUsecases, readonly contextService: ContextService, + readonly cortexUseCases: CortexUsecases, ) { - super(); + super(cortexUseCases); } - async run(passedParams: string[]): Promise { + async runCommand(passedParams: string[]): Promise { if (passedParams.length === 0) { console.error('Model ID is required'); exit(1); diff --git a/cortex-js/src/infrastructure/commanders/models/model-start.command.ts b/cortex-js/src/infrastructure/commanders/models/model-start.command.ts index e3aae3e0b..c5f3c8a57 100644 --- a/cortex-js/src/infrastructure/commanders/models/model-start.command.ts +++ b/cortex-js/src/infrastructure/commanders/models/model-start.command.ts @@ -1,9 +1,4 @@ -import { - CommandRunner, - SubCommand, - Option, - InquirerService, -} from 'nest-commander'; +import { SubCommand, Option, InquirerService } from 'nest-commander'; import ora from 'ora'; import { exit } from 'node:process'; import { ModelsCliUsecases } from '@commanders/usecases/models.cli.usecases'; @@ -16,6 +11,7 @@ import { join } from 'node:path'; import { Engines } from '../types/engine.interface'; import { checkModelCompatibility } from '@/utils/model-check'; import { EnginesUsecases } from '@/usecases/engines/engines.usecase'; +import { BaseCommand } from '../base.command'; type ModelStartOptions = { attach: boolean; @@ -31,7 +27,7 @@ type ModelStartOptions = { }, }) @SetCommandContext() -export class ModelStartCommand extends CommandRunner { +export class ModelStartCommand extends BaseCommand { constructor( private readonly inquirerService: InquirerService, private readonly cortexUsecases: CortexUsecases, @@ -40,10 +36,13 @@ export class ModelStartCommand extends CommandRunner { private readonly fileService: FileManagerService, readonly contextService: ContextService, ) { - super(); + super(cortexUsecases); } - async run(passedParams: string[], options: ModelStartOptions): Promise { + async runCommand( + passedParams: string[], + options: ModelStartOptions, + ): Promise { let modelId = passedParams[0]; const checkingSpinner = ora('Checking model...').start(); if (!modelId) { diff --git a/cortex-js/src/infrastructure/commanders/models/model-stop.command.ts b/cortex-js/src/infrastructure/commanders/models/model-stop.command.ts index af9b7d011..afdd6a2bc 100644 --- a/cortex-js/src/infrastructure/commanders/models/model-stop.command.ts +++ b/cortex-js/src/infrastructure/commanders/models/model-stop.command.ts @@ -1,8 +1,10 @@ -import { CommandRunner, SubCommand } from 'nest-commander'; +import { SubCommand } from 'nest-commander'; import { exit } from 'node:process'; import { ModelsCliUsecases } from '../usecases/models.cli.usecases'; import { SetCommandContext } from '../decorators/CommandContext'; import { ContextService } from '@/infrastructure/services/context/context.service'; +import { BaseCommand } from '../base.command'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; @SubCommand({ name: 'stop', @@ -13,15 +15,16 @@ import { ContextService } from '@/infrastructure/services/context/context.servic }, }) @SetCommandContext() -export class ModelStopCommand extends CommandRunner { +export class ModelStopCommand extends BaseCommand { constructor( private readonly modelsCliUsecases: ModelsCliUsecases, readonly contextService: ContextService, + readonly cortexUseCases: CortexUsecases, ) { - super(); + super(cortexUseCases); } - async run(passedParams: string[]): Promise { + async runCommand(passedParams: string[]): Promise { if (passedParams.length === 0) { console.error('Model ID is required'); exit(1); diff --git a/cortex-js/src/infrastructure/commanders/models/model-update.command.ts b/cortex-js/src/infrastructure/commanders/models/model-update.command.ts index ebd06bfe8..c87e8f238 100644 --- a/cortex-js/src/infrastructure/commanders/models/model-update.command.ts +++ b/cortex-js/src/infrastructure/commanders/models/model-update.command.ts @@ -1,9 +1,11 @@ -import { CommandRunner, SubCommand, Option } from 'nest-commander'; +import { SubCommand, Option } from 'nest-commander'; import { ModelsCliUsecases } from '@commanders/usecases/models.cli.usecases'; import { exit } from 'node:process'; import { SetCommandContext } from '../decorators/CommandContext'; import { UpdateModelDto } from '@/infrastructure/dtos/models/update-model.dto'; import { ContextService } from '@/infrastructure/services/context/context.service'; +import { BaseCommand } from '../base.command'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; type UpdateOptions = { model?: string; @@ -19,15 +21,19 @@ type UpdateOptions = { }, }) @SetCommandContext() -export class ModelUpdateCommand extends CommandRunner { +export class ModelUpdateCommand extends BaseCommand { constructor( private readonly modelsCliUsecases: ModelsCliUsecases, readonly contextService: ContextService, + readonly cortexUseCases: CortexUsecases, ) { - super(); + super(cortexUseCases); } - async run(passedParams: string[], option: UpdateOptions): Promise { + async runCommand( + passedParams: string[], + option: UpdateOptions, + ): Promise { const modelId = option.model; if (!modelId) { console.error('Model Id is required'); diff --git a/cortex-js/src/infrastructure/commanders/presets.command.ts b/cortex-js/src/infrastructure/commanders/presets.command.ts index 2f60c3c58..fca4e0489 100644 --- a/cortex-js/src/infrastructure/commanders/presets.command.ts +++ b/cortex-js/src/infrastructure/commanders/presets.command.ts @@ -1,23 +1,26 @@ import { FileManagerService } from '@/infrastructure/services/file-manager/file-manager.service'; import { readdirSync } from 'fs'; -import { CommandRunner, SubCommand } from 'nest-commander'; +import { SubCommand } from 'nest-commander'; import { join } from 'path'; import { SetCommandContext } from './decorators/CommandContext'; import { ContextService } from '../services/context/context.service'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; +import { BaseCommand } from './base.command'; @SubCommand({ name: 'presets', description: 'Show all available presets', }) @SetCommandContext() -export class PresetCommand extends CommandRunner { +export class PresetCommand extends BaseCommand { constructor( private readonly fileService: FileManagerService, readonly contextService: ContextService, + readonly cortexUsecases: CortexUsecases, ) { - super(); + super(cortexUsecases); } - async run(): Promise { + async runCommand(): Promise { return console.table( readdirSync( join(await this.fileService.getDataFolderPath(), `presets`), diff --git a/cortex-js/src/infrastructure/commanders/ps.command.ts b/cortex-js/src/infrastructure/commanders/ps.command.ts index a99ef62e6..bc6ab4a00 100644 --- a/cortex-js/src/infrastructure/commanders/ps.command.ts +++ b/cortex-js/src/infrastructure/commanders/ps.command.ts @@ -1,26 +1,29 @@ import ora from 'ora'; import systeminformation from 'systeminformation'; -import { CommandRunner, SubCommand } from 'nest-commander'; +import { SubCommand } from 'nest-commander'; import { PSCliUsecases } from './usecases/ps.cli.usecases'; import { SetCommandContext } from './decorators/CommandContext'; import { ContextService } from '../services/context/context.service'; import { ModelStat } from './types/model-stat.interface'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; +import { BaseCommand } from './base.command'; @SubCommand({ name: 'ps', description: 'Show running models and their status', }) @SetCommandContext() -export class PSCommand extends CommandRunner { +export class PSCommand extends BaseCommand { constructor( private readonly usecases: PSCliUsecases, - readonly contextService: ContextService, + private readonly contextService: ContextService, + private readonly cortexUsecases: CortexUsecases, ) { - super(); + super(cortexUsecases); } - async run(): Promise { + async runCommand(): Promise { const runningSpinner = ora('Running PS command...').start(); - let checkingSpinner: ora.Ora + let checkingSpinner: ora.Ora; return this.usecases .getModels() .then((models: ModelStat[]) => { @@ -29,13 +32,17 @@ export class PSCommand extends CommandRunner { }) .then(() => { checkingSpinner = ora('Checking API server...').start(); - return this.usecases.isAPIServerOnline(); + return this.cortexUsecases.isAPIServerOnline(); }) .then((isOnline) => { - checkingSpinner.succeed(isOnline ? 'API server is online' : 'API server is offline'); + checkingSpinner.succeed( + isOnline ? 'API server is online' : 'API server is offline', + ); }) .then(async () => { - const cpuUsage = (await systeminformation.currentLoad()).currentLoad.toFixed(2); + const cpuUsage = ( + await systeminformation.currentLoad() + ).currentLoad.toFixed(2); const gpusLoad = []; const gpus = await systeminformation.graphics(); for (const gpu of gpus.controllers) { @@ -45,20 +52,28 @@ export class PSCommand extends CommandRunner { }); } const memoryData = await systeminformation.mem(); - const memoryUsage = (memoryData.active / memoryData.total * 100).toFixed(2) + const memoryUsage = ( + (memoryData.active / memoryData.total) * + 100 + ).toFixed(2); const consumedTable = { 'CPU Usage': `${cpuUsage}%`, 'Memory Usage': `${memoryUsage}%`, } as { - 'CPU Usage': string, - 'Memory Usage': string, - 'VRAM'?: string, - } - - if(gpusLoad.length > 0 && gpusLoad.filter(gpu => gpu.totalVram > 0).length > 0) { - consumedTable['VRAM'] = gpusLoad.map(gpu => `${gpu.totalVram} MB`).join(', '); + 'CPU Usage': string; + 'Memory Usage': string; + VRAM?: string; + }; + + if ( + gpusLoad.length > 0 && + gpusLoad.filter((gpu) => gpu.totalVram > 0).length > 0 + ) { + consumedTable['VRAM'] = gpusLoad + .map((gpu) => `${gpu.totalVram} MB`) + .join(', '); } console.table([consumedTable]); - }) + }); } -} \ No newline at end of file +} diff --git a/cortex-js/src/infrastructure/commanders/serve.command.ts b/cortex-js/src/infrastructure/commanders/serve.command.ts deleted file mode 100644 index 4eaa7bc45..000000000 --- a/cortex-js/src/infrastructure/commanders/serve.command.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { - defaultCortexJsHost, - defaultCortexJsPort, -} from '@/infrastructure/constants/cortex'; -import { CommandRunner, SubCommand, Option } from 'nest-commander'; -import { SetCommandContext } from './decorators/CommandContext'; -import { ServeStopCommand } from './sub-commands/serve-stop.command'; -import { ContextService } from '../services/context/context.service'; -import { getApp } from '@/app'; -import chalk from 'chalk'; - -type ServeOptions = { - address?: string; - port?: number; - detach: boolean; -}; - -@SubCommand({ - name: 'serve', - description: 'Providing API endpoint for Cortex backend', - subCommands: [ServeStopCommand], -}) -@SetCommandContext() -export class ServeCommand extends CommandRunner { - constructor(readonly contextService: ContextService) { - super(); - } - - async run(passedParams: string[], options?: ServeOptions): Promise { - const host = options?.address || defaultCortexJsHost; - const port = options?.port || defaultCortexJsPort; - - return this.startServer(host, port); - } - - private async startServer(host: string, port: number) { - const app = await getApp(); - - try { - await app.listen(port, host); - console.log(chalk.blue(`Started server at http://${host}:${port}`)); - console.log( - chalk.blue(`API Playground available at http://${host}:${port}/api`), - ); - } catch { - console.error(`Failed to start server. Is port ${port} in use?`); - } - } - - @Option({ - flags: '-a, --address
', - description: 'Address to use', - }) - parseHost(value: string) { - return value; - } - - @Option({ - flags: '-p, --port ', - description: 'Port to serve the application', - }) - parsePort(value: string) { - return parseInt(value, 10); - } -} diff --git a/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts b/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts index 2142e6004..d356f054b 100644 --- a/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts +++ b/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts @@ -1,10 +1,5 @@ import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; -import { - CommandRunner, - SubCommand, - Option, - InquirerService, -} from 'nest-commander'; +import { SubCommand, Option, InquirerService } from 'nest-commander'; import { exit } from 'node:process'; import ora from 'ora'; import { ChatCliUsecases } from '@commanders/usecases/chat.cli.usecases'; @@ -16,10 +11,12 @@ import { FileManagerService } from '@/infrastructure/services/file-manager/file- import { Engines } from '../types/engine.interface'; import { checkModelCompatibility } from '@/utils/model-check'; import { EnginesUsecases } from '@/usecases/engines/engines.usecase'; +import { BaseCommand } from '../base.command'; type RunOptions = { threadId?: string; preset?: string; + chat?: boolean; }; @SubCommand({ @@ -31,7 +28,7 @@ type RunOptions = { }, description: 'Shortcut to start a model and chat', }) -export class RunCommand extends CommandRunner { +export class RunCommand extends BaseCommand { constructor( private readonly modelsCliUsecases: ModelsCliUsecases, private readonly cortexUsecases: CortexUsecases, @@ -40,10 +37,10 @@ export class RunCommand extends CommandRunner { private readonly fileService: FileManagerService, private readonly initUsecases: EnginesUsecases, ) { - super(); + super(cortexUsecases); } - async run(passedParams: string[], options: RunOptions): Promise { + async runCommand(passedParams: string[], options: RunOptions): Promise { let modelId = passedParams[0]; const checkingSpinner = ora('Checking model...').start(); if (!modelId) { @@ -93,7 +90,12 @@ export class RunCommand extends CommandRunner { return this.cortexUsecases .startCortex() .then(() => this.modelsCliUsecases.startModel(modelId, options.preset)) - .then(() => this.chatCliUsecases.chat(modelId, options.threadId)); + .then(() => { + if (options.chat) { + return this.chatCliUsecases.chat(modelId, options.threadId); + } + return; + }); } @Option({ @@ -112,6 +114,14 @@ export class RunCommand extends CommandRunner { return value; } + @Option({ + flags: '-c, --chat', + description: 'Start a chat session after starting the model', + }) + parseChat() { + return true; + } + modelInquiry = async () => { const models = await this.modelsCliUsecases.listAllModels(); if (!models.length) throw 'No models found'; diff --git a/cortex-js/src/infrastructure/commanders/sub-commands/serve-stop.command.ts b/cortex-js/src/infrastructure/commanders/sub-commands/serve-stop.command.ts index 3e8fae60c..1e1a7f4d9 100644 --- a/cortex-js/src/infrastructure/commanders/sub-commands/serve-stop.command.ts +++ b/cortex-js/src/infrastructure/commanders/sub-commands/serve-stop.command.ts @@ -1,18 +1,18 @@ -import { CORTEX_JS_STOP_API_SERVER_URL } from '@/infrastructure/constants/cortex'; -import { CommandRunner, SubCommand } from 'nest-commander'; +import { SubCommand } from 'nest-commander'; +import { BaseCommand } from '../base.command'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; @SubCommand({ name: 'stop', description: 'Stop the API server', }) -export class ServeStopCommand extends CommandRunner { - async run(): Promise { - return this.stopServer().then(() => console.log('API server stopped')); +export class ServeStopCommand extends BaseCommand { + constructor(private readonly cortexUsecases: CortexUsecases) { + super(cortexUsecases); } - - private async stopServer() { - return fetch(CORTEX_JS_STOP_API_SERVER_URL(), { - method: 'DELETE', - }).catch(() => {}); + async runCommand(): Promise { + return this.cortexUsecases + .stopApiServer() + .then(() => console.log('API server stopped')); } } diff --git a/cortex-js/src/infrastructure/commanders/telemetry.command.ts b/cortex-js/src/infrastructure/commanders/telemetry.command.ts index a32ce39a8..b4f3b3ec5 100644 --- a/cortex-js/src/infrastructure/commanders/telemetry.command.ts +++ b/cortex-js/src/infrastructure/commanders/telemetry.command.ts @@ -1,19 +1,27 @@ -import { CommandRunner, SubCommand, Option } from 'nest-commander'; +import { SubCommand, Option } from 'nest-commander'; import { TelemetryUsecases } from '@/usecases/telemetry/telemetry.usecases'; import { TelemetryOptions } from './types/telemetry-options.interface'; import { SetCommandContext } from './decorators/CommandContext'; +import { BaseCommand } from './base.command'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; @SubCommand({ name: 'telemetry', description: 'Get telemetry logs', }) @SetCommandContext() -export class TelemetryCommand extends CommandRunner { - constructor(private readonly telemetryUseCase: TelemetryUsecases) { - super(); +export class TelemetryCommand extends BaseCommand { + constructor( + private readonly telemetryUseCase: TelemetryUsecases, + readonly cortexUseCases: CortexUsecases, + ) { + super(cortexUseCases); } - async run(_input: string[], options?: TelemetryOptions): Promise { + async runCommand( + _input: string[], + options?: TelemetryOptions, + ): Promise { if (options?.type === 'crash') { try { await this.telemetryUseCase.readCrashReports((telemetryEvent) => { diff --git a/cortex-js/src/infrastructure/commanders/test/helpers.command.spec.ts b/cortex-js/src/infrastructure/commanders/test/helpers.command.spec.ts index b07b070e9..e1dc1aa04 100644 --- a/cortex-js/src/infrastructure/commanders/test/helpers.command.spec.ts +++ b/cortex-js/src/infrastructure/commanders/test/helpers.command.spec.ts @@ -5,9 +5,9 @@ import { CommandModule } from '@/command.module'; import { LogService } from '@/infrastructure/commanders/test/log.service'; import { FileManagerService } from '@/infrastructure/services/file-manager/file-manager.service'; -import axios from 'axios'; import { join } from 'path'; import { rmSync } from 'fs'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; let commandInstance: TestingModule, exitSpy: Stub, @@ -36,8 +36,13 @@ beforeAll( await fileService.writeConfigFile({ dataFolderPath: join(__dirname, 'test_data'), cortexCppHost: 'localhost', - cortexCppPort: 3929 + cortexCppPort: 3929, }); + const cortexUseCases = + await commandInstance.resolve(CortexUsecases); + jest + .spyOn(cortexUseCases, 'isAPIServerOnline') + .mockImplementation(() => Promise.resolve(true)); res(); }), ); @@ -97,7 +102,9 @@ describe('Helper commands', () => { await CommandTestFactory.run(commandInstance, ['kill']); await CommandTestFactory.run(commandInstance, ['ps']); - expect(logMock.firstCall?.args[0]).toEqual("Cortex processes stopped successfully!"); + expect(logMock.firstCall?.args[0]).toEqual( + 'Cortex processes stopped successfully!', + ); expect(tableMock.firstCall?.args[0]).toBeInstanceOf(Array); expect(tableMock.firstCall?.args[0].length).toEqual(0); }, diff --git a/cortex-js/src/infrastructure/commanders/test/models.command.spec.ts b/cortex-js/src/infrastructure/commanders/test/models.command.spec.ts index f1a6f6879..79a655163 100644 --- a/cortex-js/src/infrastructure/commanders/test/models.command.spec.ts +++ b/cortex-js/src/infrastructure/commanders/test/models.command.spec.ts @@ -5,6 +5,7 @@ import { CommandModule } from '@/command.module'; import { join } from 'path'; import { rmSync } from 'fs'; import { FileManagerService } from '@/infrastructure/services/file-manager/file-manager.service'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; let commandInstance: TestingModule; @@ -26,7 +27,11 @@ beforeAll( cortexCppHost: 'localhost', cortexCppPort: 3929, }); - + const cortexUseCases = + await commandInstance.resolve(CortexUsecases); + jest + .spyOn(cortexUseCases, 'isAPIServerOnline') + .mockImplementation(() => Promise.resolve(true)); res(); }), ); @@ -39,6 +44,7 @@ afterAll( recursive: true, force: true, }); + res(); }), ); @@ -58,11 +64,10 @@ describe('Action with models', () => { test('Empty model list', async () => { const logMock = stubMethod(console, 'table'); - await CommandTestFactory.run(commandInstance, ['models', 'list']); expect(logMock.firstCall?.args[0]).toBeInstanceOf(Array); expect(logMock.firstCall?.args[0].length).toBe(0); - }); + }, 20000); // // test( diff --git a/cortex-js/src/infrastructure/commanders/usecases/ps.cli.usecases.ts b/cortex-js/src/infrastructure/commanders/usecases/ps.cli.usecases.ts index 20d68fc01..3ed577cae 100644 --- a/cortex-js/src/infrastructure/commanders/usecases/ps.cli.usecases.ts +++ b/cortex-js/src/infrastructure/commanders/usecases/ps.cli.usecases.ts @@ -1,16 +1,12 @@ import { HttpStatus, Injectable } from '@nestjs/common'; import ora from 'ora'; -import { - CORTEX_CPP_MODELS_URL, - CORTEX_JS_HEALTH_URL, - defaultCortexJsHost, - defaultCortexJsPort, -} from '@/infrastructure/constants/cortex'; +import { CORTEX_CPP_MODELS_URL } from '@/infrastructure/constants/cortex'; import { HttpService } from '@nestjs/axios'; import { firstValueFrom } from 'rxjs'; import { ModelStat } from '@commanders/types/model-stat.interface'; import { FileManagerService } from '@/infrastructure/services/file-manager/file-manager.service'; import { Engines } from '../types/engine.interface'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; interface ModelStatResponse { object: string; @@ -21,6 +17,7 @@ export class PSCliUsecases { constructor( private readonly httpService: HttpService, private readonly fileService: FileManagerService, + private readonly cortexUsecases: CortexUsecases, ) {} /** * Get models running in the Cortex C++ server @@ -68,23 +65,6 @@ export class PSCliUsecases { }); } - /** - * Check if the Cortex API server is online - * @param host Cortex host address - * @param port Cortex port address - * @returns - */ - async isAPIServerOnline( - host: string = defaultCortexJsHost, - port: number = defaultCortexJsPort, - ): Promise { - return firstValueFrom( - this.httpService.get(CORTEX_JS_HEALTH_URL(host, port)), - ) - .then((res) => res.status === HttpStatus.OK) - .catch(() => false); - } - private formatDuration(milliseconds: number): string { const days = Math.floor(milliseconds / (1000 * 60 * 60 * 24)); const hours = Math.floor( diff --git a/cortex-js/src/infrastructure/repositories/telemetry/telemetry.repository.ts b/cortex-js/src/infrastructure/repositories/telemetry/telemetry.repository.ts index c00fe2b4f..52a514d6c 100644 --- a/cortex-js/src/infrastructure/repositories/telemetry/telemetry.repository.ts +++ b/cortex-js/src/infrastructure/repositories/telemetry/telemetry.repository.ts @@ -226,9 +226,7 @@ export class TelemetryRepositoryImpl implements TelemetryRepository { logRecords: [ { traceId: cypto.randomBytes(16).toString('hex'), - timeUnixNano: ( - BigInt(Date.now()) * BigInt(1000000) - ).toString(), + timeUnixNano: (BigInt(Date.now()) * BigInt(1000000)).toString(), body: { stringValue: body }, severityText: severity, attributes: telemetryLogAttributes, diff --git a/cortex-js/src/infrastructure/services/file-manager/file-manager.service.ts b/cortex-js/src/infrastructure/services/file-manager/file-manager.service.ts index f4fa2537c..9295ccee1 100644 --- a/cortex-js/src/infrastructure/services/file-manager/file-manager.service.ts +++ b/cortex-js/src/infrastructure/services/file-manager/file-manager.service.ts @@ -38,13 +38,13 @@ export class FileManagerService { * Get cortex configs * @returns the config object */ - async getConfig(): Promise { + async getConfig(dataFolderPath?: string): Promise { const homeDir = os.homedir(); const configPath = join(homeDir, this.configFile); - - if (!existsSync(configPath)) { - const config = this.defaultConfig(); - await this.createFolderIfNotExist(config.dataFolderPath); + const config = this.defaultConfig(); + const dataFolderPathUsed = dataFolderPath || config.dataFolderPath; + if (!existsSync(configPath) || !existsSync(dataFolderPathUsed)) { + await this.createFolderIfNotExist(dataFolderPathUsed); await this.writeConfigFile(config); return config; } @@ -59,8 +59,7 @@ export class FileManagerService { } catch (error) { console.warn('Error reading config file. Using default config.'); console.warn(error); - const config = this.defaultConfig(); - await this.createFolderIfNotExist(config.dataFolderPath); + await this.createFolderIfNotExist(dataFolderPathUsed); await this.writeConfigFile(config); return config; } @@ -194,8 +193,14 @@ export class FileManagerService { throw err; } } - readLines(filePath: string, callback: (line: string) => void) { - const fileStream = createReadStream(filePath); + readLines( + filePath: string, + callback: (line: string) => void, + start: number = 0, + ) { + const fileStream = createReadStream(filePath, { + start, + }); const rl = createInterface({ input: fileStream, crlfDelay: Infinity, diff --git a/cortex-js/src/usecases/cortex/cortex.usecases.ts b/cortex-js/src/usecases/cortex/cortex.usecases.ts index 8e5dc535c..06b7956b0 100644 --- a/cortex-js/src/usecases/cortex/cortex.usecases.ts +++ b/cortex-js/src/usecases/cortex/cortex.usecases.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@nestjs/common'; +import { HttpStatus, Injectable } from '@nestjs/common'; import { ChildProcess, fork } from 'child_process'; import { delimiter, join } from 'path'; import { CortexOperationSuccessfullyDto } from '@/infrastructure/dtos/cortex/cortex-operation-successfully.dto'; @@ -9,7 +9,10 @@ import { FileManagerService } from '@/infrastructure/services/file-manager/file- import { CORTEX_CPP_HEALTH_Z_URL, CORTEX_CPP_PROCESS_DESTROY_URL, + CORTEX_JS_HEALTH_URL, CORTEX_JS_STOP_API_SERVER_URL, + defaultCortexJsHost, + defaultCortexJsPort, } from '@/infrastructure/constants/cortex'; import { openSync } from 'fs'; @@ -39,7 +42,7 @@ export class CortexUsecases { } const engineDir = await this.fileManagerService.getCortexCppEnginePath(); - const dataFolderPath = await this.fileManagerService.getDataFolderPath() + const dataFolderPath = await this.fileManagerService.getDataFolderPath(); const writer = openSync(await this.fileManagerService.getLogPath(), 'a+'); // go up one level to get the binary folder, have to also work on windows @@ -140,4 +143,85 @@ export class CortexUsecases { }) .catch(() => false); } + + /** + * start the API server in detached mode + */ + async startServerDetached(host: string, port: number) { + const writer = openSync(await this.fileManagerService.getLogPath(), 'a+'); + const server = fork(join(__dirname, './../../main.js'), [], { + detached: true, + stdio: ['ignore', writer, writer, 'ipc'], + env: { + CORTEX_JS_HOST: host, + CORTEX_JS_PORT: port.toString(), + }, + }); + server.disconnect(); // closes the IPC channel + server.unref(); + // Await for the /healthz status ok + return new Promise((resolve, reject) => { + const TIMEOUT = 10 * 1000; + const timeout = setTimeout(() => { + clearInterval(interval); + clearTimeout(timeout); + reject(); + }, TIMEOUT); + const interval = setInterval(() => { + this.isAPIServerOnline(host, port) + .then((result) => { + if (result) { + clearInterval(interval); + clearTimeout(timeout); + resolve(true); + } + }) + .catch(reject); + }, 1000); + }); + } + /** + * Check if the Cortex API server is online + * @returns + */ + + async isAPIServerOnline(host?: string, port?: number): Promise { + const { + apiServerHost: configApiServerHost, + apiServerPort: configApiServerPort, + } = await this.fileManagerService.getConfig(); + + // for backward compatibility, we didn't have the apiServerHost and apiServerPort in the config file in the past + const apiServerHost = host || configApiServerHost || defaultCortexJsHost; + const apiServerPort = port || configApiServerPort || defaultCortexJsPort; + return firstValueFrom( + this.httpService.get(CORTEX_JS_HEALTH_URL(apiServerHost, apiServerPort)), + ) + .then((res) => res.status === HttpStatus.OK) + .catch(() => false); + } + + async stopApiServer() { + const { + apiServerHost: configApiServerHost, + apiServerPort: configApiServerPort, + } = await this.fileManagerService.getConfig(); + + // for backward compatibility, we didn't have the apiServerHost and apiServerPort in the config file in the past + const apiServerHost = configApiServerHost || defaultCortexJsHost; + const apiServerPort = configApiServerPort || defaultCortexJsPort; + await this.stopCortex(); + return fetch(CORTEX_JS_STOP_API_SERVER_URL(apiServerHost, apiServerPort), { + method: 'DELETE', + }).catch(() => {}); + } + + async updateApiServerConfig(host: string, port: number) { + const config = await this.fileManagerService.getConfig(); + await this.fileManagerService.writeConfigFile({ + ...config, + cortexCppHost: host, + cortexCppPort: port, + }); + } } diff --git a/cortex-js/src/usecases/engines/engines.usecase.ts b/cortex-js/src/usecases/engines/engines.usecase.ts index db36408d9..d69d97bbd 100644 --- a/cortex-js/src/usecases/engines/engines.usecase.ts +++ b/cortex-js/src/usecases/engines/engines.usecase.ts @@ -97,8 +97,6 @@ export class EnginesUsecases { if (!options && engine === Engines.llamaCPP) { options = await this.defaultInstallationOptions(); } - const configs = await this.fileManagerService.getConfig(); - // Ship Llama.cpp engine by default if ( !existsSync(