From d492819a5b48bbe9dea76e213762b36372731077 Mon Sep 17 00:00:00 2001 From: Louis Date: Mon, 22 Jul 2024 17:48:50 +0700 Subject: [PATCH] refactor: deprecate cortex configs --- cortex-js/src/app.module.ts | 2 - cortex-js/src/command.module.ts | 14 +--- .../src/domain/abstracts/engine.abstract.ts | 9 ++- .../domain/abstracts/extension.abstract.ts | 4 +- cortex-js/src/extensions/anthropic.engine.ts | 10 ++- cortex-js/src/extensions/groq.engine.ts | 10 ++- cortex-js/src/extensions/mistral.engine.ts | 10 ++- cortex-js/src/extensions/openai.engine.ts | 10 ++- .../commanders/configs.command.ts | 27 ------- .../commanders/configs/configs-get.command.ts | 31 ------- .../configs/configs-list.command.ts | 25 ------ .../commanders/configs/configs-set.command.ts | 60 -------------- .../commanders/cortex-command.commander.ts | 6 +- .../commanders/engines.command.ts | 14 +++- .../commanders/engines/engines-set.command.ts | 34 ++++++++ .../infrastructure/commanders/kill.command.ts | 25 ------ .../commanders/test/helpers.command.spec.ts | 8 +- .../controllers/configs.controller.spec.ts | 41 ---------- .../controllers/configs.controller.ts | 81 ------------------- .../controllers/engines.controller.spec.ts | 2 + .../controllers/engines.controller.ts | 27 +++++++ .../dtos/configs/config-update.dto.ts | 11 +-- .../dtos/engines/engines.dto.ts | 4 +- .../extensions/extension.repository.ts | 19 +++-- .../src/usecases/configs/configs.usecase.ts | 14 ++-- .../src/usecases/engines/engines.module.ts | 1 + .../src/usecases/engines/engines.usecase.ts | 42 +++++++++- 27 files changed, 188 insertions(+), 353 deletions(-) delete mode 100644 cortex-js/src/infrastructure/commanders/configs.command.ts delete mode 100644 cortex-js/src/infrastructure/commanders/configs/configs-get.command.ts delete mode 100644 cortex-js/src/infrastructure/commanders/configs/configs-list.command.ts delete mode 100644 cortex-js/src/infrastructure/commanders/configs/configs-set.command.ts create mode 100644 cortex-js/src/infrastructure/commanders/engines/engines-set.command.ts delete mode 100644 cortex-js/src/infrastructure/commanders/kill.command.ts delete mode 100644 cortex-js/src/infrastructure/controllers/configs.controller.spec.ts delete mode 100644 cortex-js/src/infrastructure/controllers/configs.controller.ts diff --git a/cortex-js/src/app.module.ts b/cortex-js/src/app.module.ts index 95b21bf49..04047d49a 100644 --- a/cortex-js/src/app.module.ts +++ b/cortex-js/src/app.module.ts @@ -29,7 +29,6 @@ import { ContextModule } from './infrastructure/services/context/context.module' import { ExtensionsModule } from './extensions/extensions.module'; import { ConfigsModule } from './usecases/configs/configs.module'; import { EnginesModule } from './usecases/engines/engines.module'; -import { ConfigsController } from './infrastructure/controllers/configs.controller'; import { EnginesController } from './infrastructure/controllers/engines.controller'; import { ResourceManagerModule } from './infrastructure/services/resources-manager/resources-manager.module'; @@ -70,7 +69,6 @@ import { ResourceManagerModule } from './infrastructure/services/resources-manag StatusController, ProcessController, EventsController, - ConfigsController, EnginesController, ], providers: [ diff --git a/cortex-js/src/command.module.ts b/cortex-js/src/command.module.ts index 7930b362a..12969f812 100644 --- a/cortex-js/src/command.module.ts +++ b/cortex-js/src/command.module.ts @@ -21,7 +21,6 @@ import { AssistantsModule } from './usecases/assistants/assistants.module'; import { MessagesModule } from './usecases/messages/messages.module'; import { FileManagerModule } from './infrastructure/services/file-manager/file-manager.module'; import { PSCommand } from './infrastructure/commanders/ps.command'; -import { KillCommand } from './infrastructure/commanders/kill.command'; import { PresetCommand } from './infrastructure/commanders/presets.command'; import { TelemetryModule } from './usecases/telemetry/telemetry.module'; import { TelemetryCommand } from './infrastructure/commanders/telemetry.command'; @@ -33,16 +32,13 @@ import { ServeStopCommand } from './infrastructure/commanders/sub-commands/serve import { ContextModule } from './infrastructure/services/context/context.module'; import { CliUsecasesModule } from './infrastructure/commanders/usecases/cli.usecases.module'; import { ExtensionsModule } from './extensions/extensions.module'; -import { ConfigsCommand } from './infrastructure/commanders/configs.command'; import { EnginesCommand } from './infrastructure/commanders/engines.command'; import { ConfigsModule } from './usecases/configs/configs.module'; import { EnginesModule } from './usecases/engines/engines.module'; -import { ConfigsGetCommand } from './infrastructure/commanders/configs/configs-get.command'; -import { ConfigsListCommand } from './infrastructure/commanders/configs/configs-list.command'; -import { ConfigsSetCommand } from './infrastructure/commanders/configs/configs-set.command'; import { EnginesListCommand } from './infrastructure/commanders/engines/engines-list.command'; import { EnginesGetCommand } from './infrastructure/commanders/engines/engines-get.command'; import { EnginesInitCommand } from './infrastructure/commanders/engines/engines-init.command'; +import { EnginesSetCommand } from './infrastructure/commanders/engines/engines-set.command'; @Module({ imports: [ @@ -73,7 +69,6 @@ import { EnginesInitCommand } from './infrastructure/commanders/engines/engines- ModelsCommand, ChatCommand, PSCommand, - KillCommand, PresetCommand, EmbeddingCommand, BenchmarkCommand, @@ -100,16 +95,11 @@ import { EnginesInitCommand } from './infrastructure/commanders/engines/engines- // Serve ServeStopCommand, - // Configs - ConfigsCommand, - ConfigsGetCommand, - ConfigsListCommand, - ConfigsSetCommand, - // Engines EnginesListCommand, EnginesGetCommand, EnginesInitCommand, + EnginesSetCommand, ], }) export class CommandModule {} diff --git a/cortex-js/src/domain/abstracts/engine.abstract.ts b/cortex-js/src/domain/abstracts/engine.abstract.ts index 9b3677a23..205c44262 100644 --- a/cortex-js/src/domain/abstracts/engine.abstract.ts +++ b/cortex-js/src/domain/abstracts/engine.abstract.ts @@ -3,6 +3,13 @@ import stream from 'stream'; import { Model, ModelSettingParams } from '../../domain/models/model.interface'; import { Extension } from './extension.abstract'; +export enum EngineStatus { + READY = 'READY', + MISSING_CONFIGURATION = 'MISSING_CONFIGURATION', + NOT_INITIALIZED = 'NOT_INITIALIZED', + ERROR = 'ERROR', +} + /** * This class should be extended by any class that represents an engine extension. * It provides methods for loading and unloading models, and for making inference requests. @@ -14,7 +21,7 @@ export abstract class EngineExtension extends Extension { transformResponse?: Function; - initalized: boolean = false; + status: EngineStatus = EngineStatus.READY; /** * Makes an inference request to the engine. diff --git a/cortex-js/src/domain/abstracts/extension.abstract.ts b/cortex-js/src/domain/abstracts/extension.abstract.ts index c3f677edf..ce62df74c 100644 --- a/cortex-js/src/domain/abstracts/extension.abstract.ts +++ b/cortex-js/src/domain/abstracts/extension.abstract.ts @@ -21,8 +21,8 @@ export abstract class Extension { /** @type {string} Extension's version. */ version?: string; - /** @type {boolean} Whether the extension is initialized or not. */ - initalized: boolean; + /** @type {string} The status of the extension. */ + status: string; /** * Called when the extension is loaded. diff --git a/cortex-js/src/extensions/anthropic.engine.ts b/cortex-js/src/extensions/anthropic.engine.ts index 916b00e49..4cc99a215 100644 --- a/cortex-js/src/extensions/anthropic.engine.ts +++ b/cortex-js/src/extensions/anthropic.engine.ts @@ -4,6 +4,7 @@ import { OAIEngineExtension } from '../domain/abstracts/oai.abstract'; import { ConfigsUsecases } from '@/usecases/configs/configs.usecase'; import { EventEmitter2 } from '@nestjs/event-emitter'; import _ from 'lodash'; +import { EngineStatus } from '@/domain/abstracts/engine.abstract'; /** * A class that implements the InferenceExtension interface from the @janhq/core package. @@ -16,7 +17,6 @@ export default class AnthropicEngineExtension extends OAIEngineExtension { productName = 'Anthropic Inference Engine'; description = 'This extension enables Anthropic chat completion API calls'; version = '0.0.1'; - initalized = true; apiKey?: string; constructor( @@ -29,6 +29,10 @@ export default class AnthropicEngineExtension extends OAIEngineExtension { eventEmmitter.on('config.updated', async (data) => { if (data.group === this.name) { this.apiKey = data.value; + this.status = + (this.apiKey?.length ?? 0) > 0 + ? EngineStatus.READY + : EngineStatus.MISSING_CONFIGURATION; } }); } @@ -38,6 +42,10 @@ export default class AnthropicEngineExtension extends OAIEngineExtension { this.name, )) as unknown as { apiKey: string }; this.apiKey = configs?.apiKey; + this.status = + (this.apiKey?.length ?? 0) > 0 + ? EngineStatus.READY + : EngineStatus.MISSING_CONFIGURATION; } override async inference( diff --git a/cortex-js/src/extensions/groq.engine.ts b/cortex-js/src/extensions/groq.engine.ts index a23b87752..5a3f1b759 100644 --- a/cortex-js/src/extensions/groq.engine.ts +++ b/cortex-js/src/extensions/groq.engine.ts @@ -2,6 +2,7 @@ import { HttpService } from '@nestjs/axios'; import { OAIEngineExtension } from '../domain/abstracts/oai.abstract'; import { ConfigsUsecases } from '@/usecases/configs/configs.usecase'; import { EventEmitter2 } from '@nestjs/event-emitter'; +import { EngineStatus } from '@/domain/abstracts/engine.abstract'; /** * A class that implements the InferenceExtension interface from the @janhq/core package. @@ -14,7 +15,6 @@ export default class GroqEngineExtension extends OAIEngineExtension { productName = 'Groq Inference Engine'; description = 'This extension enables fast Groq chat completion API calls'; version = '0.0.1'; - initalized = true; apiKey?: string; constructor( @@ -27,6 +27,10 @@ export default class GroqEngineExtension extends OAIEngineExtension { eventEmmitter.on('config.updated', async (data) => { if (data.group === this.name) { this.apiKey = data.value; + this.status = + (this.apiKey?.length ?? 0) > 0 + ? EngineStatus.READY + : EngineStatus.MISSING_CONFIGURATION; } }); } @@ -37,5 +41,9 @@ export default class GroqEngineExtension extends OAIEngineExtension { )) as unknown as { apiKey: string }; this.apiKey = configs?.apiKey; + this.status = + (this.apiKey?.length ?? 0) > 0 + ? EngineStatus.READY + : EngineStatus.MISSING_CONFIGURATION; } } diff --git a/cortex-js/src/extensions/mistral.engine.ts b/cortex-js/src/extensions/mistral.engine.ts index 44f8aa095..efafdac3f 100644 --- a/cortex-js/src/extensions/mistral.engine.ts +++ b/cortex-js/src/extensions/mistral.engine.ts @@ -2,6 +2,7 @@ import { HttpService } from '@nestjs/axios'; import { OAIEngineExtension } from '../domain/abstracts/oai.abstract'; import { ConfigsUsecases } from '@/usecases/configs/configs.usecase'; import { EventEmitter2 } from '@nestjs/event-emitter'; +import { EngineStatus } from '@/domain/abstracts/engine.abstract'; /** * A class that implements the InferenceExtension interface from the @janhq/core package. @@ -14,7 +15,6 @@ export default class MistralEngineExtension extends OAIEngineExtension { productName = 'Mistral Inference Engine'; description = 'This extension enables Mistral chat completion API calls'; version = '0.0.1'; - initalized = true; apiKey?: string; constructor( @@ -27,6 +27,10 @@ export default class MistralEngineExtension extends OAIEngineExtension { eventEmmitter.on('config.updated', async (data) => { if (data.group === this.name) { this.apiKey = data.value; + this.status = + (this.apiKey?.length ?? 0) > 0 + ? EngineStatus.READY + : EngineStatus.MISSING_CONFIGURATION; } }); } @@ -36,5 +40,9 @@ export default class MistralEngineExtension extends OAIEngineExtension { this.name, )) as unknown as { apiKey: string }; this.apiKey = configs?.apiKey; + this.status = + (this.apiKey?.length ?? 0) > 0 + ? EngineStatus.READY + : EngineStatus.MISSING_CONFIGURATION; } } diff --git a/cortex-js/src/extensions/openai.engine.ts b/cortex-js/src/extensions/openai.engine.ts index 5917d20c1..9426ba22f 100644 --- a/cortex-js/src/extensions/openai.engine.ts +++ b/cortex-js/src/extensions/openai.engine.ts @@ -2,6 +2,7 @@ import { HttpService } from '@nestjs/axios'; import { OAIEngineExtension } from '../domain/abstracts/oai.abstract'; import { ConfigsUsecases } from '@/usecases/configs/configs.usecase'; import { EventEmitter2 } from '@nestjs/event-emitter'; +import { EngineStatus } from '@/domain/abstracts/engine.abstract'; /** * A class that implements the InferenceExtension interface from the @janhq/core package. @@ -14,7 +15,6 @@ export default class OpenAIEngineExtension extends OAIEngineExtension { productName = 'OpenAI Inference Engine'; description = 'This extension enables OpenAI chat completion API calls'; version = '0.0.1'; - initalized = true; apiKey?: string; constructor( @@ -27,6 +27,10 @@ export default class OpenAIEngineExtension extends OAIEngineExtension { eventEmmitter.on('config.updated', async (data) => { if (data.group === this.name) { this.apiKey = data.value; + this.status = + (this.apiKey?.length ?? 0) > 0 + ? EngineStatus.READY + : EngineStatus.MISSING_CONFIGURATION; } }); } @@ -36,5 +40,9 @@ export default class OpenAIEngineExtension extends OAIEngineExtension { this.name, )) as unknown as { apiKey: string }; this.apiKey = configs?.apiKey; + this.status = + (this.apiKey?.length ?? 0) > 0 + ? EngineStatus.READY + : EngineStatus.MISSING_CONFIGURATION; } } diff --git a/cortex-js/src/infrastructure/commanders/configs.command.ts b/cortex-js/src/infrastructure/commanders/configs.command.ts deleted file mode 100644 index 5801891ca..000000000 --- a/cortex-js/src/infrastructure/commanders/configs.command.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { SubCommand } from 'nest-commander'; -import { SetCommandContext } from './decorators/CommandContext'; -import { ContextService } from '@/infrastructure/services/context/context.service'; -import { ConfigsGetCommand } from './configs/configs-get.command'; -import { ConfigsListCommand } from './configs/configs-list.command'; -import { ConfigsSetCommand } from './configs/configs-set.command'; -import { BaseCommand } from './base.command'; -import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; - -@SubCommand({ - name: 'configs', - description: 'Get cortex configurations', - subCommands: [ConfigsGetCommand, ConfigsListCommand, ConfigsSetCommand], -}) -@SetCommandContext() -export class ConfigsCommand extends BaseCommand { - constructor( - readonly contextService: ContextService, - readonly cortexUseCases: CortexUsecases, - ) { - super(cortexUseCases); - } - - async runCommand(): Promise { - this.command?.help(); - } -} diff --git a/cortex-js/src/infrastructure/commanders/configs/configs-get.command.ts b/cortex-js/src/infrastructure/commanders/configs/configs-get.command.ts deleted file mode 100644 index 4e815ebd8..000000000 --- a/cortex-js/src/infrastructure/commanders/configs/configs-get.command.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { CommandRunner, SubCommand } from 'nest-commander'; -import { SetCommandContext } from '../decorators/CommandContext'; -import { ContextService } from '@/infrastructure/services/context/context.service'; -import { ConfigsUsecases } from '@/usecases/configs/configs.usecase'; -import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; -import { BaseCommand } from '../base.command'; - -@SubCommand({ - name: 'get', - description: 'Get a cortex configuration', - arguments: '', - argsDescription: { - name: 'Configuration name to get', - }, -}) -@SetCommandContext() -export class ConfigsGetCommand extends BaseCommand { - constructor( - private readonly configsUsecases: ConfigsUsecases, - readonly contextService: ContextService, - readonly cortexUsecases: CortexUsecases, - ) { - super(cortexUsecases); - } - - async runCommand(passedParams: string[]): Promise { - return this.configsUsecases - .getGroupConfigs(passedParams[0]) - .then(console.table); - } -} diff --git a/cortex-js/src/infrastructure/commanders/configs/configs-list.command.ts b/cortex-js/src/infrastructure/commanders/configs/configs-list.command.ts deleted file mode 100644 index 500ef0d35..000000000 --- a/cortex-js/src/infrastructure/commanders/configs/configs-list.command.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { SubCommand } from 'nest-commander'; -import { SetCommandContext } from '../decorators/CommandContext'; -import { ContextService } from '@/infrastructure/services/context/context.service'; -import { ConfigsUsecases } from '@/usecases/configs/configs.usecase'; -import { BaseCommand } from '../base.command'; -import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; - -@SubCommand({ - name: 'list', - description: 'Get all cortex configurations', -}) -@SetCommandContext() -export class ConfigsListCommand extends BaseCommand { - constructor( - private readonly configsUsecases: ConfigsUsecases, - readonly contextService: ContextService, - readonly cortexUsecases: CortexUsecases, - ) { - super(cortexUsecases); - } - - async runCommand(): Promise { - return this.configsUsecases.getConfigs().then(console.table); - } -} diff --git a/cortex-js/src/infrastructure/commanders/configs/configs-set.command.ts b/cortex-js/src/infrastructure/commanders/configs/configs-set.command.ts deleted file mode 100644 index 4c12785cc..000000000 --- a/cortex-js/src/infrastructure/commanders/configs/configs-set.command.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { CommandRunner, SubCommand, Option } from 'nest-commander'; -import { SetCommandContext } from '../decorators/CommandContext'; -import { ContextService } from '@/infrastructure/services/context/context.service'; -import { ConfigsUsecases } from '@/usecases/configs/configs.usecase'; -import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; -import { BaseCommand } from '../base.command'; - -interface ConfigsSetOption { - key: string; - value: string; - group?: string; -} - -@SubCommand({ - name: 'set', - description: 'Set a cortex configuration', -}) -@SetCommandContext() -export class ConfigsSetCommand extends BaseCommand { - constructor( - private readonly configsUsecases: ConfigsUsecases, - readonly contextService: ContextService, - readonly cortexUsecases: CortexUsecases, - ) { - super(cortexUsecases); - } - - async runCommand( - passedParams: string[], - options: ConfigsSetOption, - ): Promise { - return this.configsUsecases - .saveConfig(options.key, options.value, options.group) - .then(() => console.log('Set configuration successfully')); - } - - @Option({ - flags: '-k, --key ', - description: 'Configuration key', - }) - parseKey(value: string) { - return value; - } - - @Option({ - flags: '-v, --value ', - description: 'Configuration value', - }) - parseValue(value: string) { - return value; - } - - @Option({ - flags: '-g, --group ', - description: 'Configuration group', - }) - parseGroup(value?: string) { - return value; - } -} diff --git a/cortex-js/src/infrastructure/commanders/cortex-command.commander.ts b/cortex-js/src/infrastructure/commanders/cortex-command.commander.ts index 086fbf36d..e1e06af33 100644 --- a/cortex-js/src/infrastructure/commanders/cortex-command.commander.ts +++ b/cortex-js/src/infrastructure/commanders/cortex-command.commander.ts @@ -4,7 +4,6 @@ import { ModelsCommand } from './models.command'; import { RunCommand } from './shortcuts/run.command'; import { ModelPullCommand } from './models/model-pull.command'; import { PSCommand } from './ps.command'; -import { KillCommand } from './kill.command'; import { PresetCommand } from './presets.command'; import { TelemetryCommand } from './telemetry.command'; import { SetCommandContext } from './decorators/CommandContext'; @@ -13,13 +12,13 @@ import { BenchmarkCommand } from './benchmark.command'; import chalk from 'chalk'; import { ContextService } from '../services/context/context.service'; import { EnginesCommand } from './engines.command'; -import { ConfigsCommand } from './configs.command'; import { defaultCortexJsHost, defaultCortexJsPort } from '../constants/cortex'; import { getApp } from '@/app'; import { FileManagerService } from '../services/file-manager/file-manager.service'; import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; import { ServeStopCommand } from './sub-commands/serve-stop.command'; import ora from 'ora'; +import { EnginesSetCommand } from './engines/engines-set.command'; type ServeOptions = { address?: string; @@ -35,14 +34,13 @@ type ServeOptions = { RunCommand, ModelPullCommand, PSCommand, - KillCommand, PresetCommand, TelemetryCommand, EmbeddingCommand, BenchmarkCommand, EnginesCommand, - ConfigsCommand, ServeStopCommand, + EnginesSetCommand, ], description: 'Cortex CLI', }) diff --git a/cortex-js/src/infrastructure/commanders/engines.command.ts b/cortex-js/src/infrastructure/commanders/engines.command.ts index a046be028..767df0ebf 100644 --- a/cortex-js/src/infrastructure/commanders/engines.command.ts +++ b/cortex-js/src/infrastructure/commanders/engines.command.ts @@ -9,6 +9,7 @@ import { ModuleRef } from '@nestjs/core'; import { EngineNamesMap } from './types/engine.interface'; import { BaseCommand } from './base.command'; import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; +import { EnginesSetCommand } from './engines/engines-set.command'; @SubCommand({ name: 'engines', @@ -22,17 +23,18 @@ export class EnginesCommand extends BaseCommand { list: EnginesListCommand, get: EnginesGetCommand, init: EnginesInitCommand, + set: EnginesSetCommand, }; constructor( readonly contextService: ContextService, - private readonly moduleRef: ModuleRef, + readonly moduleRef: ModuleRef, readonly cortexUsecases: CortexUsecases, ) { super(cortexUsecases); } - async runCommand(passedParam: string[], options: { vulkan: boolean }) { - const [parameter, command] = passedParam; + async runCommand(passedParams: string[], options: { vulkan: boolean }) { + const [parameter, command] = passedParams; if (command !== 'list' && !parameter) { console.error('Engine name is required.'); return; @@ -45,7 +47,11 @@ export class EnginesCommand extends BaseCommand { return; } const engine = invert(EngineNamesMap)[parameter] || parameter; - await this.runEngineCommand(commandClass, [engine], options); + await this.runEngineCommand( + commandClass, + [engine, ...passedParams.slice(2)], + options, + ); } private async runEngineCommand( diff --git a/cortex-js/src/infrastructure/commanders/engines/engines-set.command.ts b/cortex-js/src/infrastructure/commanders/engines/engines-set.command.ts new file mode 100644 index 000000000..a2b1067dc --- /dev/null +++ b/cortex-js/src/infrastructure/commanders/engines/engines-set.command.ts @@ -0,0 +1,34 @@ +import { SubCommand } from 'nest-commander'; +import { SetCommandContext } from '../decorators/CommandContext'; +import { ContextService } from '@/infrastructure/services/context/context.service'; +import { BaseCommand } from '../base.command'; +import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; +import { EnginesUsecases } from '@/usecases/engines/engines.usecase'; + +@SubCommand({ + name: ' set ', + description: 'Update an engine configurations', + argsDescription: { + name: 'Engine name to update', + }, +}) +@SetCommandContext() +export class EnginesSetCommand extends BaseCommand { + constructor( + readonly contextService: ContextService, + readonly cortexUsecases: CortexUsecases, + readonly engineUsecases: EnginesUsecases, + ) { + super(cortexUsecases); + } + + async runCommand(passedParams: string[]): Promise { + const engineName = passedParams[0]; + const config = passedParams[1]; + const value = passedParams[2]; + return this.engineUsecases + .updateConfigs(config, value, engineName) + .then(() => console.log('Update engine successfully')) + .catch((error) => console.error(error.message ?? error)); + } +} diff --git a/cortex-js/src/infrastructure/commanders/kill.command.ts b/cortex-js/src/infrastructure/commanders/kill.command.ts deleted file mode 100644 index 922b74171..000000000 --- a/cortex-js/src/infrastructure/commanders/kill.command.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { SubCommand } from 'nest-commander'; -import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; -import { SetCommandContext } from './decorators/CommandContext'; -import { ContextService } from '../services/context/context.service'; -import { BaseCommand } from './base.command'; - -@SubCommand({ - name: 'kill', - description: 'Kill running cortex processes', -}) -@SetCommandContext() -export class KillCommand extends BaseCommand { - constructor( - private readonly cortexUsecases: CortexUsecases, - readonly contextService: ContextService, - ) { - super(cortexUsecases); - } - async runCommand(): Promise { - return this.cortexUsecases - .stopCortex() - .then(this.cortexUsecases.stopServe) - .then(() => console.log('Cortex processes stopped successfully!')); - } -} diff --git a/cortex-js/src/infrastructure/commanders/test/helpers.command.spec.ts b/cortex-js/src/infrastructure/commanders/test/helpers.command.spec.ts index e1dc1aa04..63c8c12cd 100644 --- a/cortex-js/src/infrastructure/commanders/test/helpers.command.spec.ts +++ b/cortex-js/src/infrastructure/commanders/test/helpers.command.spec.ts @@ -94,17 +94,15 @@ describe('Helper commands', () => { // }); test( - 'Show / kill running models', + 'Show stop running models', async () => { const tableMock = stubMethod(console, 'table'); const logMock = stubMethod(console, 'log'); - await CommandTestFactory.run(commandInstance, ['kill']); + await CommandTestFactory.run(commandInstance, ['stop']); await CommandTestFactory.run(commandInstance, ['ps']); - expect(logMock.firstCall?.args[0]).toEqual( - 'Cortex processes stopped successfully!', - ); + expect(logMock.firstCall?.args[0]).toEqual('API server stopped'); expect(tableMock.firstCall?.args[0]).toBeInstanceOf(Array); expect(tableMock.firstCall?.args[0].length).toEqual(0); }, diff --git a/cortex-js/src/infrastructure/controllers/configs.controller.spec.ts b/cortex-js/src/infrastructure/controllers/configs.controller.spec.ts deleted file mode 100644 index c788da2ae..000000000 --- a/cortex-js/src/infrastructure/controllers/configs.controller.spec.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { DatabaseModule } from '../database/database.module'; -import { ExtensionModule } from '../repositories/extensions/extension.module'; -import { ModelRepositoryModule } from '../repositories/models/model.module'; -import { HttpModule } from '@nestjs/axios'; -import { DownloadManagerModule } from '@/infrastructure/services/download-manager/download-manager.module'; -import { EventEmitterModule } from '@nestjs/event-emitter'; -import { TelemetryModule } from '@/usecases/telemetry/telemetry.module'; -import { FileManagerModule } from '../services/file-manager/file-manager.module'; -import { ConfigsController } from './configs.controller'; -import { ConfigsUsecases } from '@/usecases/configs/configs.usecase'; -import { ConfigsModule } from '@/usecases/configs/configs.module'; - -describe('ConfigsController', () => { - let controller: ConfigsController; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - imports: [ - EventEmitterModule.forRoot(), - DatabaseModule, - ExtensionModule, - ModelRepositoryModule, - HttpModule, - DownloadManagerModule, - EventEmitterModule.forRoot(), - TelemetryModule, - FileManagerModule, - ConfigsModule, - ], - controllers: [ConfigsController], - providers: [ConfigsUsecases], - }).compile(); - - controller = module.get(ConfigsController); - }); - - it('should be defined', () => { - expect(controller).toBeDefined(); - }); -}); diff --git a/cortex-js/src/infrastructure/controllers/configs.controller.ts b/cortex-js/src/infrastructure/controllers/configs.controller.ts deleted file mode 100644 index 948632d5d..000000000 --- a/cortex-js/src/infrastructure/controllers/configs.controller.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { - Controller, - Get, - Post, - Body, - Param, - HttpCode, - UseInterceptors, -} from '@nestjs/common'; -import { ApiOperation, ApiParam, ApiTags, ApiResponse } from '@nestjs/swagger'; -import { TransformInterceptor } from '../interceptors/transform.interceptor'; -import { ConfigsUsecases } from '@/usecases/configs/configs.usecase'; -import { ConfigUpdateDto } from '../dtos/configs/config-update.dto'; -import { CommonResponseDto } from '../dtos/common/common-response.dto'; - -@ApiTags('Configurations') -@Controller('configs') -@UseInterceptors(TransformInterceptor) -export class ConfigsController { - constructor(private readonly configsUsecases: ConfigsUsecases) {} - - @HttpCode(200) - @ApiResponse({ - status: 200, - description: 'Ok', - type: [Object], - }) - @ApiOperation({ - summary: 'List configs', - description: - 'Lists the currently available configs, including the default and user-defined configurations.', - }) - @Get() - findAll() { - return this.configsUsecases.getConfigs(); - } - - @HttpCode(200) - @ApiResponse({ - status: 200, - description: 'Ok', - type: Object, - }) - @ApiOperation({ - summary: 'Get a config', - description: - 'Retrieves a config instance, providing basic information about the config.', - }) - @ApiParam({ - name: 'name', - required: true, - description: 'The unique identifier of the config.', - }) - @Get(':name(*)') - findOne(@Param('name') name: string) { - return this.configsUsecases.getGroupConfigs(name); - } - - @HttpCode(200) - @ApiResponse({ - status: 200, - description: 'The config has been successfully updated.', - type: CommonResponseDto, - }) - @ApiOperation({ - summary: 'Configure a model', - description: "Updates a specific configuration setting by its group and key.", - parameters: [ - { - in: 'path', - name: 'model', - required: true, - description: 'The unique identifier of the model.', - }, - ], - }) - @Post(':name(*)') - async update(@Param('name') name: string, @Body() configs: ConfigUpdateDto) { - return this.configsUsecases.saveConfig(configs.key, configs.value, name); - } -} diff --git a/cortex-js/src/infrastructure/controllers/engines.controller.spec.ts b/cortex-js/src/infrastructure/controllers/engines.controller.spec.ts index 2f488f71b..6eb91ff36 100644 --- a/cortex-js/src/infrastructure/controllers/engines.controller.spec.ts +++ b/cortex-js/src/infrastructure/controllers/engines.controller.spec.ts @@ -10,6 +10,7 @@ import { FileManagerModule } from '../services/file-manager/file-manager.module' import { EnginesController } from './engines.controller'; import { EnginesUsecases } from '@/usecases/engines/engines.usecase'; import { EnginesModule } from '@/usecases/engines/engines.module'; +import { ConfigsModule } from '@/usecases/configs/configs.module'; describe('ConfigsController', () => { let controller: EnginesController; @@ -27,6 +28,7 @@ describe('ConfigsController', () => { TelemetryModule, FileManagerModule, EnginesModule, + ConfigsModule, ], controllers: [EnginesController], providers: [EnginesUsecases], diff --git a/cortex-js/src/infrastructure/controllers/engines.controller.ts b/cortex-js/src/infrastructure/controllers/engines.controller.ts index aabae3a58..b7958bdf7 100644 --- a/cortex-js/src/infrastructure/controllers/engines.controller.ts +++ b/cortex-js/src/infrastructure/controllers/engines.controller.ts @@ -5,12 +5,15 @@ import { HttpCode, UseInterceptors, Post, + Body, + Patch, } from '@nestjs/common'; import { ApiOperation, ApiParam, ApiTags, ApiResponse } from '@nestjs/swagger'; import { TransformInterceptor } from '../interceptors/transform.interceptor'; import { EnginesUsecases } from '@/usecases/engines/engines.usecase'; import { EngineDto } from '../dtos/engines/engines.dto'; import { CommonResponseDto } from '../dtos/common/common-response.dto'; +import { ConfigUpdateDto } from '../dtos/configs/config-update.dto'; @ApiTags('Engines') @Controller('engines') @@ -81,4 +84,28 @@ export class EnginesController { message: 'Engine initialization started successfully.', }; } + + @HttpCode(200) + @ApiResponse({ + status: 200, + description: 'Ok', + type: CommonResponseDto, + }) + @ApiOperation({ + summary: 'Update the engine', + description: 'Updates the engine with configurations.', + }) + @ApiParam({ + name: 'name', + required: true, + description: 'The unique identifier of the engine.', + }) + @Patch(':name(*)') + update(@Param('name') name: string, @Body() configs: ConfigUpdateDto) { + return this.enginesUsecases.updateConfigs( + configs.config, + configs.value, + name, + ); + } } diff --git a/cortex-js/src/infrastructure/dtos/configs/config-update.dto.ts b/cortex-js/src/infrastructure/dtos/configs/config-update.dto.ts index e92bb59fc..8e31cbcb1 100644 --- a/cortex-js/src/infrastructure/dtos/configs/config-update.dto.ts +++ b/cortex-js/src/infrastructure/dtos/configs/config-update.dto.ts @@ -8,7 +8,7 @@ export class ConfigUpdateDto { }) @IsString() @IsOptional() - key: string; + config: string; // Prompt Settings @ApiProperty({ @@ -19,13 +19,4 @@ export class ConfigUpdateDto { @IsString() @IsOptional() value: string; - - @ApiProperty({ - type: String, - example: 'openai', - description: 'The name of the configuration.', - }) - @IsString() - @IsOptional() - name?: string; } diff --git a/cortex-js/src/infrastructure/dtos/engines/engines.dto.ts b/cortex-js/src/infrastructure/dtos/engines/engines.dto.ts index 4737be534..93ac8d4dc 100644 --- a/cortex-js/src/infrastructure/dtos/engines/engines.dto.ts +++ b/cortex-js/src/infrastructure/dtos/engines/engines.dto.ts @@ -42,8 +42,8 @@ export class EngineDto implements Partial { @ApiProperty({ type: String, example: true, - description: 'Whether the engine is initialized or not.', + description: 'The status of the engine.', }) @IsBoolean() - initalized?: boolean; + status?: string; } diff --git a/cortex-js/src/infrastructure/repositories/extensions/extension.repository.ts b/cortex-js/src/infrastructure/repositories/extensions/extension.repository.ts index 3ebd58e60..7a5aa4a26 100644 --- a/cortex-js/src/infrastructure/repositories/extensions/extension.repository.ts +++ b/cortex-js/src/infrastructure/repositories/extensions/extension.repository.ts @@ -11,6 +11,7 @@ import { HttpService } from '@nestjs/axios'; import LlamaCPPProvider from '@/infrastructure/providers/cortex/llamacpp.provider'; import Onnxprovider from '@/infrastructure/providers/cortex/onnx.provider'; import TensorrtLLMProvider from '@/infrastructure/providers/cortex/tensorrtllm.provider'; +import { EngineStatus } from '@/domain/abstracts/engine.abstract'; @Injectable() export class ExtensionRepositoryImpl implements ExtensionRepository { @@ -50,34 +51,40 @@ export class ExtensionRepositoryImpl implements ExtensionRepository { this.httpService, this.fileManagerService, ); - llamaCPPEngine.initalized = existsSync( + llamaCPPEngine.status = existsSync( join( await this.fileManagerService.getCortexCppEnginePath(), Engines.llamaCPP, ), - ); + ) + ? EngineStatus.READY + : EngineStatus.NOT_INITIALIZED; const onnxEngine = new Onnxprovider( this.httpService, this.fileManagerService, ); - onnxEngine.initalized = existsSync( + onnxEngine.status = existsSync( join( await this.fileManagerService.getCortexCppEnginePath(), Engines.onnx, ), - ); + ) + ? EngineStatus.READY + : EngineStatus.NOT_INITIALIZED; const tensorrtLLMEngine = new TensorrtLLMProvider( this.httpService, this.fileManagerService, ); - tensorrtLLMEngine.initalized = existsSync( + tensorrtLLMEngine.status = existsSync( join( await this.fileManagerService.getCortexCppEnginePath(), Engines.tensorrtLLM, ), - ); + ) + ? EngineStatus.READY + : EngineStatus.NOT_INITIALIZED; await llamaCPPEngine.onLoad(); await onnxEngine.onLoad(); diff --git a/cortex-js/src/usecases/configs/configs.usecase.ts b/cortex-js/src/usecases/configs/configs.usecase.ts index 4cff2e3f5..4c95b4580 100644 --- a/cortex-js/src/usecases/configs/configs.usecase.ts +++ b/cortex-js/src/usecases/configs/configs.usecase.ts @@ -13,23 +13,23 @@ export class ConfigsUsecases { /** * Save a configuration to the .cortexrc file. * @param key Configuration Key - * @param group Configuration Group where the key belongs + * @param engine The engine where the configs belongs */ async saveConfig( key: string, value: string, - group?: string, + engine?: string, ): Promise { const configs = await this.fileManagerService.getConfig(); const groupConfigs = configs[ - group as keyof typeof configs + engine as keyof typeof configs ] as unknown as object; const newConfigs = { ...configs, - ...(group + ...(engine ? { - [group]: { + [engine]: { ...groupConfigs, [key]: value, }, @@ -40,9 +40,9 @@ export class ConfigsUsecases { return this.fileManagerService .writeConfigFile(newConfigs) .then(async () => { - if (group) { + if (engine) { this.eventEmitter.emit('config.updated', { - group, + engine, key, value, }); diff --git a/cortex-js/src/usecases/engines/engines.module.ts b/cortex-js/src/usecases/engines/engines.module.ts index 3a14929a0..5ee607a8c 100644 --- a/cortex-js/src/usecases/engines/engines.module.ts +++ b/cortex-js/src/usecases/engines/engines.module.ts @@ -13,6 +13,7 @@ import { DownloadManagerModule } from '@/infrastructure/services/download-manage HttpModule, FileManagerModule, DownloadManagerModule, + ConfigsModule, ], controllers: [], providers: [EnginesUsecases], diff --git a/cortex-js/src/usecases/engines/engines.usecase.ts b/cortex-js/src/usecases/engines/engines.usecase.ts index d69d97bbd..9f707be34 100644 --- a/cortex-js/src/usecases/engines/engines.usecase.ts +++ b/cortex-js/src/usecases/engines/engines.usecase.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@nestjs/common'; +import { ForbiddenException, Injectable } from '@nestjs/common'; import { ExtensionRepository } from '@/domain/repositories/extension.interface'; import { cpSync, existsSync, mkdirSync, readdirSync } from 'fs'; @@ -20,6 +20,10 @@ import { cpuInfo } from 'cpu-instructions'; import { DownloadManagerService } from '@/infrastructure/services/download-manager/download-manager.service'; import { DownloadType } from '@/domain/models/download.interface'; import { Engines } from '@/infrastructure/commanders/types/engine.interface'; +import { CommonResponseDto } from '@/infrastructure/dtos/common/common-response.dto'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { EngineStatus } from '@/domain/abstracts/engine.abstract'; +import { ConfigsUsecases } from '../configs/configs.usecase'; @Injectable() export class EnginesUsecases { @@ -28,6 +32,8 @@ export class EnginesUsecases { private readonly fileManagerService: FileManagerService, private readonly downloadManagerService: DownloadManagerService, private readonly extensionRepository: ExtensionRepository, + private readonly configsUsecases: ConfigsUsecases, + private readonly eventEmitter: EventEmitter2, ) {} /** @@ -40,7 +46,7 @@ export class EnginesUsecases { description: engine.description, version: engine.version, productName: engine.productName, - initialized: engine.initalized, + status: engine.status?.toLowerCase(), })); } @@ -57,7 +63,7 @@ export class EnginesUsecases { description: engine.description, version: engine.version, productName: engine.productName, - initialized: engine.initalized, + status: engine.status?.toLowerCase(), } : undefined, ); @@ -144,10 +150,38 @@ export class EnginesUsecases { // Update states await this.extensionRepository.findOne(engine).then((e) => { - if (e) e.initalized = true; + if (e) e.status = EngineStatus.READY; }); }; + /** + * Update engine's configurations + * @param config Configuration Key + * @param value Configuration Value + * @param engine Configuration Group where the key belongs + */ + async updateConfigs( + config: string, + value: string, + engine: string, + ): Promise { + if (!engine || !(await this.extensionRepository.findOne(engine))) + throw new ForbiddenException('Engine not found'); + + return this.configsUsecases.saveConfig(config, value, engine); + } + + /** + * Get the configurations of an engine. + * @param engine + * @returns + */ + async getEngineConfigs(engine: string) { + if (!engine || !(await this.extensionRepository.findOne(engine))) + throw new ForbiddenException('Engine not found'); + return this.configsUsecases.getGroupConfigs(engine); + } + /** * Install CUDA Toolkit dependency (dll/so files) * @param options