Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
refactor: deprecate cortex configs
Browse files Browse the repository at this point in the history
louis-jan committed Jul 22, 2024

Verified

This commit was signed with the committer’s verified signature.
gsmet Guillaume Smet
1 parent 749cf0c commit 3e2e1c5
Showing 23 changed files with 175 additions and 322 deletions.
2 changes: 0 additions & 2 deletions cortex-js/src/app.module.ts
Original file line number Diff line number Diff line change
@@ -29,7 +29,6 @@ import { ContextModule } from './infrastructure/services/context/context.module'
import { ExtensionsModule } from './extensions/extensions.module';
import { ConfigsModule } from './usecases/configs/configs.module';
import { EnginesModule } from './usecases/engines/engines.module';
import { ConfigsController } from './infrastructure/controllers/configs.controller';
import { EnginesController } from './infrastructure/controllers/engines.controller';
import { ResourceManagerModule } from './infrastructure/services/resources-manager/resources-manager.module';

@@ -70,7 +69,6 @@ import { ResourceManagerModule } from './infrastructure/services/resources-manag
StatusController,
ProcessController,
EventsController,
ConfigsController,
EnginesController,
],
providers: [
14 changes: 2 additions & 12 deletions cortex-js/src/command.module.ts
Original file line number Diff line number Diff line change
@@ -21,7 +21,6 @@ import { AssistantsModule } from './usecases/assistants/assistants.module';
import { MessagesModule } from './usecases/messages/messages.module';
import { FileManagerModule } from './infrastructure/services/file-manager/file-manager.module';
import { PSCommand } from './infrastructure/commanders/ps.command';
import { KillCommand } from './infrastructure/commanders/kill.command';
import { PresetCommand } from './infrastructure/commanders/presets.command';
import { TelemetryModule } from './usecases/telemetry/telemetry.module';
import { TelemetryCommand } from './infrastructure/commanders/telemetry.command';
@@ -33,16 +32,13 @@ import { ServeStopCommand } from './infrastructure/commanders/sub-commands/serve
import { ContextModule } from './infrastructure/services/context/context.module';
import { CliUsecasesModule } from './infrastructure/commanders/usecases/cli.usecases.module';
import { ExtensionsModule } from './extensions/extensions.module';
import { ConfigsCommand } from './infrastructure/commanders/configs.command';
import { EnginesCommand } from './infrastructure/commanders/engines.command';
import { ConfigsModule } from './usecases/configs/configs.module';
import { EnginesModule } from './usecases/engines/engines.module';
import { ConfigsGetCommand } from './infrastructure/commanders/configs/configs-get.command';
import { ConfigsListCommand } from './infrastructure/commanders/configs/configs-list.command';
import { ConfigsSetCommand } from './infrastructure/commanders/configs/configs-set.command';
import { EnginesListCommand } from './infrastructure/commanders/engines/engines-list.command';
import { EnginesGetCommand } from './infrastructure/commanders/engines/engines-get.command';
import { EnginesInitCommand } from './infrastructure/commanders/engines/engines-init.command';
import { EnginesSetCommand } from './infrastructure/commanders/engines/engines-set.command';

@Module({
imports: [
@@ -73,7 +69,6 @@ import { EnginesInitCommand } from './infrastructure/commanders/engines/engines-
ModelsCommand,
ChatCommand,
PSCommand,
KillCommand,
PresetCommand,
EmbeddingCommand,
BenchmarkCommand,
@@ -100,16 +95,11 @@ import { EnginesInitCommand } from './infrastructure/commanders/engines/engines-
// Serve
ServeStopCommand,

// Configs
ConfigsCommand,
ConfigsGetCommand,
ConfigsListCommand,
ConfigsSetCommand,

// Engines
EnginesListCommand,
EnginesGetCommand,
EnginesInitCommand,
EnginesSetCommand,
],
})
export class CommandModule {}
9 changes: 8 additions & 1 deletion cortex-js/src/domain/abstracts/engine.abstract.ts
Original file line number Diff line number Diff line change
@@ -3,6 +3,13 @@ import stream from 'stream';
import { Model, ModelSettingParams } from '../../domain/models/model.interface';
import { Extension } from './extension.abstract';

export enum EngineStatus {
READY = 'READY',
MISSING_CONFIGURATION = 'MISSING_CONFIGURATION',
NOT_INITIALIZED = 'NOT_INITIALIZED',
ERROR = 'ERROR',
}

/**
* This class should be extended by any class that represents an engine extension.
* It provides methods for loading and unloading models, and for making inference requests.
@@ -14,7 +21,7 @@ export abstract class EngineExtension extends Extension {

transformResponse?: Function;

initalized: boolean = false;
status: EngineStatus = EngineStatus.READY;

/**
* Makes an inference request to the engine.
4 changes: 2 additions & 2 deletions cortex-js/src/domain/abstracts/extension.abstract.ts
Original file line number Diff line number Diff line change
@@ -21,8 +21,8 @@ export abstract class Extension {
/** @type {string} Extension's version. */
version?: string;

/** @type {boolean} Whether the extension is initialized or not. */
initalized: boolean;
/** @type {string} The status of the extension. */
status: string;

/**
* Called when the extension is loaded.
10 changes: 9 additions & 1 deletion cortex-js/src/extensions/anthropic.engine.ts
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@ import { OAIEngineExtension } from '../domain/abstracts/oai.abstract';
import { ConfigsUsecases } from '@/usecases/configs/configs.usecase';
import { EventEmitter2 } from '@nestjs/event-emitter';
import _ from 'lodash';
import { EngineStatus } from '@/domain/abstracts/engine.abstract';

/**
* A class that implements the InferenceExtension interface from the @janhq/core package.
@@ -16,7 +17,6 @@ export default class AnthropicEngineExtension extends OAIEngineExtension {
productName = 'Anthropic Inference Engine';
description = 'This extension enables Anthropic chat completion API calls';
version = '0.0.1';
initalized = true;
apiKey?: string;

constructor(
@@ -29,6 +29,10 @@ export default class AnthropicEngineExtension extends OAIEngineExtension {
eventEmmitter.on('config.updated', async (data) => {
if (data.group === this.name) {
this.apiKey = data.value;
this.status =
(this.apiKey?.length ?? 0) > 0
? EngineStatus.READY
: EngineStatus.MISSING_CONFIGURATION;
}
});
}
@@ -38,6 +42,10 @@ export default class AnthropicEngineExtension extends OAIEngineExtension {
this.name,
)) as unknown as { apiKey: string };
this.apiKey = configs?.apiKey;
this.status =
(this.apiKey?.length ?? 0) > 0
? EngineStatus.READY
: EngineStatus.MISSING_CONFIGURATION;
}

override async inference(
10 changes: 9 additions & 1 deletion cortex-js/src/extensions/groq.engine.ts
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@ import { HttpService } from '@nestjs/axios';
import { OAIEngineExtension } from '../domain/abstracts/oai.abstract';
import { ConfigsUsecases } from '@/usecases/configs/configs.usecase';
import { EventEmitter2 } from '@nestjs/event-emitter';
import { EngineStatus } from '@/domain/abstracts/engine.abstract';

/**
* A class that implements the InferenceExtension interface from the @janhq/core package.
@@ -14,7 +15,6 @@ export default class GroqEngineExtension extends OAIEngineExtension {
productName = 'Groq Inference Engine';
description = 'This extension enables fast Groq chat completion API calls';
version = '0.0.1';
initalized = true;
apiKey?: string;

constructor(
@@ -27,6 +27,10 @@ export default class GroqEngineExtension extends OAIEngineExtension {
eventEmmitter.on('config.updated', async (data) => {
if (data.group === this.name) {
this.apiKey = data.value;
this.status =
(this.apiKey?.length ?? 0) > 0
? EngineStatus.READY
: EngineStatus.MISSING_CONFIGURATION;
}
});
}
@@ -37,5 +41,9 @@ export default class GroqEngineExtension extends OAIEngineExtension {
)) as unknown as { apiKey: string };

this.apiKey = configs?.apiKey;
this.status =
(this.apiKey?.length ?? 0) > 0
? EngineStatus.READY
: EngineStatus.MISSING_CONFIGURATION;
}
}
10 changes: 9 additions & 1 deletion cortex-js/src/extensions/mistral.engine.ts
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@ import { HttpService } from '@nestjs/axios';
import { OAIEngineExtension } from '../domain/abstracts/oai.abstract';
import { ConfigsUsecases } from '@/usecases/configs/configs.usecase';
import { EventEmitter2 } from '@nestjs/event-emitter';
import { EngineStatus } from '@/domain/abstracts/engine.abstract';

/**
* A class that implements the InferenceExtension interface from the @janhq/core package.
@@ -14,7 +15,6 @@ export default class MistralEngineExtension extends OAIEngineExtension {
productName = 'Mistral Inference Engine';
description = 'This extension enables Mistral chat completion API calls';
version = '0.0.1';
initalized = true;
apiKey?: string;

constructor(
@@ -27,6 +27,10 @@ export default class MistralEngineExtension extends OAIEngineExtension {
eventEmmitter.on('config.updated', async (data) => {
if (data.group === this.name) {
this.apiKey = data.value;
this.status =
(this.apiKey?.length ?? 0) > 0
? EngineStatus.READY
: EngineStatus.MISSING_CONFIGURATION;
}
});
}
@@ -36,5 +40,9 @@ export default class MistralEngineExtension extends OAIEngineExtension {
this.name,
)) as unknown as { apiKey: string };
this.apiKey = configs?.apiKey;
this.status =
(this.apiKey?.length ?? 0) > 0
? EngineStatus.READY
: EngineStatus.MISSING_CONFIGURATION;
}
}
10 changes: 9 additions & 1 deletion cortex-js/src/extensions/openai.engine.ts
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@ import { HttpService } from '@nestjs/axios';
import { OAIEngineExtension } from '../domain/abstracts/oai.abstract';
import { ConfigsUsecases } from '@/usecases/configs/configs.usecase';
import { EventEmitter2 } from '@nestjs/event-emitter';
import { EngineStatus } from '@/domain/abstracts/engine.abstract';

/**
* A class that implements the InferenceExtension interface from the @janhq/core package.
@@ -14,7 +15,6 @@ export default class OpenAIEngineExtension extends OAIEngineExtension {
productName = 'OpenAI Inference Engine';
description = 'This extension enables OpenAI chat completion API calls';
version = '0.0.1';
initalized = true;
apiKey?: string;

constructor(
@@ -27,6 +27,10 @@ export default class OpenAIEngineExtension extends OAIEngineExtension {
eventEmmitter.on('config.updated', async (data) => {
if (data.group === this.name) {
this.apiKey = data.value;
this.status =
(this.apiKey?.length ?? 0) > 0
? EngineStatus.READY
: EngineStatus.MISSING_CONFIGURATION;
}
});
}
@@ -36,5 +40,9 @@ export default class OpenAIEngineExtension extends OAIEngineExtension {
this.name,
)) as unknown as { apiKey: string };
this.apiKey = configs?.apiKey;
this.status =
(this.apiKey?.length ?? 0) > 0
? EngineStatus.READY
: EngineStatus.MISSING_CONFIGURATION;
}
}
27 changes: 0 additions & 27 deletions cortex-js/src/infrastructure/commanders/configs.command.ts

This file was deleted.

This file was deleted.

This file was deleted.

Original file line number Diff line number Diff line change
@@ -4,7 +4,6 @@ import { ModelsCommand } from './models.command';
import { RunCommand } from './shortcuts/run.command';
import { ModelPullCommand } from './models/model-pull.command';
import { PSCommand } from './ps.command';
import { KillCommand } from './kill.command';
import { PresetCommand } from './presets.command';
import { TelemetryCommand } from './telemetry.command';
import { SetCommandContext } from './decorators/CommandContext';
@@ -13,13 +12,13 @@ import { BenchmarkCommand } from './benchmark.command';
import chalk from 'chalk';
import { ContextService } from '../services/context/context.service';
import { EnginesCommand } from './engines.command';
import { ConfigsCommand } from './configs.command';
import { defaultCortexJsHost, defaultCortexJsPort } from '../constants/cortex';
import { getApp } from '@/app';
import { FileManagerService } from '../services/file-manager/file-manager.service';
import { CortexUsecases } from '@/usecases/cortex/cortex.usecases';
import { ServeStopCommand } from './sub-commands/serve-stop.command';
import ora from 'ora';
import { EnginesSetCommand } from './engines/engines-set.command';

type ServeOptions = {
address?: string;
@@ -35,14 +34,13 @@ type ServeOptions = {
RunCommand,
ModelPullCommand,
PSCommand,
KillCommand,
PresetCommand,
TelemetryCommand,
EmbeddingCommand,
BenchmarkCommand,
EnginesCommand,
ConfigsCommand,
ServeStopCommand,
EnginesSetCommand,
],
description: 'Cortex CLI',
})
4 changes: 3 additions & 1 deletion cortex-js/src/infrastructure/commanders/engines.command.ts
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@ import { ModuleRef } from '@nestjs/core';
import { EngineNamesMap } from './types/engine.interface';
import { BaseCommand } from './base.command';
import { CortexUsecases } from '@/usecases/cortex/cortex.usecases';
import { EnginesSetCommand } from './engines/engines-set.command';

@SubCommand({
name: 'engines',
@@ -22,11 +23,12 @@ export class EnginesCommand extends BaseCommand {
list: EnginesListCommand,
get: EnginesGetCommand,
init: EnginesInitCommand,
set: EnginesSetCommand,
};

constructor(
readonly contextService: ContextService,
private readonly moduleRef: ModuleRef,
readonly moduleRef: ModuleRef,
readonly cortexUsecases: CortexUsecases,
) {
super(cortexUsecases);
Original file line number Diff line number Diff line change
@@ -1,31 +1,33 @@
import { CommandRunner, SubCommand } from 'nest-commander';
import { SubCommand } from 'nest-commander';
import { SetCommandContext } from '../decorators/CommandContext';
import { ContextService } from '@/infrastructure/services/context/context.service';
import { ConfigsUsecases } from '@/usecases/configs/configs.usecase';
import { CortexUsecases } from '@/usecases/cortex/cortex.usecases';
import { BaseCommand } from '../base.command';
import { CortexUsecases } from '@/usecases/cortex/cortex.usecases';
import { ConfigsUsecases } from '@/usecases/configs/configs.usecase';

@SubCommand({
name: 'get',
description: 'Get a cortex configuration',
arguments: '<name>',
name: '<name> set <config> <value>',
description: 'Update an engine configurations',
argsDescription: {
name: 'Configuration name to get',
name: 'Engine name to update',
},
})
@SetCommandContext()
export class ConfigsGetCommand extends BaseCommand {
export class EnginesSetCommand extends BaseCommand {
constructor(
private readonly configsUsecases: ConfigsUsecases,
readonly configsUsecases: ConfigsUsecases,
readonly contextService: ContextService,
readonly cortexUsecases: CortexUsecases,
) {
super(cortexUsecases);
}

async runCommand(passedParams: string[]): Promise<void> {
const engineName = passedParams[0];
const config = passedParams[1];
const value = passedParams[2];
return this.configsUsecases
.getGroupConfigs(passedParams[0])
.then(console.table);
.saveConfig(config, value, engineName)
.then(() => console.log('Update engine successfully'));
}
}
25 changes: 0 additions & 25 deletions cortex-js/src/infrastructure/commanders/kill.command.ts

This file was deleted.

Original file line number Diff line number Diff line change
@@ -94,17 +94,15 @@ describe('Helper commands', () => {
// });

test(
'Show / kill running models',
'Show stop running models',
async () => {
const tableMock = stubMethod(console, 'table');

const logMock = stubMethod(console, 'log');
await CommandTestFactory.run(commandInstance, ['kill']);
await CommandTestFactory.run(commandInstance, ['stop']);
await CommandTestFactory.run(commandInstance, ['ps']);

expect(logMock.firstCall?.args[0]).toEqual(
'Cortex processes stopped successfully!',
);
expect(logMock.firstCall?.args[0]).toEqual('API server stopped');
expect(tableMock.firstCall?.args[0]).toBeInstanceOf(Array);
expect(tableMock.firstCall?.args[0].length).toEqual(0);
},

This file was deleted.

81 changes: 0 additions & 81 deletions cortex-js/src/infrastructure/controllers/configs.controller.ts

This file was deleted.

27 changes: 27 additions & 0 deletions cortex-js/src/infrastructure/controllers/engines.controller.ts
Original file line number Diff line number Diff line change
@@ -5,12 +5,15 @@ import {
HttpCode,
UseInterceptors,
Post,
Body,
Patch,
} from '@nestjs/common';
import { ApiOperation, ApiParam, ApiTags, ApiResponse } from '@nestjs/swagger';
import { TransformInterceptor } from '../interceptors/transform.interceptor';
import { EnginesUsecases } from '@/usecases/engines/engines.usecase';
import { EngineDto } from '../dtos/engines/engines.dto';
import { CommonResponseDto } from '../dtos/common/common-response.dto';
import { ConfigUpdateDto } from '../dtos/configs/config-update.dto';

@ApiTags('Engines')
@Controller('engines')
@@ -81,4 +84,28 @@ export class EnginesController {
message: 'Engine initialization started successfully.',
};
}

@HttpCode(200)
@ApiResponse({
status: 200,
description: 'Ok',
type: CommonResponseDto,
})
@ApiOperation({
summary: 'Update the engine',
description: 'Updates the engine with configurations.',
})
@ApiParam({
name: 'name',
required: true,
description: 'The unique identifier of the engine.',
})
@Patch(':name(*)')
update(@Param('name') name: string, @Body() configs: ConfigUpdateDto) {
return this.enginesUsecases.updateConfigs(
configs.config,
configs.value,
name,
);
}
}
11 changes: 1 addition & 10 deletions cortex-js/src/infrastructure/dtos/configs/config-update.dto.ts
Original file line number Diff line number Diff line change
@@ -8,7 +8,7 @@ export class ConfigUpdateDto {
})
@IsString()
@IsOptional()
key: string;
config: string;

// Prompt Settings
@ApiProperty({
@@ -19,13 +19,4 @@ export class ConfigUpdateDto {
@IsString()
@IsOptional()
value: string;

@ApiProperty({
type: String,
example: 'openai',
description: 'The name of the configuration.',
})
@IsString()
@IsOptional()
name?: string;
}
4 changes: 2 additions & 2 deletions cortex-js/src/infrastructure/dtos/engines/engines.dto.ts
Original file line number Diff line number Diff line change
@@ -42,8 +42,8 @@ export class EngineDto implements Partial<Extension> {
@ApiProperty({
type: String,
example: true,
description: 'Whether the engine is initialized or not.',
description: 'The status of the engine.',
})
@IsBoolean()
initalized?: boolean;
status?: string;
}
Original file line number Diff line number Diff line change
@@ -11,6 +11,7 @@ import { HttpService } from '@nestjs/axios';
import LlamaCPPProvider from '@/infrastructure/providers/cortex/llamacpp.provider';
import Onnxprovider from '@/infrastructure/providers/cortex/onnx.provider';
import TensorrtLLMProvider from '@/infrastructure/providers/cortex/tensorrtllm.provider';
import { EngineStatus } from '@/domain/abstracts/engine.abstract';

@Injectable()
export class ExtensionRepositoryImpl implements ExtensionRepository {
@@ -50,34 +51,40 @@ export class ExtensionRepositoryImpl implements ExtensionRepository {
this.httpService,
this.fileManagerService,
);
llamaCPPEngine.initalized = existsSync(
llamaCPPEngine.status = existsSync(
join(
await this.fileManagerService.getCortexCppEnginePath(),
Engines.llamaCPP,
),
);
)
? EngineStatus.READY
: EngineStatus.NOT_INITIALIZED;

const onnxEngine = new Onnxprovider(
this.httpService,
this.fileManagerService,
);
onnxEngine.initalized = existsSync(
onnxEngine.status = existsSync(
join(
await this.fileManagerService.getCortexCppEnginePath(),
Engines.onnx,
),
);
)
? EngineStatus.READY
: EngineStatus.NOT_INITIALIZED;

const tensorrtLLMEngine = new TensorrtLLMProvider(
this.httpService,
this.fileManagerService,
);
tensorrtLLMEngine.initalized = existsSync(
tensorrtLLMEngine.status = existsSync(
join(
await this.fileManagerService.getCortexCppEnginePath(),
Engines.tensorrtLLM,
),
);
)
? EngineStatus.READY
: EngineStatus.NOT_INITIALIZED;

await llamaCPPEngine.onLoad();
await onnxEngine.onLoad();
66 changes: 63 additions & 3 deletions cortex-js/src/usecases/engines/engines.usecase.ts
Original file line number Diff line number Diff line change
@@ -20,6 +20,9 @@ import { cpuInfo } from 'cpu-instructions';
import { DownloadManagerService } from '@/infrastructure/services/download-manager/download-manager.service';
import { DownloadType } from '@/domain/models/download.interface';
import { Engines } from '@/infrastructure/commanders/types/engine.interface';
import { CommonResponseDto } from '@/infrastructure/dtos/common/common-response.dto';
import { EventEmitter2 } from '@nestjs/event-emitter';
import { EngineStatus } from '@/domain/abstracts/engine.abstract';

@Injectable()
export class EnginesUsecases {
@@ -28,6 +31,7 @@ export class EnginesUsecases {
private readonly fileManagerService: FileManagerService,
private readonly downloadManagerService: DownloadManagerService,
private readonly extensionRepository: ExtensionRepository,
private readonly eventEmitter: EventEmitter2,
) {}

/**
@@ -40,7 +44,7 @@ export class EnginesUsecases {
description: engine.description,
version: engine.version,
productName: engine.productName,
initialized: engine.initalized,
status: engine.status?.toLowerCase(),
}));
}

@@ -57,7 +61,7 @@ export class EnginesUsecases {
description: engine.description,
version: engine.version,
productName: engine.productName,
initialized: engine.initalized,
status: engine.status?.toLowerCase(),
}
: undefined,
);
@@ -144,10 +148,66 @@ export class EnginesUsecases {

// Update states
await this.extensionRepository.findOne(engine).then((e) => {
if (e) e.initalized = true;
if (e) e.status = EngineStatus.READY;
});
};

/**
* Update engine's configurations
* @param key Configuration Key
* @param value Configuration Value
* @param engine Configuration Group where the key belongs
*/
async updateConfigs(
key: string,
value: string,
engine: string,
): Promise<CommonResponseDto> {
const configs = await this.fileManagerService.getConfig();

const groupConfigs = configs[
engine as keyof typeof configs
] as unknown as object;
const newConfigs = {
...configs,
...(engine
? {
[engine]: {
...groupConfigs,
[key]: value,
},
}
: {}),
};

return this.fileManagerService
.writeConfigFile(newConfigs)
.then(async () => {
if (engine) {
this.eventEmitter.emit('config.updated', {
engine,
key,
value,
});
}
})
.then(() => {
return {
message: 'The config has been successfully updated.',
};
});
}

/**
* Get the configurations of an engine.
* @param engine
* @returns
*/
async getEngineConfigs(engine: string) {
const configs = await this.fileManagerService.getConfig();
return configs[engine as keyof typeof configs] as unknown as object;
}

/**
* Install CUDA Toolkit dependency (dll/so files)
* @param options

0 comments on commit 3e2e1c5

Please sign in to comment.