feat: cortex benchmark command

cortex-js/package.json

@@ -48,11 +48,13 @@
     "decompress": "^4.2.1",
     "js-yaml": "^4.1.0",
     "nest-commander": "^3.13.0",
+    "openai": "^4.50.0",
     "readline": "^1.3.0",
     "reflect-metadata": "^0.2.0",
     "rxjs": "^7.8.1",
     "sqlite": "^5.1.1",
     "sqlite3": "^5.1.7",
+    "systeminformation": "^5.22.10",
     "typeorm": "^0.3.20",
     "ulid": "^2.3.0",
     "update-notifier": "^5.0.0",

cortex-js/src/command.module.ts

@@ -28,6 +28,7 @@ import { PSCommand } from './infrastructure/commanders/ps.command';
 import { KillCommand } from './infrastructure/commanders/kill.command';
 import { PresetCommand } from './infrastructure/commanders/presets.command';
 import { EmbeddingCommand } from './infrastructure/commanders/embeddings.command';
+import { BenchmarkCommand } from './infrastructure/commanders/benchmark.command';
 
 @Module({
   imports: [
@@ -56,6 +57,7 @@ import { EmbeddingCommand } from './infrastructure/commanders/embeddings.command
     KillCommand,
     PresetCommand,
     EmbeddingCommand,
+    BenchmarkCommand,
 
     // Questions
     InitRunModeQuestions,

cortex-js/src/file-manager/file-manager.service.ts

@@ -12,6 +12,7 @@ export class FileManagerService {
   private modelFolderName = 'models';
   private presetFolderName = 'presets';
   private extensionFoldername = 'extensions';
+  private benchmarkFoldername = 'benchmark';
   private cortexCppFolderName = 'cortex-cpp';
 
   /**
@@ -116,4 +117,14 @@ export class FileManagerService {
     const dataFolderPath = await this.getDataFolderPath();
     return join(dataFolderPath, this.extensionFoldername);
   }
+
+  /**
+   * Get the benchmark folder path
+   * Usually it is located at the home directory > cortex > extensions
+   * @returns the path to the extensions folder
+   */
+  async getBenchmarkPath(): Promise<string> {
+    const dataFolderPath = await this.getDataFolderPath();
+    return join(dataFolderPath, this.benchmarkFoldername);
+  }
 }

cortex-js/src/infrastructure/commanders/benchmark.command.ts

@@ -0,0 +1,18 @@
+import { CommandRunner, SubCommand } from 'nest-commander';
+import { BenchmarkCliUsecases } from './usecases/benchmark.cli.usecases';
+
+@SubCommand({
+  name: 'benchmark',
+  subCommands: [],
+  description:
+    'Benchmark and analyze the performance of a specific AI model using a variety of system resources',
+})
+export class BenchmarkCommand extends CommandRunner {
+  constructor(private readonly benchmarkUsecases: BenchmarkCliUsecases) {
+    super();
+  }
+
+  async run(): Promise<void> {
+    return this.benchmarkUsecases.benchmark();
+  }
+}

cortex-js/src/infrastructure/commanders/chat.command.ts

@@ -6,8 +6,9 @@ import {
 } from 'nest-commander';
 import { ChatCliUsecases } from './usecases/chat.cli.usecases';
 import { exit } from 'node:process';
-import { ModelStat, PSCliUsecases } from './usecases/ps.cli.usecases';
+import { PSCliUsecases } from './usecases/ps.cli.usecases';
 import { ModelsUsecases } from '@/usecases/models/models.usecases';
+import { ModelStat } from './types/model-stat.interface';
 
 type ChatOptions = {
   threadId?: string;

cortex-js/src/infrastructure/commanders/cortex-command.commander.ts

@@ -10,6 +10,7 @@ import { KillCommand } from './kill.command';
 import pkg from '@/../package.json';
 import { PresetCommand } from './presets.command';
 import { EmbeddingCommand } from './embeddings.command';
+import { BenchmarkCommand } from './benchmark.command';
 
 interface CortexCommandOptions {
   version: boolean;
@@ -26,6 +27,7 @@ interface CortexCommandOptions {
     KillCommand,
     PresetCommand,
     EmbeddingCommand,
+    BenchmarkCommand,
   ],
   description: 'Cortex CLI',
 })

cortex-js/src/infrastructure/commanders/embeddings.command.ts

@@ -5,9 +5,10 @@ import {
   SubCommand,
 } from 'nest-commander';
 import { ModelsUsecases } from '@/usecases/models/models.usecases';
-import { ModelStat, PSCliUsecases } from './usecases/ps.cli.usecases';
+import { PSCliUsecases } from './usecases/ps.cli.usecases';
 import { ChatCliUsecases } from './usecases/chat.cli.usecases';
 import { inspect } from 'util';
+import { ModelStat } from './types/model-stat.interface';
 
 interface EmbeddingCommandOptions {
   encoding_format?: string;

cortex-js/src/infrastructure/commanders/types/benchmark-config.interface.ts

@@ -0,0 +1,28 @@
+import { ChatCompletionMessageParam } from 'openai/resources';
+
+export interface BenchmarkConfig {
+  api: {
+    base_url: string;
+    api_key: string;
+    parameters: {
+      messages: ChatCompletionMessageParam[];
+      model: string;
+      stream?: boolean;
+      max_tokens?: number;
+      stop?: string[];
+      frequency_penalty?: number;
+      presence_penalty?: number;
+      temperature?: number;
+      top_p?: number;
+    };
+  };
+  prompts?: {
+    min: number;
+    max: number;
+    samples: number;
+  };
+  output: string;
+  concurrency: number;
+  num_rounds: number;
+  hardware: string[];
+}

cortex-js/src/infrastructure/commanders/types/model-stat.interface.ts

@@ -0,0 +1,8 @@
+export interface ModelStat {
+  modelId: string;
+  engine?: string;
+  duration?: string;
+  status: string;
+  vram?: string;
+  ram?: string;
+}