Merge branch 'dev' into feat/cortex-publish-launchpad

cortex-cpp/CMakeLists.txt

@@ -4,6 +4,13 @@ project(cortex-cpp C CXX)
 
 # Build using CMAKE-JS
 if(DEFINED CMAKE_JS_INC)
+  if(WIN32)
+  add_definitions(
+    -DV8_COMPRESS_POINTERS
+    -DV8_REVERSE_JSARGS
+    -DV8_COMPRESS_POINTERS_IN_ISOLATE_CAGE
+  )
+  endif()
   include_directories(${CMAKE_JS_INC})
 endif()
 
@@ -76,7 +83,19 @@ if(DEFINED CMAKE_JS_INC)
 
   add_library(${PROJECT_NAME} SHARED addon.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/utils/cpuid/cpu_info.cc
+    ${CMAKE_JS_SRC}
   )
+
+  if(WIN32)
+    target_link_libraries(${PROJECT_NAME}
+      PRIVATE
+      msvcprt.lib
+      msvcrt.lib
+      vcruntime.lib
+      ucrt.lib
+      ${CMAKE_JS_LIB}
+    )
+  endif()
 else() # Official build
   add_executable(${PROJECT_NAME} main.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/utils/cpuid/cpu_info.cc

cortex-js/package.json

@@ -54,7 +54,7 @@
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.1",
     "cli-progress": "^3.12.0",
-    "cortex-cpp": "0.4.25",
+    "cortex-cpp": "0.4.34",
     "cortexso-node": "^0.0.4",
     "cpu-instructions": "^0.0.11",
     "decompress": "^4.2.1",

cortex-js/src/extensions/anthropic.engine.ts

@@ -93,7 +93,7 @@ export default class AnthropicEngineExtension extends OAIEngineExtension {
       return {
         choices: [
           {
-            delta: {
+            message: {
               content: data.content[0].text,
             },
           },

cortex-js/src/index.ts

@@ -5,6 +5,7 @@ import {
 } from '@/infrastructure/constants/cortex';
 import { getApp } from './app';
 import chalk from 'chalk';
+import { CortexUsecases } from './usecases/cortex/cortex.usecases';
 
 /**
  * Start the API server
@@ -17,6 +18,8 @@ export async function start(host?: string, port?: number) {
 
   try {
     await app.listen(sPort, sHost);
+    const cortexUsecases = await app.resolve(CortexUsecases);
+    await cortexUsecases.startCortex();
     console.log(chalk.blue(`Started server at http://${sHost}:${sPort}`));
     console.log(
       chalk.blue(`API Playground available at http://${sHost}:${sPort}/api`),

cortex-js/src/infrastructure/commanders/chat.command.ts

@@ -19,12 +19,13 @@ import { Engines } from './types/engine.interface';
 import { join } from 'path';
 import { EnginesUsecases } from '@/usecases/engines/engines.usecase';
 import { FileManagerService } from '../services/file-manager/file-manager.service';
-import { isLocalModel } from '@/utils/normalize-model-id';
+import { isLocalModel, isRemoteEngine } from '@/utils/normalize-model-id';
 
 type ChatOptions = {
   threadId?: string;
   message?: string;
   attach: boolean;
+  preset?: string;
 };
 
 @SubCommand({
@@ -89,6 +90,7 @@ export class ChatCommand extends BaseCommand {
     const engine = existingModel.engine || Engines.llamaCPP;
     // Pull engine if not exist
     if (
+      !isRemoteEngine(engine) &&
       !existsSync(join(await this.fileService.getCortexCppEnginePath(), engine))
     ) {
       await this.initUsecases.installEngine(undefined, 'latest', engine);
@@ -106,7 +108,7 @@ export class ChatCommand extends BaseCommand {
     );
     return this.cortexUsecases
       .startCortex()
-      .then(() => this.modelsCliUsecases.startModel(modelId))
+      .then(() => this.modelsCliUsecases.startModel(modelId, options.preset))
       .then(() =>
         this.chatCliUsecases.chat(
           modelId,
@@ -156,4 +158,12 @@ export class ChatCommand extends BaseCommand {
   parseAttach() {
     return true;
   }
+
+  @Option({
+    flags: '-p, --preset <preset>',
+    description: 'Apply a chat preset to the chat session',
+  })
+  parsePreset(value: string) {
+    return value;
+  }
 }

cortex-js/src/infrastructure/commanders/cortex-command.commander.ts

@@ -1,3 +1,4 @@
+import pkg from '@/../package.json';
 import { RootCommand, CommandRunner, Option } from 'nest-commander';
 import { ChatCommand } from './chat.command';
 import { ModelsCommand } from './models.command';
@@ -18,13 +19,15 @@ import { FileManagerService } from '../services/file-manager/file-manager.servic
 import { CortexUsecases } from '@/usecases/cortex/cortex.usecases';
 import { ServeStopCommand } from './sub-commands/serve-stop.command';
 import ora from 'ora';
+import { printSlogan } from '@/utils/logo';
 import { EnginesSetCommand } from './engines/engines-set.command';
 
 type ServeOptions = {
   address?: string;
   port?: number;
   logs?: boolean;
   dataFolder?: string;
+  version?: boolean;
 };
 
 @RootCommand({
@@ -58,8 +61,15 @@ export class CortexCommand extends CommandRunner {
     const host = options?.address || defaultCortexJsHost;
     const port = options?.port || defaultCortexJsPort;
     const showLogs = options?.logs || false;
+    const showVersion = options?.version || false;
     const dataFolderPath = options?.dataFolder;
-
+    if (showVersion) {
+      printSlogan();
+      console.log('\n');
+      console.log(`Cortex CLI - v${pkg.version}`);
+      console.log(chalk.blue(`Github: ${pkg.homepage}`));
+      return;
+    }
     return this.startServer(host, port, showLogs, dataFolderPath);
   }
 
@@ -114,13 +124,15 @@ export class CortexCommand extends CommandRunner {
         apiServerPort: port,
         dataFolderPath: dataFolderPath || config.dataFolderPath,
       });
+      process.exit(1);
     } catch (e) {
       console.error(e);
       // revert the data folder path if it was set
       await this.fileManagerService.writeConfigFile({
         ...config,
       });
       console.error(`Failed to start server. Is port ${port} in use?`);
+      process.exit(0);
     }
   }
 
@@ -155,4 +167,12 @@ export class CortexCommand extends CommandRunner {
   parseDataFolder(value: string) {
     return value;
   }
+
+  @Option({
+    flags: '-v, --version',
+    description: 'Show version',
+  })
+  parseVersion() {
+    return true;
+  }
 }

cortex-js/src/infrastructure/commanders/models.command.ts

@@ -1,30 +1,86 @@
-import { CommandRunner, SubCommand } from 'nest-commander';
+import { SubCommand } from 'nest-commander';
 import { ModelStartCommand } from './models/model-start.command';
 import { ModelGetCommand } from './models/model-get.command';
 import { ModelListCommand } from './models/model-list.command';
 import { ModelStopCommand } from './models/model-stop.command';
-import { ModelPullCommand } from './models/model-pull.command';
 import { ModelRemoveCommand } from './models/model-remove.command';
 import { ModelUpdateCommand } from './models/model-update.command';
-import { RunCommand } from './shortcuts/run.command';
 import { BaseCommand } from './base.command';
+import { CortexUsecases } from '@/usecases/cortex/cortex.usecases';
+import { ModuleRef } from '@nestjs/core';
+import { ModelPullCommand } from './models/model-pull.command';
 
 @SubCommand({
   name: 'models',
   subCommands: [
-    ModelPullCommand,
     ModelStartCommand,
     ModelStopCommand,
     ModelListCommand,
     ModelGetCommand,
     ModelRemoveCommand,
     ModelUpdateCommand,
-    RunCommand,
   ],
   description: 'Subcommands for managing models',
 })
 export class ModelsCommand extends BaseCommand {
-  async runCommand(): Promise<void> {
-    this.command?.help();
+  constructor(
+    private readonly moduleRef: ModuleRef,
+    readonly cortexUsecases: CortexUsecases,
+  ) {
+    super(cortexUsecases);
+  }
+
+  commandMap: { [key: string]: any } = {
+    pull: ModelPullCommand,
+    start: ModelStartCommand,
+    stop: ModelStopCommand,
+    list: ModelListCommand,
+    get: ModelGetCommand,
+    remove: ModelRemoveCommand,
+    update: ModelUpdateCommand,
+  };
+  async runCommand(passedParam: string[], options: any) {
+    const [parameter, command, ...restParams] = passedParam;
+    if (command !== 'list' && !parameter) {
+      console.error('Model id is required.');
+      return;
+    }
+
+    // Handle the commands accordingly
+    const commandClass = this.commandMap[command as string];
+    if (!commandClass) {
+      this.command?.help();
+      return;
+    }
+    const modelId = parameter;
+    await this.runModelCommand(
+      commandClass,
+      [modelId, ...(restParams || [])],
+      options,
+    );
+  }
+
+  private async runModelCommand(
+    commandClass: any,
+    params: string[] = [],
+    options?: any,
+  ) {
+    const commandInstance = this.moduleRef.get(commandClass, { strict: false });
+    if (commandInstance) {
+      await commandInstance.run(params, options);
+    } else {
+      console.error('Command not found.');
+    }
+  }
+
+  help() {
+    console.log('Usage: cortex models <subcommand|parameter> [subcommand]');
+    console.log('Commands:');
+    console.log('  <model_id> start  - Start a model by ID');
+    console.log('  <model_id> stop  - Stop a model by ID');
+    console.log('  list - List all available models');
+    console.log('  <model_id> get  - Get model details by ID');
+    console.log('  <model_id> remove  - Remove a model by ID');
+    console.log('  <model_id> update  - Update a model by ID');
   }
 }

cortex-js/src/infrastructure/commanders/models/model-start.command.ts

@@ -12,6 +12,7 @@ import { Engines } from '../types/engine.interface';
 import { checkModelCompatibility } from '@/utils/model-check';
 import { EnginesUsecases } from '@/usecases/engines/engines.usecase';
 import { BaseCommand } from '../base.command';
+import { isRemoteEngine } from '@/utils/normalize-model-id';
 
 type ModelStartOptions = {
   attach: boolean;
@@ -72,6 +73,7 @@ export class ModelStartCommand extends BaseCommand {
     const engine = existingModel.engine || Engines.llamaCPP;
     // Pull engine if not exist
     if (
+      !isRemoteEngine(engine) &&
       !existsSync(join(await this.fileService.getCortexCppEnginePath(), engine))
     ) {
       await this.initUsecases.installEngine(undefined, 'latest', engine);

cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts

@@ -12,6 +12,7 @@ import { Engines } from '../types/engine.interface';
 import { checkModelCompatibility } from '@/utils/model-check';
 import { EnginesUsecases } from '@/usecases/engines/engines.usecase';
 import { BaseCommand } from '../base.command';
+import { isLocalModel, isRemoteEngine } from '@/utils/normalize-model-id';
 
 type RunOptions = {
   threadId?: string;
@@ -69,11 +70,7 @@ export class RunCommand extends BaseCommand {
 
     // Second check if model is available
     const existingModel = await this.modelsCliUsecases.getModel(modelId);
-    if (
-      !existingModel ||
-      !Array.isArray(existingModel.files) ||
-      /^(http|https):\/\/[^/]+\/.*/.test(existingModel.files[0])
-    ) {
+    if (!existingModel || !isLocalModel(existingModel.files)) {
       checkingSpinner.fail(`Model is not available`);
       process.exit(1);
     }
@@ -82,6 +79,7 @@ export class RunCommand extends BaseCommand {
     const engine = existingModel.engine || Engines.llamaCPP;
     // Pull engine if not exist
     if (
+      !isRemoteEngine(engine) &&
       !existsSync(join(await this.fileService.getCortexCppEnginePath(), engine))
     ) {
       await this.initUsecases.installEngine(undefined, 'latest', engine);

cortex-js/src/infrastructure/commanders/types/engine.interface.ts

@@ -17,3 +17,10 @@ export const EngineNamesMap: {
   [Engines.onnx]: 'onnx',
   [Engines.tensorrtLLM]: 'tensorrt-llm',
 };
+
+export const RemoteEngines: Engines[] = [
+  Engines.groq,
+  Engines.mistral,
+  Engines.openai,
+  Engines.anthropic,
+];

cortex-js/src/usecases/chat/chat.usecases.ts

@@ -50,7 +50,10 @@ export class ChatUsecases {
       throw new Error(`No engine found with name: ${model.engine}`);
     }
     try {
-      return await engine.inference(createChatDto, headers);
+      return await engine.inference(
+        { ...createChatDto, engine: model.engine },
+        headers,
+      );
     } catch (error) {
       await this.telemetryUseCases.createCrashReport(
         error,

cortex-js/src/utils/normalize-model-id.ts

@@ -1,5 +1,9 @@
 import { ModelArtifact } from '@/domain/models/model.interface';
 import { getGpuInfo } from './cuda';
+import {
+  Engines,
+  RemoteEngines,
+} from '@/infrastructure/commanders/types/engine.interface';
 
 export const normalizeModelId = (modelId: string): string => {
   return modelId.replace(':main', '').replace(/[:/]/g, '-');
@@ -15,6 +19,10 @@ export const isLocalModel = (
   );
 };
 
+export const isRemoteEngine = (engine: string): boolean => {
+  return RemoteEngines.includes(engine as Engines);
+};
+
 /**
  * Parse the model hub engine branch
  * @param branch