diff --git a/cortex-js/package.json b/cortex-js/package.json index 7cb44f2a5..ef20491ce 100644 --- a/cortex-js/package.json +++ b/cortex-js/package.json @@ -11,7 +11,7 @@ "scripts": { "dev": "nest dev", "build": "nest build && cpx \"cpuinfo/bin/**\" dist/bin", - "build:binary": "run-script-os && cpx \"cpuinfo/bin/**\" dist/bin", + "build:binary": "npx -q patch-package && run-script-os && cpx \"cpuinfo/bin/**\" dist/bin", "build:binary:windows": "bun build --compile ./src/command.ts --outfile cortex.exe --external @nestjs/microservices --external @nestjs/websockets/socket-module --external class-transformer/storage", "build:binary:linux": "bun build --compile ./src/command.ts --outfile cortex --external @nestjs/microservices --external @nestjs/websockets/socket-module --external class-transformer/storage", "build:binary:macos": "bun build --compile ./src/command.ts --outfile cortex --external @nestjs/microservices --external @nestjs/websockets/socket-module --external class-transformer/storage", @@ -31,8 +31,7 @@ "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", "test:e2e": "jest --config ./test/jest-e2e.json", "typeorm": "typeorm-ts-node-esm", - "build:dev": "npx nest build && chmod +x ./dist/src/command.js && npm link", - "postinstall": "npx patch-package" + "build:dev": "npx nest build && chmod +x ./dist/src/command.js && npm link" }, "dependencies": { "@huggingface/gguf": "^0.1.5", diff --git a/cortex-js/src/command.ts b/cortex-js/src/command.ts index d4978311b..e85d7050f 100644 --- a/cortex-js/src/command.ts +++ b/cortex-js/src/command.ts @@ -1,4 +1,4 @@ -#!/usr/bin/env node +#!/usr/bin/env node --no-warnings import { CommandFactory } from 'nest-commander'; import { CommandModule } from './command.module'; import { TelemetryUsecases } from './usecases/telemetry/telemetry.usecases'; diff --git a/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts b/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts index bddf5ee7c..c2df6259b 100644 --- a/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts +++ b/cortex-js/src/infrastructure/commanders/shortcuts/run.command.ts @@ -9,6 +9,11 @@ import { exit } from 'node:process'; import { ChatCliUsecases } from '@commanders/usecases/chat.cli.usecases'; import { ModelsCliUsecases } from '@commanders/usecases/models.cli.usecases'; import { ModelNotFoundException } from '@/infrastructure/exception/model-not-found.exception'; +import { existsSync } from 'fs'; +import { join } from 'path'; +import { FileManagerService } from '@/infrastructure/services/file-manager/file-manager.service'; +import { InitCliUsecases } from '../usecases/init.cli.usecases'; +import { Engines } from '../types/engine.interface'; type RunOptions = { threadId?: string; @@ -30,6 +35,8 @@ export class RunCommand extends CommandRunner { private readonly cortexUsecases: CortexUsecases, private readonly chatCliUsecases: ChatCliUsecases, private readonly inquirerService: InquirerService, + private readonly fileService: FileManagerService, + private readonly initUsecases: InitCliUsecases, ) { super(); } @@ -57,6 +64,33 @@ export class RunCommand extends CommandRunner { }); } + // Second check if model is available + const existingModel = await this.modelsCliUsecases.getModel(modelId); + if ( + !existingModel || + !Array.isArray(existingModel.files) || + /^(http|https):\/\/[^/]+\/.*/.test(existingModel.files[0]) + ) { + console.error('Model is not available. Please pull the model first.'); + process.exit(1); + } + + const engine = existingModel.engine || 'cortex.llamacpp'; + // Pull engine if not exist + if ( + !existsSync(join(await this.fileService.getCortexCppEnginePath(), engine)) + ) { + await this.initUsecases.installEngine( + await this.initUsecases.defaultInstallationOptions(), + 'latest', + engine, + ); + } + if (engine === Engines.onnx && process.platform !== 'win32') { + console.error('The ONNX engine does not support this OS yet.'); + process.exit(1); + } + return this.cortexUsecases .startCortex(false) .then(() => this.modelsCliUsecases.startModel(modelId, options.preset)) diff --git a/cortex-js/src/usecases/cortex/cortex.usecases.ts b/cortex-js/src/usecases/cortex/cortex.usecases.ts index 04dafd0b3..b1210471b 100644 --- a/cortex-js/src/usecases/cortex/cortex.usecases.ts +++ b/cortex-js/src/usecases/cortex/cortex.usecases.ts @@ -1,6 +1,6 @@ import { Injectable } from '@nestjs/common'; import { ChildProcess, spawn } from 'child_process'; -import { join } from 'path'; +import { delimiter, join } from 'path'; import { CortexOperationSuccessfullyDto } from '@/infrastructure/dtos/cortex/cortex-operation-successfully.dto'; import { HttpService } from '@nestjs/axios'; @@ -11,6 +11,7 @@ import { CORTEX_CPP_HEALTH_Z_URL, CORTEX_CPP_PROCESS_DESTROY_URL, } from '@/infrastructure/constants/cortex'; +import { Engines } from '@/infrastructure/commanders/types/engine.interface'; @Injectable() export class CortexUsecases { @@ -44,6 +45,11 @@ export class CortexUsecases { throw new Error('The engine is not available, please run "cortex init".'); } + const cortexCPPPath = join( + await this.fileManagerService.getDataFolderPath(), + 'cortex-cpp', + ); + // go up one level to get the binary folder, have to also work on windows this.cortexProcess = spawn(cortexCppPath, args, { detached: !attach, @@ -52,6 +58,11 @@ export class CortexUsecases { env: { ...process.env, CUDA_VISIBLE_DEVICES: '0', + PATH: (process.env.PATH || '').concat(delimiter, cortexCPPPath), + LD_LIBRARY_PATH: (process.env.LD_LIBRARY_PATH || '').concat( + delimiter, + cortexCPPPath, + ), // // Vulkan - Support 1 device at a time for now // ...(executableOptions.vkVisibleDevices?.length > 0 && { // GGML_VULKAN_DEVICE: executableOptions.vkVisibleDevices[0],