From 99388f44186112e5655c1ca82a2af0bc7bd5d446 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 16 Jan 2025 11:52:59 +0700 Subject: [PATCH 1/9] chore: log engine variant symlink error --- extensions/engine-management-extension/src/node/index.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/extensions/engine-management-extension/src/node/index.ts b/extensions/engine-management-extension/src/node/index.ts index cf26d9290f..b1305d6fb6 100644 --- a/extensions/engine-management-extension/src/node/index.ts +++ b/extensions/engine-management-extension/src/node/index.ts @@ -138,9 +138,11 @@ const symlinkEngines = async () => { await mkdir(path.join(symlinkEnginePath, variant), { recursive: true, - }).catch(console.error) + }).catch((error) => log(JSON.stringify(error))) - await symlink(targetVariantPath, symlinkVariantPath).catch(console.error) + await symlink(targetVariantPath, symlinkVariantPath).catch((error) => + log(JSON.stringify(error)) + ) console.log(`Symlink created: ${targetVariantPath} -> ${symlinkEnginePath}`) } } From ae7a6c5d64f385f1bc160df1da23f1157c089290 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 16 Jan 2025 12:36:07 +0700 Subject: [PATCH 2/9] chore: update symbolic type --- .../src/node/index.ts | 4 ++-- .../src/node/index.ts | 20 +++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/extensions/engine-management-extension/src/node/index.ts b/extensions/engine-management-extension/src/node/index.ts index b1305d6fb6..31ad90ed27 100644 --- a/extensions/engine-management-extension/src/node/index.ts +++ b/extensions/engine-management-extension/src/node/index.ts @@ -140,8 +140,8 @@ const symlinkEngines = async () => { recursive: true, }).catch((error) => log(JSON.stringify(error))) - await symlink(targetVariantPath, symlinkVariantPath).catch((error) => - log(JSON.stringify(error)) + await symlink(targetVariantPath, symlinkVariantPath, 'junction').catch( + (error) => log(JSON.stringify(error)) ) console.log(`Symlink created: ${targetVariantPath} -> ${symlinkEnginePath}`) } diff --git a/extensions/inference-cortex-extension/src/node/index.ts b/extensions/inference-cortex-extension/src/node/index.ts index dda92fbcff..5d969701e3 100644 --- a/extensions/inference-cortex-extension/src/node/index.ts +++ b/extensions/inference-cortex-extension/src/node/index.ts @@ -1,5 +1,10 @@ import path from 'path' -import { appResourcePath, getJanDataFolderPath, log, SystemInformation } from '@janhq/core/node' +import { + appResourcePath, + getJanDataFolderPath, + log, + SystemInformation, +} from '@janhq/core/node' import { ProcessWatchdog } from './watchdog' import { readdir, symlink } from 'fs/promises' @@ -19,12 +24,9 @@ function run(systemInfo?: SystemInformation): Promise { let binaryName = `cortex-server${process.platform === 'win32' ? '.exe' : ''}` const binPath = path.join(__dirname, '..', 'bin') await createEngineSymlinks(binPath) - + const executablePath = path.join(binPath, binaryName) - const sharedPath = path.join( - appResourcePath(), - 'shared' - ) + const sharedPath = path.join(appResourcePath(), 'shared') // Execute the binary log(`[CORTEX]:: Spawn cortex at path: ${executablePath}`) @@ -63,7 +65,7 @@ function run(systemInfo?: SystemInformation): Promise { /** * Create symlinks for the engine shared libraries - * @param binPath + * @param binPath */ async function createEngineSymlinks(binPath: string) { const sharedPath = path.join(appResourcePath(), 'shared') @@ -72,7 +74,9 @@ async function createEngineSymlinks(binPath: string) { if (sharedLibFile.endsWith('.dll') || sharedLibFile.endsWith('.so')) { const targetDllPath = path.join(sharedPath, sharedLibFile) const symlinkDllPath = path.join(binPath, sharedLibFile) - await symlink(targetDllPath, symlinkDllPath).catch(console.error) + await symlink(targetDllPath, symlinkDllPath, 'junction').catch( + console.error + ) console.log(`Symlink created: ${targetDllPath} -> ${symlinkDllPath}`) } } From f5e2712172fdbd701511d5c62d0f6b14a5cd0746 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 16 Jan 2025 13:25:27 +0700 Subject: [PATCH 3/9] chore: symlink dll should be type file instead of junction --- extensions/inference-cortex-extension/src/node/index.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/extensions/inference-cortex-extension/src/node/index.ts b/extensions/inference-cortex-extension/src/node/index.ts index 5d969701e3..2638cd69d4 100644 --- a/extensions/inference-cortex-extension/src/node/index.ts +++ b/extensions/inference-cortex-extension/src/node/index.ts @@ -74,9 +74,7 @@ async function createEngineSymlinks(binPath: string) { if (sharedLibFile.endsWith('.dll') || sharedLibFile.endsWith('.so')) { const targetDllPath = path.join(sharedPath, sharedLibFile) const symlinkDllPath = path.join(binPath, sharedLibFile) - await symlink(targetDllPath, symlinkDllPath, 'junction').catch( - console.error - ) + await symlink(targetDllPath, symlinkDllPath, 'file').catch(console.error) console.log(`Symlink created: ${targetDllPath} -> ${symlinkDllPath}`) } } From a4649fa9cc64bf5e4d13ef11c1620bfed6a71f6f Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 16 Jan 2025 15:26:15 +0700 Subject: [PATCH 4/9] chore: bump engine version to 0.1.46 --- extensions/engine-management-extension/rolldown.config.mjs | 4 ++-- extensions/inference-cortex-extension/download.bat | 2 +- extensions/inference-cortex-extension/download.sh | 2 +- extensions/inference-cortex-extension/rolldown.config.mjs | 2 +- extensions/inference-cortex-extension/src/node/index.ts | 4 +++- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/extensions/engine-management-extension/rolldown.config.mjs b/extensions/engine-management-extension/rolldown.config.mjs index b59d395d1d..7e9a2d1567 100644 --- a/extensions/engine-management-extension/rolldown.config.mjs +++ b/extensions/engine-management-extension/rolldown.config.mjs @@ -13,7 +13,7 @@ export default defineConfig([ NODE: JSON.stringify(`${pkgJson.name}/${pkgJson.node}`), API_URL: JSON.stringify('http://127.0.0.1:39291'), SOCKET_URL: JSON.stringify('ws://127.0.0.1:39291'), - CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.43'), + CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.46'), DEFAULT_REMOTE_ENGINES: JSON.stringify(engines), DEFAULT_REMOTE_MODELS: JSON.stringify(models), }, @@ -26,7 +26,7 @@ export default defineConfig([ file: 'dist/node/index.cjs.js', }, define: { - CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.43'), + CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.46'), }, }, { diff --git a/extensions/inference-cortex-extension/download.bat b/extensions/inference-cortex-extension/download.bat index 00e37cd192..1dfdfd4495 100644 --- a/extensions/inference-cortex-extension/download.bat +++ b/extensions/inference-cortex-extension/download.bat @@ -2,7 +2,7 @@ set BIN_PATH=./bin set SHARED_PATH=./../../electron/shared set /p CORTEX_VERSION=<./bin/version.txt -set ENGINE_VERSION=0.1.43 +set ENGINE_VERSION=0.1.46 @REM Download cortex.llamacpp binaries set DOWNLOAD_URL=https://github.com/janhq/cortex.llamacpp/releases/download/v%ENGINE_VERSION%/cortex.llamacpp-%ENGINE_VERSION%-windows-amd64 diff --git a/extensions/inference-cortex-extension/download.sh b/extensions/inference-cortex-extension/download.sh index 9d35bd117e..cf3ad372dd 100755 --- a/extensions/inference-cortex-extension/download.sh +++ b/extensions/inference-cortex-extension/download.sh @@ -2,7 +2,7 @@ # Read CORTEX_VERSION CORTEX_VERSION=$(cat ./bin/version.txt) -ENGINE_VERSION=0.1.43 +ENGINE_VERSION=0.1.46 CORTEX_RELEASE_URL="https://github.com/janhq/cortex.cpp/releases/download" ENGINE_DOWNLOAD_URL="https://github.com/janhq/cortex.llamacpp/releases/download/v${ENGINE_VERSION}/cortex.llamacpp-${ENGINE_VERSION}" CUDA_DOWNLOAD_URL="https://github.com/janhq/cortex.llamacpp/releases/download/v${ENGINE_VERSION}" diff --git a/extensions/inference-cortex-extension/rolldown.config.mjs b/extensions/inference-cortex-extension/rolldown.config.mjs index c20bf7ed00..cebfa71904 100644 --- a/extensions/inference-cortex-extension/rolldown.config.mjs +++ b/extensions/inference-cortex-extension/rolldown.config.mjs @@ -111,7 +111,7 @@ export default defineConfig([ SETTINGS: JSON.stringify(defaultSettingJson), CORTEX_API_URL: JSON.stringify('http://127.0.0.1:39291'), CORTEX_SOCKET_URL: JSON.stringify('ws://127.0.0.1:39291'), - CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.43'), + CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.46'), }, }, { diff --git a/extensions/inference-cortex-extension/src/node/index.ts b/extensions/inference-cortex-extension/src/node/index.ts index 2638cd69d4..c93da671cd 100644 --- a/extensions/inference-cortex-extension/src/node/index.ts +++ b/extensions/inference-cortex-extension/src/node/index.ts @@ -74,7 +74,9 @@ async function createEngineSymlinks(binPath: string) { if (sharedLibFile.endsWith('.dll') || sharedLibFile.endsWith('.so')) { const targetDllPath = path.join(sharedPath, sharedLibFile) const symlinkDllPath = path.join(binPath, sharedLibFile) - await symlink(targetDllPath, symlinkDllPath, 'file').catch(console.error) + await symlink(targetDllPath, symlinkDllPath, 'file').catch((error) => + log(JSON.stringify(error)) + ) console.log(`Symlink created: ${targetDllPath} -> ${symlinkDllPath}`) } } From 687b034b064710fc0a942c2e67cc0c9e80e8e7b8 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 16 Jan 2025 16:04:28 +0700 Subject: [PATCH 5/9] fix: file symlink does not work on windows for cuda dependencies --- .../inference-cortex-extension/download.bat | 4 ++-- .../inference-cortex-extension/download.sh | 5 +++-- .../src/node/index.ts | 20 ------------------- 3 files changed, 5 insertions(+), 24 deletions(-) diff --git a/extensions/inference-cortex-extension/download.bat b/extensions/inference-cortex-extension/download.bat index 1dfdfd4495..fc359557c7 100644 --- a/extensions/inference-cortex-extension/download.bat +++ b/extensions/inference-cortex-extension/download.bat @@ -19,8 +19,8 @@ call .\node_modules\.bin\download %DOWNLOAD_URL%-avx.tar.gz -e --strip 1 -o %SHA call .\node_modules\.bin\download %DOWNLOAD_URL%-avx2.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx2/v%ENGINE_VERSION% call .\node_modules\.bin\download %DOWNLOAD_URL%-avx512.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-avx512/v%ENGINE_VERSION% call .\node_modules\.bin\download %DOWNLOAD_URL%-vulkan.tar.gz -e --strip 1 -o %SHARED_PATH%/engines/cortex.llamacpp/windows-amd64-vulkan/v%ENGINE_VERSION% -call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cuda-12-0-windows-amd64.tar.gz -e --strip 1 -o %SHARED_PATH% -call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cuda-11-7-windows-amd64.tar.gz -e --strip 1 -o %SHARED_PATH% +call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cuda-12-0-windows-amd64.tar.gz -e --strip 1 -o %BIN_PATH% +call .\node_modules\.bin\download %CUDA_DOWNLOAD_URL%/cuda-11-7-windows-amd64.tar.gz -e --strip 1 -o %BIN_PATH% move %BIN_PATH%\cortex-server-beta.exe %BIN_PATH%\cortex-server.exe del %BIN_PATH%\cortex-beta.exe diff --git a/extensions/inference-cortex-extension/download.sh b/extensions/inference-cortex-extension/download.sh index cf3ad372dd..0c21e19872 100755 --- a/extensions/inference-cortex-extension/download.sh +++ b/extensions/inference-cortex-extension/download.sh @@ -6,6 +6,7 @@ ENGINE_VERSION=0.1.46 CORTEX_RELEASE_URL="https://github.com/janhq/cortex.cpp/releases/download" ENGINE_DOWNLOAD_URL="https://github.com/janhq/cortex.llamacpp/releases/download/v${ENGINE_VERSION}/cortex.llamacpp-${ENGINE_VERSION}" CUDA_DOWNLOAD_URL="https://github.com/janhq/cortex.llamacpp/releases/download/v${ENGINE_VERSION}" +BIN_PATH=./bin SHARED_PATH="../../electron/shared" # Detect platform OS_TYPE=$(uname) @@ -28,8 +29,8 @@ if [ "$OS_TYPE" == "Linux" ]; then download "${ENGINE_DOWNLOAD_URL}-linux-amd64-noavx-cuda-12-0.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-noavx-cuda-12-0/v${ENGINE_VERSION}" 1 download "${ENGINE_DOWNLOAD_URL}-linux-amd64-noavx-cuda-11-7.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-noavx-cuda-11-7/v${ENGINE_VERSION}" 1 download "${ENGINE_DOWNLOAD_URL}-linux-amd64-vulkan.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-vulkan/v${ENGINE_VERSION}" 1 - download "${CUDA_DOWNLOAD_URL}/cuda-12-0-linux-amd64.tar.gz" -e --strip 1 -o "${SHARED_PATH}" 1 - download "${CUDA_DOWNLOAD_URL}/cuda-11-7-linux-amd64.tar.gz" -e --strip 1 -o "${SHARED_PATH}" 1 + download "${CUDA_DOWNLOAD_URL}/cuda-12-0-linux-amd64.tar.gz" -e --strip 1 -o "${BIN_PATH}" 1 + download "${CUDA_DOWNLOAD_URL}/cuda-11-7-linux-amd64.tar.gz" -e --strip 1 -o "${BIN_PATH}" 1 elif [ "$OS_TYPE" == "Darwin" ]; then # macOS downloads diff --git a/extensions/inference-cortex-extension/src/node/index.ts b/extensions/inference-cortex-extension/src/node/index.ts index c93da671cd..08d83ad726 100644 --- a/extensions/inference-cortex-extension/src/node/index.ts +++ b/extensions/inference-cortex-extension/src/node/index.ts @@ -23,7 +23,6 @@ function run(systemInfo?: SystemInformation): Promise { let gpuVisibleDevices = systemInfo?.gpuSetting?.gpus_in_use.join(',') ?? '' let binaryName = `cortex-server${process.platform === 'win32' ? '.exe' : ''}` const binPath = path.join(__dirname, '..', 'bin') - await createEngineSymlinks(binPath) const executablePath = path.join(binPath, binaryName) const sharedPath = path.join(appResourcePath(), 'shared') @@ -63,25 +62,6 @@ function run(systemInfo?: SystemInformation): Promise { }) } -/** - * Create symlinks for the engine shared libraries - * @param binPath - */ -async function createEngineSymlinks(binPath: string) { - const sharedPath = path.join(appResourcePath(), 'shared') - const sharedLibFiles = await readdir(sharedPath) - for (const sharedLibFile of sharedLibFiles) { - if (sharedLibFile.endsWith('.dll') || sharedLibFile.endsWith('.so')) { - const targetDllPath = path.join(sharedPath, sharedLibFile) - const symlinkDllPath = path.join(binPath, sharedLibFile) - await symlink(targetDllPath, symlinkDllPath, 'file').catch((error) => - log(JSON.stringify(error)) - ) - console.log(`Symlink created: ${targetDllPath} -> ${symlinkDllPath}`) - } - } -} - /** * Every module should have a dispose function * This will be called when the extension is unloaded and should clean up any resources From 713926e68c5e6c8873b01efca64c10e789c4a6ed Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 16 Jan 2025 16:18:32 +0700 Subject: [PATCH 6/9] chore: attempt to stop model before switching engine variant --- .../inference-cortex-extension/src/node/index.ts | 12 ------------ web/screens/Settings/Engines/LocalEngineSettings.tsx | 5 +++++ 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/extensions/inference-cortex-extension/src/node/index.ts b/extensions/inference-cortex-extension/src/node/index.ts index 08d83ad726..420c84b6e3 100644 --- a/extensions/inference-cortex-extension/src/node/index.ts +++ b/extensions/inference-cortex-extension/src/node/index.ts @@ -71,18 +71,6 @@ function dispose() { watchdog?.terminate() } -function addEnvPaths(dest: string) { - // Add engine path to the PATH and LD_LIBRARY_PATH - if (process.platform === 'win32') { - process.env.PATH = (process.env.PATH || '').concat(path.delimiter, dest) - } else { - process.env.LD_LIBRARY_PATH = (process.env.LD_LIBRARY_PATH || '').concat( - path.delimiter, - dest - ) - } -} - /** * Cortex process info */ diff --git a/web/screens/Settings/Engines/LocalEngineSettings.tsx b/web/screens/Settings/Engines/LocalEngineSettings.tsx index d62e2a6728..4f62127839 100644 --- a/web/screens/Settings/Engines/LocalEngineSettings.tsx +++ b/web/screens/Settings/Engines/LocalEngineSettings.tsx @@ -26,6 +26,7 @@ import { formatDownloadPercentage } from '@/utils/converter' import ExtensionSetting from '../ExtensionSetting' import DeleteEngineVariant from './DeleteEngineVariant' +import { useActiveModel } from '@/hooks/useActiveModel' const os = () => { switch (PLATFORM) { case 'win32': @@ -52,6 +53,7 @@ const LocalEngineSettings = ({ engine }: { engine: InferenceEngine }) => { const [installingEngines, setInstallingEngines] = useState< Map >(new Map()) + const { stopModel } = useActiveModel() const isEngineUpdated = latestReleasedEngine && @@ -104,6 +106,7 @@ const LocalEngineSettings = ({ engine }: { engine: InferenceEngine }) => { const handleEngineUpdate = useCallback( (event: { id: string; type: DownloadEvent; percent: number }) => { + stopModel() mutateInstalledEngines() mutateDefaultEngineVariant() // Backward compatible support - cortex.cpp returns full variant file name @@ -138,6 +141,7 @@ const LocalEngineSettings = ({ engine }: { engine: InferenceEngine }) => { }) }, [ + stopModel, mutateDefaultEngineVariant, mutateInstalledEngines, setInstallingEngines, @@ -153,6 +157,7 @@ const LocalEngineSettings = ({ engine }: { engine: InferenceEngine }) => { }, [handleEngineUpdate]) const handleChangeVariant = (e: string) => { + stopModel() setSelectedVariants(e) setDefaultEngineVariant(engine, { variant: e, From 589fc53030018f87d3e21329d4713d5fe5c84b02 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 16 Jan 2025 16:28:05 +0700 Subject: [PATCH 7/9] fix: race condition between model stop and engine update --- web/screens/Settings/Engines/LocalEngineSettings.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web/screens/Settings/Engines/LocalEngineSettings.tsx b/web/screens/Settings/Engines/LocalEngineSettings.tsx index 4f62127839..b7cf3acdd1 100644 --- a/web/screens/Settings/Engines/LocalEngineSettings.tsx +++ b/web/screens/Settings/Engines/LocalEngineSettings.tsx @@ -105,8 +105,8 @@ const LocalEngineSettings = ({ engine }: { engine: InferenceEngine }) => { }, [defaultEngineVariant]) const handleEngineUpdate = useCallback( - (event: { id: string; type: DownloadEvent; percent: number }) => { - stopModel() + async (event: { id: string; type: DownloadEvent; percent: number }) => { + await stopModel().catch(console.info) mutateInstalledEngines() mutateDefaultEngineVariant() // Backward compatible support - cortex.cpp returns full variant file name @@ -156,8 +156,8 @@ const LocalEngineSettings = ({ engine }: { engine: InferenceEngine }) => { } }, [handleEngineUpdate]) - const handleChangeVariant = (e: string) => { - stopModel() + const handleChangeVariant = async (e: string) => { + await stopModel().catch(console.info) setSelectedVariants(e) setDefaultEngineVariant(engine, { variant: e, From 5298d7584dea66b426005c8780427161f7e2bda0 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 16 Jan 2025 16:30:51 +0700 Subject: [PATCH 8/9] chore: typo for universal download --- web/containers/Layout/BottomPanel/DownloadingState/index.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/containers/Layout/BottomPanel/DownloadingState/index.tsx b/web/containers/Layout/BottomPanel/DownloadingState/index.tsx index dc9ffca892..1c3419aab7 100644 --- a/web/containers/Layout/BottomPanel/DownloadingState/index.tsx +++ b/web/containers/Layout/BottomPanel/DownloadingState/index.tsx @@ -32,12 +32,12 @@ export default function DownloadingState() { {Object.values(downloadStates)?.length > 0 && (