From 8ccf02a26d650725a5a6a3011f9035ed07b4080d Mon Sep 17 00:00:00 2001 From: Louis Date: Tue, 31 Dec 2024 19:25:47 +0700 Subject: [PATCH 1/9] fix: bump llama.cpp engine hotfix version to address mode load issue on some Intel machines --- extensions/inference-cortex-extension/download.bat | 2 +- extensions/inference-cortex-extension/download.sh | 2 +- extensions/inference-cortex-extension/rollup.config.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extensions/inference-cortex-extension/download.bat b/extensions/inference-cortex-extension/download.bat index 0e7eef20e5..a167a9cde5 100644 --- a/extensions/inference-cortex-extension/download.bat +++ b/extensions/inference-cortex-extension/download.bat @@ -2,7 +2,7 @@ set BIN_PATH=./bin set SHARED_PATH=./../../electron/shared set /p CORTEX_VERSION=<./bin/version.txt -set ENGINE_VERSION=0.1.42 +set ENGINE_VERSION=0.1.42-hotfix @REM Download cortex.llamacpp binaries set DOWNLOAD_URL=https://github.com/janhq/cortex.llamacpp/releases/download/v%ENGINE_VERSION%/cortex.llamacpp-%ENGINE_VERSION%-windows-amd64 diff --git a/extensions/inference-cortex-extension/download.sh b/extensions/inference-cortex-extension/download.sh index b0f3b36e35..f5db760308 100755 --- a/extensions/inference-cortex-extension/download.sh +++ b/extensions/inference-cortex-extension/download.sh @@ -2,7 +2,7 @@ # Read CORTEX_VERSION CORTEX_VERSION=$(cat ./bin/version.txt) -ENGINE_VERSION=0.1.42 +ENGINE_VERSION=0.1.42-hotfix CORTEX_RELEASE_URL="https://github.com/janhq/cortex.cpp/releases/download" ENGINE_DOWNLOAD_URL="https://github.com/janhq/cortex.llamacpp/releases/download/v${ENGINE_VERSION}/cortex.llamacpp-${ENGINE_VERSION}" CUDA_DOWNLOAD_URL="https://github.com/janhq/cortex.llamacpp/releases/download/v${ENGINE_VERSION}" diff --git a/extensions/inference-cortex-extension/rollup.config.ts b/extensions/inference-cortex-extension/rollup.config.ts index 266281a756..6cfe982b42 100644 --- a/extensions/inference-cortex-extension/rollup.config.ts +++ b/extensions/inference-cortex-extension/rollup.config.ts @@ -120,7 +120,7 @@ export default [ SETTINGS: JSON.stringify(defaultSettingJson), CORTEX_API_URL: JSON.stringify('http://127.0.0.1:39291'), CORTEX_SOCKET_URL: JSON.stringify('ws://127.0.0.1:39291'), - CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.42'), + CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.42-hotfix'), }), // Allow json resolution json(), From f6ca24256a98743ac439d51280fcef58d5d60d22 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 2 Jan 2025 08:52:51 +0700 Subject: [PATCH 2/9] fix: should not disable Vulkan support option --- web/screens/Settings/Advanced/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/screens/Settings/Advanced/index.tsx b/web/screens/Settings/Advanced/index.tsx index 52aafba834..3dbb56a86f 100644 --- a/web/screens/Settings/Advanced/index.tsx +++ b/web/screens/Settings/Advanced/index.tsx @@ -417,7 +417,7 @@ const Advanced = () => { )} {/* Vulkan for AMD GPU/ APU and Intel Arc GPU */} - {!isMac && gpuList.length > 0 && experimentalEnabled && ( + {!isMac && experimentalEnabled && (
From 27e40c35d895aa858e407f7b7f8f0b1706cfd2aa Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 2 Jan 2025 13:03:16 +0700 Subject: [PATCH 3/9] chore: add cpu_threads settings in cortex extension --- extensions/inference-cortex-extension/bin/version.txt | 2 +- .../resources/default_settings.json | 10 ++++++++++ extensions/inference-cortex-extension/src/index.ts | 10 ++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/extensions/inference-cortex-extension/bin/version.txt b/extensions/inference-cortex-extension/bin/version.txt index 238d6e882a..3c25d39659 100644 --- a/extensions/inference-cortex-extension/bin/version.txt +++ b/extensions/inference-cortex-extension/bin/version.txt @@ -1 +1 @@ -1.0.7 +1.0.8-rc1 diff --git a/extensions/inference-cortex-extension/resources/default_settings.json b/extensions/inference-cortex-extension/resources/default_settings.json index 1e5ec8db68..31586fbe64 100644 --- a/extensions/inference-cortex-extension/resources/default_settings.json +++ b/extensions/inference-cortex-extension/resources/default_settings.json @@ -18,6 +18,16 @@ "placeholder": "4" } }, + { + "key": "cpu_threads", + "title": "CPU Threads", + "description": "The number of threads to use for inferencing (CPU MODE ONLY)", + "controllerType": "input", + "controllerProps": { + "value": "", + "placeholder": "4" + } + }, { "key": "flash_attn", "title": "Flash Attention enabled", diff --git a/extensions/inference-cortex-extension/src/index.ts b/extensions/inference-cortex-extension/src/index.ts index 4cc322436d..5ba28ac57e 100644 --- a/extensions/inference-cortex-extension/src/index.ts +++ b/extensions/inference-cortex-extension/src/index.ts @@ -43,6 +43,7 @@ export enum Settings { flash_attn = 'flash_attn', cache_type = 'cache_type', use_mmap = 'use_mmap', + cpu_threads = 'cpu_threads', } /** @@ -66,6 +67,7 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine { flash_attn: boolean = true use_mmap: boolean = true cache_type: string = 'f16' + cpu_threads?: number /** * The URL for making inference requests. @@ -105,6 +107,10 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine { this.flash_attn = await this.getSetting(Settings.flash_attn, true) this.use_mmap = await this.getSetting(Settings.use_mmap, true) this.cache_type = await this.getSetting(Settings.cache_type, 'f16') + const threads_number = Number( + await this.getSetting(Settings.cpu_threads, '') + ) + if (!Number.isNaN(threads_number)) this.cpu_threads = threads_number this.queue.add(() => this.clean()) @@ -150,6 +156,9 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine { this.cache_type = value as string } else if (key === Settings.use_mmap && typeof value === 'boolean') { this.use_mmap = value as boolean + } else if (key === Settings.cpu_threads && typeof value === 'string') { + const threads_number = Number(value) + if (!Number.isNaN(threads_number)) this.cpu_threads = threads_number } } @@ -207,6 +216,7 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine { flash_attn: this.flash_attn, cache_type: this.cache_type, use_mmap: this.use_mmap, + ...(this.cpu_threads ? { cpu_threads: this.cpu_threads } : {}), }, timeout: false, signal, From 33bb35cb99027daf4f74261507c432f803063d13 Mon Sep 17 00:00:00 2001 From: Faisal Amir Date: Thu, 2 Jan 2025 15:16:34 +0700 Subject: [PATCH 4/9] fix: enable default opt-in analytic (#4387) --- web/helpers/atoms/Setting.atom.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/helpers/atoms/Setting.atom.ts b/web/helpers/atoms/Setting.atom.ts index 3568d87d0a..bd398f1e78 100644 --- a/web/helpers/atoms/Setting.atom.ts +++ b/web/helpers/atoms/Setting.atom.ts @@ -47,7 +47,7 @@ export const spellCheckAtom = atomWithStorage( ) export const productAnalyticAtom = atomWithStorage( PRODUCT_ANALYTIC, - false, + true, undefined, { getOnInit: true } ) From 538a6bf923cee968758802c5cc92271b7d5f168b Mon Sep 17 00:00:00 2001 From: Faisal Amir Date: Thu, 2 Jan 2025 21:04:09 +0700 Subject: [PATCH 5/9] fix: send event opt out (#4390) * fux: send event opt out from button deny * chore: fix copy * chore: update placeholder --- .../resources/default_settings.json | 4 ++-- web/containers/Layout/index.tsx | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/extensions/inference-cortex-extension/resources/default_settings.json b/extensions/inference-cortex-extension/resources/default_settings.json index 31586fbe64..6a0dcd4a0b 100644 --- a/extensions/inference-cortex-extension/resources/default_settings.json +++ b/extensions/inference-cortex-extension/resources/default_settings.json @@ -21,11 +21,11 @@ { "key": "cpu_threads", "title": "CPU Threads", - "description": "The number of threads to use for inferencing (CPU MODE ONLY)", + "description": "The number of CPU threads to use (when in CPU mode)", "controllerType": "input", "controllerProps": { "value": "", - "placeholder": "4" + "placeholder": "Number of CPU threads" } }, { diff --git a/web/containers/Layout/index.tsx b/web/containers/Layout/index.tsx index 29fda70de8..38d45ebd09 100644 --- a/web/containers/Layout/index.tsx +++ b/web/containers/Layout/index.tsx @@ -123,6 +123,7 @@ const BaseLayout = () => { if (isAllowed) { posthog.opt_in_capturing() } else { + posthog.capture('user_opt_out', { timestamp: new Date() }) posthog.opt_out_capturing() } } From 786d528abbea24c7386bc4302840d954d06f8c78 Mon Sep 17 00:00:00 2001 From: Faisal Amir Date: Fri, 3 Jan 2025 11:05:11 +0700 Subject: [PATCH 6/9] fix: revert back product analytic with default opt-out (#4394) --- web/containers/Layout/index.tsx | 1 - web/helpers/atoms/Setting.atom.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/web/containers/Layout/index.tsx b/web/containers/Layout/index.tsx index 38d45ebd09..29fda70de8 100644 --- a/web/containers/Layout/index.tsx +++ b/web/containers/Layout/index.tsx @@ -123,7 +123,6 @@ const BaseLayout = () => { if (isAllowed) { posthog.opt_in_capturing() } else { - posthog.capture('user_opt_out', { timestamp: new Date() }) posthog.opt_out_capturing() } } diff --git a/web/helpers/atoms/Setting.atom.ts b/web/helpers/atoms/Setting.atom.ts index bd398f1e78..3568d87d0a 100644 --- a/web/helpers/atoms/Setting.atom.ts +++ b/web/helpers/atoms/Setting.atom.ts @@ -47,7 +47,7 @@ export const spellCheckAtom = atomWithStorage( ) export const productAnalyticAtom = atomWithStorage( PRODUCT_ANALYTIC, - true, + false, undefined, { getOnInit: true } ) From 4b6eceb339430ed3cd55e3425a59e5923eede49a Mon Sep 17 00:00:00 2001 From: Louis Date: Fri, 3 Jan 2025 12:54:21 +0700 Subject: [PATCH 7/9] fix: max_tokens revert back to 8192 automatically when creating a new thread --- web/hooks/useCreateNewThread.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/hooks/useCreateNewThread.ts b/web/hooks/useCreateNewThread.ts index 9291577a0d..126fa1a940 100644 --- a/web/hooks/useCreateNewThread.ts +++ b/web/hooks/useCreateNewThread.ts @@ -98,7 +98,7 @@ export const useCreateNewThread = () => { // Use ctx length by default const overriddenParameters = { max_tokens: !isLocalEngine(defaultModel?.engine) - ? (defaultModel?.parameters.token_limit ?? 8192) + ? (defaultModel?.parameters.max_tokens ?? 8192) : defaultContextLength, } From 3cd4c9852af4e0d376bf0ad3ac084aa13b9429be Mon Sep 17 00:00:00 2001 From: Louis Date: Mon, 6 Jan 2025 14:24:12 +0700 Subject: [PATCH 8/9] chore: bump latest cortex release 1.0.8 (#4405) --- extensions/inference-cortex-extension/bin/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/inference-cortex-extension/bin/version.txt b/extensions/inference-cortex-extension/bin/version.txt index 3c25d39659..b0f3d96f87 100644 --- a/extensions/inference-cortex-extension/bin/version.txt +++ b/extensions/inference-cortex-extension/bin/version.txt @@ -1 +1 @@ -1.0.8-rc1 +1.0.8 From e3d44c9e7bd9a55d6df2a4fb450cc18a587fa1ca Mon Sep 17 00:00:00 2001 From: Louis Date: Mon, 6 Jan 2025 20:43:16 +0700 Subject: [PATCH 9/9] chore: resolve hotfix into dev (#4409) --- extensions/inference-cortex-extension/download.bat | 2 +- extensions/inference-cortex-extension/download.sh | 2 +- extensions/inference-cortex-extension/rollup.config.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extensions/inference-cortex-extension/download.bat b/extensions/inference-cortex-extension/download.bat index a167a9cde5..0e7eef20e5 100644 --- a/extensions/inference-cortex-extension/download.bat +++ b/extensions/inference-cortex-extension/download.bat @@ -2,7 +2,7 @@ set BIN_PATH=./bin set SHARED_PATH=./../../electron/shared set /p CORTEX_VERSION=<./bin/version.txt -set ENGINE_VERSION=0.1.42-hotfix +set ENGINE_VERSION=0.1.42 @REM Download cortex.llamacpp binaries set DOWNLOAD_URL=https://github.com/janhq/cortex.llamacpp/releases/download/v%ENGINE_VERSION%/cortex.llamacpp-%ENGINE_VERSION%-windows-amd64 diff --git a/extensions/inference-cortex-extension/download.sh b/extensions/inference-cortex-extension/download.sh index f5db760308..b0f3b36e35 100755 --- a/extensions/inference-cortex-extension/download.sh +++ b/extensions/inference-cortex-extension/download.sh @@ -2,7 +2,7 @@ # Read CORTEX_VERSION CORTEX_VERSION=$(cat ./bin/version.txt) -ENGINE_VERSION=0.1.42-hotfix +ENGINE_VERSION=0.1.42 CORTEX_RELEASE_URL="https://github.com/janhq/cortex.cpp/releases/download" ENGINE_DOWNLOAD_URL="https://github.com/janhq/cortex.llamacpp/releases/download/v${ENGINE_VERSION}/cortex.llamacpp-${ENGINE_VERSION}" CUDA_DOWNLOAD_URL="https://github.com/janhq/cortex.llamacpp/releases/download/v${ENGINE_VERSION}" diff --git a/extensions/inference-cortex-extension/rollup.config.ts b/extensions/inference-cortex-extension/rollup.config.ts index 6cfe982b42..266281a756 100644 --- a/extensions/inference-cortex-extension/rollup.config.ts +++ b/extensions/inference-cortex-extension/rollup.config.ts @@ -120,7 +120,7 @@ export default [ SETTINGS: JSON.stringify(defaultSettingJson), CORTEX_API_URL: JSON.stringify('http://127.0.0.1:39291'), CORTEX_SOCKET_URL: JSON.stringify('ws://127.0.0.1:39291'), - CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.42-hotfix'), + CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.42'), }), // Allow json resolution json(),