Sync release 0.5.13 into dev

extensions/inference-cortex-extension/bin/version.txt

@@ -1 +1 @@
-1.0.7
+1.0.8

extensions/inference-cortex-extension/resources/default_settings.json

@@ -18,6 +18,16 @@
       "placeholder": "4"
     }
   },
+  {
+    "key": "cpu_threads",
+    "title": "CPU Threads",
+    "description": "The number of CPU threads to use (when in CPU mode)",
+    "controllerType": "input",
+    "controllerProps": {
+      "value": "",
+      "placeholder": "Number of CPU threads"
+    }
+  },
   {
     "key": "flash_attn",
     "title": "Flash Attention enabled",

extensions/inference-cortex-extension/src/index.ts

@@ -43,6 +43,7 @@ export enum Settings {
   flash_attn = 'flash_attn',
   cache_type = 'cache_type',
   use_mmap = 'use_mmap',
+  cpu_threads = 'cpu_threads',
 }
 
 /**
@@ -66,6 +67,7 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
   flash_attn: boolean = true
   use_mmap: boolean = true
   cache_type: string = 'f16'
+  cpu_threads?: number
 
   /**
    * The URL for making inference requests.
@@ -105,6 +107,10 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
     this.flash_attn = await this.getSetting<boolean>(Settings.flash_attn, true)
     this.use_mmap = await this.getSetting<boolean>(Settings.use_mmap, true)
     this.cache_type = await this.getSetting<string>(Settings.cache_type, 'f16')
+    const threads_number = Number(
+      await this.getSetting<string>(Settings.cpu_threads, '')
+    )
+    if (!Number.isNaN(threads_number)) this.cpu_threads = threads_number
 
     this.queue.add(() => this.clean())
 
@@ -150,6 +156,9 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
       this.cache_type = value as string
     } else if (key === Settings.use_mmap && typeof value === 'boolean') {
       this.use_mmap = value as boolean
+    } else if (key === Settings.cpu_threads && typeof value === 'string') {
+      const threads_number = Number(value)
+      if (!Number.isNaN(threads_number)) this.cpu_threads = threads_number
     }
   }
 
@@ -207,6 +216,7 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
             flash_attn: this.flash_attn,
             cache_type: this.cache_type,
             use_mmap: this.use_mmap,
+            ...(this.cpu_threads ? { cpu_threads: this.cpu_threads } : {}),
           },
           timeout: false,
           signal,

web/hooks/useCreateNewThread.ts

@@ -5,12 +5,12 @@
   ExtensionTypeEnum,
   Thread,
   ThreadAssistantInfo,
   ThreadState,
   AssistantTool,
   Model,
   Assistant,
 } from '@janhq/core'
 import { atom, useAtom, useAtomValue, useSetAtom } from 'jotai'
 
 import { useDebouncedCallback } from 'use-debounce'
 
@@ -33,7 +33,7 @@
   threadsAtom,
   updateThreadAtom,
   setThreadModelParamsAtom,
   isGeneratingResponseAtom,
   createNewThreadAtom,
 } from '@/helpers/atoms/Thread.atom'
 
@@ -98,7 +98,7 @@
     // Use ctx length by default
     const overriddenParameters = {
       max_tokens: !isLocalEngine(defaultModel?.engine)
-        ? (defaultModel?.parameters.token_limit ?? 8192)
+        ? (defaultModel?.parameters.max_tokens ?? 8192)
         : defaultContextLength,
     }
 
@@ -136,19 +136,19 @@
     //TODO: Why do we have thread list then thread states? Should combine them
     try {
       const createdThread = await persistNewThread(thread, assistantInfo)
       if (!createdThread) throw 'Thread created failed.'
       createNewThread(createdThread)
 
       setSelectedModel(defaultModel)
       setThreadModelParams(createdThread.id, {
         ...defaultModel?.settings,
         ...defaultModel?.parameters,
         ...overriddenSettings,
       })
 
       // Delete the file upload state
       setFileUpload(undefined)
       setActiveThread(createdThread)
     } catch (ex) {
       return toaster({
         title: 'Thread created failed.',
@@ -159,7 +159,7 @@
   }
 
   const updateThreadExtension = (thread: Thread) => {
     return extensionManager
       .get<ConversationalExtension>(ExtensionTypeEnum.Conversational)
       ?.modifyThread(thread)
   }
@@ -168,7 +168,7 @@
     threadId: string,
     assistant: ThreadAssistantInfo
   ) => {
     return extensionManager
       .get<ConversationalExtension>(ExtensionTypeEnum.Conversational)
       ?.modifyThreadAssistant(threadId, assistant)
   }
@@ -205,13 +205,13 @@
       .get<ConversationalExtension>(ExtensionTypeEnum.Conversational)
       ?.createThread(thread)
       .then(async (thread) => {
         await extensionManager
           .get<ConversationalExtension>(ExtensionTypeEnum.Conversational)
           ?.createThreadAssistant(thread.id, assistantInfo)
           .catch(console.error)
         return thread
       })
       .catch(() => undefined)
   }
 
   return {

web/screens/Settings/Advanced/index.tsx

@@ -78,7 +78,7 @@
   const selectedGpu = gpuList
     .filter((x) => gpusInUse.includes(x.id))
     .map((y) => {
       return y['name']
     })
 
   /**
@@ -87,7 +87,7 @@
    * there is also a case where state update persist everytime user type in the input
    */
   const updatePullOptions = useDebouncedCallback(
     () => configurePullOptions(),
     300
   )
   /**
@@ -417,7 +417,7 @@
         )}
 
         {/* Vulkan for AMD GPU/ APU and Intel Arc GPU */}
-        {!isMac && gpuList.length > 0 && experimentalEnabled && (
+        {!isMac && experimentalEnabled && (
           <div className="flex w-full flex-col items-start justify-between gap-4 border-b border-[hsla(var(--app-border))] py-4 first:pt-0 last:border-none sm:flex-row">
             <div className="space-y-1">
               <div className="flex gap-x-2">