fix: add fallback as default endpoint for inference engine

janhq · Apr 10, 2024 · db05524 · db05524
1 parent 2931a46
commit db05524
Show file tree

Hide file tree

Showing 9 changed files with 52 additions and 12 deletions.
diff --git a/core/src/browser/extensions/engines/OAIEngine.ts b/core/src/browser/extensions/engines/OAIEngine.ts
@@ -102,7 +102,7 @@ export abstract class OAIEngine extends AIEngine {
         events.emit(MessageEvent.OnMessageUpdate, message)
       },
       error: async (err: any) => {
-        console.error(`Inference error:`, err)
+        console.error(`Inference error:`, JSON.stringify(err))
         if (this.isCancelled || message.content.length) {
           message.status = MessageStatus.Stopped
           events.emit(MessageEvent.OnMessageUpdate, message)

diff --git a/extensions/inference-groq-extension/resources/settings.json b/extensions/inference-groq-extension/resources/settings.json
@@ -5,7 +5,7 @@
     "description": "The endpoint to use for chat completions. See the [Groq documentation](https://console.groq.com/docs/openai) for more information.",
     "controllerType": "input",
     "controllerProps": {
-      "placeholder": "Chat Completions Endpoint",
+      "placeholder": "https://api.groq.com/openai/v1/chat/completions",
       "value": "https://api.groq.com/openai/v1/chat/completions"
     }
   },

diff --git a/extensions/inference-groq-extension/src/index.ts b/extensions/inference-groq-extension/src/index.ts
@@ -6,7 +6,7 @@
  * @module inference-groq-extension/src/index
  */
 
-import { RemoteOAIEngine } from '@janhq/core'
+import { RemoteOAIEngine, SettingComponentProps } from '@janhq/core'
 
 declare const SETTINGS: Array<any>
 declare const MODELS: Array<any>
@@ -43,7 +43,17 @@ export default class JanInferenceGroqExtension extends RemoteOAIEngine {
     if (key === Settings.apiKey) {
       this.apiKey = value as string
     } else if (key === Settings.chatCompletionsEndPoint) {
-      this.inferenceUrl = value as string
+      if (typeof value !== 'string') return
+
+      if (value.trim().length === 0) {
+        SETTINGS.forEach((setting) => {
+          if (setting.key === Settings.chatCompletionsEndPoint) {
+            this.inferenceUrl = setting.controllerProps.value as string
+          }
+        })
+      } else {
+        this.inferenceUrl = value
+      }
     }
   }
 }
diff --git a/extensions/inference-mistral-extension/resources/settings.json b/extensions/inference-mistral-extension/resources/settings.json
@@ -5,7 +5,7 @@
     "description": "The endpoint to use for chat completions. See the [Mistral API documentation](https://docs.mistral.ai/api/#operation/createChatCompletion) for more information.",
     "controllerType": "input",
     "controllerProps": {
-      "placeholder": "Chat Completions Endpoint",
+      "placeholder": "https://api.mistral.ai/v1/chat/completions",
       "value": "https://api.mistral.ai/v1/chat/completions"
     }
   },

diff --git a/extensions/inference-mistral-extension/src/index.ts b/extensions/inference-mistral-extension/src/index.ts
@@ -42,7 +42,17 @@ export default class JanInferenceMistralExtension extends RemoteOAIEngine {
     if (key === Settings.apiKey) {
       this.apiKey = value as string
     } else if (key === Settings.chatCompletionsEndPoint) {
-      this.inferenceUrl = value as string
+      if (typeof value !== 'string') return
+
+      if (value.trim().length === 0) {
+        SETTINGS.forEach((setting) => {
+          if (setting.key === Settings.chatCompletionsEndPoint) {
+            this.inferenceUrl = setting.controllerProps.value as string
+          }
+        })
+      } else {
+        this.inferenceUrl = value
+      }
     }
   }
 }
diff --git a/extensions/inference-openai-extension/resources/settings.json b/extensions/inference-openai-extension/resources/settings.json
@@ -5,7 +5,7 @@
     "description": "The endpoint to use for chat completions. See the [OpenAI API documentation](https://platform.openai.com/docs/api-reference/chat/create) for more information.",
     "controllerType": "input",
     "controllerProps": {
-      "placeholder": "Chat Completions Endpoint",
+      "placeholder": "https://api.openai.com/v1/chat/completions",
       "value": "https://api.openai.com/v1/chat/completions"
     }
   },

diff --git a/extensions/inference-openai-extension/src/index.ts b/extensions/inference-openai-extension/src/index.ts
@@ -6,7 +6,7 @@
  * @module inference-openai-extension/src/index
  */
 
-import { RemoteOAIEngine } from '@janhq/core'
+import { RemoteOAIEngine, SettingComponentProps } from '@janhq/core'
 
 declare const SETTINGS: Array<any>
 declare const MODELS: Array<any>
@@ -43,7 +43,17 @@ export default class JanInferenceOpenAIExtension extends RemoteOAIEngine {
     if (key === Settings.apiKey) {
       this.apiKey = value as string
     } else if (key === Settings.chatCompletionsEndPoint) {
-      this.inferenceUrl = value as string
+      if (typeof value !== 'string') return
+
+      if (value.trim().length === 0) {
+        SETTINGS.forEach((setting) => {
+          if (setting.key === Settings.chatCompletionsEndPoint) {
+            this.inferenceUrl = setting.controllerProps.value as string
+          }
+        })
+      } else {
+        this.inferenceUrl = value
+      }
     }
   }
 }
diff --git a/extensions/inference-triton-trtllm-extension/resources/settings.json b/extensions/inference-triton-trtllm-extension/resources/settings.json
@@ -5,7 +5,7 @@
     "description": "The endpoint to use for chat completions.",
     "controllerType": "input",
     "controllerProps": {
-      "placeholder": "Chat Completions Endpoint",
+      "placeholder": "http://localhost:8000/v2/models/tensorrt_llm_bls/generate",
       "value": "http://localhost:8000/v2/models/tensorrt_llm_bls/generate"
     }
   },

diff --git a/extensions/inference-triton-trtllm-extension/src/index.ts b/extensions/inference-triton-trtllm-extension/src/index.ts
@@ -6,7 +6,7 @@
  * @module inference-nvidia-triton-trt-llm-extension/src/index
  */
 
-import { RemoteOAIEngine } from '@janhq/core'
+import { RemoteOAIEngine, SettingComponentProps } from '@janhq/core'
 
 declare const SETTINGS: Array<any>
 enum Settings {
@@ -43,7 +43,17 @@ export default class JanInferenceTritonTrtLLMExtension extends RemoteOAIEngine {
     if (key === Settings.apiKey) {
       this.apiKey = value as string
     } else if (key === Settings.chatCompletionsEndPoint) {
-      this.inferenceUrl = value as string
+      if (typeof value !== 'string') return
+
+      if (value.trim().length === 0) {
+        SETTINGS.forEach((setting) => {
+          if (setting.key === Settings.chatCompletionsEndPoint) {
+            this.inferenceUrl = setting.controllerProps.value as string
+          }
+        })
+      } else {
+        this.inferenceUrl = value
+      }
     }
   }
 }