Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: remote engine management #4364

Merged
merged 31 commits into from
Jan 14, 2025
Merged
Changes from 1 commit
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
b7a86a0
feat: remote engine management
urmauur Dec 30, 2024
68e05ee
chore: fix linter issue
urmauur Dec 30, 2024
323700f
chore: remove unused imports
louis-menlo Jan 2, 2025
22e9db0
fix: populate engines, models and legacy settings (#4403)
louis-menlo Jan 6, 2025
b62db41
fix: check exist path before reading
louis-menlo Jan 6, 2025
3eb4295
fix: engines and models persist - race condition
louis-menlo Jan 6, 2025
3e48008
chore: update issue state
urmauur Jan 6, 2025
dc84557
test: update test cases
louis-menlo Jan 6, 2025
7fd8950
chore: bring back Cortex extension settings
louis-menlo Jan 6, 2025
e108f49
chore: setup button gear / plus based apikey
urmauur Jan 6, 2025
3fa8f41
chore: fix remote engine from welcome screen
urmauur Jan 8, 2025
7fd0dd6
chore: resolve linter issue
urmauur Jan 8, 2025
0df0983
chore: support request headers template
louis-menlo Jan 8, 2025
db75c9b
chore: update engines using header_template instead of api_key_template
louis-menlo Jan 8, 2025
a50582a
chore: update models on changes
louis-menlo Jan 8, 2025
5d8d4b6
fix: anthropic response template
louis-menlo Jan 8, 2025
f47392b
chore: fix welcome screen and debounce update value input
urmauur Jan 8, 2025
0468afb
chore: update engines list on changes
louis-menlo Jan 8, 2025
8e6ed11
chore: update engines list on change
louis-menlo Jan 8, 2025
67625df
chore: update desc form add modal remote engines
urmauur Jan 8, 2025
c11ec3c
chore: bump cortex version to latest RC
louis-menlo Jan 13, 2025
06188a2
chore: fix linter
louis-menlo Jan 13, 2025
a15d2ba
fix: transform payload of Anthropic and OpenAI
louis-menlo Jan 13, 2025
9094d02
fix: typo
urmauur Jan 14, 2025
5d56d1e
fix: openrouter model id for auto routing
louis-menlo Jan 14, 2025
dffb37d
chore: remove remote engine URL setting
louis-menlo Jan 14, 2025
fe67931
chore: add cohere engine and model support
louis-menlo Jan 14, 2025
a54e0f6
fix: should not clean on app launch - models list display issue
louis-menlo Jan 14, 2025
3511583
fix: local engine check logic
louis-menlo Jan 14, 2025
ca55df3
chore: bump app version to latest release 0.5.13
louis-menlo Jan 14, 2025
a63322b
test: fix failed tests
louis-menlo Jan 14, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix: should not clean on app launch - models list display issue
  • Loading branch information
louis-menlo committed Jan 14, 2025
commit a54e0f66fdf862a8e72a90c9e5a18c610373c814
4 changes: 2 additions & 2 deletions extensions/inference-cortex-extension/src/@types/global.d.ts
Original file line number Diff line number Diff line change
@@ -2,5 +2,5 @@ declare const NODE: string
declare const CORTEX_API_URL: string
declare const CORTEX_SOCKET_URL: string
declare const CORTEX_ENGINE_VERSION: string
declare const SETTINGS: object[]
declare const MODELS: object[]
declare const SETTINGS: any
declare const MODELS: any
2 changes: 0 additions & 2 deletions extensions/inference-cortex-extension/src/index.ts
Original file line number Diff line number Diff line change
@@ -111,8 +111,6 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine {
)
if (!Number.isNaN(threads_number)) this.cpu_threads = threads_number

this.queue.add(() => this.clean())

// Run the process watchdog
const systemInfo = await systemInformation()
this.queue.add(() => executeOnMain(NODE, 'run', systemInfo))
1 change: 0 additions & 1 deletion web/containers/ErrorMessage/index.tsx
Original file line number Diff line number Diff line change
@@ -2,7 +2,6 @@ import {
EngineManager,
ErrorCode,
InferenceEngine,
MessageStatus,
ThreadMessage,
} from '@janhq/core'

Original file line number Diff line number Diff line change
@@ -1,25 +1,29 @@
import { Tooltip, Button, Badge } from '@janhq/joi'

import { useAtom } from 'jotai'
import { useAtom, useAtomValue } from 'jotai'

import { useActiveModel } from '@/hooks/useActiveModel'

import { toGibibytes } from '@/utils/converter'

import { isLocalEngine } from '@/utils/modelEngine'

import { installedEnginesAtom } from '@/helpers/atoms/Engines.atom'
import { serverEnabledAtom } from '@/helpers/atoms/LocalServer.atom'

const TableActiveModel = () => {
const { activeModel, stateModel, stopModel } = useActiveModel()
const engines = useAtomValue(installedEnginesAtom)

const [serverEnabled, setServerEnabled] = useAtom(serverEnabledAtom)

return (
<div className="w-1/2">
<div className="overflow-hidden border-b border-[hsla(var(--app-border))]">
<table className="w-full px-8">
{activeModel && isLocalEngine(activeModel.engine) ? (
{activeModel &&
engines &&
isLocalEngine(engines, activeModel.engine) ? (
<tbody>
<tr>
<td
11 changes: 8 additions & 3 deletions web/containers/ModelDropdown/index.tsx
Original file line number Diff line number Diff line change
@@ -31,7 +31,6 @@ import SetupRemoteModel from '@/containers/SetupRemoteModel'
import { useCreateNewThread } from '@/hooks/useCreateNewThread'
import useDownloadModel from '@/hooks/useDownloadModel'
import { modelDownloadStateAtom } from '@/hooks/useDownloadState'
import { useGetEngines } from '@/hooks/useEngineManagement'

import useRecommendedModel from '@/hooks/useRecommendedModel'

@@ -95,7 +94,7 @@ const ModelDropdown = ({
const searchInputRef = useRef<HTMLInputElement>(null)
const configuredModels = useAtomValue(configuredModelsAtom)

const featuredModel = configuredModels.filter(
const featuredModels = configuredModels.filter(
(x) =>
manualRecommendationModel.includes(x.id) &&
x.metadata?.tags?.includes('Featured') &&
@@ -379,6 +378,12 @@ const ModelDropdown = ({
<ScrollArea className="h-[calc(100%-90px)] w-full">
{engineList
.filter((e) => e.type === searchFilter)
.filter(
(e) =>
e.type === 'remote' ||
e.name === InferenceEngine.cortex_llamacpp ||
filteredDownloadedModels.some((e) => e.engine === e.name)
)
.map((engine, i) => {
const isConfigured =
engine.type === 'local' ||
@@ -450,7 +455,7 @@ const ModelDropdown = ({
showModel &&
!searchText.length && (
<ul className="pb-2">
{featuredModel.map((model) => {
{featuredModels.map((model) => {
const isDownloading = downloadingModels.some(
(md) => md === model.id
)
5 changes: 4 additions & 1 deletion web/containers/Providers/ModelHandler.tsx
Original file line number Diff line number Diff line change
@@ -34,6 +34,7 @@ import {
deleteMessageAtom,
subscribedGeneratingMessageAtom,
} from '@/helpers/atoms/ChatMessage.atom'
import { installedEnginesAtom } from '@/helpers/atoms/Engines.atom'
import { downloadedModelsAtom } from '@/helpers/atoms/Model.atom'
import {
updateThreadWaitingForResponseAtom,
@@ -74,6 +75,7 @@ export default function ModelHandler() {
const activeModelParams = useAtomValue(getActiveThreadModelParamsAtom)
const activeModelParamsRef = useRef(activeModelParams)
const setTokenSpeed = useSetAtom(tokenSpeedAtom)
const engines = useAtomValue(installedEnginesAtom)

useEffect(() => {
activeThreadRef.current = activeThread
@@ -241,7 +243,8 @@ export default function ModelHandler() {
} else if (
message.status === MessageStatus.Error &&
activeModelRef.current?.engine &&
isLocalEngine(activeModelRef.current.engine)
engines &&
isLocalEngine(engines, activeModelRef.current.engine)
) {
;(async () => {
if (
2 changes: 0 additions & 2 deletions web/containers/SetupRemoteModel/index.tsx
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
import { useState, useEffect } from 'react'

import { InferenceEngine } from '@janhq/core'

import { Button } from '@janhq/joi'
2 changes: 1 addition & 1 deletion web/helpers/atoms/Engines.atom.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { EngineConfig, Engines } from '@janhq/core'
import { Engines } from '@janhq/core'
import { atom } from 'jotai'

/**
4 changes: 0 additions & 4 deletions web/hooks/useEngines.ts
Original file line number Diff line number Diff line change
@@ -2,11 +2,7 @@ import { useCallback, useEffect } from 'react'

import {
ExtensionTypeEnum,
Model,
ModelEvent,
ModelExtension,
events,
ModelManager,
EngineEvent,
EngineManagementExtension,
Engines,
7 changes: 3 additions & 4 deletions web/hooks/useModels.ts
Original file line number Diff line number Diff line change
@@ -7,14 +7,13 @@ import {
ModelExtension,
events,
ModelManager,
InferenceEngine,
} from '@janhq/core'

import { useSetAtom } from 'jotai'

import { useDebouncedCallback } from 'use-debounce'

import { isLocalEngine } from '@/utils/modelEngine'

import { extensionManager } from '@/extension'

import {
@@ -43,7 +42,7 @@ const useModels = () => {
const remoteModels = ModelManager.instance()
.models.values()
.toArray()
.filter((e) => !isLocalEngine(e.engine))
.filter((e) => e.engine !== InferenceEngine.cortex_llamacpp)
const toUpdate = [
...localModels,
...remoteModels.filter(
@@ -84,7 +83,7 @@ const useModels = () => {
...downloadedModels,
...cachedModels.filter(
(e) =>
!isLocalEngine(e.engine) &&
e.engine !== InferenceEngine.cortex_llamacpp &&
!downloadedModels.some((g: Model) => g.id === e.id)
),
])
7 changes: 4 additions & 3 deletions web/hooks/useStarterScreen.ts
Original file line number Diff line number Diff line change
@@ -18,12 +18,13 @@
const remoteEngines =
engines &&
Object.entries(engines)
.filter(([key]) => !isLocalEngine(key as InferenceEngine))
.filter(([key]) => !isLocalEngine(engines, key as InferenceEngine))
.flatMap(([_, engineArray]) => engineArray as EngineConfig)

Check warning on line 22 in web/hooks/useStarterScreen.ts

GitHub Actions / test-on-macos

'_' is defined but never used

Check warning on line 22 in web/hooks/useStarterScreen.ts

GitHub Actions / test-on-windows-pr

'_' is defined but never used

Check warning on line 22 in web/hooks/useStarterScreen.ts

GitHub Actions / test-on-ubuntu

'_' is defined but never used

Check warning on line 22 in web/hooks/useStarterScreen.ts

GitHub Actions / coverage-check

'_' is defined but never used

const isDownloadALocalModel = useMemo(
() => downloadedModels.some((x) => isLocalEngine(x.engine)),
[downloadedModels]
() =>
downloadedModels.some((x) => engines && isLocalEngine(engines, x.engine)),
[engines, downloadedModels]
)

const isAnyRemoteModelConfigured = useMemo(
4 changes: 2 additions & 2 deletions web/screens/Settings/Engines/index.tsx
Original file line number Diff line number Diff line change
@@ -23,7 +23,7 @@ const Engines = () => {
</h6>
{engines &&
Object.entries(engines).map(([key]) => {
if (!isLocalEngine(key as InferenceEngine)) return
if (!isLocalEngine(engines, key as InferenceEngine)) return
return (
<LocalEngineItems engine={key as InferenceEngine} key={key} />
)
@@ -40,7 +40,7 @@ const Engines = () => {
</div>
{engines &&
Object.entries(engines).map(([key, values]) => {
if (isLocalEngine(key as InferenceEngine)) return
if (isLocalEngine(engines, key as InferenceEngine)) return
return (
<RemoteEngineItems
engine={key as InferenceEngine}
10 changes: 6 additions & 4 deletions web/screens/Settings/MyModels/MyModelList/index.tsx
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@ import { memo, useState } from 'react'

import { Model } from '@janhq/core'
import { Badge, Button, Tooltip, useClickOutside } from '@janhq/joi'
import { useAtom } from 'jotai'
import { useAtom, useAtomValue } from 'jotai'
import {
MoreVerticalIcon,
PlayIcon,
@@ -18,6 +18,7 @@ import { toGibibytes } from '@/utils/converter'

import { isLocalEngine } from '@/utils/modelEngine'

import { installedEnginesAtom } from '@/helpers/atoms/Engines.atom'
import { serverEnabledAtom } from '@/helpers/atoms/LocalServer.atom'

type Props = {
@@ -31,6 +32,7 @@ const MyModelList = ({ model }: Props) => {
const { deleteModel } = useDeleteModel()
const [more, setMore] = useState(false)
const [serverEnabled, setServerEnabled] = useAtom(serverEnabledAtom)
const engines = useAtomValue(installedEnginesAtom)

const [menu, setMenu] = useState<HTMLDivElement | null>(null)
const [toggle, setToggle] = useState<HTMLDivElement | null>(null)
@@ -54,14 +56,14 @@ const MyModelList = ({ model }: Props) => {
<h6
className={twMerge(
'font-medium lg:line-clamp-1 lg:min-w-[280px] lg:max-w-[280px]',
!isLocalEngine(model.engine) &&
!isLocalEngine(engines, model.engine) &&
'max-w-none text-[hsla(var(--text-secondary))]'
)}
title={model.name}
>
{model.name}
</h6>
{isLocalEngine(model.engine) && (
{isLocalEngine(engines, model.engine) && (
<div className="flex gap-x-8">
<p
className="line-clamp-1 text-[hsla(var(--text-secondary))] lg:min-w-[160px] lg:max-w-[160px] xl:max-w-none"
@@ -74,7 +76,7 @@ const MyModelList = ({ model }: Props) => {
</div>
</div>

{isLocalEngine(model.engine) && (
{isLocalEngine(engines, model.engine) && (
<div className="flex gap-x-4">
<div className="md:min-w-[90px] md:max-w-[90px]">
<Badge theme="secondary" className="sm:mr-8">
5 changes: 4 additions & 1 deletion web/screens/Thread/ThreadCenterPanel/ChatInput/index.tsx
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { useEffect, useRef, useState } from 'react'

import { InferenceEngine, MessageStatus } from '@janhq/core'
import { InferenceEngine } from '@janhq/core'

import { TextArea, Button, Tooltip, useClickOutside, Badge } from '@janhq/joi'
import { useAtom, useAtomValue } from 'jotai'
@@ -35,6 +35,7 @@ import RichTextEditor from './RichTextEditor'
import { showRightPanelAtom } from '@/helpers/atoms/App.atom'
import { experimentalFeatureEnabledAtom } from '@/helpers/atoms/AppConfig.atom'
import { activeAssistantAtom } from '@/helpers/atoms/Assistant.atom'
import { installedEnginesAtom } from '@/helpers/atoms/Engines.atom'
import { selectedModelAtom } from '@/helpers/atoms/Model.atom'
import { spellCheckAtom } from '@/helpers/atoms/Setting.atom'
import {
@@ -63,6 +64,7 @@ const ChatInput = () => {
const textareaRef = useRef<HTMLTextAreaElement>(null)
const fileInputRef = useRef<HTMLInputElement>(null)
const imageInputRef = useRef<HTMLInputElement>(null)
const engines = useAtomValue(installedEnginesAtom)
const experimentalFeature = useAtomValue(experimentalFeatureEnabledAtom)
const isBlockingSend = useAtomValue(isBlockingSendAtom)
const activeAssistant = useAtomValue(activeAssistantAtom)
@@ -95,6 +97,7 @@ const ChatInput = () => {
}

const isModelSupportRagAndTools = isLocalEngine(
engines,
selectedModel?.engine as InferenceEngine
)

4 changes: 3 additions & 1 deletion web/screens/Thread/ThreadRightPanel/index.tsx
Original file line number Diff line number Diff line change
@@ -39,6 +39,7 @@ import Tools from './Tools'

import { experimentalFeatureEnabledAtom } from '@/helpers/atoms/AppConfig.atom'
import { activeAssistantAtom } from '@/helpers/atoms/Assistant.atom'
import { installedEnginesAtom } from '@/helpers/atoms/Engines.atom'
import { selectedModelAtom } from '@/helpers/atoms/Model.atom'
import {
activeThreadAtom,
@@ -60,12 +61,13 @@ const ThreadRightPanel = () => {
const [activeTabThreadRightPanel, setActiveTabThreadRightPanel] = useAtom(
activeTabThreadRightPanelAtom
)
const engines = useAtomValue(installedEnginesAtom)
const { updateThreadMetadata } = useCreateNewThread()
const experimentalFeature = useAtomValue(experimentalFeatureEnabledAtom)

const isModelSupportRagAndTools =
selectedModel?.engine === InferenceEngine.openai ||
isLocalEngine(selectedModel?.engine as InferenceEngine)
isLocalEngine(engines, selectedModel?.engine as InferenceEngine)

const setEngineParamsUpdate = useSetAtom(engineParamsUpdateAtom)
const { stopModel } = useActiveModel()
13 changes: 4 additions & 9 deletions web/utils/modelEngine.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { EngineManager, InferenceEngine, LocalOAIEngine } from '@janhq/core'
import { Engines, InferenceEngine } from '@janhq/core'

export const getLogoEngine = (engine: InferenceEngine) => {
switch (engine) {
@@ -38,15 +38,10 @@ export const getLogoEngine = (engine: InferenceEngine) => {
* @param engine
* @returns
*/
export const isLocalEngine = (engine?: string) => {
if (!engine) return false
export const isLocalEngine = (engines?: Engines, engine?: string) => {
if (!engines || !engine) return false

const engineObj = EngineManager.instance().get(engine)
if (!engineObj) return false
return (
Object.getPrototypeOf(engineObj).constructor.__proto__.name ===
LocalOAIEngine.name
)
return engines[engine as InferenceEngine]?.[0]?.type === 'local'
}

export const getTitleByEngine = (engine: InferenceEngine) => {
Loading