chore: jan can see

core/src/types/inference/inferenceEntity.ts

@@ -1,3 +1,5 @@
+import { ContentType, ContentValue } from '../message'
+
 /**
  * The role of the author of this message.
  */
@@ -13,7 +15,25 @@ export enum ChatCompletionRole {
  */
 export type ChatCompletionMessage = {
   /** The contents of the message. **/
-  content?: string
+  content?: ChatCompletionMessageContent
   /** The role of the author of this message. **/
   role: ChatCompletionRole
 }
+
+export type ChatCompletionMessageContent =
+  | string
+  | (ChatCompletionMessageContentText & ChatCompletionMessageContentImage)[]
+
+export enum ChatCompletionMessageContentType {
+  Text = 'text',
+  Image = 'image_url',
+}
+
+export type ChatCompletionMessageContentText = {
+  type: ChatCompletionMessageContentType
+  text: string
+}
+export type ChatCompletionMessageContentImage = {
+  type: ChatCompletionMessageContentType
+  image_url: { url: string }
+}

core/src/types/model/modelEntity.ts

@@ -92,6 +92,11 @@ export type Model = {
    * The model engine.
    */
   engine: InferenceEngine
+
+  /**
+   * Is multimodal or not.
+   */
+  visionModel?: boolean
 }
 
 export type ModelMetadata = {

extensions/inference-nitro-extension/bin/version.txt

@@ -1 +1 @@
-0.1.27
+0.1.29

web/containers/Providers/Jotai.tsx

@@ -9,6 +9,7 @@ type Props = {
 }
 
 export const currentPromptAtom = atom<string>('')
+export const currentFileAtom = atom<File | undefined | null>(undefined)
 export const appDownloadProgress = atom<number>(-1)
 export const searchAtom = atom<string>('')
 

web/hooks/useSendChatMessage.ts

@@ -20,10 +20,15 @@ import { useAtom, useAtomValue, useSetAtom } from 'jotai'
 import { ulid } from 'ulid'
 
 import { selectedModelAtom } from '@/containers/DropdownListSidebar'
-import { currentPromptAtom } from '@/containers/Providers/Jotai'
+import {
+  currentFileAtom,
+  currentPromptAtom,
+} from '@/containers/Providers/Jotai'
 
 import { toaster } from '@/containers/Toast'
 
+import { getBase64 } from '@/utils/base64'
+
 import { useActiveModel } from './useActiveModel'
 
 import { extensionManager } from '@/extension/ExtensionManager'
@@ -57,6 +62,7 @@ export default function useSendChatMessage() {
   const threadStates = useAtomValue(threadStatesAtom)
   const updateThreadInitSuccess = useSetAtom(updateThreadInitSuccessAtom)
   const activeModelParams = useAtomValue(getActiveThreadModelRuntimeParamsAtom)
+  const getUploadedImage = useAtomValue(currentFileAtom)
 
   useEffect(() => {
     modelRef.current = activeModel
@@ -181,6 +187,8 @@ export default function useSendChatMessage() {
     const prompt = currentPrompt.trim()
     setCurrentPrompt('')
 
+    const base64Image = await getBase64(getUploadedImage)
+
     const messages: ChatCompletionMessage[] = [
       activeThread.assistants[0]?.instructions,
     ]
@@ -201,10 +209,24 @@ export default function useSendChatMessage() {
           .concat([
             {
               role: ChatCompletionRole.User,
-              content: prompt,
+              content: selectedModel
+                ? [
+                    {
+                      type: ContentType.Text,
+                      text: prompt,
+                    },
+                    {
+                      type: ContentType.Image,
+                      image_url: {
+                        url: base64Image,
+                      },
+                    },
+                  ]
+                : prompt,
             } as ChatCompletionMessage,
           ])
       )
+
     const msgId = ulid()
 
     const modelRequest = selectedModel ?? activeThread.assistants[0].model

web/screens/Chat/index.tsx

@@ -1,17 +1,21 @@
 import { ChangeEvent, Fragment, KeyboardEvent, useEffect, useRef } from 'react'
 
+import { FolderIcon } from '@heroicons/react/24/solid'
 import { EventName, MessageStatus, events } from '@janhq/core'
 import { Button, Textarea } from '@janhq/uikit'
 
-import { useAtom, useAtomValue } from 'jotai'
+import { useAtom, useAtomValue, useSetAtom } from 'jotai'
 
 import { StopCircle } from 'lucide-react'
 import { twMerge } from 'tailwind-merge'
 
 import LogoMark from '@/containers/Brand/Logo/Mark'
 
 import ModelStart from '@/containers/Loader/ModelStart'
-import { currentPromptAtom } from '@/containers/Providers/Jotai'
+import {
+  currentFileAtom,
+  currentPromptAtom,
+} from '@/containers/Providers/Jotai'
 
 import { MainViewState } from '@/constants/screens'
 
@@ -53,11 +57,13 @@ const ChatScreen = () => {
 
   const activeThreadId = useAtomValue(getActiveThreadIdAtom)
   const [isWaitingToSend, setIsWaitingToSend] = useAtom(waitingToSendMessage)
+  const setUploadedFile = useSetAtom(currentFileAtom)
 
   const showing = useAtomValue(showRightSideBarAtom)
 
   const textareaRef = useRef<HTMLTextAreaElement>(null)
   const modelRef = useRef(activeModel)
+  const fileInputRef = useRef<HTMLInputElement | null>(null)
 
   useEffect(() => {
     modelRef.current = activeModel
@@ -102,6 +108,16 @@ const ChatScreen = () => {
     events.emit(EventName.OnInferenceStopped, {})
   }
 
+  /**
+   * Handles the change event of the extension file input element by setting the file name state.
+   * Its to be used to display the extension file name of the selected file.
+   * @param event - The change event object.
+   */
+  const handleFileChange = (event: React.ChangeEvent<HTMLInputElement>) => {
+    const file = event.target.files?.[0]
+    if (file) setUploadedFile(file)
+  }
+
   return (
     <div className="flex h-full w-full">
       <div className="flex h-full w-60 flex-shrink-0 flex-col overflow-y-auto border-r border-border bg-background dark:bg-background/50">
@@ -167,6 +183,17 @@ const ChatScreen = () => {
                 onPromptChange(e)
               }
             />
+            {/* {activeModel?.visionModel && ( */}
+            <input
+              type="file"
+              style={{ display: 'none' }}
+              ref={fileInputRef}
+              onChange={handleFileChange}
+            />
+            <Button onClick={() => fileInputRef.current?.click()}>
+              <FolderIcon className="h-6 w-6" />
+            </Button>
+            {/* )} */}
             {messages[messages.length - 1]?.status !== MessageStatus.Pending ? (
               <Button
                 size="lg"

web/utils/base64.ts

@@ -0,0 +1,18 @@
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export const getBase64 = async (file: any): Promise<string> => {
+  return new Promise((resolve, reject) => {
+    let fileInfo
+    // Make new FileReader
+    const reader = new FileReader()
+
+    // Convert the file to base64 text
+    reader.readAsDataURL(file)
+
+    // on reader load somthing...
+    reader.onload = () => {
+      // Make a fileInfo Object
+      const baseURL = reader.result
+      resolve(baseURL as string)
+    }
+  })
+}