images in chat

client/src/Chat.tsx

@@ -1,55 +1,75 @@
-import { useState } from "react";
+import { useRef, useState } from "react";
 import { useParams } from "react-router-dom";
 import { useMutation } from "@tanstack/react-query";
 import { Input } from "@/components/ui/input";
 import { Button } from "@/components/ui/button";
+import { ImageIcon } from "lucide-react";
 import "./App.css";
+import path from "path";
 
 type TextResponse = {
     text: string;
     user: string;
+    attachments?: { url: string; contentType: string; title: string }[];
 };
 
 export default function Chat() {
     const { agentId } = useParams();
     const [input, setInput] = useState("");
     const [messages, setMessages] = useState<TextResponse[]>([]);
+    const [selectedFile, setSelectedFile] = useState<File | null>(null);
+    const fileInputRef = useRef<HTMLInputElement>(null);
 
     const mutation = useMutation({
         mutationFn: async (text: string) => {
+            const formData = new FormData();
+            formData.append("text", text);
+            formData.append("userId", "user");
+            formData.append("roomId", `default-room-${agentId}`);
+
+            if (selectedFile) {
+                formData.append("file", selectedFile);
+            }
+
             const res = await fetch(`/api/${agentId}/message`, {
                 method: "POST",
-                headers: {
-                    "Content-Type": "application/json",
-                },
-                body: JSON.stringify({
-                    text,
-                    userId: "user",
-                    roomId: `default-room-${agentId}`,
-                }),
+                body: formData,
             });
             return res.json() as Promise<TextResponse[]>;
         },
         onSuccess: (data) => {
             setMessages((prev) => [...prev, ...data]);
+            setSelectedFile(null);
         },
     });
 
     const handleSubmit = async (e: React.FormEvent) => {
         e.preventDefault();
-        if (!input.trim()) return;
+        if (!input.trim() && !selectedFile) return;
 
         // Add user message immediately to state
         const userMessage: TextResponse = {
             text: input,
             user: "user",
+            attachments: selectedFile ? [{ url: URL.createObjectURL(selectedFile), contentType: selectedFile.type, title: selectedFile.name }] : undefined,
         };
         setMessages((prev) => [...prev, userMessage]);
 
         mutation.mutate(input);
         setInput("");
     };
 
+    const handleFileSelect = () => {
+        fileInputRef.current?.click();
+    };
+
+    const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+        const file = e.target.files?.[0];
+        if (file && file.type.startsWith('image/')) {
+            setSelectedFile(file);
+        }
+    };
+
     return (
         <div className="flex flex-col h-screen max-h-screen w-full">
             <div className="flex-1 min-h-0 overflow-y-auto p-4">
@@ -72,6 +92,21 @@ export default function Chat() {
                                     }`}
                                 >
                                     {message.text}
+                                    {message.attachments?.map((attachment, i) => (
+                                        attachment.contentType.startsWith('image/') && (
+                                            <img
+                                                key={i}
+                                                src={message.user === "user"
+                                                    ? attachment.url
+                                                    : attachment.url.startsWith('http')
+                                                        ? attachment.url
+                                                        : `http://localhost:3000/media/generated/${attachment.url.split('/').pop()}`
+                                                }
+                                                alt={attachment.title || "Attached image"}
+                                                className="mt-2 max-w-full rounded-lg"
+                                            />
+                                        )
+                                    ))}
                                 </div>
                             </div>
                         ))
@@ -86,17 +121,38 @@ export default function Chat() {
             <div className="border-t p-4 bg-background">
                 <div className="max-w-3xl mx-auto">
                     <form onSubmit={handleSubmit} className="flex gap-2">
+                        <input
+                            type="file"
+                            ref={fileInputRef}
+                            onChange={handleFileChange}
+                            accept="image/*"
+                            className="hidden"
+                        />
                         <Input
                             value={input}
                             onChange={(e) => setInput(e.target.value)}
                             placeholder="Type a message..."
                             className="flex-1"
                             disabled={mutation.isPending}
                         />
+                        <Button
+                            type="button"
+                            variant="outline"
+                            size="icon"
+                            onClick={handleFileSelect}
+                            disabled={mutation.isPending}
+                        >
+                            <ImageIcon className="h-4 w-4" />
+                        </Button>
                         <Button type="submit" disabled={mutation.isPending}>
                             {mutation.isPending ? "..." : "Send"}
                         </Button>
                     </form>
+                    {selectedFile && (
+                        <div className="mt-2 text-sm text-muted-foreground">
+                            Selected file: {selectedFile.name}
+                        </div>
+                    )}
                 </div>
             </div>
         </div>

packages/client-direct/package.json

@@ -17,6 +17,7 @@
     "multer": "1.4.5-lts.1"
   },
   "devDependencies": {
+    "@types/multer": "^1.4.12",
     "tsup": "8.3.5"
   },
   "scripts": {

packages/client-direct/src/index.ts

@@ -1,8 +1,13 @@
 import bodyParser from "body-parser";
 import cors from "cors";
 import express, { Request as ExpressRequest } from "express";
-import multer, { File } from "multer";
-import { elizaLogger, generateCaption, generateImage } from "@elizaos/core";
+import multer from "multer";
+import {
+    elizaLogger,
+    generateCaption,
+    generateImage,
+    Media,
+} from "@elizaos/core";
 import { composeContext } from "@elizaos/core";
 import { generateMessageResponse } from "@elizaos/core";
 import { messageCompletionFooter } from "@elizaos/core";
@@ -19,7 +24,23 @@ import { settings } from "@elizaos/core";
 import { createApiRouter } from "./api.ts";
 import * as fs from "fs";
 import * as path from "path";
-const upload = multer({ storage: multer.memoryStorage() });
+
+const storage = multer.diskStorage({
+    destination: (req, file, cb) => {
+        const uploadDir = path.join(process.cwd(), "data", "uploads");
+        // Create the directory if it doesn't exist
+        if (!fs.existsSync(uploadDir)) {
+            fs.mkdirSync(uploadDir, { recursive: true });
+        }
+        cb(null, uploadDir);
+    },
+    filename: (req, file, cb) => {
+        const uniqueSuffix = `${Date.now()}-${Math.round(Math.random() * 1e9)}`;
+        cb(null, `${uniqueSuffix}-${file.originalname}`);
+    },
+});
+
+const upload = multer({ storage });
 
 export const messageHandlerTemplate =
     // {{goals}}
@@ -66,12 +87,22 @@ export class DirectClient {
         this.app.use(bodyParser.json());
         this.app.use(bodyParser.urlencoded({ extended: true }));
 
+        // Serve both uploads and generated images
+        this.app.use(
+            "/media/uploads",
+            express.static(path.join(process.cwd(), "/data/uploads"))
+        );
+        this.app.use(
+            "/media/generated",
+            express.static(path.join(process.cwd(), "/generatedImages"))
+        );
+
         const apiRouter = createApiRouter(this.agents, this);
         this.app.use(apiRouter);
 
         // Define an interface that extends the Express Request interface
         interface CustomRequest extends ExpressRequest {
-            file: File;
+            file?: Express.Multer.File;
         }
 
         // Update the route handler to use CustomRequest instead of express.Request
@@ -128,6 +159,7 @@ export class DirectClient {
 
         this.app.post(
             "/:agentId/message",
+            upload.single("file"),
             async (req: express.Request, res: express.Response) => {
                 const agentId = req.params.agentId;
                 const roomId = stringToUuid(
@@ -162,9 +194,29 @@ export class DirectClient {
                 const text = req.body.text;
                 const messageId = stringToUuid(Date.now().toString());
 
+                const attachments: Media[] = [];
+                if (req.file) {
+                    const filePath = path.join(
+                        process.cwd(),
+                        "agent",
+                        "data",
+                        "uploads",
+                        req.file.filename
+                    );
+                    attachments.push({
+                        id: Date.now().toString(),
+                        url: filePath,
+                        title: req.file.originalname,
+                        source: "direct",
+                        description: `Uploaded file: ${req.file.originalname}`,
+                        text: "",
+                        contentType: req.file.mimetype,
+                    });
+                }
+
                 const content: Content = {
                     text,
-                    attachments: [],
+                    attachments,
                     source: "direct",
                     inReplyTo: undefined,
                 };

packages/plugin-node/src/actions/describe-image.ts

@@ -0,0 +1,137 @@
+import {
+    Action,
+    IAgentRuntime,
+    Memory,
+    State,
+    HandlerCallback,
+    composeContext,
+    generateObject,
+    ActionExample,
+    ModelClass,
+    elizaLogger,
+    ServiceType,
+    IImageDescriptionService,
+} from "@elizaos/core";
+import { getFileLocationTemplate } from "../templates";
+import { FileLocationResultSchema, isFileLocationResult } from "../types";
+
+export const describeImage: Action = {
+    name: "DESCRIBE_IMAGE",
+    similes: ["DESCRIBE_PICTURE", "EXPLAIN_PICTURE", "EXPLAIN_IMAGE"],
+    validate: async (_runtime: IAgentRuntime, _message: Memory) => {
+        return true;
+    },
+    description: "Describe an image",
+    handler: async (
+        runtime: IAgentRuntime,
+        message: Memory,
+        state: State,
+        _options: { [key: string]: unknown },
+        callback?: HandlerCallback
+    ): Promise<boolean> => {
+        // Create context with attachments and URL
+        const getFileLocationContext = composeContext({
+            state,
+            template: getFileLocationTemplate,
+        });
+
+        const fileLocationResultObject = await generateObject({
+            runtime,
+            context: getFileLocationContext,
+            modelClass: ModelClass.SMALL,
+            schema: FileLocationResultSchema,
+            stop: ["\n"],
+        });
+
+        if (!isFileLocationResult(fileLocationResultObject?.object)) {
+            elizaLogger.error("Failed to generate file location");
+            return false;
+        }
+
+        const { fileLocation } = fileLocationResultObject.object;
+
+        const { description } = await runtime
+            .getService<IImageDescriptionService>(ServiceType.IMAGE_DESCRIPTION)
+            .describeImage(fileLocation);
+
+        runtime.messageManager.createMemory({
+            userId: message.agentId,
+            agentId: message.agentId,
+            roomId: message.roomId,
+            content: {
+                text: description,
+            },
+        });
+
+        callback({
+            text: description,
+        });
+
+        return true;
+    },
+    examples: [
+        [
+            {
+                user: "{{user1}}",
+                content: {
+                    text: "Can you describe this image for me?",
+                },
+            },
+            {
+                user: "{{user2}}",
+                content: {
+                    text: "Let me analyze this image for you...",
+                    action: "DESCRIBE_IMAGE",
+                },
+            },
+            {
+                user: "{{user2}}",
+                content: {
+                    text: "I see an orange tabby cat sitting on a windowsill. The cat appears to be relaxed and looking out the window at birds flying by. The lighting suggests it's a sunny afternoon.",
+                },
+            },
+        ],
+        [
+            {
+                user: "{{user1}}",
+                content: {
+                    text: "What's in this picture?",
+                },
+            },
+            {
+                user: "{{user2}}",
+                content: {
+                    text: "I'll take a look at that image...",
+                    action: "DESCRIBE_IMAGE",
+                },
+            },
+            {
+                user: "{{user2}}",
+                content: {
+                    text: "The image shows a modern kitchen with stainless steel appliances. There's a large island counter in the center with marble countertops. The cabinets are white with sleek handles, and there's pendant lighting hanging above the island.",
+                },
+            },
+        ],
+        [
+            {
+                user: "{{user1}}",
+                content: {
+                    text: "Could you tell me what this image depicts?",
+                },
+            },
+            {
+                user: "{{user2}}",
+                content: {
+                    text: "I'll describe this image for you...",
+                    action: "DESCRIBE_IMAGE",
+                },
+            },
+            {
+                user: "{{user2}}",
+                content: {
+                    text: "This is a scenic mountain landscape at sunset. The peaks are snow-capped and reflected in a calm lake below. The sky is painted in vibrant oranges and purples, with a few wispy clouds catching the last rays of sunlight.",
+                },
+            },
+        ],
+    ] as ActionExample[][],
+} as Action;