From f9208b0a4029796e6414f06119d528d4ab45c6d2 Mon Sep 17 00:00:00 2001 From: Ahmad Bilal Date: Mon, 25 Nov 2024 16:23:31 +0500 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=93=A6=20NEW:=20Types?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../baseai/src/dev/routes/beta/pipes/run.ts | 4 +- packages/baseai/types/pipe.ts | 48 ++++++++++++++++++- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/packages/baseai/src/dev/routes/beta/pipes/run.ts b/packages/baseai/src/dev/routes/beta/pipes/run.ts index ddb88673..867c07b2 100644 --- a/packages/baseai/src/dev/routes/beta/pipes/run.ts +++ b/packages/baseai/src/dev/routes/beta/pipes/run.ts @@ -4,7 +4,7 @@ import { dlog } from '@/dev/utils/dlog'; import { handleStreamingResponse } from '@/dev/utils/provider-handlers/streaming-response-handler'; import { logger } from '@/utils/logger-utils'; import { Hono } from 'hono'; -import { schemaMessage, VariablesSchema } from 'types/pipe'; +import { schemaMessage, schemaPipeMessage, VariablesSchema } from 'types/pipe'; import { z } from 'zod'; // Schema definitions @@ -31,7 +31,7 @@ const PipeSchema = z.object({ status: z.string(), meta: MetaSchema, model: ModelSchema, - messages: z.array(schemaMessage), + messages: z.array(schemaPipeMessage), functions: z.array(z.unknown()).default([]), memorysets: z.array(z.string().trim().min(1)).default([]), variables: VariablesSchema diff --git a/packages/baseai/types/pipe.ts b/packages/baseai/types/pipe.ts index bc18a65a..1c9dd560 100644 --- a/packages/baseai/types/pipe.ts +++ b/packages/baseai/types/pipe.ts @@ -13,7 +13,52 @@ import type { } from './model'; import type { PipeTool } from './tools'; +const contentTypeSchema = z.object({ + type: z.string(), + text: z.string().optional(), + image_url: z + .object({ + url: z.string(), + detail: z.string().optional() + }) + .optional() +}); + export const schemaMessage = z + .object({ + role: z.enum(['system', 'user', 'assistant', 'function', 'tool']), + content: z + .union([z.string(), z.array(contentTypeSchema), z.null()]) + .optional(), + tool_call_id: z.string().optional(), + name: z.string().optional(), + tool_calls: z + .array( + z.object({ + id: z.string(), + type: z.string(), + function: z.record(z.unknown()) + }) + ) + .optional() + }) + .refine( + ({ content, role, tool_calls }) => { + // If content is null, role isn't assistant and tool_calls is not present. + // then the schema is invalid + // because the message content is null and its not an assistant tool call + const isSchemaInvalid = + content === null && role !== 'assistant' && !tool_calls; + + if (isSchemaInvalid) return false; + return true; + }, + { + message: 'Message content cannot be empty.' + } + ); + +export const schemaPipeMessage = z .object({ role: z.enum(['system', 'user', 'assistant', 'function', 'tool']), content: z.string().nullable(), @@ -46,6 +91,7 @@ export const schemaMessage = z ); export type Message = z.infer; +export type PipeMessage = z.infer; export const VariableSchema = z.object({ name: z.string(), @@ -98,7 +144,7 @@ export interface Pipe { stop: string[]; tool_choice: ToolChoice; parallel_tool_calls: boolean; - messages: Message[]; + messages: PipeMessage[]; variables: VariablesI; tools: PipeTool[]; memory: { From a2cd424aa3e5a01055a63f46481d9dcaa0c9cab2 Mon Sep 17 00:00:00 2001 From: Ahmad Bilal Date: Mon, 25 Nov 2024 16:23:44 +0500 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=93=A6=20NEW:=20Get=20message=20conte?= =?UTF-8?q?nt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dev/utils/thread/get-message-content.ts | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 packages/baseai/src/dev/utils/thread/get-message-content.ts diff --git a/packages/baseai/src/dev/utils/thread/get-message-content.ts b/packages/baseai/src/dev/utils/thread/get-message-content.ts new file mode 100644 index 00000000..011f4879 --- /dev/null +++ b/packages/baseai/src/dev/utils/thread/get-message-content.ts @@ -0,0 +1,36 @@ +import type { Message } from 'types/pipe'; + +export function getMessageContent(message: Message): string | null { + // Tool calls have no content + if (!message?.content) return null; + + // If content is a string, return it + if (typeof message.content === 'string') { + return message.content; + } + + /** + * If content is an array, find the text content part and return its text + * + * 1. Image messages have text and image content objects + * {"type": "text", "text": "What’s in this image?"}, + * { + * "type": "image_url", + * "image_url": { + * "url": "", + * }, + * }, + * + * 2. Audio messages always have text and audio content objects + * content: [ + * {type: 'text', text: 'What is in this recording?'}, + * {type: 'input_audio', input_audio: {data: base64str, format: 'wav'}}, + * ]; + */ + + if (Array.isArray(message.content)) { + return message.content.find(item => item.type === 'text')?.text || null; + } + + return null; +} From c96bcb83bf1c55991a648eb99f8ebcf983853279 Mon Sep 17 00:00:00 2001 From: Ahmad Bilal Date: Mon, 25 Nov 2024 16:23:59 +0500 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=93=A6=20NEW:=20Vision=20support?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/dev/utils/thread/process-messages.ts | 32 +++++++++++++++---- packages/baseai/src/utils/memory/lib.ts | 3 +- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/packages/baseai/src/dev/utils/thread/process-messages.ts b/packages/baseai/src/dev/utils/thread/process-messages.ts index 4bd0fc7b..8b4c4e59 100644 --- a/packages/baseai/src/dev/utils/thread/process-messages.ts +++ b/packages/baseai/src/dev/utils/thread/process-messages.ts @@ -78,21 +78,41 @@ function replaceVarsInMessagesWithVals({ // 1- message.content is empty // 2- message.role is 'assistant' // 3- message.tool_calls is an array of tool calls requested by LLM. + + // 1. If tool call or no content, return msg as is const isAssistantToolCall = !message.content && message.role === 'assistant' && message.tool_calls?.length; + if (isAssistantToolCall || !message.content) return message; + + // 2. If the content is an array, replace variables in each item + if (Array.isArray(message.content)) { + const updatedContent = message.content.map(contentItem => ({ + ...contentItem, - // Since no content to replace variables in, return the message as is. - if (isAssistantToolCall) return message; - if (!message.content) return message; + text: contentItem.text?.replace( + variableRegex, + (match, varName) => { + const trimmedVarName = varName.trim(); // Trim any extra spaces - // Replace variables in the message content + // If the variable exists in the map, replace with its value; otherwise, leave the placeholder intact + return variablesMap.get(trimmedVarName) || match; + } + ) + })); + return { + ...message, + content: updatedContent + }; + } + + // 3. If content is a string, replace variables in it const updatedContent = message.content.replace( variableRegex, (match, varName) => { - const trimmedVarName = varName.trim(); // Trim any extra spaces - // If the variable exists in the map, replace with its value; otherwise, leave the placeholder intact + const trimmedVarName = varName.trim(); + return variablesMap.get(trimmedVarName) || match; } ); diff --git a/packages/baseai/src/utils/memory/lib.ts b/packages/baseai/src/utils/memory/lib.ts index a567ceb0..ad3d77fe 100644 --- a/packages/baseai/src/utils/memory/lib.ts +++ b/packages/baseai/src/utils/memory/lib.ts @@ -20,6 +20,7 @@ import { loadConfig } from '../config/config-handler'; import { logger } from '../logger-utils'; import { generateLocalEmbeddings } from './generate-local-embeddings'; import { getOpenAIEmbeddings } from './generate-openai-embeddings'; +import { getMessageContent } from '@/dev/utils/thread/get-message-content'; export async function checkDirectoryExists(directoryPath: string) { try { @@ -155,7 +156,7 @@ export const addContextFromMemory = async ({ const lastUserMsg = [...messages] .reverse() .find(m => m.role === 'user'); - const userPrompt = lastUserMsg?.content; + const userPrompt = getMessageContent(lastUserMsg!); // If there is no user prompt, return the messages. if (!userPrompt) return;