From fd3ac82b621f9a37f69e6c966ce4ba0744f62d88 Mon Sep 17 00:00:00 2001 From: jonmatthis Date: Tue, 2 Jul 2024 09:03:49 -0400 Subject: [PATCH 1/5] fixnig context message bug --- .../services/discord-context-prompt.service.ts | 17 +++++++++-------- .../discord/services/discord-message.service.ts | 5 ++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/interfaces/discord/services/discord-context-prompt.service.ts b/src/interfaces/discord/services/discord-context-prompt.service.ts index 4b7d16d..cd71570 100644 --- a/src/interfaces/discord/services/discord-context-prompt.service.ts +++ b/src/interfaces/discord/services/discord-context-prompt.service.ts @@ -186,11 +186,12 @@ export class DiscordContextPromptService { ), ); - return instructionMessages - .map((message: Message) => + const extractedMessages: string[] = await Promise.all( + instructionMessages.map((message: Message) => this._messageService.extractMessageContentAsString(message), - ) - .join('\n'); + ), + ); + return extractedMessages.join('\n'); } public isBotInstructionEmoji(emoji: GuildEmoji | ReactionEmoji): boolean { @@ -213,17 +214,17 @@ export class DiscordContextPromptService { if (pinnedMessages.size === 0) { return ''; } - let pinnedMessagesContent = `MESSAGES PINNED IN CHANNEL ${channel.name}:\n\n`; + let pinnedMessagesContent = `CHANNEL '${channel.name}' PINNED MESSAGES START:\n\n`; let pinnedMessageCount = 0; for (const message of pinnedMessages.values()) { - pinnedMessagesContent += `BEGIN PINNED MESSAGE ${pinnedMessageCount++}:\n\n`; + pinnedMessagesContent += `PINNED MESSAGE [${pinnedMessageCount++}] START:\n\n`; const content = await this._messageService.extractMessageContentAsString(message); pinnedMessagesContent += content; - pinnedMessagesContent += `END PINNED MESSAGE ${pinnedMessageCount++}:\n\n`; + pinnedMessagesContent += `PINNED MESSAGE [${pinnedMessageCount++}] END\n\n`; - pinnedMessagesContent += '\nEND PINNED MESSAGES'; + pinnedMessagesContent += `CHANNEL '${channel.name}' PINNED MESSAGES END\n\n`; } return pinnedMessagesContent; } diff --git a/src/interfaces/discord/services/discord-message.service.ts b/src/interfaces/discord/services/discord-message.service.ts index 8f4ccbc..be4db9f 100644 --- a/src/interfaces/discord/services/discord-message.service.ts +++ b/src/interfaces/discord/services/discord-message.service.ts @@ -195,14 +195,13 @@ export class DiscordMessageService { public async extractMessageContentAsString(discordMessage: Message) { const { humanInputText, attachmentText, imageURLs } = await this.extractMessageContent(discordMessage); - let fullText = `BEGIN MESSAGE CONTENT:\n\n ${humanInputText}\n\n END MESSAGE CONTENT\n\n`; + let fullText = `BEGIN MESSAGE ${discordMessage.id}: \n\n ${humanInputText}\n\n END MESSAGE ${discordMessage.id}\n\n`; if (attachmentText) { fullText += attachmentText; } if (imageURLs.length > 0) { - fullText += '\n\nBEGIN IMAGE URLS:\n\n'; + fullText += '\n\nIMAGE URLS:\n\n'; fullText += imageURLs.join('\n'); - fullText += '\n\nEND IMAGE URLS\n\n'; } return fullText; } From 7394da512ff6ba9886aa517671fd2791abe976b2 Mon Sep 17 00:00:00 2001 From: jonmatthis Date: Tue, 2 Jul 2024 09:44:45 -0400 Subject: [PATCH 2/5] handle codeblocks better when streaming --- .../services/discord-attachment.service.ts | 2 +- .../discord-context-prompt.service.ts | 2 +- .../services/discord-message.service.ts | 33 +++++++++++++++++-- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/src/interfaces/discord/services/discord-attachment.service.ts b/src/interfaces/discord/services/discord-attachment.service.ts index b4135dc..7d7d0ea 100644 --- a/src/interfaces/discord/services/discord-attachment.service.ts +++ b/src/interfaces/discord/services/discord-attachment.service.ts @@ -223,7 +223,7 @@ export class DiscordAttachmentService { const simpleUrl = attachment.url.split('?')[0]; return { ...response, - text: `> ${fileType} file URL: ${simpleUrl}\n\n\`\`\`\n\nBEGIN ${response.decorator}\n\n${response.rawText}\n\nEND ${response.decorator}\n\n\`\`\``, + text: `> ${fileType} file URL: ${simpleUrl}\n\nBEGIN ${response.decorator}\n\n${response.rawText}\n\nEND ${response.decorator}\n\n`, }; } } diff --git a/src/interfaces/discord/services/discord-context-prompt.service.ts b/src/interfaces/discord/services/discord-context-prompt.service.ts index cd71570..d8e28cb 100644 --- a/src/interfaces/discord/services/discord-context-prompt.service.ts +++ b/src/interfaces/discord/services/discord-context-prompt.service.ts @@ -52,7 +52,7 @@ export class DiscordContextPromptService { categoryInstructions, channelTopic, channelPinnedInstructions, - ].join('\n\n'); + ].join('\n-----\n'); } catch (error) { this.logger.error( `Failed to get context instructions with error: ${error}`, diff --git a/src/interfaces/discord/services/discord-message.service.ts b/src/interfaces/discord/services/discord-message.service.ts index be4db9f..b899845 100644 --- a/src/interfaces/discord/services/discord-message.service.ts +++ b/src/interfaces/discord/services/discord-message.service.ts @@ -5,6 +5,25 @@ import { DiscordContextRouteService } from './discord-context-route.service'; import { DiscordAttachmentService } from './discord-attachment.service'; import { OpenaiChatService } from '../../../core/ai/openai/openai-chat.service'; +/** + * Checks if the given text ends with an unclosed code block. + * @param text The text to check + * @returns True if the text ends with an unclosed code block, false otherwise + */ +function hasUnclosedCodeBlock(text: string): boolean { + const codeBlockPattern = /```/g; + let match; + let codeBlockCount = 0; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + while ((match = codeBlockPattern.exec(text)) !== null) { + codeBlockCount++; + } + + // A code block is unclosed if the count of ``` is odd + return codeBlockCount % 2 !== 0; +} + @Injectable() export class DiscordMessageService { private readonly maxMessageLength = 2000 * 0.85; // discord max message length is 2000 characters (and * 0.85 to be safe) @@ -140,6 +159,7 @@ export class DiscordMessageService { continue; } fullAiTextResponse += incomingTextChunk; + const inCodeBlock = hasUnclosedCodeBlock(fullAiTextResponse); // If the proposed text is less than the max message length, just add it to the current text if ( @@ -147,15 +167,22 @@ export class DiscordMessageService { maxMessageLength ) { currentReplyMessageText += incomingTextChunk; - await currentReplyMessage.edit(currentReplyMessageText); + let replyMessageToSend = currentReplyMessageText; + if (inCodeBlock) { + replyMessageToSend += '\n```\n'; + } + await currentReplyMessage.edit(replyMessageToSend); } else { // Otherwise, split the message and start a new one this.logger.debug( 'Reply message too long, splitting into multiple messages', ); - const continuingFromString = `> continuing from \`...${currentReplyMessageText.slice( + let continuingFromString = `> continuing from '...${currentReplyMessageText.slice( -50, - )}...\`\n\n`; + )}'...\n\n`; + if (inCodeBlock) { + continuingFromString += '```\n'; + } replyMessages.push( await currentReplyMessage.reply(continuingFromString), From f7fc0b366d5caf3f7efa9c9ed774320dace0905a Mon Sep 17 00:00:00 2001 From: jonmatthis Date: Tue, 2 Jul 2024 15:12:41 -0400 Subject: [PATCH 3/5] md output works but overwrites on each message --- .../services/discord-attachment.service.ts | 21 ++----- .../discord-context-prompt.service.ts | 7 +-- .../services/discord-message.service.ts | 10 +--- .../services/discord-persistence.service.ts | 57 +++++++++++++++---- 4 files changed, 52 insertions(+), 43 deletions(-) diff --git a/src/interfaces/discord/services/discord-attachment.service.ts b/src/interfaces/discord/services/discord-attachment.service.ts index 7d7d0ea..a59e7b1 100644 --- a/src/interfaces/discord/services/discord-attachment.service.ts +++ b/src/interfaces/discord/services/discord-attachment.service.ts @@ -98,11 +98,7 @@ export class DiscordAttachmentService { decorator: `AUDIO TRANSCRIPT: ${attachment.name}`, verboseOutput: transcriptionResponse, }; - return this.formatResponse( - rawResponse, - mime.lookup(attachment.name), - attachment, - ); + return this.formatResponse(rawResponse); } catch (error) { this.logger.error( `Error processing audio attachment: ${error.message || error}`, @@ -151,11 +147,7 @@ export class DiscordAttachmentService { rawText: textFileContent, decorator: `TEXT ATTACHMENT: ${attachment.name}`, }; - return this.formatResponse( - rawResponse, - mime.lookup(attachment.name), - attachment, - ); + return this.formatResponse(rawResponse); } catch { return false; } @@ -215,15 +207,10 @@ export class DiscordAttachmentService { } } - private formatResponse( - response: any, - fileType: string, - attachment: Attachment, - ) { - const simpleUrl = attachment.url.split('?')[0]; + private formatResponse(response: any) { return { ...response, - text: `> ${fileType} file URL: ${simpleUrl}\n\nBEGIN ${response.decorator}\n\n${response.rawText}\n\nEND ${response.decorator}\n\n`, + text: `\nBEGIN ${response.decorator}\n\n${response.rawText}\n\n`, }; } } diff --git a/src/interfaces/discord/services/discord-context-prompt.service.ts b/src/interfaces/discord/services/discord-context-prompt.service.ts index d8e28cb..efd3554 100644 --- a/src/interfaces/discord/services/discord-context-prompt.service.ts +++ b/src/interfaces/discord/services/discord-context-prompt.service.ts @@ -214,17 +214,12 @@ export class DiscordContextPromptService { if (pinnedMessages.size === 0) { return ''; } - let pinnedMessagesContent = `CHANNEL '${channel.name}' PINNED MESSAGES START:\n\n`; + let pinnedMessagesContent = `CHANNEL '${channel.name}' PINNED MESSAGE INSTRUCTIONS:\n\n`; - let pinnedMessageCount = 0; for (const message of pinnedMessages.values()) { - pinnedMessagesContent += `PINNED MESSAGE [${pinnedMessageCount++}] START:\n\n`; const content = await this._messageService.extractMessageContentAsString(message); pinnedMessagesContent += content; - pinnedMessagesContent += `PINNED MESSAGE [${pinnedMessageCount++}] END\n\n`; - - pinnedMessagesContent += `CHANNEL '${channel.name}' PINNED MESSAGES END\n\n`; } return pinnedMessagesContent; } diff --git a/src/interfaces/discord/services/discord-message.service.ts b/src/interfaces/discord/services/discord-message.service.ts index b899845..558d217 100644 --- a/src/interfaces/discord/services/discord-message.service.ts +++ b/src/interfaces/discord/services/discord-message.service.ts @@ -222,7 +222,7 @@ export class DiscordMessageService { public async extractMessageContentAsString(discordMessage: Message) { const { humanInputText, attachmentText, imageURLs } = await this.extractMessageContent(discordMessage); - let fullText = `BEGIN MESSAGE ${discordMessage.id}: \n\n ${humanInputText}\n\n END MESSAGE ${discordMessage.id}\n\n`; + let fullText = `${humanInputText}\n\n`; if (attachmentText) { fullText += attachmentText; } @@ -237,16 +237,10 @@ export class DiscordMessageService { discordMessage: Message, respondToChannelOrMessage?: Message | TextBasedChannel, ) { - let humanInputText = discordMessage.content; + const humanInputText = discordMessage.content; let attachmentText = ''; const imageURLs = []; if (discordMessage.attachments.size > 0) { - if (humanInputText.length > 0) { - humanInputText = - 'BEGIN TEXT FROM HUMAN INPUT:\n\n' + - humanInputText + - '\n\nEND TEXT FROM HUMAN INPUT\n\n'; - } for (const [, attachment] of discordMessage.attachments) { if (attachment.contentType.split('/')[0] == 'image') { imageURLs.push( diff --git a/src/interfaces/discord/services/discord-persistence.service.ts b/src/interfaces/discord/services/discord-persistence.service.ts index caf1d37..8f010c0 100644 --- a/src/interfaces/discord/services/discord-persistence.service.ts +++ b/src/interfaces/discord/services/discord-persistence.service.ts @@ -10,15 +10,17 @@ import { TextChannel, } from 'discord.js'; import { AiChatDocument } from '../../../core/database/collections/ai-chats/ai-chat.schema'; -import { CoupletDocument } from '../../../core/database/collections/couplets/couplet.schema'; +import { DiscordAttachmentService } from './discord-attachment.service'; @Injectable() export class DiscordPersistenceService { private readonly logger = new Logger(DiscordPersistenceService.name); + constructor( private readonly _coupletService: CoupletsService, private readonly _aiChatsService: AiChatsService, private readonly _messageService: MessagesService, + private readonly _attachementService: DiscordAttachmentService, ) {} public async persistInteraction( @@ -78,7 +80,9 @@ export class DiscordPersistenceService { aiChatId, await this._aiChatsService.findOne(aiChatId), discordMessage.channel as TextChannel, - couplet, + discordMessage.content, + attachmentText, + fullAiTextResponse, ); } catch (error) { this.logger.error(`Error persisting interaction: ${error}`); @@ -89,23 +93,47 @@ export class DiscordPersistenceService { aiChatId: string, aiChatDocument: AiChatDocument, channel: TextChannel, - couplet: CoupletDocument, + humanMessageContent: string, + humanAttachmentContent: string, + aiFullMessageReponse: string, ) { this.logger.debug( `Attaching chat document to oldest message in thread ${aiChatId}`, ); - // remove the _id field from the document - const aiChatAsJson = JSON.parse(JSON.stringify(aiChatDocument, null, 2)); - delete aiChatAsJson._id; - delete aiChatAsJson.__v; - const oldestMessage = await this._findOldestMessage(channel); - const aiChatAttachmentName = `chat-${aiChatId}.json`; - const aiChatAttachment = new AttachmentBuilder( - Buffer.from(JSON.stringify(aiChatDocument, null, 2)), - { name: aiChatAttachmentName }, + const aiChatAttachmentName = `chat-${aiChatId}.md`; + let chatAttachmentText = ''; + + // Check if the attachment already exists + const existingAttachment = oldestMessage.attachments.find( + (attachment) => attachment.name === aiChatAttachmentName, ); + if (!existingAttachment) { + // Initialize new chat document if it does not exist + const aiChatAsJson = JSON.parse(JSON.stringify(aiChatDocument, null, 2)); + const contextInstructions = aiChatAsJson.contextInstructions; + delete aiChatAsJson.contextInstructions; + delete aiChatAsJson._id; + delete aiChatAsJson.__v; + chatAttachmentText = + '```\n' + JSON.stringify(aiChatAsJson, null, 2) + '\n```\n'; + chatAttachmentText += '\n___\n'; + chatAttachmentText += '\nCONTEXT INSTRUCTIONS/SYSTEM PROMPT:\n\n```\n'; + chatAttachmentText += `${contextInstructions}\n`; + chatAttachmentText += '```\n'; + } else { + // Fetch the existing attachment content + const chatAttachmentPath = + await this._attachementService.downloadAttachment(existingAttachment); + } + + // Append the new HumanMessage and AIMessage + chatAttachmentText += `**HUMAN MESSAGE:**\n\n${humanMessageContent}\n\n`; + if (humanAttachmentContent) { + chatAttachmentText += `**ATTACHMENTS:**\n\n${humanAttachmentContent}\n\n`; + } + chatAttachmentText += `**AI MESSAGE:**\n\n${aiFullMessageReponse}\n\n`; const existingAttachmentsCollection = oldestMessage.attachments.filter( (attachment) => attachment.name !== aiChatAttachmentName, ); @@ -114,6 +142,11 @@ export class DiscordPersistenceService { existingAttachmentsCollection.values(), ); + const aiChatAttachment = new AttachmentBuilder( + Buffer.from(chatAttachmentText), + { name: aiChatAttachmentName }, + ); + await oldestMessage.edit({ content: oldestMessage.content, files: [aiChatAttachment, ...existingAttachments], From 54b5eed756a8a5eae8bb77af40d29f092640ccc9 Mon Sep 17 00:00:00 2001 From: jonmatthis Date: Thu, 4 Jul 2024 15:00:25 -0400 Subject: [PATCH 4/5] md output works but overwrites on each message --- .../discord/services/discord-attachment.service.ts | 9 ++++++++- .../discord/services/discord-persistence.service.ts | 9 +++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/interfaces/discord/services/discord-attachment.service.ts b/src/interfaces/discord/services/discord-attachment.service.ts index a59e7b1..5cbb78a 100644 --- a/src/interfaces/discord/services/discord-attachment.service.ts +++ b/src/interfaces/discord/services/discord-attachment.service.ts @@ -2,7 +2,6 @@ import { Injectable, Logger } from '@nestjs/common'; import { Attachment } from 'discord.js'; import { OpenaiAudioService } from '../../../core/ai/openai/openai-audio.service'; import axios from 'axios'; -import * as mime from 'mime-types'; // Ensure to import mime-types import * as path from 'path'; import * as fs from 'fs'; import { createReadStream, createWriteStream } from 'fs'; @@ -183,6 +182,14 @@ export class DiscordAttachmentService { }; } + public async getAttachmentText(attachment: Attachment): Promise { + const attachmentPath = await this.downloadAttachment(attachment); + try { + return fs.readFileSync(attachmentPath, 'utf8'); + } catch (err) { + console.error(err); + } + } public async downloadAttachment(attachment: Attachment): Promise { this.logger.log('Downloading attachment:', attachment.name); try { diff --git a/src/interfaces/discord/services/discord-persistence.service.ts b/src/interfaces/discord/services/discord-persistence.service.ts index 8f010c0..be275bf 100644 --- a/src/interfaces/discord/services/discord-persistence.service.ts +++ b/src/interfaces/discord/services/discord-persistence.service.ts @@ -105,11 +105,11 @@ export class DiscordPersistenceService { let chatAttachmentText = ''; // Check if the attachment already exists - const existingAttachment = oldestMessage.attachments.find( + const existingChatPersistenceAttachment = oldestMessage.attachments.find( (attachment) => attachment.name === aiChatAttachmentName, ); - if (!existingAttachment) { + if (!existingChatPersistenceAttachment) { // Initialize new chat document if it does not exist const aiChatAsJson = JSON.parse(JSON.stringify(aiChatDocument, null, 2)); const contextInstructions = aiChatAsJson.contextInstructions; @@ -124,8 +124,9 @@ export class DiscordPersistenceService { chatAttachmentText += '```\n'; } else { // Fetch the existing attachment content - const chatAttachmentPath = - await this._attachementService.downloadAttachment(existingAttachment); + chatAttachmentText = await this._attachementService.getAttachmentText( + existingChatPersistenceAttachment, + ); } // Append the new HumanMessage and AIMessage From d9792e54565f53bc7a09d1c2e587b16367d73850 Mon Sep 17 00:00:00 2001 From: jonmatthis Date: Thu, 11 Jul 2024 10:59:54 -0400 Subject: [PATCH 5/5] fix message persistence --- .../discord/services/discord-attachment.service.ts | 7 ++----- .../discord/services/discord-persistence.service.ts | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/interfaces/discord/services/discord-attachment.service.ts b/src/interfaces/discord/services/discord-attachment.service.ts index 5cbb78a..41ed43d 100644 --- a/src/interfaces/discord/services/discord-attachment.service.ts +++ b/src/interfaces/discord/services/discord-attachment.service.ts @@ -184,12 +184,9 @@ export class DiscordAttachmentService { public async getAttachmentText(attachment: Attachment): Promise { const attachmentPath = await this.downloadAttachment(attachment); - try { - return fs.readFileSync(attachmentPath, 'utf8'); - } catch (err) { - console.error(err); - } + return await fs.promises.readFile(attachmentPath, 'utf8'); } + public async downloadAttachment(attachment: Attachment): Promise { this.logger.log('Downloading attachment:', attachment.name); try { diff --git a/src/interfaces/discord/services/discord-persistence.service.ts b/src/interfaces/discord/services/discord-persistence.service.ts index be275bf..22bc0cc 100644 --- a/src/interfaces/discord/services/discord-persistence.service.ts +++ b/src/interfaces/discord/services/discord-persistence.service.ts @@ -121,7 +121,7 @@ export class DiscordPersistenceService { chatAttachmentText += '\n___\n'; chatAttachmentText += '\nCONTEXT INSTRUCTIONS/SYSTEM PROMPT:\n\n```\n'; chatAttachmentText += `${contextInstructions}\n`; - chatAttachmentText += '```\n'; + chatAttachmentText += '```\n---\n## CONVERSATION\n\n'; } else { // Fetch the existing attachment content chatAttachmentText = await this._attachementService.getAttachmentText(