Merge pull request #36 from freemocap/jon/fix-various

Jon/fix various

src/interfaces/discord/services/discord-attachment.service.ts

@@ -2,7 +2,6 @@ import { Injectable, Logger } from '@nestjs/common';
 import { Attachment } from 'discord.js';
 import { OpenaiAudioService } from '../../../core/ai/openai/openai-audio.service';
 import axios from 'axios';
-import * as mime from 'mime-types'; // Ensure to import mime-types
 import * as path from 'path';
 import * as fs from 'fs';
 import { createReadStream, createWriteStream } from 'fs';
@@ -98,11 +97,7 @@ export class DiscordAttachmentService {
         decorator: `AUDIO TRANSCRIPT: ${attachment.name}`,
         verboseOutput: transcriptionResponse,
       };
-      return this.formatResponse(
-        rawResponse,
-        mime.lookup(attachment.name),
-        attachment,
-      );
+      return this.formatResponse(rawResponse);
     } catch (error) {
       this.logger.error(
         `Error processing audio attachment: ${error.message || error}`,
@@ -151,11 +146,7 @@ export class DiscordAttachmentService {
         rawText: textFileContent,
         decorator: `TEXT ATTACHMENT: ${attachment.name}`,
       };
-      return this.formatResponse(
-        rawResponse,
-        mime.lookup(attachment.name),
-        attachment,
-      );
+      return this.formatResponse(rawResponse);
     } catch {
       return false;
     }
@@ -191,6 +182,11 @@ export class DiscordAttachmentService {
     };
   }
 
+  public async getAttachmentText(attachment: Attachment): Promise<string> {
+    const attachmentPath = await this.downloadAttachment(attachment);
+    return await fs.promises.readFile(attachmentPath, 'utf8');
+  }
+
   public async downloadAttachment(attachment: Attachment): Promise<string> {
     this.logger.log('Downloading attachment:', attachment.name);
     try {
@@ -215,15 +211,10 @@ export class DiscordAttachmentService {
     }
   }
 
-  private formatResponse(
-    response: any,
-    fileType: string,
-    attachment: Attachment,
-  ) {
-    const simpleUrl = attachment.url.split('?')[0];
+  private formatResponse(response: any) {
     return {
       ...response,
-      text: `> ${fileType} file URL: ${simpleUrl}\n\n\`\`\`\n\nBEGIN ${response.decorator}\n\n${response.rawText}\n\nEND ${response.decorator}\n\n\`\`\``,
+      text: `\nBEGIN ${response.decorator}\n\n${response.rawText}\n\n`,
     };
   }
 }

src/interfaces/discord/services/discord-context-prompt.service.ts

@@ -52,7 +52,7 @@ export class DiscordContextPromptService {
         categoryInstructions,
         channelTopic,
         channelPinnedInstructions,
-      ].join('\n\n');
+      ].join('\n-----\n');
     } catch (error) {
       this.logger.error(
         `Failed to get context instructions with error: ${error}`,
@@ -186,11 +186,12 @@ export class DiscordContextPromptService {
       ),
     );
 
-    return instructionMessages
-      .map((message: Message) =>
+    const extractedMessages: string[] = await Promise.all(
+      instructionMessages.map((message: Message) =>
         this._messageService.extractMessageContentAsString(message),
-      )
-      .join('\n');
+      ),
+    );
+    return extractedMessages.join('\n');
   }
 
   public isBotInstructionEmoji(emoji: GuildEmoji | ReactionEmoji): boolean {
@@ -213,17 +214,12 @@ export class DiscordContextPromptService {
     if (pinnedMessages.size === 0) {
       return '';
     }
-    let pinnedMessagesContent = `MESSAGES PINNED IN CHANNEL ${channel.name}:\n\n`;
+    let pinnedMessagesContent = `CHANNEL '${channel.name}' PINNED MESSAGE INSTRUCTIONS:\n\n`;
 
-    let pinnedMessageCount = 0;
     for (const message of pinnedMessages.values()) {
-      pinnedMessagesContent += `BEGIN PINNED MESSAGE ${pinnedMessageCount++}:\n\n`;
       const content =
         await this._messageService.extractMessageContentAsString(message);
       pinnedMessagesContent += content;
-      pinnedMessagesContent += `END PINNED MESSAGE ${pinnedMessageCount++}:\n\n`;
-
-      pinnedMessagesContent += '\nEND PINNED MESSAGES';
     }
     return pinnedMessagesContent;
   }

src/interfaces/discord/services/discord-message.service.ts

@@ -5,6 +5,25 @@ import { DiscordContextRouteService } from './discord-context-route.service';
 import { DiscordAttachmentService } from './discord-attachment.service';
 import { OpenaiChatService } from '../../../core/ai/openai/openai-chat.service';
 
+/**
+ * Checks if the given text ends with an unclosed code block.
+ * @param text The text to check
+ * @returns True if the text ends with an unclosed code block, false otherwise
+ */
+function hasUnclosedCodeBlock(text: string): boolean {
+  const codeBlockPattern = /```/g;
+  let match;
+  let codeBlockCount = 0;
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  while ((match = codeBlockPattern.exec(text)) !== null) {
+    codeBlockCount++;
+  }
+
+  // A code block is unclosed if the count of ```  is odd
+  return codeBlockCount % 2 !== 0;
+}
+
 @Injectable()
 export class DiscordMessageService {
   private readonly maxMessageLength = 2000 * 0.85; // discord max message length is 2000 characters (and * 0.85 to be safe)
@@ -140,22 +159,30 @@ export class DiscordMessageService {
           continue;
         }
         fullAiTextResponse += incomingTextChunk;
+        const inCodeBlock = hasUnclosedCodeBlock(fullAiTextResponse);
 
         // If the proposed text is less than the max message length, just add it to the current text
         if (
           currentReplyMessageText.length + incomingTextChunk.length <
           maxMessageLength
         ) {
           currentReplyMessageText += incomingTextChunk;
-          await currentReplyMessage.edit(currentReplyMessageText);
+          let replyMessageToSend = currentReplyMessageText;
+          if (inCodeBlock) {
+            replyMessageToSend += '\n```\n';
+          }
+          await currentReplyMessage.edit(replyMessageToSend);
         } else {
           // Otherwise, split the message and start a new one
           this.logger.debug(
             'Reply message too long, splitting into multiple messages',
           );
-          const continuingFromString = `> continuing from \`...${currentReplyMessageText.slice(
+          let continuingFromString = `> continuing from '...${currentReplyMessageText.slice(
             -50,
-          )}...\`\n\n`;
+          )}'...\n\n`;
+          if (inCodeBlock) {
+            continuingFromString += '```\n';
+          }
 
           replyMessages.push(
             await currentReplyMessage.reply(continuingFromString),
@@ -195,14 +222,13 @@ export class DiscordMessageService {
   public async extractMessageContentAsString(discordMessage: Message<boolean>) {
     const { humanInputText, attachmentText, imageURLs } =
       await this.extractMessageContent(discordMessage);
-    let fullText = `BEGIN MESSAGE CONTENT:\n\n ${humanInputText}\n\n END MESSAGE CONTENT\n\n`;
+    let fullText = `${humanInputText}\n\n`;
     if (attachmentText) {
       fullText += attachmentText;
     }
     if (imageURLs.length > 0) {
-      fullText += '\n\nBEGIN IMAGE URLS:\n\n';
+      fullText += '\n\nIMAGE URLS:\n\n';
       fullText += imageURLs.join('\n');
-      fullText += '\n\nEND IMAGE URLS\n\n';
     }
     return fullText;
   }
@@ -211,16 +237,10 @@ export class DiscordMessageService {
     discordMessage: Message<boolean>,
     respondToChannelOrMessage?: Message<boolean> | TextBasedChannel,
   ) {
-    let humanInputText = discordMessage.content;
+    const humanInputText = discordMessage.content;
     let attachmentText = '';
     const imageURLs = [];
     if (discordMessage.attachments.size > 0) {
-      if (humanInputText.length > 0) {
-        humanInputText =
-          'BEGIN TEXT FROM HUMAN INPUT:\n\n' +
-          humanInputText +
-          '\n\nEND TEXT FROM HUMAN INPUT\n\n';
-      }
       for (const [, attachment] of discordMessage.attachments) {
         if (attachment.contentType.split('/')[0] == 'image') {
           imageURLs.push(

src/interfaces/discord/services/discord-persistence.service.ts

@@ -10,15 +10,17 @@ import {
   TextChannel,
 } from 'discord.js';
 import { AiChatDocument } from '../../../core/database/collections/ai-chats/ai-chat.schema';
-import { CoupletDocument } from '../../../core/database/collections/couplets/couplet.schema';
+import { DiscordAttachmentService } from './discord-attachment.service';
 
 @Injectable()
 export class DiscordPersistenceService {
   private readonly logger = new Logger(DiscordPersistenceService.name);
+
   constructor(
     private readonly _coupletService: CoupletsService,
     private readonly _aiChatsService: AiChatsService,
     private readonly _messageService: MessagesService,
+    private readonly _attachementService: DiscordAttachmentService,
   ) {}
 
   public async persistInteraction(
@@ -78,7 +80,9 @@ export class DiscordPersistenceService {
         aiChatId,
         await this._aiChatsService.findOne(aiChatId),
         discordMessage.channel as TextChannel,
-        couplet,
+        discordMessage.content,
+        attachmentText,
+        fullAiTextResponse,
       );
     } catch (error) {
       this.logger.error(`Error persisting interaction: ${error}`);
@@ -89,23 +93,48 @@ export class DiscordPersistenceService {
     aiChatId: string,
     aiChatDocument: AiChatDocument,
     channel: TextChannel,
-    couplet: CoupletDocument,
+    humanMessageContent: string,
+    humanAttachmentContent: string,
+    aiFullMessageReponse: string,
   ) {
     this.logger.debug(
       `Attaching chat document to oldest message in thread ${aiChatId}`,
     );
-    // remove the _id field from the document
-    const aiChatAsJson = JSON.parse(JSON.stringify(aiChatDocument, null, 2));
-    delete aiChatAsJson._id;
-    delete aiChatAsJson.__v;
-
     const oldestMessage = await this._findOldestMessage(channel);
-    const aiChatAttachmentName = `chat-${aiChatId}.json`;
-    const aiChatAttachment = new AttachmentBuilder(
-      Buffer.from(JSON.stringify(aiChatDocument, null, 2)),
-      { name: aiChatAttachmentName },
+    const aiChatAttachmentName = `chat-${aiChatId}.md`;
+    let chatAttachmentText = '';
+
+    // Check if the attachment already exists
+    const existingChatPersistenceAttachment = oldestMessage.attachments.find(
+      (attachment) => attachment.name === aiChatAttachmentName,
     );
 
+    if (!existingChatPersistenceAttachment) {
+      // Initialize new chat document if it does not exist
+      const aiChatAsJson = JSON.parse(JSON.stringify(aiChatDocument, null, 2));
+      const contextInstructions = aiChatAsJson.contextInstructions;
+      delete aiChatAsJson.contextInstructions;
+      delete aiChatAsJson._id;
+      delete aiChatAsJson.__v;
+      chatAttachmentText =
+        '```\n' + JSON.stringify(aiChatAsJson, null, 2) + '\n```\n';
+      chatAttachmentText += '\n___\n';
+      chatAttachmentText += '\nCONTEXT INSTRUCTIONS/SYSTEM PROMPT:\n\n```\n';
+      chatAttachmentText += `${contextInstructions}\n`;
+      chatAttachmentText += '```\n---\n## CONVERSATION\n\n';
+    } else {
+      // Fetch the existing attachment content
+      chatAttachmentText = await this._attachementService.getAttachmentText(
+        existingChatPersistenceAttachment,
+      );
+    }
+
+    // Append the new HumanMessage and AIMessage
+    chatAttachmentText += `**HUMAN MESSAGE:**\n\n${humanMessageContent}\n\n`;
+    if (humanAttachmentContent) {
+      chatAttachmentText += `**ATTACHMENTS:**\n\n${humanAttachmentContent}\n\n`;
+    }
+    chatAttachmentText += `**AI MESSAGE:**\n\n${aiFullMessageReponse}\n\n`;
     const existingAttachmentsCollection = oldestMessage.attachments.filter(
       (attachment) => attachment.name !== aiChatAttachmentName,
     );
@@ -114,6 +143,11 @@ export class DiscordPersistenceService {
       existingAttachmentsCollection.values(),
     );
 
+    const aiChatAttachment = new AttachmentBuilder(
+      Buffer.from(chatAttachmentText),
+      { name: aiChatAttachmentName },
+    );
+
     await oldestMessage.edit({
       content: oldestMessage.content,
       files: [aiChatAttachment, ...existingAttachments],