diff --git a/src/interfaces/discord/discord.module.ts b/src/interfaces/discord/discord.module.ts index 027aa70..d608a0a 100644 --- a/src/interfaces/discord/discord.module.ts +++ b/src/interfaces/discord/discord.module.ts @@ -4,18 +4,18 @@ import { NecordModule } from 'necord'; import { DiscordConfigService } from './services/discord-config.service'; import { DiscordEventService } from './services/events/discord-event.service'; -import { DiscordChatService } from './services/threads/discord-chat.service'; +import { DiscordChatService } from './services/chats/discord-chat.service'; import { GcpModule } from '../../core/gcp/gcp.module'; import { UsersModule } from '../../core/database/collections/users/users.module'; import { ChatbotModule } from '../../core/chatbot/chatbot.module'; import { AiChatsModule } from '../../core/database/collections/ai-chats/ai-chats.module'; import { CoupletsModule } from '../../core/database/collections/couplets/couplets.module'; import { MessagesModule } from '../../core/database/collections/messages/messages.module'; -import { DiscordContextService } from './services/threads/discord-context.service'; +import { DiscordContextService } from './services/chats/discord-context.service'; import { DiscordMongodbService } from './services/discord-mongodb.service'; -import { DiscordMessageService } from './services/threads/discord-message.service'; +import { DiscordMessageService } from './services/chats/discord-message.service'; import { OpenaiModule } from '../../core/ai/openai/openai.module'; -import { DiscordAttachmentService } from './services/threads/discord-attachment.service'; +import { DiscordAttachmentService } from './services/chats/discord-attachment.service'; import { DiscordOnMessageService } from './services/events/discord-on-message.service'; @Module({ diff --git a/src/interfaces/discord/services/chats/discord-attachment.service.ts b/src/interfaces/discord/services/chats/discord-attachment.service.ts new file mode 100644 index 0000000..1595ec5 --- /dev/null +++ b/src/interfaces/discord/services/chats/discord-attachment.service.ts @@ -0,0 +1,176 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { Attachment } from 'discord.js'; +import { OpenaiAudioService } from '../../../../core/ai/openai/openai-audio.service'; +import axios from 'axios'; +import * as mime from 'mime-types'; // Ensure to import mime-types +import * as path from 'path'; +import * as fs from 'fs'; +import { createReadStream, createWriteStream } from 'fs'; +import { promisify } from 'util'; +import * as stream from 'stream'; + +@Injectable() +export class DiscordAttachmentService { + private readonly logger = new Logger(DiscordAttachmentService.name); + constructor(private readonly _openaiAudioService: OpenaiAudioService) {} + + async handleAttachment(attachment: Attachment) { + const tempFilePath = ''; + try { + const tempFilePath = await this._downloadAttachment(attachment); + const mimeType = mime.lookup(attachment.name); + + if (mimeType?.startsWith('audio/')) { + return await this.handleAudioAttachment(tempFilePath, attachment); + } else if (mimeType?.startsWith('video/')) { + return await this.handleVideoAttachment(attachment); + } else if (path.extname(attachment.name).toLowerCase() === '.zip') { + return await this.handleZipAttachment(attachment); + } else { + // Default to handling as a text file if we don't recognize the file type + return await this.handleTextAttachment(tempFilePath, attachment); + } + + // TODO - handle PDF, docx, and other complex text-type attachments + // TODO - handle image attachments -> would need add `OpenaiImageService` `to OpenaiModule` + // TODO - handle other attachments? + // TODO - parse text attachements into json if possible? i.e. .md (by heading/bullet point), .csv, .toml, .yaml, etc + } catch (error) { + this.logger.error(`Error handling attachment: ${error}`); + return null; + } finally { + try { + // Clean up temp file, if it's still around + await fs.promises.unlink(tempFilePath); + } catch {} + } + } + + private async handleAudioAttachment( + audioFilePath: string, + attachment: Attachment, + ) { + this.logger.log('Processing audio attachment:', attachment.name); + + try { + const transcriptionResponse = + await this._openaiAudioService.createAudioTranscription({ + file: createReadStream(audioFilePath), + model: 'whisper-1', + language: 'en', + response_format: 'verbose_json', + temperature: 0, + }); + + this.logger.log( + `Transcription: \n\n ${JSON.stringify( + transcriptionResponse.text, + null, + 2, + )}`, + ); + + const rawResponse = { + type: 'transcript', + rawText: transcriptionResponse.text, + decorator: `AUDIO TRANSCRIPT: ${attachment.name}`, + verboseOutput: transcriptionResponse, + }; + return this.formatResponse( + rawResponse, + mime.lookup(attachment.name), + attachment, + ); + } catch (error) { + this.logger.error( + `Error processing audio attachment: ${error.message || error}`, + ); + throw error; + } finally { + } + } + + private async handleTextAttachment( + tempFilePath: string, + attachment: Attachment, + ) { + try { + const textFileContent = await fs.promises.readFile(tempFilePath, 'utf-8'); + this.logger.log('Processing text attachment:', attachment.name); + const rawResponse = { + type: 'text_file', + rawText: textFileContent, + decorator: `TEXT ATTACHMENT: ${attachment.name}`, + }; + return this.formatResponse( + rawResponse, + mime.lookup(attachment.name), + attachment, + ); + } catch { + return false; + } + } + + private async handleVideoAttachment(attachment: Attachment) { + // Add Video processing logic here - basically, strip the audio and treat it as an audio attachment + this.logger.log('Processing video attachment:', attachment.name); + // Example return format (adjust according to your actual logic) + return { + type: 'transcript', + rawText: 'Example video content', + decorator: `VIDEO TRANSCRIPT: ${attachment.name}`, + }; + } + + private async handleZipAttachment(attachment: Attachment) { + // Add Zip processing logic here - basically, unzip it and process each internal file as a separate attachment + this.logger.log('Processing zip attachment:', attachment.name); + // Example return format (adjust according to your actual logic) + return { + type: 'zip', + rawText: 'Example zip content', + decorator: `ZIP ATTACHMENT: ${attachment.name}`, + }; + } + + private async _downloadAttachment(attachment: Attachment): Promise { + this.logger.log('Downloading attachment:', attachment.name); + try { + const tempDirectoryPath = path.join(__dirname, 'temp'); + const tempFilePath = path.join( + tempDirectoryPath, + `tempfile-${path.basename(attachment.name)}`, + ); + await fs.promises.mkdir(tempDirectoryPath, { recursive: true }); + const response = await axios({ + method: 'get', + url: attachment.url, + responseType: 'stream', + }); + const writer = createWriteStream(tempFilePath); + response.data.pipe(writer); + await promisify(stream.finished)(writer); + return tempFilePath; + } catch (error) { + this.logger.error(`Error downloading attachment: ${error}`); + throw error; + } + } + + private formatResponse( + response: any, + fileType: string, + attachment: Attachment, + ) { + const simpleUrl = attachment.url.split('?')[0]; + return { + ...response, + text: `> ${fileType} file URL: ${simpleUrl}\n\n\`\`\`\n\nBEGIN ${response.decorator}\n\n${response.rawText}\n\nEND ${response.decorator}\n\n\`\`\``, + }; + } + + private extractFileType(filename: string | undefined): string { + return filename?.split('.').pop() || ''; + } +} diff --git a/src/interfaces/discord/services/threads/discord-chat.service.ts b/src/interfaces/discord/services/chats/discord-chat.service.ts similarity index 100% rename from src/interfaces/discord/services/threads/discord-chat.service.ts rename to src/interfaces/discord/services/chats/discord-chat.service.ts diff --git a/src/interfaces/discord/services/threads/discord-context.service.ts b/src/interfaces/discord/services/chats/discord-context.service.ts similarity index 100% rename from src/interfaces/discord/services/threads/discord-context.service.ts rename to src/interfaces/discord/services/chats/discord-context.service.ts diff --git a/src/interfaces/discord/services/threads/discord-message.service.ts b/src/interfaces/discord/services/chats/discord-message.service.ts similarity index 93% rename from src/interfaces/discord/services/threads/discord-message.service.ts rename to src/interfaces/discord/services/chats/discord-message.service.ts index 1de1080..49e1832 100644 --- a/src/interfaces/discord/services/threads/discord-message.service.ts +++ b/src/interfaces/discord/services/chats/discord-message.service.ts @@ -195,10 +195,23 @@ export class DiscordMessageService { respondToChannelOrMessage && attachmentResponse.type === 'transcript' ) { - await this.sendChunkedMessage( + const replyMessages = await this.sendChunkedMessage( respondToChannelOrMessage, attachmentText, ); + const verboseJsonBuffer = Buffer.from( + JSON.stringify(attachmentResponse.verboseOutput, null, 4), + 'utf-8', + ); + await replyMessages[replyMessages.length - 1].edit({ + content: replyMessages[replyMessages.length - 1].content, + files: [ + { + attachment: verboseJsonBuffer, + name: `message-${discordMessage.id}-transcription.json`, + }, + ], + }); } attachmentText += 'END TEXT FROM ATTACHMENTS'; } diff --git a/src/interfaces/discord/services/events/discord-on-message.service.ts b/src/interfaces/discord/services/events/discord-on-message.service.ts index c9d1100..a4eea57 100644 --- a/src/interfaces/discord/services/events/discord-on-message.service.ts +++ b/src/interfaces/discord/services/events/discord-on-message.service.ts @@ -1,10 +1,10 @@ import { Injectable, Logger } from '@nestjs/common'; import { AiChatsService } from '../../../../core/database/collections/ai-chats/ai-chats.service'; import { Message, ThreadChannel } from 'discord.js'; -import { DiscordMessageService } from '../threads/discord-message.service'; +import { DiscordMessageService } from '../chats/discord-message.service'; import { ChatbotManagerService } from '../../../../core/chatbot/chatbot-manager.service'; import { AiChatDocument } from '../../../../core/database/collections/ai-chats/ai-chat.schema'; -import { DiscordContextService } from '../threads/discord-context.service'; +import { DiscordContextService } from '../chats/discord-context.service'; import { UsersService } from '../../../../core/database/collections/users/users.service'; import { OpenaiChatService } from '../../../../core/ai/openai/openai-chat.service'; diff --git a/src/interfaces/discord/services/threads/discord-attachment.service.ts b/src/interfaces/discord/services/threads/discord-attachment.service.ts deleted file mode 100644 index 12b1bde..0000000 --- a/src/interfaces/discord/services/threads/discord-attachment.service.ts +++ /dev/null @@ -1,196 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { Attachment } from 'discord.js'; -import { OpenaiAudioService } from '../../../../core/ai/openai/openai-audio.service'; -import axios from 'axios'; -import * as path from 'path'; -import * as fs from 'fs'; -import { createReadStream, createWriteStream } from 'fs'; -import { promisify } from 'util'; -import * as stream from 'stream'; - -@Injectable() -export class DiscordAttachmentService { - constructor( - private readonly _logger: Logger, - private readonly _openaiAudioService: OpenaiAudioService, - ) {} - - async handleAttachment(attachment: Attachment) { - const fileType = this.extractFileType(attachment.name); - let attachmentResponse; - - if (this.isAudio(fileType)) { - attachmentResponse = await this.processAudioAttachment( - attachment, - fileType, - ); - } else if (this.isVideo(fileType)) { - attachmentResponse = await this.processVideoAttachment( - attachment, - fileType, - ); - } else if (this.isText(fileType)) { - attachmentResponse = await this.processTextAttachment( - attachment, - fileType, - ); - } else if (fileType === 'zip') { - attachmentResponse = await this.processZipAttachment( - attachment, - fileType, - ); - } else { - this._logger.log('Unsupported file type:', fileType); - } - - return attachmentResponse - ? this.formatResponse(attachmentResponse, fileType, attachment) - : null; - } - - processAudioAttachment(attachment: Attachment, fileType: string) { - if (!this.isAudio(fileType)) { - throw new Error(`Unsupported file type: ${fileType}`); - } - return this.handleAudioAttachment(attachment).then((response) => ({ - type: 'transcript', - rawText: response.text, - Decorator: `AUDIO TRANSCRIPT: ${attachment.name}`, - })); - } - - processVideoAttachment(attachment: Attachment, fileType: string) { - if (!this.isVideo(fileType)) { - throw new Error(`Unsupported file type: ${fileType}`); - } - return this.handleAudioAttachment(attachment).then((response) => ({ - // assuming audio extraction from video - type: 'transcript', - rawText: response.text, - Decorator: `VIDEO TRANSCRIPT: ${attachment.name}`, - })); - } - - processTextAttachment(attachment: Attachment, fileType: string) { - if (!this.isText(fileType)) { - throw new Error(`Unsupported file type: ${fileType}`); - } - return this.handleTextAttachment(attachment).then((rawText) => ({ - type: 'file_text', - rawText, - Decorator: `TEXT ATTACHMENT: ${attachment.name}`, - })); - } - - processZipAttachment(attachment: Attachment, fileType: string) { - if (fileType !== 'zip') { - throw new Error(`Unsupported file type: ${fileType}`); - } - return this.handleZipAttachment(attachment).then((rawText) => ({ - type: 'zip', - rawText, - Decorator: `ZIP ATTACHMENT: ${attachment.name}`, - })); - } - - formatResponse(response: any, fileType: string, attachment: Attachment) { - const simpleUrl = attachment.url.split('?')[0]; - return { - ...response, - text: `> File URL: ${simpleUrl}\n\n\`\`\`\n\nBEGIN ${response.Decorator}\n\n${response.rawText}\n\nEND ${response.Decorator}\n\n\`\`\``, - }; - } - - isAudio(fileType: string) { - return ['mp3', 'wav', 'ogg'].includes(fileType); - } - - isVideo(fileType: string) { - return ['mp4', 'avi', 'mkv'].includes(fileType); - } - - isText(fileType: string) { - return ['txt', 'md', 'pdf'].includes(fileType); - } - - extractFileType(filename: string | undefined): string { - return filename?.split('.').pop() || ''; - } - - private async _downloadAttachment(attachment: Attachment): Promise { - this._logger.log('Processing audio attachment:', attachment.name); - try { - // Define temp directory and file paths - const tempDirectoryPath = path.join(__dirname, 'temp'); - const tempFilePath = path.join( - tempDirectoryPath, - `tempfile-${path.basename(attachment.name)}`, - ); - - // Ensure temp directory exists - await fs.promises.mkdir(tempDirectoryPath, { recursive: true }); - - // Download the attachment and save to file - const response = await axios({ - method: 'get', - url: attachment.url, - responseType: 'stream', - }); - - const writer = createWriteStream(tempFilePath); - response.data.pipe(writer); - await promisify(stream.finished)(writer); - - return tempFilePath; - } catch (error) { - this._logger.error(`Error downloading attachment: ${error}`); - } - } - - private async handleAudioAttachment(attachment: Attachment) { - this._logger.log('Processing audio attachment:', attachment.name); - const audioFilePath = await this._downloadAttachment(attachment); - - try { - // Download attachment - - // Process the downloaded file for transcription - const transcriptionResponse = - await this._openaiAudioService.createAudioTranscription({ - file: createReadStream(audioFilePath), - model: 'whisper-1', - language: 'en', - response_format: 'verbose_json', - temperature: 0, - }); - - this._logger.log( - `Transcription: ${JSON.stringify(transcriptionResponse, null, 2)}`, - ); - return { ...transcriptionResponse, audioFilePath }; - } catch (error) { - this._logger.error( - `Error processing audio attachment: ${error.message || error}`, - ); - throw error; - } finally { - //delete the audio file - await fs.promises.unlink(audioFilePath); - } - } - - private async handleVideoAttachment(attachment: Attachment) { - // Video processing logic here - this._logger.log('Processing video attachment:', attachment.name); - } - - private async handleTextAttachment(attachment: Attachment) { - // Text processing logic here - this._logger.log('Processing text attachment:', attachment.name); - } - - private async handleZipAttachment(attachment: Attachment) { - // Zip processing logic here - unzip and process each file according to its type. - this._logger.log('Processing zip attachment:', attachment.name); - } -}