From f46230b0edc3f024d07e850adc58e2dfa26f233f Mon Sep 17 00:00:00 2001
From: WFH Brian <brian@plexm.com>
Date: Mon, 2 Dec 2024 16:49:12 -0500
Subject: [PATCH] Enhance SmartThread and related components to support
 improved context input including internal embedded links

- Added functionality to handle internal embedded links (![[link]]) in SmartThread, allowing for inline processing and context extraction.
- Updated message rendering in components to display embedded links correctly.
- Refactored chat input handling to improve user experience with context suggestions.
- Introduced new utility functions for detecting and extracting internal embedded links.
- Updated tests to cover new embedded link functionality and ensure reliability.
---
 smart-chats/components/message.js        |   1 +
 smart-chats/components/thread.js         |  23 +--
 smart-chats/smart_thread.js              | 178 +++++++++++------------
 smart-chats/utils/internal_links.js      |  16 ++
 smart-chats/utils/internal_links.test.js |  31 +++-
 5 files changed, 134 insertions(+), 115 deletions(-)

diff --git a/smart-chats/components/message.js b/smart-chats/components/message.js
index 790750d0..5579bd86 100644
--- a/smart-chats/components/message.js
+++ b/smart-chats/components/message.js
@@ -16,6 +16,7 @@ export function build_html(message, opts = {}) {
           // return `<img src="${part.image_url.url}" alt="Chat image" class="sc-message-image"/>`;
           return ' ![[' + part.input.image_path + ']] ';
         }
+        if(part.type === 'text' && part.input?.key?.length) return ' ![[' + part.input.key + ']] ';
         if(part.type === 'text' && part.text?.length) return part.text;
       }).join('\n')
     : message.content
diff --git a/smart-chats/components/thread.js b/smart-chats/components/thread.js
index 6bbc591e..5ea66c5c 100644
--- a/smart-chats/components/thread.js
+++ b/smart-chats/components/thread.js
@@ -30,7 +30,7 @@ export function build_html(thread, opts = {}) {
       </div>
       <div class="sc-config-error-notice" style="display: none;"></div>
       <div class="sc-chat-form">
-        <textarea class="sc-chat-input" placeholder="Try &quot;Based on my notes&quot; or &quot;Summarize [[this note]]&quot; or &quot;Important tasks in /folder/&quot;"></textarea>
+        <textarea class="sc-chat-input" placeholder="Use @ to add context. Try &quot;Based on my notes&quot; or &quot;Summarize [[this note]]&quot; or &quot;Important tasks in /folder/&quot;"></textarea>
         <div class="sc-btn-container">
           <span id="sc-abort-button" style="display: none;">${this.get_icon_html('square')}</span>
           <button class="send-button" id="sc-send-button">
@@ -160,27 +160,8 @@ function handle_chat_input_keydown(e, thread, chat_input, opts) {
     chat_input.value = '';
     return;
   }
+  opts.handle_chat_input_keydown(e, chat_input);
 
-  if (!["/", "@", "[", "!"].includes(e.key)) return;
-  
-  const pos = chat_input.selectionStart;
-  if (e.key === "[" && chat_input.value[pos - 1] === "[" && opts.open_file_suggestion_modal) {
-    setTimeout(() => opts.open_file_suggestion_modal(), 10);
-    return;
-  }
-  
-  if (e.key === "/" && (!pos || [" ", "\n"].includes(chat_input.value[pos - 1])) && opts.open_folder_suggestion_modal) {
-    setTimeout(() => opts.open_folder_suggestion_modal(), 10);
-    return;
-  }
-  
-  if (e.key === "@" && (!pos || [" ", "\n"].includes(chat_input.value[pos - 1])) && opts.open_system_prompt_modal) {
-    setTimeout(() => opts.open_system_prompt_modal(), 10);
-  }
-
-  if (e.key === "!" && (!pos || [" ", "\n"].includes(chat_input.value[pos - 1])) && opts.open_image_suggestion_modal) {
-    setTimeout(() => opts.open_image_suggestion_modal(), 10);
-  }
 }
 
 /**
diff --git a/smart-chats/smart_thread.js b/smart-chats/smart_thread.js
index 115a23d6..6b4238ea 100644
--- a/smart-chats/smart_thread.js
+++ b/smart-chats/smart_thread.js
@@ -1,7 +1,12 @@
 import { SmartSource } from "smart-sources";
 import { render as thread_template } from "./components/thread.js";
 import { contains_folder_reference, extract_folder_references } from "./utils/folder_references";
-import { contains_internal_link, extract_internal_links } from "./utils/internal_links";
+import {
+  contains_internal_link,
+  extract_internal_links,
+  contains_internal_embedded_link,
+  extract_internal_embedded_links
+} from "./utils/internal_links";
 import { contains_self_referential_keywords } from "./utils/self_referential_keywords";
 import { contains_system_prompt_ref, extract_system_prompt_ref } from "./utils/system_prompts";
 import { contains_markdown_image, extract_markdown_images } from "./utils/markdown_images";
@@ -107,115 +112,101 @@ export class SmartThread extends SmartSource {
       const new_msg_data = {
         thread_key: this.key,
         role: 'user',
-        content: [],
+        content: [{
+          type: 'text',
+          text: content.trim(),
+        }],
+        context: {},
       };
-      const context = {};
       const language = this.env.settings?.language || 'en';
-      // Handle system prompt references (@"system prompt") FIRST
-      if (contains_system_prompt_ref(content)) {
-        const { mentions, content: content_after_refs } = extract_system_prompt_ref(content);
-        context.system_prompt_refs = mentions;
-        content = content_after_refs; // remove system prompt references from content
-      }
-  
-      // Handle internal links ([[link]])
-      if (contains_internal_link(content)) {
-        const internal_links = extract_internal_links(content);
-        context.internal_links = internal_links.map(link => {
-          return this.env.smart_sources?.fs?.get_link_target_path(link, '/') || link;
-        });
-      }
-  
-      // Handle folder references (/folder/ or /folder/subfolder/)
-      if (contains_folder_reference(content)) {
-        const folders = Object.keys(this.env.smart_sources.fs.folders);
-        const folder_refs = extract_folder_references(folders, content);
-        context.folder_refs = folder_refs;
-      }
-  
-      // Handle self-referential keywords
-      if(contains_self_referential_keywords(content, language)){
-        context.has_self_ref = true;
-      }
-      
-      // Handle markdown images LAST to preserve all processed text content
-      if (contains_markdown_image(content)) {
-        console.log('contains_markdown_image', content);
-        if (!this.chat_model.model_config.multimodal) {
-          console.warn("Current model does not support multimodal (image) content");
-          throw new Error("⚠️ Current model does not support multimodal (image) content");
+
+      // INLINE PROCESSING
+      // Handle internal embedded links (![[link]])
+      for(let i = 0; i < new_msg_data.content.length; i++){
+        const part = new_msg_data.content[i];
+        if(part.type !== 'text' || !part.text) continue;
+        if (contains_internal_embedded_link(part.text)) {
+          const internal_links = extract_internal_embedded_links(part.text);
+          for(const [full_match, link_path] of internal_links){
+            const [before, after] = part.text.split(full_match);
+            const embedded_part = {};
+            const is_image = ['png', 'jpg', 'jpeg'].some(ext => link_path.endsWith(ext));
+            if(is_image){
+              embedded_part.type = 'image_url';
+              embedded_part.input = {
+                image_path: link_path,
+              };
+            } else {
+              embedded_part.type = 'text';
+              embedded_part.input = {
+                key: this.env.smart_sources.fs.get_link_target_path(link_path, '/'),
+              };
+            }
+            part.text = after;
+            if(typeof before === 'string' && before.trim().length) new_msg_data.content.splice(
+              i, 
+              0, 
+              {
+                type: 'text',
+                text: before,
+              },
+              embedded_part
+            );
+          }
         }
+      }
   
-        const images = extract_markdown_images(content);
-        if (images.length > 0) {
-          images.forEach(image => {
-            image.image_path = this.env.smart_sources.fs.get_link_target_path(image.image_path, '/');
-            const [before, after] = content.split(image.full_match);
-            if(typeof before === 'string' && before.trim().length) new_msg_data.content.push({
-              type: 'text',
-              text: before,
-            });
-            new_msg_data.content.push({
-              type: 'image_url',
-              image_url: null,
-              input: image,
-            });
-            content = after;
+      // CONTEXT PROCESSING
+      for(let i=0; i < new_msg_data.content.length; i++){
+        const part = new_msg_data.content[i];
+        if(part.type !== 'text' || !part.text) continue;
+        // Handle internal links ([[link]])
+        if (contains_internal_link(part.text)) {
+          const internal_links = extract_internal_links(part.text);
+          new_msg_data.context.internal_links = internal_links.map(link => {
+            console.log('link', link);
+            return this.env.smart_sources?.fs?.get_link_target_path(link, '/') || link;
           });
         }
+        // Handle folder references (/folder/ or /folder/subfolder/)
+        if (contains_folder_reference(part.text)) {
+          const folders = Object.keys(this.env.smart_sources.fs.folders);
+          const folder_refs = extract_folder_references(folders, part.text);
+          new_msg_data.context.folder_refs = folder_refs;
+        }
+    
+        // Handle self-referential keywords
+        if(contains_self_referential_keywords(part.text, language)){
+          new_msg_data.context.has_self_ref = true;
+        }
       }
   
-      if (typeof content === 'string'){
-        new_msg_data.content.push({
-          type: 'text',
-          text: content.trim(),
-        });
-      } else new_msg_data.content = content;
-
-      if(Object.keys(context).length > 0){
-        console.log('creating system message with context', context);
-        new_msg_data.context = context;
-        await this.create_system_message(context);
-      }
       // Create a new SmartMessage for the user's message
       await this.env.smart_messages.create_or_update(new_msg_data);
     } catch (error) {
       console.error("Error in handle_message_from_user:", error);
     }
   }
-  // handle creating system message
-  async create_system_message(context) {
-    const system_message = {
-      role: "system",
-      content: [],
-      thread_key: this.key,
-    };
-    /**
-     * Build system message
-     */
-    // Combine all context into a single system message
-    if (Array.isArray(context.system_prompt_refs) && context.system_prompt_refs.length > 0) {
-      context.system_prompt_refs.forEach(key => {
-        const system_prompt = {
-          type: 'text',
-          input: {
-            key,
-          },
-        };
-        system_message.content.push(system_prompt);
-      });
-      // remove system_prompt_refs from context
-      context.system_prompt_refs = `added to system message ${system_message.id}`;
-    }
-    if (context?.has_self_ref || context?.folder_refs) {
-      const system_prompt = {
+  async add_system_message(system_message){
+    if(typeof system_message === 'string'){
+      system_message = {
         type: 'text',
-        // replace this text with `key` to default system message for lookup
-        text: `- Answer based on the context from lookup!\n- The context may be referred to as notes.`,
+        text: system_message,
       };
-      system_message.content.push(system_prompt);
     }
-    await this.env.smart_messages.create_or_update(system_message);
+    if(!system_message.type) system_message.type = 'text';
+    // if last message is a system message, update it
+    const last_msg = this.messages[this.messages.length - 1];
+    if(last_msg?.role === 'system'){
+      last_msg.content.push(system_message);
+      last_msg.render();
+    } else {
+      await this.env.smart_messages.create_or_update({
+        role: 'system',
+        content: [system_message],
+        thread_key: this.key,
+      });
+    }
   }
 
   /**
@@ -226,6 +217,7 @@ export class SmartThread extends SmartSource {
   async handle_message_from_chat_model(response, opts = {}) {
     const choices = response.choices;
     const response_id = response.id;
+    if(!response_id) return [];
     const msg_items = await Promise.all(choices.map(async (choice, index) => {
       const msg_data = {
         ...(choice?.message || choice), // fallback on full choice to handle non-message choices
diff --git a/smart-chats/utils/internal_links.js b/smart-chats/utils/internal_links.js
index c7abbcdf..f0fdc223 100644
--- a/smart-chats/utils/internal_links.js
+++ b/smart-chats/utils/internal_links.js
@@ -14,6 +14,22 @@ export function extract_internal_links(user_input) {
   }
   return matches;
 }
+
+export function contains_internal_embedded_link(user_input) {
+  if (user_input.indexOf("![") === -1) return false;
+  if (user_input.indexOf("]") === -1) return false;
+  return true;
+}
+export function extract_internal_embedded_links(user_input) {
+  const matches = [];
+  const regex = /[!]\[\[(.*?)\]\]/g;
+  let match;
+  while ((match = regex.exec(user_input)) !== null) {
+    matches.push(match);
+  }
+  return matches;
+}
+
 // slower
 export function extract_internal_links2(user_input) {
   const matches = user_input.match(/\[\[(.*?)\]\]/g);
diff --git a/smart-chats/utils/internal_links.test.js b/smart-chats/utils/internal_links.test.js
index 1c89738d..208f5511 100644
--- a/smart-chats/utils/internal_links.test.js
+++ b/smart-chats/utils/internal_links.test.js
@@ -1,5 +1,11 @@
 import test from 'ava';
-import { contains_internal_link, extract_internal_links, extract_internal_links2 } from './internal_links.js';
+import {
+  contains_internal_link,
+  extract_internal_links,
+  extract_internal_links2,
+  contains_internal_embedded_link,
+  extract_internal_embedded_links
+} from './internal_links.js';
 // contains_internal_link
 test('returns true if user input contains internal link', t => {
   const userInput = 'This is a [[link]]';
@@ -68,3 +74,26 @@ test.serial('benchmark extract_internal_links2 performance', t => {
   console.log(`extract_internal_links2 took ${total_time.toFixed(2)}ms for ${iterations} iterations`);
   t.pass();
 });
+
+// contains_internal_embedded_link
+test('returns true if user input contains internal embedded link', t => {
+  const userInput = 'This is a ![[link]]';
+  t.true(contains_internal_embedded_link(userInput));
+});
+test('returns false if user input does not contain internal embedded link', t => {
+  const userInput = 'This is a [[link]]';
+  t.false(contains_internal_embedded_link(userInput));
+});
+test('extracts internal embedded links', t => {
+  const userInput = 'This is a ![[link]] and ![[another link]]';
+  const links = extract_internal_embedded_links(userInput);
+  t.deepEqual(links[0][1], 'link');
+  t.deepEqual(links[1][1], 'another link');
+});
+test('does not extract links from non-embedded links', t => {
+  const userInput = 'This is a [[link]] and ![[another link]]';
+  const links = extract_internal_embedded_links(userInput);
+  t.deepEqual(links[0][1], 'another link');
+});
+
+// FUTURE: should handle display text (e.g. ![[link|display text]])
\ No newline at end of file