diff --git a/Rakefile b/Rakefile index 06ef59a..f8c9a71 100644 --- a/Rakefile +++ b/Rakefile @@ -189,3 +189,105 @@ Rake::RDocTask.new do |rdoc| rdoc.rdoc_files.include "lib/workflows/topic_model_trainer_workflow.rb" rdoc.rdoc_files.include "lib/workflows/topic_model_trainer_workflowtest.rb" end + +desc "Build all images" +task "build-all" do + ALL_IMAGES.each do |image| + Rake::Task["build/#{image}"].invoke + end +end + +desc "Tag all images" +task "tag-all" do + ALL_IMAGES.each do |image| + Rake::Task["tag/#{image}"].invoke + end +end + +desc "Push all images" +task "push-all" do + ALL_IMAGES.each do |image| + Rake::Task["push/#{image}"].invoke + end +end + +Rake::RDocTask.new do |rdoc| + rdoc.title = "flowbots v0.1" + rdoc.rdoc_dir = "#{APP_ROOT}/doc" + rdoc.options += [ + "-w", + "2", + "-H", + "-A", + "-f", + "darkfish", # This bit + "-m", + "README.md", + "--visibility", + "nodoc", + "--markup", + "markdown" + ] + rdoc.rdoc_files.include "README.md" + rdoc.rdoc_files.include "LICENSE" + rdoc.rdoc_files.include "exe/flowbots" + + rdoc.rdoc_files.include "lib/api.rb" + rdoc.rdoc_files.include "lib/cli.rb" + rdoc.rdoc_files.include "lib/flowbots.rb" + rdoc.rdoc_files.include "lib/helper.rb" + rdoc.rdoc_files.include "lib/logging.rb" + rdoc.rdoc_files.include "lib/tasks.rb" + rdoc.rdoc_files.include "lib/ui.rb" + rdoc.rdoc_files.include "lib/workflows.rb" + + rdoc.rdoc_files.include "lib/integrations/flowise.rb" + + rdoc.rdoc_files.include "lib/processors/GrammarProcessor.rb" + rdoc.rdoc_files.include "lib/processors/NLPProcessor.rb" + rdoc.rdoc_files.include "lib/processors/TextProcessor.rb" + rdoc.rdoc_files.include "lib/processors/TextSegmentProcessor.rb" + rdoc.rdoc_files.include "lib/processors/TextTaggerProcessor.rb" + rdoc.rdoc_files.include "lib/processors/TextTokenizeProcessor.rb" + rdoc.rdoc_files.include "lib/processors/TopicModelProcessor.rb" + + rdoc.rdoc_files.include "lib/tasks/accumulate_filtered_segments_task.rb" + rdoc.rdoc_files.include "lib/tasks/display_results_task.rb" + rdoc.rdoc_files.include "lib/tasks/file_loader_task.rb" + rdoc.rdoc_files.include "lib/tasks/filter_segments_task.rb" + rdoc.rdoc_files.include "lib/tasks/llm_analysis_task.rb" + rdoc.rdoc_files.include "lib/tasks/load_text_files_task.rb" + rdoc.rdoc_files.include "lib/tasks/nlp_analysis_task.rb" + rdoc.rdoc_files.include "lib/tasks/preprocess_text_file_task.rb" + rdoc.rdoc_files.include "lib/tasks/text_segment_task.rb" + rdoc.rdoc_files.include "lib/tasks/text_tagger_task.rb" + rdoc.rdoc_files.include "lib/tasks/text_tokenize_task.rb" + rdoc.rdoc_files.include "lib/tasks/tokenize_segments_task.rb" + rdoc.rdoc_files.include "lib/tasks/topic_modeling_task.rb" + rdoc.rdoc_files.include "lib/tasks/train_topic_model_task.rb" + + rdoc.rdoc_files.include "lib/components/ExceptionAgent.rb" + rdoc.rdoc_files.include "lib/components/ExceptionHandler.rb" + rdoc.rdoc_files.include "lib/components/FileLoader.rb" + rdoc.rdoc_files.include "lib/components/OhmModels.rb" + rdoc.rdoc_files.include "lib/components/WorkflowAgent.rb" + rdoc.rdoc_files.include "lib/components/WorkflowOrchestrator.rb" + rdoc.rdoc_files.include "lib/components/word_salad.rb" + + rdoc.rdoc_files.include "lib/grammars/markdown_yaml.rb" + + rdoc.rdoc_files.include "lib/utils/command.rb" + rdoc.rdoc_files.include "lib/utils/transcribe.rb" + rdoc.rdoc_files.include "lib/utils/tts.rb" + rdoc.rdoc_files.include "lib/utils/writefile.rb" + + rdoc.rdoc_files.include "lib/workflows/text_processing_workflow.rb" + rdoc.rdoc_files.include "lib/workflows/topic_model_trainer_workflow.rb" + rdoc.rdoc_files.include "lib/workflows/topic_model_trainer_workflowtest.rb" +end + +Gokdok::Dokker.new do |gd| + gd.remote_path = "" # Put into the root directory + gd.repo_url = "git@github.com:b08x/flowbots.git" + gd.doc_home = "#{APP_ROOT}/doc" +end diff --git a/compressed_prompt_test.rb b/compressed_prompt_test.rb index 3d8bc08..c9029d0 100644 --- a/compressed_prompt_test.rb +++ b/compressed_prompt_test.rb @@ -1,20 +1,20 @@ **Evaluation Test:** -Now, to evaluate the effectiveness of the condensed prompt, I will employ various criteria: +Now, to evaluate the effectiveness of the condensed prompt, I will employ various criteria: -- Clarity: Does the optimized version provide a clear and concise explanation of the benefits of using a visual representation? Does it eliminate ambiguity and ensure that the message is easily understandable? +- Clarity: Does the optimized version provide a clear and concise explanation of the benefits of using a visual representation? Does it eliminate ambiguity and ensure that the message is easily understandable? -Evaluation: The optimized version excels in clarity. By elaborating on the advantages of the visual representation, it provides a more explicit explanation. The use of phrases like "elucidating the inherent sequential nature" and "providing insights into the flow of data" offers a precise understanding of how visuals aid in comprehension and communication. The message is straightforward and free from ambiguity, making it easily comprehensible to the target audience. +Evaluation: The optimized version excels in clarity. By elaborating on the advantages of the visual representation, it provides a more explicit explanation. The use of phrases like "elucidating the inherent sequential nature" and "providing insights into the flow of data" offers a precise understanding of how visuals aid in comprehension and communication. The message is straightforward and free from ambiguity, making it easily comprehensible to the target audience. -- Completeness: Does the optimized version retain all the crucial information from the original prompt? Does it provide a comprehensive understanding of the topic without omitting relevant details? +- Completeness: Does the optimized version retain all the crucial information from the original prompt? Does it provide a comprehensive understanding of the topic without omitting relevant details? -Evaluation: The optimized version effectively maintains the completeness of information. It expands on the ideas presented in the original prompt, offering a more detailed yet concise explanation. The addition of phrases like "distinct process stages" and "shared understanding of the system's architecture" ensures that the full scope of the visual representation's utility is conveyed. No relevant details have been omitted, and the optimized version successfully captures the core message of the original prompt. +Evaluation: The optimized version effectively maintains the completeness of information. It expands on the ideas presented in the original prompt, offering a more detailed yet concise explanation. The addition of phrases like "distinct process stages" and "shared understanding of the system's architecture" ensures that the full scope of the visual representation's utility is conveyed. No relevant details have been omitted, and the optimized version successfully captures the core message of the original prompt. -- Eliciting Desired Responses: Will the optimized version be more effective in eliciting the desired response from the AI assistant or model? Does it provide a clear directive, enabling the AI to generate a more accurate and contextually appropriate response? +- Eliciting Desired Responses: Will the optimized version be more effective in eliciting the desired response from the AI assistant or model? Does it provide a clear directive, enabling the AI to generate a more accurate and contextually appropriate response? -Evaluation: The optimized version is designed to elicit a more focused and accurate response from the AI assistant. By providing additional context and clarity, the AI has a better understanding of the specific benefits attributed to the visual representation. The use of phrases like "graphical depiction," "shared understanding," and "communication of complex ideas" offers a clear framework for the AI to generate a response that aligns with the prompt's intent. The optimized version reduces potential ambiguity and enhances the likelihood of receiving a contextually relevant and high-quality response from the AI. +Evaluation: The optimized version is designed to elicit a more focused and accurate response from the AI assistant. By providing additional context and clarity, the AI has a better understanding of the specific benefits attributed to the visual representation. The use of phrases like "graphical depiction," "shared understanding," and "communication of complex ideas" offers a clear framework for the AI to generate a response that aligns with the prompt's intent. The optimized version reduces potential ambiguity and enhances the likelihood of receiving a contextually relevant and high-quality response from the AI. -Overall Conclusion: +Overall Conclusion: Based on the evaluation test, the optimized version of the prompt demonstrates superior effectiveness compared to the original. It achieves a higher standard of clarity by providing explicit and detailed explanations while maintaining the completeness of the information conveyed. The optimized version is also tailored to elicit more accurate and contextually appropriate responses from AI assistants or models, ensuring a more productive and efficient interaction. This comprehensive test underscores the value of careful prompt design and analysis, highlighting the potential for enhanced AI performance and output quality. diff --git a/doc/BoxUI.html b/doc/BoxUI.html new file mode 100644 index 0000000..dab09f6 --- /dev/null +++ b/doc/BoxUI.html @@ -0,0 +1,271 @@ + + + + + + +module BoxUI - flowbots v0.1 + + + + + + + + + + + + + + + + +
+

+ module BoxUI +

+ +
+ +

Add this to your ui.rb file or create a new file called box_ui.rb

+ +
+ +
+ + + + + +
+
+

Public Class Methods

+
+ +
+
+ side_by_side_boxes(text1, text2, title1: "Box 1", title2: "Box 2") + click to toggle source +
+ +
+ + +
+ 
# File lib/ui.rb, line 154
+def side_by_side_boxes(text1, text2, title1: "Box 1", title2: "Box 2")
+  screen_width = TTY::Screen.width
+  screen_height = TTY::Screen.height
+  box_width = (screen_width / 2) - 2
+  box_height = screen_height - 4  # Leave some space for prompts
+
+  box1 = create_scrollable_box(text1, box_width, box_height, title1)
+  box2 = create_scrollable_box(text2, box_width, box_height, title2)
+
+  display_boxes(box1, box2, box_height)
+end
+
+
+ + +
+ +
+ +
+
+

Private Class Methods

+
+ +
+
+ create_scrollable_box(text, width, height, title) + click to toggle source +
+ +
+ + +
+ 
# File lib/ui.rb, line 168
+def create_scrollable_box(text, width, height, title)
+  lines = text.split("\n")
+  total_pages = (lines.length.to_f / (height - 2)).ceil
+  {
+    title: title,
+    lines: lines,
+    width: width,
+    height: height,
+    total_pages: total_pages,
+    current_page: 1
+  }
+end
+
+
+ + +
+ +
+
+ display_boxes(box1, box2, box_height) + click to toggle source +
+ +
+ + +
+ 
# File lib/ui.rb, line 181
+def display_boxes(box1, box2, box_height)
+  loop do
+    system('clear') || system('cls')
+    print_boxes(box1, box2, box_height)
+    print_navigation_info(box1, box2)
+
+    input = STDIN.getch
+    case input.downcase
+    when 'q'
+      break
+    when 'a'
+      box1[:current_page] = [1, box1[:current_page] - 1].max
+    when 'd'
+      box1[:current_page] = [box1[:total_pages], box1[:current_page] + 1].min
+    when 'j'
+      box2[:current_page] = [1, box2[:current_page] - 1].max
+    when 'l'
+      box2[:current_page] = [box2[:total_pages], box2[:current_page] + 1].min
+    end
+  end
+end
+
+
+ + +
+ +
+
+ print_boxes(box1, box2, box_height) + click to toggle source +
+ +
+ + + +
+ + +
+ +
+
+ print_navigation_info(box1, box2) + click to toggle source +
+ +
+ + + +
+ + +
+ +
+ +
+
+ + + + diff --git a/doc/Flowbots/UI.html b/doc/Flowbots/UI.html new file mode 100644 index 0000000..699512a --- /dev/null +++ b/doc/Flowbots/UI.html @@ -0,0 +1,285 @@ + + + + + + +module Flowbots::UI - flowbots v0.1 + + + + + + + + + + + + + + + + +
+

+ module Flowbots::UI +

+ +
+ +
+ +
+ + + + + +
+
+

Public Instance Methods

+
+ +
+
+ exception(text) + click to toggle source +
+ +
+ + +
+ 
# File lib/ui.rb, line 53
+def exception(text)
+  ui.framed do
+    ui.failed "Exception Message" do
+      ui.puts text, glyph: "💡"
+    end
+  end
+end
+
+
+ + +
+ +
+
+ header() + click to toggle source +
+ +
+ + +
+ 
# File lib/ui.rb, line 89
+def header
+  ui.space
+  ui.h1 "UI: Message Types"
+  ui.space
+end
+
+
+ + +
+ +
+
+ info(text) + click to toggle source +
+ +
+ + +
+ 
# File lib/ui.rb, line 46
+def info(text)
+  header
+  ui.framed do
+      ui.puts text, glyph: "💡"
+  end
+end
+
+
+ + +
+ +
+
+ prompt() + click to toggle source +
+ +
+ + +
+ 
# File lib/ui.rb, line 24
+def prompt
+  @prompt = TTY::Prompt.new(enable_color: true, active_color: :cyan)
+end
+
+
+ + +
+ +
+
+ response(response) + click to toggle source +
+ +
+ + +
+ 
# File lib/ui.rb, line 61
+def response(response)
+  ui.space
+  ui.h1 "UI: Text Line Animation"
+  ui.space
+
+  response.each_line do |line|
+    input = line.chomp
+    unless input.nil?
+      input.each_char do |char|
+        print "\e[34m#{char}\e[0m"
+        ui.message char
+        sleep 0.02
+      end
+      puts "\n"
+    end
+    res = line.chomp
+    unless res.nil?
+      puts "\e[32m#{res.strip.chomp}\e[0m"
+      puts "\n"
+    end
+    cap = line.chomp
+    puts "\e[33m#{cap.strip.chomp}\e[0m\n" unless cap.nil?
+  end
+  puts "\n"
+  sleep 3
+  ui.space
+end
+
+
+ + +
+ +
+
+ say(type, statement) + click to toggle source +
+ +
+ + +
+ 
# File lib/ui.rb, line 28
+def say(type, statement)
+  prompt
+  type = :ok if type.nil?
+  case type
+  when :ok
+    @prompt.ok(statement)
+    logger.info statement
+  when :warn
+    @prompt.warn(statement)
+    logger.warn statement
+  when :error
+    @prompt.error(statement)
+    logger.fatal statement
+  else
+    PASTEL.say(statement)
+  end
+end
+
+
+ + +
+ +
+ +
+
+ + + + diff --git a/doc/PreprocessTextFileTask.html b/doc/PreprocessTextFileTask.html new file mode 100644 index 0000000..21e23c8 --- /dev/null +++ b/doc/PreprocessTextFileTask.html @@ -0,0 +1,293 @@ + + + + + + +class PreprocessTextFileTask - flowbots v0.1 + + + + + + + + + + + + + + + + +
+

+ class PreprocessTextFileTask +

+ +
+ +

This task preprocesses a text file, extracting metadata and content.

+ +
+ +
+ + + + + +
+
+

Public Instance Methods

+
+ +
+
+ execute() + click to toggle source +
+ +
+

Executes the task.

+ +

@return [void]

+ +
+ 
# File lib/tasks/preprocess_text_file_task.rb, line 9
+def execute
+  logger.info "Starting PreprocessTextFileTask"
+
+  @textfile = retrieve_current_textfile
+
+  logger.debug "File content: #{@textfile.content[0..500]}..." # Log first 200 characters
+
+  begin
+    grammar_processor = Flowbots::GrammarProcessor.new("markdown_yaml")
+    parse_result = grammar_processor.parse(@textfile.content)
+    logger.debug "Parse result: #{parse_result.inspect}"
+
+    if parse_result
+      content = parse_result[:markdown_content]
+      metadata = extract_metadata(parse_result[:yaml_front_matter])
+      store_preprocessed_data(content, metadata)
+      logger.info "Successfully preprocessed file with custom grammar"
+    else
+      logger.error "Failed to parse the document with custom grammar"
+      @textfile.update(preprocessed_content: "")
+      @textfile.update(metadata: {})
+      @textfile.save
+    end
+  rescue StandardError => e
+    logger.error "Error in grammar processing: #{e.message}"
+    logger.error e.backtrace.join("\n")
+    Flowbots::UI.exception("#{e.message}")
+    @textfile.update(preprocessed_content: "")
+    @textfile.update(metadata: {})
+    @textfile.save
+  end
+
+  logger.info "PreprocessTextFileTask completed"
+end
+
+
+ + +
+ +
+ +
+
+

Private Instance Methods

+
+ +
+
+ extract_metadata(yaml_front_matter) + click to toggle source +
+ +
+

Extracts metadata from the YAML front matter.

+ +

@param yaml_front_matter [String] The YAML front matter string.

+ +

@return [Hash] The extracted metadata.

+ +
+ 
# File lib/tasks/preprocess_text_file_task.rb, line 68
+def extract_metadata(yaml_front_matter)
+  return {} if yaml_front_matter.empty?
+
+  begin
+    YAML.safe_load(yaml_front_matter)
+  rescue StandardError => e
+    logger.error "Error parsing YAML front matter: #{e.message}"
+    {}
+  end
+end
+
+
+ + +
+ +
+
+ normalize_text(text) + click to toggle source +
+ +
+

Normalizes the given text by converting it to lowercase and removing non-alphanumeric characters.

+ +

@param text [String] The text to normalize.

+ +

@return [String] The normalized text.

+ +
+ 
# File lib/tasks/preprocess_text_file_task.rb, line 51
+def normalize_text(text)
+  text.downcase.gsub(/[^a-z0-9\s]/i, "")
+end
+
+
+ + +
+ +
+
+ retrieve_current_textfile() + click to toggle source +
+ +
+

Retrieves the current Textfile object from Redis.

+ +

@return [Textfile] The Textfile object representing the current file.

+ +
+ 
# File lib/tasks/preprocess_text_file_task.rb, line 58
+def retrieve_current_textfile
+  textfile_id = Jongleur::WorkerTask.class_variable_get(:@@redis).get("current_textfile_id")
+  Textfile[textfile_id]
+end
+
+
+ + +
+ +
+
+ store_preprocessed_data(content, metadata) + click to toggle source +
+ +
+

Stores the preprocessed content and metadata in the database.

+ +

@param content [String] The preprocessed content. @param metadata [Hash] The extracted metadata.

+ +

@return [void]

+ +
+ 
# File lib/tasks/preprocess_text_file_task.rb, line 85
+def store_preprocessed_data(content, metadata)
+  # redis = Jongleur::WorkerTask.class_variable_get(:@@redis)
+  @textfile.update(preprocessed_content: content)
+  @textfile.update(metadata:)
+  @textfile.save
+  # redis.set("preprocessed_content", content)
+  # redis.set("file_metadata", metadata.to_json)
+  logger.debug "Stored preprocessed content (first 100 chars): #{content[0..100]}"
+  logger.debug "Stored metadata: #{metadata.inspect}"
+end
+
+
+ + +
+ +
+ +
+
+ + + + diff --git a/doc/TextTaggerTask.html b/doc/TextTaggerTask.html index 7c16e6c..9b77dc6 100644 --- a/doc/TextTaggerTask.html +++ b/doc/TextTaggerTask.html @@ -53,18 +53,18 @@

- +
- - + + - + - - + + - +
- - + + - - - + + + - +
- - + + - + - - + + - +
- - + + - + - - + +
- + 
# File lib/general_task_agent.rb, line 63
@@ -152,7 +152,7 @@ 
Public Instance Methods
- + 
# File lib/general_task_agent.rb, line 67
@@ -214,7 +214,7 @@ 
Private Instance Methods
- + 
# File lib/general_task_agent.rb, line 76
@@ -328,7 +328,7 @@ 
Private Instance Methods
- + 
# File lib/general_task_agent.rb, line 80
@@ -352,4 +352,3 @@ 
Private Instance Methods

   
Generated by RDoc 6.4.0.
   
Based on Darkfish by Michael Granger.
 
-
diff --git a/doc/TrainTopicModelTask.html b/doc/TrainTopicModelTask.html
index 70c9851..6f3dfeb 100644
--- a/doc/TrainTopicModelTask.html
+++ b/doc/TrainTopicModelTask.html
@@ -53,20 +53,20 @@
- +
- - + + - - - + + + - +
- - + + - - - + + +
- +
@@ -158,7 +158,7 @@

Attributes

- +
@@ -343,4 +343,3 @@

Private Instance Methods

Generated by RDoc 6.4.0.

Based on Darkfish by Michael Granger. - diff --git a/doc/WorkflowOrchestrator.html b/doc/WorkflowOrchestrator.html index 3fcfc57..3868e76 100644 --- a/doc/WorkflowOrchestrator.html +++ b/doc/WorkflowOrchestrator.html @@ -53,20 +53,20 @@

- +
- - + + - - - + + +
- +