diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..329631a --- /dev/null +++ b/.env.example @@ -0,0 +1,2 @@ +COHERE_API_KEY= +GEMINI_API_KEY= diff --git a/.gitignore b/.gitignore index 4e0f123..f32c288 100644 --- a/.gitignore +++ b/.gitignore @@ -2,9 +2,29 @@ /.yardoc /_yardoc/ /coverage/ -/doc/ /pkg/ /site/ /spec/reports/ /tmp/ /Gemfile.lock +/docker/data/ +.env +/docker/.env/ +/log/ +/vendor/ +/docker/*.pem +/docs/agenta.json +.venv/ +/docker/data.tar.gz +flowbots.json +/docker/docker-compose.yml +/models/ +/examples/test/ +/data/ +/.bold/ +/.examples/ +/.vendor/ +/.docs/ +/exception_reports/ +*.text.json +.gh_pages diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..1b7acb7 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "nano-bots"] + path = nano-bots + url = git@github.com:b08x/nano-bots.git diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..5e3d646 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,131 @@ +# Use Ruby 3.3 as the base image +FROM ruby:3.3-slim + +# Create a non-root user to run the app +RUN useradd -s /bin/bash -m flowbots + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + --no-install-recommends \ + apt-transport-https \ + apt-utils \ + build-essential \ + ca-certificates \ + cmake \ + curl \ + dialog \ + exiftool \ + git \ + gnupg \ + gnuplot \ + gpg-agent \ + graphviz \ + libcairo2-dev \ + libczmq-dev \ + libffi-dev \ + libfftw3-dev \ + libgdbm-dev \ + libgmp-dev \ + libgsl-dev \ + liblink-grammar-dev \ + libmagick++-dev \ + libmariadb-dev-compat \ + libmariadb-dev \ + libncurses5-dev \ + libopenblas-dev \ + libplot2c2 \ + libpoppler-glib-dev \ + libpq-dev \ + libreadline-dev \ + libreoffice \ + libsqlite3-dev \ + libssl-dev \ + libtamuanova-0.2 \ + libxml2-dev \ + libxslt1-dev \ + libyaml-dev \ + libzmq3-dev \ + link-grammar \ + lsb-release \ + minisat \ + neovim \ + openssl \ + pandoc \ + pdftk \ + pkg-config \ + plotutils \ + poppler-utils \ + postgresql-client \ + python3 \ + python3-link-grammar \ + python3-pip \ + python3.11-venv \ + rsync \ + ruby-psych \ + software-properties-common \ + sqlite3 \ + tesseract-ocr \ + tidy \ + tzdata \ + wget \ + zip \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* + +# Set the working directory in the container +WORKDIR /app + +ARG USE_TRF=False +ARG USE_BOOKNLP=False + +RUN python3 -m venv .venv && \ + . /app/.venv/bin/activate && \ + echo "[[ -f /app/.venv ]] && cd /app && . /app/.venv/bin/activate" >> /home/flowbots/.bashrc && \ + echo "gem: --user-instal --no-document" >> /home/flowbots/.gemrc && \ + pip3 install -U setuptools wheel && \ + pip3 install -U spacy 'pdfminer.six[image]' && \ + python3 -m spacy download en_core_web_lg && \ + python -c "import sys, importlib.util as util; 1 if util.find_spec('nltk') else sys.exit(); import nltk; nltk.download('punkt')" + +RUN if [ "${USE_TRF}" = "True"]; then \ + . /app/.venv/bin/activate && \ + python3 -m spacy download en_core_web_trf \ + ; fi + +RUN if [ "${USE_BOOKNLP}" = "True"]; then \ + . /app/.venv/bin/activate && \ + pip3 install -U transformers booknlp \ + ; fi + +# Copy only the Gemfile and requirements.txt +COPY Gemfile ./ + +# Copy the rest of the application code +# Copy only the specified directories and files +COPY bin/ ./bin/ +COPY examples/ ./examples/ +COPY exe/ ./exe/ +COPY lib/ ./lib/ +COPY nano-bots/ ./nano-bots/ +COPY flowbots.json . + +# Set environment variables +ENV LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 + +# Create necessary directories +RUN mkdir -p log models workspace + +RUN chown -R flowbots:flowbots /app + +USER flowbots + +ENV PATH="/home/flowbots/.local/share/gem/ruby/3.3.0/bin:$PATH" +ENV PATH="/app/.venv/bin:$PATH" + +RUN bundle lock --add-platform x86_64-linux && \ + bundle config build.redic --with-cxx="clang++" --with-cflags="-std=c++0x" && \ + bundle install + +# Set the default command (can be overridden) +CMD . .venv/bin/activate && exec bash diff --git a/Gemfile b/Gemfile index 345c34f..3295bbd 100644 --- a/Gemfile +++ b/Gemfile @@ -1,16 +1,76 @@ source "https://rubygems.org" -gemspec - +# gemspec +gem "algorithms", "~> 1.0" +gem "ansi_palette", "~> 0.0.1" +gem "chroma-db", "~> 0.7.0" +gem "cli-ui", "~> 2.2" +gem "dotenv", "~> 3.1" +gem "groq", "~> 0.3.1" +gem "highline", "~> 3.0" +gem "jongleur", "~> 1.1" +gem "json", "~> 2.7" +gem "jsonl", "~> 0.1.5" +gem "kramdown", "~> 2.4" +gem "langchainrb", "~> 0.13.5" +gem "lingua", "~> 0.6.2" +gem "mimemagic", "~> 0.4.3" gem "minitest", "~> 5.11" gem "minitest-rg", "~> 5.3" +gem "nano-bots", "~> 3.4" +gem "natty-ui", "~> 0.10.0" +gem "ohm", "~> 3.1" +gem "ohm-contrib", "~> 3.0" +gem "open3", "~> 0.2.1" +gem "open4", "~> 1.3" +gem "parallel", "~> 1.25" +gem "pastel", "~> 0.8.0" +gem "pdf-reader", "~> 2.12" +gem "pg", "~> 1.5" +gem "pgvector", "~> 0.3.1" +gem "pragmatic_segmenter", "~> 0.3.23" +gem "pragmatic_tokenizer", "~> 3.2" gem "pry", "~> 0.14.2" gem "pry-doc", "~> 1.5" +gem "pry-stack_explorer", "~> 0.6.1" gem "rake", "~> 13.0" gem "rb-readline", "~> 0.5.5" -gem "rubocop", "1.64.1" +gem "redis", "~> 5.2" +gem "rubocop", "~> 1.64" gem "rubocop-minitest", "0.35.0" gem "rubocop-packaging", "0.5.2" gem "rubocop-performance", "1.21.1" gem "rubocop-rake", "0.6.0" -gem "ruby-lsp", "~> 0.17.4" \ No newline at end of file +gem "ruby-lsp", "~> 0.17.4" +gem "ruby-spacy", "~> 0.2.2" +gem "sequel", "~> 5.82" +gem "solargraph", "~> 0.48.0" +gem "stream_lines", "~> 0.4.1" +gem "thor", "~> 1.2" +gem "timeout", "~> 0.4.1" +gem "tomoto", "~> 0.4.0" +gem "tool_tailor", "~> 0.2.1" +gem "treetop", "~> 1.6" +gem "tty-box", "~> 0.7.0" +gem "tty-markdown", "~> 0.7.2" +gem "tty-prompt" +gem "tty-screen", "~> 0.8.2" +gem "tty-spinner", "~> 0.9.3" +gem "tty-table", "~> 0.12.0" +gem "wordnet", "~> 1.2" +gem "wordnet-defaultdb", "~> 2.0" +gem "yaml", "~> 0.3.0" + +gem "polyglot", "~> 0.3.5" + +gem "engtagger", "~> 0.4.1" + +gem "scalpel", "~> 0.2.1" + +#gem "gokdok", "~> 0.4.2" + +gem "shale", "~> 1.1" + +gem "paint", "~> 2.3" + +gem "color", "~> 1.8" diff --git a/LICENSE.txt b/LICENSE similarity index 100% rename from LICENSE.txt rename to LICENSE diff --git a/README.md b/README.md index 9737c02..c51b3ec 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,270 @@ -# flowbots +# Flowbots -[![Gem Version](https://img.shields.io/gem/v/flowbots)](https://rubygems.org/gems/flowbots) -[![Gem Downloads](https://img.shields.io/gem/dt/flowbots)](https://www.ruby-toolbox.com/projects/flowbots) -[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/b08x/flowbots/ci.yml)](https://github.com/b08x/flowbots/actions/workflows/ci.yml) -[![Code Climate maintainability](https://img.shields.io/codeclimate/maintainability/b08x/flowbots)](https://codeclimate.com/github/b08x/flowbots) +Flowbots is an advanced text processing and analysis system that combines the power of [ruby-nano-bots](https://github.com/icebaker/ruby-nano-bots.git), [workflow orchestration](https://gitlab.com/RedFred7/Jongleur.git), and natural language processing to provide a flexible and powerful tool for document analysis and topic modeling. -TODO: Description of this gem goes here. +## Features ---- +- Text processing workflows for individual files and batch processing +- Advanced NLP methods including tokenization, part-of-speech tagging, and named entity recognition +- Topic modeling with dynamic model training and inference +- Flexible workflow system using Jongleur for task orchestration +- Redis-based data persistence using Ohm models +- Custom nano-bot cartridges for specialized AI-powered tasks +- Robust error handling and logging system -- [Quick start](#quick-start) -- [Support](#support) -- [License](#license) -- [Code of conduct](#code-of-conduct) -- [Contribution guide](#contribution-guide) +## System Architecture -## Quick start +### Class Diagram -``` -gem install flowbots -``` +```mermaid +classDiagram + class CLI { + +version() + +workflows() + +train_topic_model(folder) + +process_text(file) + } + + class Workflows { + -prompt: TTY::Prompt + +list_and_select() + +run(workflow_name) + -get_workflows() + -display_workflows(workflows) + -select_workflow(workflows) + -extract_workflow_description(file) + } + + class WorkflowOrchestrator { + -agents: Map + +add_agent(role, cartridge_file) + +define_workflow(workflow_definition) + +run_workflow() + } + + class WorkflowAgent { + -role: String + -state: Map + -bot: NanoBot + +process(input) + +save_state() + +load_state() + } + + class Task { + <> + +execute() + } + + class TextProcessingWorkflow { + -input_file_path: String + -orchestrator: WorkflowOrchestrator + +run() + } + + class TopicModelTrainerWorkflow { + -input_folder_path: String + -orchestrator: WorkflowOrchestrator + +run() + } + + class TextProcessor { + <> + +process(text) + } + + class NLPProcessor { + -nlp_model: Object + +process(segment, options) + } + + class TopicModelProcessor { + -model_path: String + -model: Object + -model_params: Map + +load_or_create_model() + +train_model(documents, iterations) + +infer_topics(document) + } + + class FileLoader { + -file_data: Textfile + +initialize(file_path) + } + + class Textfile { + +path: String + +name: String + +content: String + +preprocessed_content: String + +metadata: Map + +topics: Set~Topic~ + +segments: List~Segment~ + +lemmas: List~Lemma~ + } + + class Segment { + +text: String + +tokens: List + +tagged: Map + +words: List~Word~ + } -```ruby -require "flowbots" + class Word { + +word: String + +pos: String + +tag: String + +dep: String + +ner: String + } + + class Topic { + +name: String + +description: String + +vector: List + } + + CLI --> Workflows : uses + Workflows --> TextProcessingWorkflow : runs + Workflows --> TopicModelTrainerWorkflow : runs + TextProcessingWorkflow --> WorkflowOrchestrator : uses + TopicModelTrainerWorkflow --> WorkflowOrchestrator : uses + WorkflowOrchestrator --> WorkflowAgent : manages + WorkflowOrchestrator --> Task : executes + Task <|-- FileLoaderTask + Task <|-- PreprocessTextFileTask + Task <|-- TextSegmentTask + Task <|-- TokenizeSegmentsTask + Task <|-- NlpAnalysisTask + Task <|-- TopicModelingTask + Task <|-- LlmAnalysisTask + Task <|-- DisplayResultsTask + TextProcessor <|-- NLPProcessor + TextProcessor <|-- TopicModelProcessor + NlpAnalysisTask --> NLPProcessor : uses + TopicModelingTask --> TopicModelProcessor : uses + FileLoaderTask --> FileLoader : uses + Textfile "1" *-- "many" Segment + Segment "1" *-- "many" Word + Textfile "1" *-- "many" Topic + Textfile "1" *-- "many" Lemma ``` -## Support +## Project Structure + +The Flowbots project is organized into several key directories: + +- `/lib`: Main application code + - `/components`: Core system components + - `/processors`: Text and NLP processors + - `/tasks`: Individual workflow tasks + - `/workflows`: Workflow definitions + - `/ohm`: Ohm model definitions + - `/utils`: Utility functions and classes +- `/nano-bots/cartridges`: Nano-bot cartridge definitions +- `/test`: Test files and test helpers +- `/log`: Log files + +## Key Components + +1. **CLI**: The main entry point for user interaction, allowing users to select and run workflows. +2. **WorkflowOrchestrator**: Manages the execution of workflows and their constituent tasks. +3. **Task Processors**: Specialized classes for text processing, NLP analysis, and topic modeling. +4. **Ohm Models**: Data persistence layer for storing document information and workflow states. +5. **NanoBot Integration**: Utilizes nano-bot cartridges for specialized AI-powered tasks. +6. **Logging System**: Comprehensive logging for debugging and monitoring. + + +# Detailed Operation + +## 1. Workflow Initialization + +When a user selects a workflow through the CLI, the system initializes the chosen workflow (e.g., TextProcessingWorkflow or TopicModelTrainerWorkflow). The WorkflowOrchestrator sets up the task graph based on the workflow definition. + +## 2. Task Execution + +The WorkflowOrchestrator executes tasks in the defined order. Each task follows a similar pattern: + +1. Retrieve necessary data from Redis or Ohm models. +2. Process the data using specialized processors (e.g., NLPProcessor, TopicModelProcessor). +3. Store the results back in Redis (for temporary storage) or Ohm models (for persistence). + +## 3. Data Flow + +- Redis is used for storing temporary data and passing information between tasks. This includes file IDs, current batch information, and intermediate processing results. +- Ohm models, backed by Redis, are used for persistent storage of document information, segments, tokens, and analysis results. + +## 4. NLP and Topic Modeling + +- The NlpAnalysisTask uses the ruby-spacy gem to perform tasks like tokenization, part-of-speech tagging, and named entity recognition. +- The TopicModelingTask uses the tomoto gem to implement topic modeling algorithms. + +## 5. LLM Integration + +The LlmAnalysisTask integrates with external language models through the NanoBot system. This allows for high-level analysis and insights generation based on the processed text data. + +## 6. Error Handling and Logging + +Each task and the WorkflowOrchestrator include error handling mechanisms. Errors are caught, logged, and in some cases, trigger the ExceptionAgent for detailed error analysis. + +## 7. Batch Processing + +For the TopicModelTrainerWorkflow, files are processed in batches. The WorkflowOrchestrator manages the batch state, ensuring all files in a batch are processed before moving to the next batch. + +## 8. Result Presentation + +The DisplayResultsTask formats the analysis results and presents them to the user through the CLI. This may include summaries, topic distributions, and insights generated by the LLM. + +## Key Interactions + +1. **CLI <-> WorkflowOrchestrator**: The CLI initiates workflow execution and receives final results. +2. **WorkflowOrchestrator <-> Tasks**: The orchestrator manages task execution order and handles task results. +3. **Tasks <-> Redis**: Tasks use Redis for short-term storage and inter-task communication. +4. **Tasks <-> Ohm Models**: Tasks interact with Ohm models for persistent storage of document data and analysis results. +5. **NLP and Topic Modeling Tasks <-> External Libraries**: These tasks utilize external Ruby gems for specialized processing. +6. **LlmAnalysisTask <-> NanoBot**: This task interacts with the NanoBot system to leverage external language models. + +This architecture allows Flowbots to process text data through a series of specialized tasks, each building upon the results of previous tasks, to provide comprehensive text analysis and insights. + + +# Ruby Gems Used in Flowbots + +## Workflow and Task Management + +- [jongleur](https://rubygems.org/gems/jongleur): Core component for defining and executing task workflows, providing workflow orchestration and task management capabilities. + +## Data Persistence + +- [ohm](https://rubygems.org/gems/ohm): Object-hash mapping for Redis, used as the data persistence layer for storing document information and workflow states. + +## Parallel Processing + +- [parallel](https://rubygems.org/gems/parallel): Enables parallel processing, with potential use for parallel execution of tasks (not prominently used in the current implementation). + +## Development and Debugging + +- [pry](https://rubygems.org/gems/pry) and [pry-stack_explorer](https://rubygems.org/gems/pry-stack_explorer): Enhanced REPL and debugging tools for development and debugging purposes. + +## Natural Language Processing + +- [ruby-spacy](https://rubygems.org/gems/ruby-spacy): Ruby bindings for the Spacy NLP library, used for Natural Language Processing tasks. +- [lingua](https://rubygems.org/gems/lingua): Provides additional natural language detection and processing capabilities. +- [pragmatic_segmenter](https://rubygems.org/gems/pragmatic_segmenter): Used for text segmentation, splitting text into meaningful segments. +- [pragmatic_tokenizer](https://rubygems.org/gems/pragmatic_tokenizer): Handles text tokenization, breaking text into individual tokens. + +## Command-Line Interface -If you want to report a bug, or have ideas, feedback or questions about the gem, [let me know via GitHub issues](https://github.com/b08x/flowbots/issues/new) and I will do my best to provide a helpful answer. Happy hacking! +- [thor](https://rubygems.org/gems/thor): Used for building command-line interfaces, specifically for creating the CLI for Flowbots. -## License +## Parsing and Data Handling -The gem is available as open source under the terms of the [MIT License](LICENSE.txt). +- [treetop](https://rubygems.org/gems/treetop): A parsing expression grammar (PEG) parser generator, used for custom grammar parsing, particularly for Markdown with YAML front matter. +- [yaml](https://rubygems.org/gems/yaml): Handles YAML parsing and generation, particularly for configuration files and document front matter. -## Code of conduct +## Terminal Output Formatting -Everyone interacting in this project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](CODE_OF_CONDUCT.md). +- [tty-box](https://rubygems.org/gems/tty-box), [tty-cursor](https://rubygems.org/gems/tty-cursor), [tty-prompt](https://rubygems.org/gems/tty-prompt), [tty-screen](https://rubygems.org/gems/tty-screen), [tty-spinner](https://rubygems.org/gems/tty-spinner), [tty-table](https://rubygems.org/gems/tty-table): Various terminal output formatting and interaction tools for creating rich command-line interfaces and displaying formatted output. -## Contribution guide +## Topic Modeling -Pull requests are welcome! +- [tomoto](https://rubygems.org/gems/tomoto): Used for implementing topic modeling algorithms. diff --git a/Rakefile b/Rakefile index 53cbdf4..3414f41 100644 --- a/Rakefile +++ b/Rakefile @@ -1,107 +1,198 @@ -require "bundler/gem_tasks" -require "rake/testtask" -require "rubocop/rake_task" +#!/usr/bin/env ruby -Rake::TestTask.new(:test) do |t| - t.libs << "test" - t.libs << "lib" - t.test_files = FileList["test/**/*_test.rb"] -end +lib_dir = File.expand_path(File.join(__dir__, "lib")) +$LOAD_PATH.unshift lib_dir unless $LOAD_PATH.include?(lib_dir) -RuboCop::RakeTask.new +APP_ROOT = __dir__ -task default: %i[test rubocop] +### Task: rdoc +require "rake" +require "rake/testtask" +require "rdoc/task" +require "gokdok" + +OWNER = "b08x".freeze +ALL_IMAGES = %w[ + flowbots +].each(&:freeze).freeze + +BASE_IMAGES = ALL_IMAGES.map do |name| + base_image_name, base_image_tag = nil + IO.foreach("Dockerfile") do |line| + break if base_image_name && base_image_tag + + case line + when /BASE_IMAGE_TAG=(\h+)/ + base_image_tag = Regexp.last_match(1) + when /BASE_IMAGE_TAG=latest/ + base_image_tag = "latest" + when /\AFROM\s+([^:]+)/ + base_image_name = Regexp.last_match(1) + end + end + [ + name, + [base_image_name, base_image_tag].join(":") + ] +end.to_h -# == "rake release" enhancements ============================================== +DOCKER_FLAGS = ENV.fetch("DOCKER_FLAGS", nil) -Rake::Task["release"].enhance do - puts "Don't forget to publish the release on GitHub!" - system "open https://github.com/b08x/flowbots/releases" -end +TAG_LENGTH = 12 -task :disable_overcommit do - ENV["OVERCOMMIT_DISABLE"] = "1" +def git_revision + `git rev-parse HEAD`.chomp end -Rake::Task[:build].enhance [:disable_overcommit] - -task :verify_gemspec_files do - git_files = `git ls-files -z`.split("\x0") - gemspec_files = Gem::Specification.load("flowbots.gemspec").files.sort - ignored_by_git = gemspec_files - git_files - next if ignored_by_git.empty? - - raise <<~ERROR - - The `spec.files` specified in flowbots.gemspec include the following files - that are being ignored by git. Did you forget to add them to the repo? If - not, you may need to delete these files or modify the gemspec to ensure - that they are not included in the gem by mistake: - - #{ignored_by_git.join("\n").gsub(/^/, ' ')} - - ERROR +def tag_from_commit_sha1 + git_revision[...TAG_LENGTH] end -Rake::Task[:build].enhance [:verify_gemspec_files] +ALL_IMAGES.each do |image| + revision_tag = tag_from_commit_sha1 -# == "rake bump" tasks ======================================================== + desc "Pull the base image for #{OWNER}/#{image} image" + task "pull/base_image/#{image}" do + base_image = BASE_IMAGES[image] + sh "docker pull #{base_image}" + end -task bump: %w[bump:bundler bump:ruby bump:year] + desc "Build #{OWNER}/#{image} image" + task "build/#{image}" => "pull/base_image/#{image}" do + sh "docker build #{DOCKER_FLAGS} --rm --force-rm -t #{OWNER}/notebook-#{image}:latest ." + end -namespace :bump do - task :bundler do - sh "bundle update --bundler" + desc "Make #{OWNER}/#{image} image" + task "make/#{image}" do + sh "docker build #{DOCKER_FLAGS} --rm --force-rm -t #{OWNER}/notebook-#{image}:latest ." end - task :ruby do - replace_in_file "flowbots.gemspec", /ruby_version = .*">= (.*)"/ => RubyVersions.lowest - replace_in_file ".rubocop.yml", /TargetRubyVersion: (.*)/ => RubyVersions.lowest - replace_in_file ".github/workflows/ci.yml", /ruby: (\[.+\])/ => RubyVersions.all.inspect + desc "Tag #{OWNER}/#{image} image" + task "tag/#{image}" => "build/#{image}" do + sh "docker tag #{OWNER}/notebook-#{image}:latest #{OWNER}/notebook-#{image}:#{revision_tag}" end - task :year do - replace_in_file "LICENSE.txt", /\(c\) (\d+)/ => Date.today.year.to_s + desc "Push #{OWNER}/#{image} image" + task "push/#{image}" => "tag/#{image}" do + sh "docker push #{OWNER}/notebook-#{image}:latest" + sh "docker push #{OWNER}/notebook-#{image}:#{revision_tag}" end end -require "date" -require "open-uri" -require "yaml" - -def replace_in_file(path, replacements) - contents = File.read(path) - orig_contents = contents.dup - replacements.each do |regexp, text| - raise "Can't find #{regexp} in #{path}" unless regexp.match?(contents) - - contents.gsub!(regexp) do |match| - match[regexp, 1] = text - match - end +desc "Build all images" +task "build-all" do + ALL_IMAGES.each do |image| + Rake::Task["build/#{image}"].invoke end - File.write(path, contents) if contents != orig_contents end -module RubyVersions - class << self - def lowest - all.first - end +desc "Tag all images" +task "tag-all" do + ALL_IMAGES.each do |image| + Rake::Task["tag/#{image}"].invoke + end +end - def all - patches = versions.values_at(:stable, :security_maintenance).compact.flatten - sorted_minor_versions = patches.map { |p| p[/\d+\.\d+/] }.sort_by(&:to_f) - [*sorted_minor_versions, "head"] - end +desc "Push all images" +task "push-all" do + ALL_IMAGES.each do |image| + Rake::Task["push/#{image}"].invoke + end +end - private +Rake::RDocTask.new do |rdoc| + rdoc.title = "flowbots v0.1" + rdoc.rdoc_dir = "#{APP_ROOT}/doc" + rdoc.options += [ + "-w", + "2", + "-H", + "-A", + "-f", + "darkfish", # This bit + "-m", + "README.md", + "--visibility", + "nodoc", + "--markup", + "markdown" + ] + rdoc.rdoc_files.include "README.md" + rdoc.rdoc_files.include "LICENSE" + rdoc.rdoc_files.include "exe/flowbots" + + rdoc.rdoc_files.include "lib/api.rb" + rdoc.rdoc_files.include "lib/cli.rb" + rdoc.rdoc_files.include "lib/example.rb" + rdoc.rdoc_files.include "lib/flowbots.rb" + rdoc.rdoc_files.include "lib/general_task_agent.rb" + rdoc.rdoc_files.include "lib/helper.rb" + + rdoc.rdoc_files.include "lib/logging.rb" + rdoc.rdoc_files.include "lib/ui.rb" + rdoc.rdoc_files.include "lib/tasks.rb" + rdoc.rdoc_files.include "lib/workflows.rb" + + rdoc.rdoc_files.include "lib/components/BatchProcessor.rb" + rdoc.rdoc_files.include "lib/components/ExceptionAgent.rb" + rdoc.rdoc_files.include "lib/components/ExceptionHandler.rb" + rdoc.rdoc_files.include "lib/components/FileDiscovery.rb" + rdoc.rdoc_files.include "lib/components/FileLoader.rb" + rdoc.rdoc_files.include "lib/components/InputRetrieval.rb" + rdoc.rdoc_files.include "lib/components/OhmModels.rb" + rdoc.rdoc_files.include "lib/components/RedisKeys.rb" + rdoc.rdoc_files.include "lib/components/word_salad.rb" + rdoc.rdoc_files.include "lib/components/WorkflowAgent.rb" + rdoc.rdoc_files.include "lib/components/WorkflowOrchestrator.rb" + + rdoc.rdoc_files.include "lib/flowbots/errors.rb" + + rdoc.rdoc_files.include "lib/grammars/markdown_yaml.rb" + + rdoc.rdoc_files.include "lib/integrations/flowise.rb" + + rdoc.rdoc_files.include "lib/pipelines/unified_file_processing.rb" + + rdoc.rdoc_files.include "lib/processors/GrammarProcessor.rb" + rdoc.rdoc_files.include "lib/processors/NLPProcessor.rb" + rdoc.rdoc_files.include "lib/processors/TextProcessor.rb" + rdoc.rdoc_files.include "lib/processors/TextSegmentProcessor.rb" + rdoc.rdoc_files.include "lib/processors/TextTaggerProcessor.rb" + rdoc.rdoc_files.include "lib/processors/TextTokenizeProcessor.rb" + rdoc.rdoc_files.include "lib/processors/TopicModelProcessor.rb" + + rdoc.rdoc_files.include "lib/tasks/accumulate_filtered_segments_task.rb" + rdoc.rdoc_files.include "lib/tasks/display_results_task.rb" + rdoc.rdoc_files.include "lib/tasks/file_loader_task.rb" + rdoc.rdoc_files.include "lib/tasks/filter_segments_task.rb" + rdoc.rdoc_files.include "lib/tasks/llm_analysis_task.rb" + rdoc.rdoc_files.include "lib/tasks/load_file_object_task.rb" + rdoc.rdoc_files.include "lib/tasks/load_text_files_task.rb" + rdoc.rdoc_files.include "lib/tasks/nlp_analysis_task.rb" + rdoc.rdoc_files.include "lib/tasks/preprocess_file_object_task.rb" + rdoc.rdoc_files.include "lib/tasks/text_segment_task.rb" + rdoc.rdoc_files.include "lib/tasks/text_tagger_task.rb" + rdoc.rdoc_files.include "lib/tasks/text_tokenize_task.rb" + rdoc.rdoc_files.include "lib/tasks/tokenize_segments_task.rb" + rdoc.rdoc_files.include "lib/tasks/topic_modeling_task.rb" + rdoc.rdoc_files.include "lib/tasks/train_topic_model_task.rb" + + rdoc.rdoc_files.include "lib/ui/base.rb" + rdoc.rdoc_files.include "lib/ui/box.rb" + rdoc.rdoc_files.include "lib/ui/scrollable_box.rb" + + rdoc.rdoc_files.include "lib/utils/command.rb" + rdoc.rdoc_files.include "lib/utils/transcribe.rb" + rdoc.rdoc_files.include "lib/utils/tts.rb" + rdoc.rdoc_files.include "lib/utils/writefile.rb" + + rdoc.rdoc_files.include "lib/workflows/text_processing_workflow.rb" + rdoc.rdoc_files.include "lib/workflows/topic_model_trainer_workflow.rb" + rdoc.rdoc_files.include "lib/workflows/topic_model_trainer_workflowtest.rb" +end - def versions - @_versions ||= begin - yaml = URI.open("https://raw.githubusercontent.com/ruby/www.ruby-lang.org/HEAD/_data/downloads.yml") - YAML.safe_load(yaml, symbolize_names: true) - end - end - end +Gokdok::Dokker.new do |gd| + gd.remote_path = "" # Put into the root directory + gd.repo_url = "git@github.com:b08x/flowbots.git" + gd.doc_home = "#{APP_ROOT}/doc" end diff --git a/assets/anthropic/workbench/agent_persona_template001.py b/assets/anthropic/workbench/agent_persona_template001.py new file mode 100644 index 0000000..1e6860e --- /dev/null +++ b/assets/anthropic/workbench/agent_persona_template001.py @@ -0,0 +1,98 @@ +import anthropic + +client = anthropic.Anthropic( + # defaults to os.environ.get("ANTHROPIC_API_KEY") + api_key="my_api_key", +) + +# Replace placeholders like {{PERSONA_DESCRIPTION}} with real values, +# because the SDK does not support variables. +message = client.messages.create( + model="claude-3-opus-20240229", + max_tokens=1000, + temperature=0.3, + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Your task is to create a template for generating AI Agent Persona prompts based on the provided persona description. The template should be based on functional and generative grammars, as well as semiotic features, and should correlate these elements with the pragmatic intentions identified in the example.\n\nHere is the persona description:\n\n{{PERSONA_DESCRIPTION}}\n\n\nFirst, analyze the persona description, paying close attention to the functional and generative grammars used, as well as any semiotic features present. Identify the key elements that contribute to the overall characterization of the AI agent, such as their name, area of expertise, skills, and notable qualities.\n\nNext, determine the pragmatic intentions behind the persona description. Consider what the description aims to convey about the AI agent's capabilities, personality, and role.\n\nBased on your analysis, create a template for generating similar AI Agent Persona prompts. The template should include placeholders for the key elements identified, such as:\n- AI agent name\n- Area of expertise or focus\n- Specific skills and proficiencies\n- Notable qualities or characteristics\n- Pragmatic intentions or role of the AI agent\n\nWhen creating the template, focus on the functional and generative grammars that align with the pragmatic intentions you identified. Use these grammatical structures to ensure that the generated prompts effectively convey the desired characteristics and capabilities of the AI agent.\n\nPlease provide your template inside