From 4f70a8ad34fb39a567dfffd668bc73a9fd3eca5f Mon Sep 17 00:00:00 2001 From: Fred Zimmerman Date: Sat, 24 Aug 2024 23:49:51 -0400 Subject: [PATCH] feat: Add MultiContextProcessor class This commit introduces the MultiContextProcessor class, which is designed to handle the processing of multiple contexts with a selected prompt group and output settings. The MultiContextProcessor class provides the following functionality: Processes multiple context files grouped by name. Reads the content of each file and processes it with the selected prompt group. Organizes the results into a dictionary where keys are group names and values are lists of results for each file in the group. Saves the results to separate text files named according to the group and result index. This class enhances the flexibility and modularity of the code by separating the prompt building, launching, and result saving logic for multi-context processing. Signed-off-by: Fred Zimmerman --- .../Builders/Codexes2PartsOfTheBook.py | 16 +++++++++++-- .../classes/Codexes/Builders/PromptGroups.py | 4 +++- .../resources/prompts/user_prompts_dict.json | 9 +++++++ Codexes2Gemini/ui/streamlit_ui.py | 24 +++++++++++++------ Codexes2Gemini/ui/utils.py | 19 +++++++++++++++ 5 files changed, 62 insertions(+), 10 deletions(-) create mode 100644 Codexes2Gemini/ui/utils.py diff --git a/Codexes2Gemini/classes/Codexes/Builders/Codexes2PartsOfTheBook.py b/Codexes2Gemini/classes/Codexes/Builders/Codexes2PartsOfTheBook.py index 89d397d..bc88efd 100644 --- a/Codexes2Gemini/classes/Codexes/Builders/Codexes2PartsOfTheBook.py +++ b/Codexes2Gemini/classes/Codexes/Builders/Codexes2PartsOfTheBook.py @@ -3,6 +3,7 @@ import logging import os import traceback +import uuid from importlib import resources from time import sleep from typing import List @@ -162,8 +163,19 @@ def process_codex_to_book_part(self, plan: PromptGroups): f"Output for prompt {i + 1} does not meet desired length of {plan.minimum_required_output_tokens}. Discarding.") if satisfactory_results: - self.logger.info(f"Returning satisfactory results of {self.count_tokens(satisfactory_results)}") - + self.logger.info(f"Returning satisfactory results of {self.count_tokens(satisfactory_results)}") + + if plan.require_json_output: + output_data = { + "text": response.text, + "prompt_feedback": response.prompt_feedback, + # ... (Add other relevant data to the dictionary) ... + } + json_output_path = f"{plan.thisdoc_dir}/{plan.output_file_path}_{str(uuid.uuid4())[:6]}.json" + with open(json_output_path, 'w') as json_file: + json.dump(output_data, json_file, indent=4) + self.logger.info(f"JSON output saved to {json_output_path}") + st.info(f"JSON output saved to {json_output_path}") else: self.logger.warning("No satisfactory results were generated.") satisfactory_results = "No satisfactory results were generated." diff --git a/Codexes2Gemini/classes/Codexes/Builders/PromptGroups.py b/Codexes2Gemini/classes/Codexes/Builders/PromptGroups.py index b04b653..3b9a3e8 100644 --- a/Codexes2Gemini/classes/Codexes/Builders/PromptGroups.py +++ b/Codexes2Gemini/classes/Codexes/Builders/PromptGroups.py @@ -31,7 +31,8 @@ def __init__(self, context: str = "", context_file_paths: List[str] = None, user selected_system_instruction_keys: List[str] = None, selected_user_prompt_values: List[str] = None, selected_user_prompts_dict: Dict[str, Any] = None, - config_file: str = None, use_all_user_keys: bool = False, add_system_prompt: str = "") -> None: + config_file: str = None, use_all_user_keys: bool = False, add_system_prompt: str = "", + require_json_output=False) -> None: self.logger = logging.getLogger(__name__) self.logger.setLevel(log_level) @@ -59,6 +60,7 @@ def __init__(self, context: str = "", context_file_paths: List[str] = None, user self.list_of_user_keys_to_use = list_of_user_keys_to_use or [] self.continuation_prompts = continuation_prompts self.output_file_path = output_file_base_name + self.require_json_output = require_json_output self.number_to_run = number_to_run self.minimum_required_output = minimum_required_output self.minimum_required_output_tokens = minimum_required_output_tokens diff --git a/Codexes2Gemini/resources/prompts/user_prompts_dict.json b/Codexes2Gemini/resources/prompts/user_prompts_dict.json index 3b35e34..cc9b066 100644 --- a/Codexes2Gemini/resources/prompts/user_prompts_dict.json +++ b/Codexes2Gemini/resources/prompts/user_prompts_dict.json @@ -186,6 +186,15 @@ "book highlights" ] }, + "outline2prompts": { + "prompt": "Expand this outline for a book into the following objects:\n) A document that may be used as context for model requests. This contains factual information about plot, characters, themes and other elements that will help ensure continuity.\n2) System instructions to guide the model in how to create responses.\n3) A series of user prompts which each generate a chapter in the book. Each prompt can be expanded additional details and specific actions for each character. You can also introduce new characters or locations as the story progresses. Remember to maintain consistency with the themes, plot twists, and overall tone of the book.", + "tags": [ + "book outline", + "AI writing", + "style", + "prompts" + ] + }, "page_by_page_analysis": { "prompt": "Now do a page-by-page analysis of the uploaded manuscript. Award +1 for each positive, -1 for each negative, calculate totals and net.", "tags": [ diff --git a/Codexes2Gemini/ui/streamlit_ui.py b/Codexes2Gemini/ui/streamlit_ui.py index 8024751..f74364a 100644 --- a/Codexes2Gemini/ui/streamlit_ui.py +++ b/Codexes2Gemini/ui/streamlit_ui.py @@ -39,6 +39,7 @@ from Codexes2Gemini.classes.Utilities.utilities import configure_logger from Codexes2Gemini.classes.user_space import UserSpace, save_user_space, load_user_space from Codexes2Gemini import __version__, __announcements__ +from Codexes2Gemini.ui.multi_context_page import MultiContextUI as MCU logger = configure_logger("DEBUG") @@ -449,6 +450,7 @@ def multiplan_builder(user_space: UserSpace): maximum_output_tokens = 10000000 minimum_required_output = False minimum_required_output_tokens = 50 + require_json_output = st.checkbox("Require JSON Output", value=False) with st.expander("Set Output Destinations"): thisdoc_dir = st.text_input("Output directory", value=os.path.join(os.getcwd(), 'output', 'c2g')) @@ -477,6 +479,7 @@ def multiplan_builder(user_space: UserSpace): "minimum_required_output": minimum_required_output, "minimum_required_output_tokens": minimum_required_output_tokens, "log_level": log_level, + "require_json_output": require_json_output }) st.session_state.multiplan.append(st.session_state.current_plan) st.success(f"Plan '{plan_name}' added to multiplan") @@ -556,6 +559,9 @@ def run_multiplan(multiplan, user_space): 'complete_system_instruction': plan['complete_system_instruction'], 'selected_system_instructions': plan['selected_system_instruction_keys'], 'selected_user_prompts_dict': plan['selected_user_prompts_dict'], + 'user_prompts_dict': plan['user_prompts_dict'], + 'log_level': plan['log_level'], + 'require_json_output': plan['require_json_output'], } try: @@ -898,15 +904,19 @@ def run_streamlit_app(): user_space = UserSpace() save_user_space(user_space) - tab1, tab2, tab4 = st.tabs(["Create Build Plans", "Run Saved Plans", "UserSpace"]) - - with tab1: + # Create pages using st.sidebar.selectbox + page = st.sidebar.selectbox( + "Select a page", + ["Create Build Plans", "Run Saved Plans", "Multi-Context Processing", "UserSpace"], + ) + if page == "Create Build Plans": multiplan_builder(user_space) - - with tab2: + elif page == "Run Saved Plans": upload_build_plan() - - with tab4: + elif page == "Multi-Context Processing": + multi_context_app = MCU(user_space) + multi_context_app.render() + elif page == "UserSpace": user_space_app(user_space) diff --git a/Codexes2Gemini/ui/utils.py b/Codexes2Gemini/ui/utils.py new file mode 100644 index 0000000..26f76ff --- /dev/null +++ b/Codexes2Gemini/ui/utils.py @@ -0,0 +1,19 @@ +import json +from importlib import resources + +import streamlit as st + + +def filter_dict(dictionary, filter_text): + return {k: v for k, v in dictionary.items() if + filter_text.lower() in k.lower() or ( + isinstance(v, dict) and filter_text.lower() in v.get('prompt', '').lower())} + + +def load_json_file(file_name): + try: + with resources.files('Codexes2Gemini.resources.prompts').joinpath(file_name).open('r') as file: + return json.load(file) + except Exception as e: + st.error(f"Error loading JSON file: {e}") + return {}