UI tabs replaced with top-down scroll UX. Data and zip generation rem…

…oved and uses the new graph-data-generator package (#18)

Pipfile

@@ -4,23 +4,11 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-pytest = "*"
-streamlit = "*"
-streamlit-extras = "*"
-pyperclip = "*"
-streamlit-agraph = "*"
-lorem-text = "*"
-faker = "*"
 streamlit-player = "*"
-requests = "*"
-pandas = "*"
-random-address = "*"
-openai = "*"
-numpy = "*"
-certifi = "*"
+graph-data-generator = "*"
+streamlit-extras = "*"
 
 [dev-packages]
-mock-generators = {editable = true, path = "."}
 
 [requires]
 python_version = "3.11"

README.md

@@ -1,17 +1,9 @@
 # MOCK GRAPH DATA GENERATOR
-This is a prototype app for generating mock graph data for [Neo4j](https://neo4j.com/) database instances.
-
-The app uses [Streamlit](https://streamlit.io/) to create and manage the UI interface.
-
-
-## Recommendations
-Connect with a Chromium browser. Known issues when using with Safari, especially with interfacing with arrows and the data-importer.
+Applet with Streamlit UI interface to conveniently use and test the graph-data-generator package.
 
+## Install Poetry
+This applet uses [Poetry](https://python-poetry.org) for dependency management.
 
 ## Running
-Locally
-```
-pipenv shell
-pipenv sync
-pipenv run streamlit run mock_generators/app.py
-```
+`poetry update`
+`poetry run streamlit run graph_data_generator_streamlit/app.py`

graph_data_generator_streamlit/app.py

@@ -0,0 +1,26 @@
+import streamlit as st
+from tabs.instructions_tab import instructions_tab
+from tabs.design_tab import design_tab
+from tabs.generate_tab import generate_tab
+from tabs.data_importer_tab import data_importer_tab
+import logging
+import sys
+
+# SETUP
+st.set_page_config(layout="wide")
+logging.getLogger().setLevel(logging.DEBUG)
+logging.info(f'App Started')
+
+instructions_tab()
+
+st.markdown("-------------")
+st.markdown("**① DESIGN**")
+design_tab()
+
+st.markdown("-------------")
+st.markdown("**② GENERATE**")
+generate_tab()
+
+st.markdown("-------------")
+st.markdown("**③ IMPORT**")
+data_importer_tab()

graph_data_generator_streamlit/tabs/data_importer_tab.py

@@ -0,0 +1,11 @@
+import streamlit as st
+import streamlit.components.v1 as components
+from streamlit_extras.colored_header import colored_header
+
+def data_importer_tab():
+    is_local = st.checkbox("Use HTTP", value=False, help="Select Use HTTP if connecting with a local Neo4j instance.")
+
+    if is_local == True:
+        components.iframe("http://data-importer.graphapp.io/", height=1000, scrolling=False)    
+    else:
+        components.iframe("https://data-importer.graphapp.io/", height=1000, scrolling=False)

graph_data_generator_streamlit/tabs/design_tab.py

@@ -0,0 +1,138 @@
+import streamlit as st
+import streamlit.components.v1 as components
+from graph_data_generator import generators, GeneratorType, Generator
+
+import os
+import logging
+import datetime
+import json
+import sys
+import io
+
+def load_string(filepath: str, default=None):
+    if os.path.isfile(filepath) == False and os.access(filepath, os.R_OK) == False:
+         with io.open(os.path.join(filepath), 'r') as _:
+            logging.info(f"file_utils.py: No file at {filepath}")
+            return default
+    with open(filepath, 'r') as f:
+        return f.read()
+
+def filtered_generators(
+        search_term: str, 
+        type_filter: str,
+        generators: dict[str, Generator]):
+
+    def passes_search(search_term: str, 
+                      generator: Generator):
+        if search_term is None:
+            return True
+        if search_term != "" and search_term.lower() not in generator.name.lower() and search_term.lower() not in generator.description.lower():
+            return False
+        return True
+
+    def passes_type_filter(type_filter: str,
+                           generator: Generator):
+        if type_filter != "All" and type_filter != generator.type.to_string():
+            return False
+        return True
+
+    return [generator for key, generator in sorted(generators.items(), key=lambda gen:(gen[1].name)) if passes_search(search_term, generator) and passes_type_filter(type_filter, generator)]
+
+
+def design_tab():
+    st.markdown(
+        """
+        Use the arrows app to quickly design your mock data. When ready, click on the `Download/Export` button, select the `JSON` tab, then copy the .JSON data to the **② Generate** section
+        """
+    )
+    c1, c2 = st.columns([8,2])
+    with c1:
+        components.iframe("https://arrows.app", height=1000, scrolling=False)
+    with c2:
+        search_term = st.text_input("Search Generators by keyword", "", help="Generators are functions for creating mock data.")
+        # st.write(generators)
+        type_filter = st.radio("Filter Generator outputs by type", ["All", "String", "Bool", "Integer", "Float", "Function", "Datetime", "Assignment"])
+
+        total_count = len(generators)
+        display_generators = filtered_generators(search_term, type_filter, generators)
+        count = len(display_generators)
+        st.write(f"Displaying {count} of {total_count} generators:")
+        for generator in display_generators:
+        # for _, generator in sorted(generators.items(), key=lambda gen:(gen[1].name)):
+
+        #     # Filtering
+        #     if type_filter != "All" and type_filter != generator.type.to_string():
+        #         continue
+
+        #     if search_term != "" and search_term.lower() not in generator.name.lower() and search_term.lower() not in generator.description.lower():
+        #         continue
+
+        #     # Don't have a vertical / horizontal scroll container, so limit display
+        #     if index > 10:
+        #         continue
+
+            # try:
+            #     # Check that we can load code first
+            #     code_filepath = generator.code_url
+            #     code_file = load_string(code_filepath)
+            # except:
+            #     logging.error(f"Could not load generator code from {code_filepath}: {sys.exc_info()[0]}") 
+            #     continue
+
+            with st.expander(generator.name):
+            # st.markdown("------------------")
+                st.write(generator.name)
+                st.write(f"\n {generator.description}")
+                # st.write(f'Generator Code:\n')
+                # st.markdown(f'```python\n{code_file}\n```')
+                args = generator.args
+                arg_inputs = []
+                for arg in args:
+                    if arg.type == GeneratorType.STRING:
+                        arg_inputs.append(st.text_input(
+                            label=arg.label, 
+                            value = arg.default,
+                            key = f'{generator.name}_{arg.label}',
+                            placeholder = f'{arg.hint}',
+                            help = f'{arg.description}'
+                            ))
+                    if arg.type == GeneratorType.INT or arg.type == GeneratorType.FLOAT:
+                        arg_inputs.append(st.number_input(
+                            label= arg.label,
+                            value= arg.default,
+                            key = f'{generator.name}_{arg.label}'
+                            ))
+                    if arg.type == GeneratorType.BOOL:
+                        arg_inputs.append(st.radio(
+                            label=arg.label,
+                            index=arg.default,
+                            key = f'{generator.name}_{arg.label}'
+                        ))
+                    if arg.type == GeneratorType.DATETIME:
+                        arg_inputs.append(st.date_input(
+                            label=arg.label,
+                            value=datetime.datetime.fromisoformat(arg.default),
+                            key = f'{generator.name}_{arg.label}'
+                        ))
+                # if c.button("Generate Example Output", key=f"run_{generator.name}"):
+                #     try:
+                #         module = __import__(generator.import_url(), fromlist=['generate'])
+                #         # logging.info(f'arg_inputs: {arg_inputs}')
+
+                #         # TODO: Need a fake list of Node Mappings to test out assignment generators
+                #         result = module.generate(arg_inputs)
+                #         c.write(f'Output: {result}')
+                #     except:
+                #         c.error(f"Problem running generator {generator.name}: {sys.exc_info()[0]}")
+
+                # Property Code
+                # name = generator.name
+                args = arg_inputs
+                obj = {
+                    generator.gid: args
+                }
+
+                json_string = json.dumps(obj, default=str)
+
+                st.write('Copy & paste below as a node/relationship property value in arrows.app')
+                st.code(f'{json_string}')

graph_data_generator_streamlit/tabs/generate_tab.py

@@ -0,0 +1,55 @@
+# Now the new Generate Tab
+
+import streamlit as st
+from graph_data_generator import generators
+import graph_data_generator as gdg
+
+def generate_tab():
+
+    # c1, c2 = st.tabs(["Copy & Paste", "Import File"])
+    # with c1:
+
+    st.write("Copy & Paste Arrows.app .JSON file")
+    filename = st.text_input("Name of file", value="mock_data")
+    txt = st.text_area("Paste arrows.app JSON here", height=500, help="Click out of the text area to generate the .zip file.")
+    if txt is not None and txt != "":
+
+        try:
+            zip = gdg.generate(txt, enable_logging=True)
+            if zip is None:
+                st.warning('Unexpected problem generating file. Try an alternate JSON input')
+            else:
+                st.download_button(
+                    label = "Download .zip file",
+                    data = zip,
+                    file_name = f"{filename}.zip",
+                    mime = "text/plain"
+                )
+        except Exception as e:
+            st.error(e)
+
+    # with c2:
+    #     uploaded_file = st.file_uploader("Upload an arrows JSON file", type="json")
+    #     if uploaded_file is not None:
+    #         # To convert to a string based IO:
+    #         stringio = StringIO(uploaded_file.getvalue().decode("utf-8"))
+    #         # To read file as string:
+    #         current_file = stringio.read()
+
+    #         # Save to session state
+    #         st.session_state[MAPPINGS] = Mapping.empty()
+
+    #         name = uploaded_file.name.split(".")[0]
+    #         if current_file is not None:
+    #             # TODO: Verfiy file is valid arrows JSON
+    #             generators = st.session_state[GENERATORS]
+    #             mapping = mapping_from_json(
+    #                 current_file, 
+    #                 generators)
+    #             zip = generate_zip(mapping)
+    #             st.download_button(
+    #                 label = "Download .zip file",
+    #                 data = zip,
+    #                 file_name = f"{name}.zip",
+    #                 mime = "text/plain"
+    #             )

graph_data_generator_streamlit/tabs/instructions_tab.py

@@ -0,0 +1,16 @@
+from streamlit_player import st_player
+import streamlit as st
+
+def instructions_tab():
+    st.title("Graph Data Generator App")
+    st.markdown(
+        """
+        This app is a central collection tools built around the [graph-data-generator](https://pypi.org/project/graph-data-generator/) package for generating .csvs of interconnected mock data that can be imported into databases, including a [Neo4j](https://neo4j.com) graph database.
+        
+        NOTES: 
+        - Chromium browser recommended for best experience.
+        - Each tool may require independent logins with first use.
+    """)
+    # url = st.secrets["VIDEO_TUTORIAL_URL"]
+    # Disabling until updated video is available
+    # st_player(url, height=600)