reformatted the whole project to avoid code smells

Co-authored-by: leondaniel22 <[email protected]>
Signed-off-by: Amir-Hussein-OTH <[email protected]>

src/application/res/app.py

@@ -40,12 +40,11 @@
 class SimpleSearchForm(FlaskForm):
     searchValue = StringField('Search Value', validators=[DataRequired()])
     submit = SubmitField('Search')
-    #values for pagination
+    # values for pagination
     currentPageSS = IntegerField('currentPageSS')
     resultsPerPageSS = IntegerField('resultsPerPageSS')
 
 
-
 class AdvancedEntryForm(FlaskForm):
     # Get all available fields for the specified index
     all_fields = os_manager.get_all_fields(index_name=index_name)
@@ -84,12 +83,11 @@ class AdvancedSearchForm(FlaskForm):
     entry = FieldList(FormField(AdvancedEntryForm), min_entries=1)
     # Submit button for submitting the form
     submit = SubmitField('Submit')
-    #values for pagination
+    # values for pagination
     currentPage = IntegerField('currentPage')
     resultsPerPage = IntegerField('resultsPerPage')
 
 
-
 def renderResult(input):
     # Extract the relevant list from the dictionary
     hits_list = input['hits']['hits']

src/application/res/backend/configuration.py

@@ -38,10 +38,9 @@ def get_config_values():
         elif key == "only_selected_tags":
             options[key] = config.getboolean('General', key, fallback=value)
         elif key == "file_types":
-            tmp = config.get('General',key,fallback=value)
+            tmp = config.get('General', key, fallback=value)
             options[key] = tmp.split(";")
         else:
             options[key] = config.get('General', key, fallback=value)
 
     return options
-

src/application/res/backend/opensearch_api.py

@@ -47,8 +47,8 @@ def _connect_to_open_search(self):
         # Create the client with SSL/TLS and hostname verification disabled
         # Port on which the OpenSearch node runs
         load_dotenv()  # load the environment
-        #print(os.getenv("OS_PASSWORD"))
-        auth = (os.getenv("OS_USER"),os.getenv("OS_PASSWORD") ) # credentials are loaded from the .env file
+        # print(os.getenv("OS_PASSWORD"))
+        auth = (os.getenv("OS_USER"), os.getenv("OS_PASSWORD"))  # credentials are loaded from the .env file
         self._client = OpenSearch(
             hosts=[{'host': self._host, 'port': self._port}],  # Host and port to connect with
             http_auth=auth,  # Credentials
@@ -101,7 +101,6 @@ def get_all_fields(self, index_name: str) -> list[str]:
             print(f"Error occurred while retrieving fields for index '{index_name}': {str(e)}")
             return None
 
-
     def get_datatype(self, index_name: str, field_name: str) -> str:
         """Get the datatype of a specific field for a specific index.
 
@@ -298,7 +297,7 @@ def simple_search(self, index_name: str, search_text: str, page: int = 0, page_s
         """
         fields = self.get_all_fields(index_name)
         query = {
-            'size' : self.search_size,
+            'size': self.search_size,
             "query": {
                 "bool": {
                     "should": []
@@ -341,12 +340,12 @@ def advanced_search(self, index_name: str, search_info: dict, page: int = 0, pag
                 search_content = search_info[search_field]['search_content']
                 operator = search_info[search_field]['operator']
                 weight = search_info[search_field]['weight']
-                sub_queries.append(self._get_sub_query(data_type, operator, search_field, weight,search_content))
+                sub_queries.append(self._get_sub_query(data_type, operator, search_field, weight, search_content))
         if sub_queries:
             query = self._get_query(sub_queries, self.search_size)
         else:
             query = {"query": {"exists": {"field": " "}}}
-        print("Advanced_query:",query)
+        print("Advanced_query:", query)
 
         response = self._client.search(
             body=query,
@@ -356,7 +355,6 @@ def advanced_search(self, index_name: str, search_info: dict, page: int = 0, pag
         )
         return response
 
-
     @staticmethod
     def _get_query(sub_queries: list[tuple], search_size) -> dict:
         """
@@ -366,8 +364,8 @@ def _get_query(sub_queries: list[tuple], search_size) -> dict:
         the value 'must' or 'must_not'.
         :return: Returns a query that can be used to search in OpenSearch.
         """
-        #The default size is 10, now it goes to 100 for example!
-        query = {'size' : search_size,'query': {'bool': {}}}
+        # The default size is 10, now it goes to 100 for example!
+        query = {'size': search_size, 'query': {'bool': {}}}
         for sub_query, functionality in sub_queries:
             if functionality not in query['query']['bool']:
                 query['query']['bool'][functionality] = [sub_query]
@@ -376,7 +374,7 @@ def _get_query(sub_queries: list[tuple], search_size) -> dict:
         return query
 
     @staticmethod
-    def _get_sub_query(data_type: str, operator: str, search_field: str, weight: str,search_content: any) -> tuple:
+    def _get_sub_query(data_type: str, operator: str, search_field: str, weight: str, search_content: any) -> tuple:
         """Returns a subquery that can be used to create a complete query.
 
           Args:

src/application/res/backend/os_dashboard_api.py

@@ -21,7 +21,6 @@ def _convert_time(updated_at_ts):
     return updated_at_human_readable
 
 
-
 class OSDashboardManager:
     """
      Class for managing the connection to the OpenSearch-Dashboard detects new visualizations.
@@ -74,8 +73,8 @@ def get_iframes(self):
                 visualization_id = obj['id']
                 encoded_id = urllib.parse.quote(visualization_id)
                 title = obj['attributes']['title'].capitalize()
-                #vis_state = json.loads(obj['attributes']['visState'])
-                #vis_type = vis_state['type'].capitalize()
+                # vis_state = json.loads(obj['attributes']['visState'])
+                # vis_type = vis_state['type'].capitalize()
 
                 # get and convert last update time
                 updated_at_human_readable = _convert_time(obj['updated_at'])
@@ -86,7 +85,7 @@ def get_iframes(self):
                 # Add the data to the iframe_data list
                 iframe_data.append({
                     'title': title,
-                    #'type': vis_type,
+                    # 'type': vis_type,
 
                     'updated_at': updated_at_human_readable,
                     'iframe_code': iframe_code

src/application/res/config.ini

@@ -1,9 +1,9 @@
 [General]
-localhost=False
-index_name=amoscore
-search_size=24
-limit=100
-selected_tags=FileName;FileSize;FileType
-only_selected_tags=False
-only_new_data=False
-file_types=XML;JPEG
+localhost = False
+index_name = amoscore
+search_size = 24
+limit = 100
+selected_tags = FileName;FileSize;FileType
+only_selected_tags = False
+only_new_data = False
+file_types = XML;JPEG

src/application/res/import-script/graphql_queries.py

@@ -1,7 +1,6 @@
 from graphql_query import Argument, Operation, Query, Field
 
 
-
 class FilterFunction:
     def __init__(self, tag: str, value: str, operation: str, data_type: str):
         """Creates a new object of class FilterFunction.
@@ -111,7 +110,6 @@ def _get_selected_tags(self) -> Argument:
         )
         return selected_tags
 
-
     def _get_arguments(self, filter_functions: Argument, sort_functions: Argument, selected_tags: Argument):
         """Returns all necessary arguments for the GraphQL query.
 
@@ -184,5 +182,3 @@ def generate_tag_query(self) -> str:
 
         # Return the rendered version of the executed query as a string
         return operation.render()
-
-

src/application/res/import-script/import_pipeline.py

@@ -15,7 +15,6 @@
 from backend.configuration import get_config_values
 
 
-
 def create_managers(localhost: bool = False):
     """ This function creates the managers to handle the APIs to the MetaDataHub and the OpenSearch Node
 
@@ -47,7 +46,8 @@ def extract_metadata_tags_from_mdh(mdh_manager: MetaDataHubManager, amount_of_ta
 
 
 def extract_data_from_mdh(mdh_manager: MetaDataHubManager, latest_timestamp: str = False, limit: int = False,
-        offset: int = False, selected_tags: list = None, file_type: str = False) -> tuple[list[dict], int]:
+                          offset: int = False, selected_tags: list = None, file_type: str = False) -> tuple[
+    list[dict], int]:
     """
     Extract data from the MetaDataHub.
 
@@ -269,17 +269,18 @@ def execute_pipeline(import_control: ImportControl):
         metadata_tags = {}
         for field in fields_in_os:
             metadata_tags[field] = os_manager.get_datatype(index_name=index_name, field_name=field)
-    else: # this is executed if it is the first import
+    else:  # this is executed if it is the first import
         mdh_tags = extract_metadata_tags_from_mdh(mdh_manager=mdh_manager)
         metadata_tags = modify_metadata_tags(mdh_tags=mdh_tags)  # modify the datatypes so they fit in OpenSearch
 
-    #file_types = ["XML","JPEG", "TXT"] #TODO
+    # file_types = ["XML","JPEG", "TXT"] #TODO
     limit = int(limit / len(file_types))
 
     for file_type in file_types:
-
         # extract the data from the mdh
-        mdh_data, files_amount = extract_data_from_mdh(mdh_manager=mdh_manager, limit=limit, latest_timestamp=latest_timestamp, selected_tags=selected_tags, file_type=file_type)
+        mdh_data, files_amount = extract_data_from_mdh(mdh_manager=mdh_manager, limit=limit,
+                                                       latest_timestamp=latest_timestamp, selected_tags=selected_tags,
+                                                       file_type=file_type)
 
         # get the amount of files that exist in the mdh core
         files_in_mdh = mdh_manager.get_total_files_count()

src/application/res/import-script/mdh_api.py

@@ -83,7 +83,8 @@ def _generate_metadata_tag_query(self, amount_of_tags: int = 900):
         self._write_file(self.format_query(gql_query.generate_tag_query()))
         return gql_query
 
-    def _generate_data_query(self, timestamp: str = False, filters: list = None, limit: int = False, offset: int = False, selected_tags: list = None, file_type: str = False):
+    def _generate_data_query(self, timestamp: str = False, filters: list = None, limit: int = False,
+                             offset: int = False, selected_tags: list = None, file_type: str = False):
         filter_functions = []
         if timestamp:
             f = FilterFunction(tag="MdHTimestamp", value=timestamp, operation="GREATER", data_type="TS")
@@ -93,7 +94,7 @@ def _generate_data_query(self, timestamp: str = False, filters: list = None, lim
             if "SourceFile" not in selected_tags:
                 selected_tags.append("SourceFile")
 
-        if file_type: 
+        if file_type:
             t = FilterFunction(tag="FileType", value=file_type, operation="EQUAL", data_type="STR")
             filter_functions.append(t)
 
@@ -125,9 +126,11 @@ def download_metadata_tags(self, amount_of_tags: int = 900):
         for core in mdh.core.main.get():
             self.metadata_tags = mdh.core.main.execute(core, self._request_path_file)
 
-    def download_data(self, timestamp: str = False, limit: int = False, offset: int = False, selected_tags: list = None, file_type: str = False):
+    def download_data(self, timestamp: str = False, limit: int = False, offset: int = False, selected_tags: list = None,
+                      file_type: str = False):
         """ download the data from a request and store it """
-        self._generate_data_query(timestamp=timestamp, limit=limit, offset=offset, selected_tags=selected_tags, file_type=file_type)
+        self._generate_data_query(timestamp=timestamp, limit=limit, offset=offset, selected_tags=selected_tags,
+                                  file_type=file_type)
         for core in mdh.core.main.get():
             self.data = mdh.core.main.execute(core, self._request_path_file)
 

src/application/res/import-script/request.gql

@@ -1,20 +1,20 @@
 query {
-  mdhSearch(
-    filterFunctions: []
-    sortFunctions: []
-    selectedTags: []
-    filterLogicOption: AND
-    limit: 3435
-  ) {
-    totalFilesCount
-    returnedFilesCount
-    instanceName
-    queryStatusAsText
-    files {
-      metadata {
-        name
-        value
-      }
+    mdhSearch(
+        filterFunctions: []
+        sortFunctions: []
+        selectedTags: []
+        filterLogicOption: AND
+        limit: 3435
+    ) {
+        totalFilesCount
+        returnedFilesCount
+        instanceName
+        queryStatusAsText
+        files {
+            metadata {
+                name
+                value
+            }
+        }
     }
-  }
-}
+}