Hdf5 browser (#1175)

* Removing log statement

* Creating prototype hdf5 browser for parameter space wizard

* Update HDF5 browser to query user-entered path

* Formatting user-entered path for web service call, and changing mime-type based on HDF5 group/dataset

* Fixing double clicking and path appending bug in HDF5 browser

* Formatting fix

* Fixing bug to browse up a directory on button click

* Removing unnecessary log

* Removing "save project data" checkbox from HDF5 browser tab

* Removing blank space at bottom of HDF5 browser tab

* Adding visible 'HDF5 Browser' tab identifier when HDF5 filetype is selected

* Removing unnecessary filetype selector from HDF5 browser

* Creating new html template for hdf5 browser and removing unnecessary hostname labels

* Adding HDF5 browser for input/output, and storing table paths in model

Adding one HDF5 browser for input table, one HDF5 browser for output table, and storing the paths to those two tables in the couch model doc. These paths will then be used to retrieve those input/output tables, and combine them to make the model.

* Fixing HDF5 pathing in react HDF5 browser

* Creating web service that combines input and output tables for model creation

* Enabling continue button coloring in HDF5 table selection

* Adding model naming tab for hdf5 model creation

* Fixing highlighting in hdf5 table selection and indexing error in input/output column header

* Adding separate tab titles for input/output selection

* Addressing PR comments

packages/slycat/web/server/engine.py

@@ -158,6 +158,9 @@ def abspath(path):
   dispatcher.connect("post-remotes", "/remotes", slycat.web.server.handlers.post_remotes, conditions={"method" : ["POST"]})
   dispatcher.connect("post-remotes-smb", "/remotes/smb", slycat.web.server.handlers.post_remotes_smb, conditions={"method" : ["POST"]})
   dispatcher.connect("post-smb-browse", "/smb/remotes/:hostname/browse{path:.*}", slycat.web.server.handlers.post_smb_browse, conditions={"method" : ["POST"]})
+  dispatcher.connect("post-hdf5-browse", "/hdf5/browse/:path/:pid/:mid", slycat.web.server.handlers.post_browse_hdf5, conditions={"method" : ["POST"]})
+  dispatcher.connect("post-hdf5-table", "/hdf5/table/:path/:pid/:mid", slycat.web.server.handlers.post_hdf5_table, conditions={"method" : ["POST"]})
+  dispatcher.connect("post-combine-hdf5-tables", "/hdf5/combine/:mid", slycat.web.server.handlers.post_combine_hdf5_tables, conditions={"method": ["POST"]})
   dispatcher.connect("put-model-arrayset-array", "/models/:mid/arraysets/:aid/arrays/:array", slycat.web.server.handlers.put_model_arrayset_array, conditions={"method" : ["PUT"]})
   dispatcher.connect("put-model-arrayset-data", "/models/:mid/arraysets/:aid/data", slycat.web.server.handlers.put_model_arrayset_data, conditions={"method" : ["PUT"]})
   dispatcher.connect("put-model-arrayset", "/models/:mid/arraysets/:aid", slycat.web.server.handlers.put_model_arrayset, conditions={"method" : ["PUT"]})

packages/slycat/web/server/handlers.py

@@ -2702,7 +2702,6 @@ def post_remotes():
         msg = "login could not save session for remote host"
     return {"sid": sid, "status": True, "msg": msg}
 
-
 @cherrypy.tools.json_in(on=True)
 @cherrypy.tools.json_out(on=True)
 def post_remotes_smb():
@@ -2759,6 +2758,142 @@ def post_smb_browse(hostname, path):
     with slycat.web.server.smb.get_session(sid) as session:
         return session.browse(path=path)
 
+def post_hdf5_table(path, pid, mid):
+    """
+        Takes a user selected path inside an HDF5 file, and stores the
+        table at that path as either inputs or outputs for the model. 
+
+        path {string} -- path to table inside of HDF5 file
+    """
+    # Need to find the HDF5 stored on Slycat server, so we can query it for the path.
+    path = path.replace('-', '/')
+    database = slycat.web.server.database.couchdb.connect()
+    model = database.get("model", mid)
+    project = database.get("project", pid)
+    did = model['project_data'][0]
+    project_data = database.get("project_data", did)
+    file_name = project_data['hdf5_name']
+    hdf5_path = cherrypy.request.app.config["slycat-web-server"]["data-store"] + "/" + file_name
+    h5 = h5py.File(hdf5_path, 'r')
+    table = list(h5[path])
+    column_headers = list(h5[path].dims[1][0])
+    headers = []
+    attributes = []
+    dimensions = [{"name": "row", "type": "int64", "begin": 0, "end": len(table[0])}]
+
+    if 'hdf5-inputs' not in model:
+        model['hdf5-inputs'] = path
+    else:
+        model['hdf5-outputs'] = path
+    slycat.web.server.authentication.require_project_writer(project)
+    database.save(model)
+
+    cherrypy.response.status = "200 Project updated."
+
+def post_combine_hdf5_tables(mid):
+    """
+        Combines user selected input/output tables inside of an HDF5 file into one table.
+        Uses the hdf5-input and hdf5-output paths in the model to find the tables in the HDF5 file. 
+    """
+
+    database = slycat.web.server.database.couchdb.connect()
+    model = database.get("model", mid)
+    project = model['project']
+    slycat.web.server.authentication.require_project_writer(project)
+    input_path = '/' + model['hdf5-inputs']
+    output_path = '/' + model['hdf5-outputs']
+    did = model['project_data'][0]
+    project_data = database.get("project_data", did)
+    file_name = project_data['hdf5_name']
+    hdf5_path = cherrypy.request.app.config["slycat-web-server"]["data-store"] + "/" + file_name
+    h5 = h5py.File(hdf5_path, 'r')
+
+    # Getting indices for input/output columns
+    unformatted_input = list(h5[input_path])
+    input_column_headers_indices = [i for i in range(0, len(unformatted_input[0]))]
+    unformatted_output = list(h5[output_path])
+    output_column_headers_indices = [i for i in range(len(unformatted_input[0]), (len(unformatted_input[0]) + len(unformatted_output[0])))]
+
+    slycat.web.server.put_model_parameter(database, model, 'input-columns', input_column_headers_indices, True)
+    slycat.web.server.put_model_parameter(database, model, 'output-columns', output_column_headers_indices, True)
+
+    combined_dataset = []
+    output_headers = []
+    input_headers = []
+    attributes = []
+
+    column_headers_input = list(h5[input_path].dims[1][0])
+    column_headers_output = list(h5[output_path].dims[1][0])
+
+    # Once we have column headers, this is how we can get/store them. 
+    for i, column in enumerate(column_headers_input):
+        input_headers.append(str(column.decode('utf-8')))
+        attributes.append({"name": str(column.decode('utf-8')), "type": str(type(unformatted_input[0][i])).split('numpy.')[1].split("'>")[0]})
+    for j, column in enumerate(column_headers_output):
+        output_headers.append(str(column.decode('utf-8')))
+        attributes.append({"name": str(column.decode('utf-8')), "type": str(type(unformatted_output[0][j])).split('numpy.')[1].split("'>")[0]})
+
+    combined_headers = numpy.concatenate((input_headers, output_headers), axis=0)
+    combined_headers = combined_headers.tolist()
+    combined_data = numpy.concatenate((unformatted_input, unformatted_output), axis=1)
+    combined_data = numpy.transpose(combined_data)
+
+    combined_data = combined_data.tolist()
+
+    for row in combined_data:
+        combined_dataset.append(numpy.asarray(row))
+
+    dimensions = [{"name": "row", "type": "int64", "begin": 0, "end": len(combined_dataset[0])}]
+
+    array_index = 0    
+    slycat.web.server.put_model_arrayset(database, model, 'data-table', input)
+    slycat.web.server.put_model_array(database, model, 'data-table', 0, attributes, dimensions)
+    slycat.web.server.put_model_arrayset_data(database, model, 'data-table', "%s/.../..." % array_index, combined_data)
+
+    cherrypy.response.status = "200 Project updated."
+
+def post_browse_hdf5(path, pid, mid):
+    """
+        Given a path inside of an HDF5 file, builds out the current tree at that path.
+        Formats the tree structure to conform with remote/hdf5 browser requirements.
+    """
+    def allkeys_single_level(obj, tree_structure):
+        path = obj.name # This is current top level path
+        # Need to include all these fields because we are repurposing the remote file browser, which expects all these
+        tree_structure['path'] = path
+        tree_structure['name'] = []
+        tree_structure['sizes'] = []
+        tree_structure['types'] = []
+        tree_structure['mtimes'] = []
+        tree_structure['mime-types'] = []
+        all_items = obj.items()
+        for key, value in all_items:
+            # key will be all the sub groups and datasets in the current path
+            tree_structure['name'].append(key)
+            tree_structure['sizes'].append(0)
+            tree_structure['mtimes'].append('2024')
+            if isinstance(value, h5py.Group):
+                tree_structure['mime-types'].append('application/x-directory')
+                tree_structure['types'].append('d')
+            else:
+                tree_structure['mime-types'].append('file')
+                tree_structure['types'].append('f')
+        return tree_structure
+    # Need to find the HDF5 stored on Slycat server, so we can query it for the path.
+    path = path.replace('-', '/')
+    database = slycat.web.server.database.couchdb.connect()
+    model = database.get("model", mid)
+    did = model['project_data'][0]
+    project_data = database.get("project_data", did)
+    file_name = project_data['hdf5_name']
+    hdf5_path = cherrypy.request.app.config["slycat-web-server"]["data-store"] + "/" + file_name
+    h5 = h5py.File(hdf5_path, 'r')
+    tree_structure = {}
+    current_level = allkeys_single_level(h5[path], tree_structure)
+    json_payload = json.dumps(current_level)
+
+    return json_payload
+
 @cherrypy.tools.json_out(on=True)
 def get_remotes(hostname):
     """

packages/slycat/web/server/upload.py

@@ -245,6 +245,10 @@ def numeric_order(x):
       # adding this check for backwards compatibility 
       # new way self._aids[0] is the file name being added to the model and hdf5
       # self._aids[1] is the name of the file being pushed to the project_data data object
+        if len(self._aids) > 1:
+          if '.h5' in self._aids[1] or '.hdf5' in self._aids[1]:
+            slycat.web.server.plugin.manager.parsers[self._parser]["parse"](database, model, self._input,
+                                                                files, self._aids, **self._kwargs)
         if isinstance(self._aids[0], list):
           slycat.web.server.plugin.manager.parsers[self._parser]["parse"](database, model, self._input,
                                                                           files, self._aids[0], **self._kwargs)