Skip to content

Commit

Permalink
Hdf5 cleanup (#1186)
Browse files Browse the repository at this point in the history 
* Only store hdf5 project data temporarily for parsing unless specified by the user

* Extending HDF5 support to remote and server files

Allows you to create models using remote HDF5 files and slycat server HDF5 files
  • Loading branch information
@Spurs20
Spurs20 authored Nov 6, 2024
1 parent 6b8c0ab commit c71d1be
Show file tree
Hide file tree
Showing 6 changed files with 50 additions and 20 deletions.
7 changes: 5 additions & 2 deletions packages/slycat/web/server/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,8 +135,11 @@ def parse_existing_file(database, parser, input, attachment, model, aid):
:return: not used
"""
kwargs = {}
aids = []
aids.append(aid)
if not isinstance(aid, list):
aids = []
aids.append(aid)
else:
aids = aid
try:
slycat.web.server.plugin.manager.parsers[parser]["parse"](database, model, input, attachment,
aids, **kwargs)
Expand Down
2 changes: 2 additions & 0 deletions packages/slycat/web/server/handlers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -443,6 +443,8 @@ def put_project_csv_data(pid, file_key, parser, mid, aids):
cherrypy.log.error(str(e))

model = database.get("model", mid)
if isinstance(aids, str):
aids = aids.split(',')
slycat.web.server.parse_existing_file(database, parser, True, attachment, model, aids)
return {"Status": "Success"}

Expand Down
10 changes: 5 additions & 5 deletions packages/slycat/web/server/upload.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,13 +249,13 @@ def numeric_order(x):
if '.h5' in self._aids[1] or '.hdf5' in self._aids[1]:
slycat.web.server.plugin.manager.parsers[self._parser]["parse"](database, model, self._input,
files, self._aids, **self._kwargs)
if isinstance(self._aids[0], list):
slycat.web.server.plugin.manager.parsers[self._parser]["parse"](database, model, self._input,
files, self._aids[0], **self._kwargs)
elif isinstance(self._aids[0], list):
slycat.web.server.plugin.manager.parsers[self._parser]["parse"](database, model, self._input,
files, self._aids[0], **self._kwargs)
else:
slycat.web.server.plugin.manager.parsers[self._parser]["parse"](database, model, self._input,
files, self._aids, **self._kwargs)
if model["model-type"] == "parameter-image" and self.useProjectData == True:
files, self._aids, **self._kwargs)
if model["model-type"] == "parameter-image" and self.useProjectData == True and '.h5' not in self._aids[1] and '.hdf5' not in self._aids[1]:
# Use project data
slycat.web.server.handlers.create_project_data(self._mid, self._aids, files)
except Exception as e:
Expand Down
16 changes: 11 additions & 5 deletions web-server/plugins/slycat-hdf5-parser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@ def parse_file(file, model, database):

def parse(database, model, input, files, aids, **kwargs):
# Read HDF5 file
slycat.web.server.handlers.create_project_data(model['_id'], aids, files)
start = time.time()

# parsed = [parse_file(file, model, database) for file in files]
Expand All @@ -89,10 +88,17 @@ def parse(database, model, input, files, aids, **kwargs):
# slycat.web.server.put_model_arrayset_data(database, model, aid, "%s/.../..." % array_index, combined_data)

end = time.time()
model = database.get("model", model['_id'])
slycat.web.server.put_model_parameter(database, model, "error-messages", "")
model["db_creation_time"] = (end - start)
database.save(model)
database = slycat.web.server.database.couchdb.connect()
model = database.get("model", model["_id"])
with slycat.web.server.database.couchdb.db_lock:
model["db_creation_time"] = (end - start)
database.save(model)
# If the user selected to not save project data, this will only be temporary during the wizard to parse the HDF5 file
slycat.web.server.handlers.create_project_data(model['_id'], aids, files)
slycat.web.server.put_model_parameter(database, model, "error-messages", "")
# model["db_creation_time"] = (end - start)
# model = database.get("model", model['_id'])
# database_curr.save(model)

def register_slycat_plugin(context):
context.register_parser("slycat-hdf5-parser", "HDF5", ["table"], parse)
29 changes: 24 additions & 5 deletions web-server/plugins/slycat-parameter-image/js/wizard-ui.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ function constructor(params) {
file_key: component.selected_file(),
parser: component.parser(),
mid: component.model._id(),
aids: "data-table",
aids: [["data-table"], component.current_aids()],

success: function (response) {
var data = JSON.stringify(response);
Expand Down Expand Up @@ -225,8 +225,7 @@ function constructor(params) {
component.cancel = function () {
component.smb_wizard_login_root.unmount();
if (component.model._id()) {
client
.get_project_data_in_model_fetch({
client.get_project_data_in_model_fetch({
mid: component.model._id(),
})
.then((did) => {
Expand Down Expand Up @@ -352,8 +351,8 @@ function constructor(params) {
};

component.existing_table = function () {
var fileName = component.selected_file;
component.current_aids = fileName();
var fileName = component.selected_file();
component.current_aids(fileName);
component.get_server_files();
};

Expand Down Expand Up @@ -561,6 +560,26 @@ function constructor(params) {
mid: component.model._id(),
success : (results) =>
{
if (component.model._id() && component.useProjectData() == false) {
client.get_project_data_in_model_fetch({
mid: component.model._id(),
})
.then((did) => {
// if the data id isn't empty
// delete model first
client.delete_model_fetch({ mid: component.model._id() }).then(() => {
// Get list of model ids project data is used in
if (did.length >= 1) {
client.get_project_data_parameter_fetch({ did: did, param: "mid" }).then((models) => {
// if there are no more models using that project data, delete it
if (models && models.length === 0) {
client.delete_project_data_fetch({ did: did });
}
});
}
});
});
}
component.finish();
}
})
Expand Down
6 changes: 3 additions & 3 deletions web-server/plugins/slycat-parameter-image/wizard-ui.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,10 @@ <h3 class="modal-title">New Parameter Space Model</h3>
<ul class="nav nav-pills">
<li class="nav-item" data-bind="css:{active:tab() == 0}"><a class="nav-link">Locate Data</a></li>
<li class="nav-item" data-bind="css:{active:tab() == 1}, visible: ps_type() == 'local'"><a class="nav-link">Upload Table</a></li>
<li class="nav-item" data-bind="css:{active:tab() == 6}, visible: ps_type() == 'local' && parser() == 'slycat-hdf5-parser'"><a class="nav-link">Select HDF5 Inputs</a></li>
<li class="nav-item" data-bind="css:{active:tab() == 7}, visible: ps_type() == 'local' && parser() == 'slycat-hdf5-parser'"><a class="nav-link">Select HDF5 Outputs</a></li>
<li class="nav-item" data-bind="css:{active:tab() == 2}, visible: ps_type() == 'remote' || ps_type() == 'smb'"><a class="nav-link">Choose Host</a></li>
<li class="nav-item" data-bind="css:{active:tab() == 3}, visible: ps_type() == 'remote' || ps_type() == 'smb'"><a class="nav-link">Select Table</a></li>
<li class="nav-item" data-bind="css:{active:tab() == 6}, visible: (ps_type() == 'local' || ps_type() == 'remote' || ps_type() == 'server') && parser() == 'slycat-hdf5-parser'"><a class="nav-link">Select HDF5 Inputs</a></li>
<li class="nav-item" data-bind="css:{active:tab() == 7}, visible: (ps_type() == 'local' || ps_type() == 'remote' || ps_type() == 'server') && parser() == 'slycat-hdf5-parser'"><a class="nav-link">Select HDF5 Outputs</a></li>
<li class="nav-item" data-bind="css:{active:tab() == 4}, visible: parser() != 'slycat-hdf5-parser'"><a class="nav-link">Select Columns</a></li>
<li class="nav-item" data-bind="css:{active:tab() == 5}"><a class="nav-link">Name Model</a></li>
</ul>
Expand Down Expand Up @@ -127,7 +127,7 @@ <h3 class="modal-title">New Parameter Space Model</h3>
</div>

<div data-bind="visible:tab() == 6">
<div class="hdf5-wizard-input-browse" data-bind="visible: ps_type() == 'local'">HDF5 Inputs</div>
<div class="hdf5-wizard-input-browse" data-bind="visible: ps_type() == 'local' || ps_type() == 'remote' || ps_type() == 'server'">HDF5 Inputs</div>
<div class="slycat-remote-browser-flex-layout" data-bind="visible:ps_type() == 'local'">
<label class="alert alert-danger slycat-big-scrolling-alert" role="alert"
data-bind="visible:error_messages().length > 0, text: error_messages()"></label>
Expand Down

0 comments on commit c71d1be

Please sign in to comment.