From 368df181bafb253f69ce296ac1b327981446e48f Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 6 Nov 2023 15:00:33 -0500 Subject: [PATCH 01/11] use latest dandi dev branch --- environments/environment-Windows.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environments/environment-Windows.yml b/environments/environment-Windows.yml index 2ec71c529..62ca451b0 100644 --- a/environments/environment-Windows.yml +++ b/environments/environment-Windows.yml @@ -18,5 +18,6 @@ dependencies: - flask-cors === 3.0.10 - flask_restx == 1.1.0 - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[full] + - dandi @ git+https://github.com/dandi/dandi-cli.git@main - pytest == 7.2.2 - pytest-cov == 4.1.0 From d6fa3a872c2c327e7486f1d77446097cfbc5d949 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 6 Nov 2023 15:10:00 -0500 Subject: [PATCH 02/11] dandi uses master not main --- environments/environment-Windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/environment-Windows.yml b/environments/environment-Windows.yml index 62ca451b0..f9a02d974 100644 --- a/environments/environment-Windows.yml +++ b/environments/environment-Windows.yml @@ -18,6 +18,6 @@ dependencies: - flask-cors === 3.0.10 - flask_restx == 1.1.0 - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[full] - - dandi @ git+https://github.com/dandi/dandi-cli.git@main + - dandi @ git+https://github.com/dandi/dandi-cli.git@master - pytest == 7.2.2 - pytest-cov == 4.1.0 From 3684bb3c9a29aa0295011f94dbc6faeac60bd262 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 6 Nov 2023 20:31:53 -0500 Subject: [PATCH 03/11] include dandi explicitly in pyinstaller --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c08c52266..13ce918c9 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "build:mac": "npm run build && npm run build:flask && npm run build:electron:mac", "build:linux": "npm run build && npm run build:flask && npm run build:electron:linux", "build:flask": "python -m PyInstaller nwb-guide.spec --log-level DEBUG --clean --noconfirm --distpath ./build/flask", - "build:flask:spec:base": "pyi-makespec --name nwb-guide --onedir --collect-data jsonschema_specifications --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ndx_dandi_icephys --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils --hidden-import email_validator ./pyflask/app.py", + "build:flask:spec:base": "pyi-makespec --name nwb-guide --onedir --collect-data jsonschema_specifications --collect-all dandi --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ndx_dandi_icephys --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils --hidden-import email_validator ./pyflask/app.py", "build:flask:spec": "npm run build:flask:spec:base && python prepare_pyinstaller_spec.py", "build:electron:win": "electron-builder build --win --publish never", "build:electron:mac": "electron-builder build --mac --publish never", From 30c1ed3f1e4f82e873f6d24e9b83e0b41e87b2b2 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 6 Nov 2023 20:32:27 -0500 Subject: [PATCH 04/11] include in spec --- nwb-guide.spec | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nwb-guide.spec b/nwb-guide.spec index 8b4f5649d..8db13407b 100644 --- a/nwb-guide.spec +++ b/nwb-guide.spec @@ -11,6 +11,8 @@ datas = [('./paths.config.json', '.'), ('./package.json', '.')] binaries = [] hiddenimports = ['scipy._distributor_init', 'scipy._lib.messagestream', 'scipy._lib._ccallback', 'scipy._lib._testutils', 'email_validator'] datas += collect_data_files('jsonschema_specifications') +tmp_ret = collect_all('dandi') +datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] tmp_ret = collect_all('nwbinspector') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] tmp_ret = collect_all('neuroconv') From 759ef4ed2fc4d3b46c13b515f042074e6088fb7b Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 7 Nov 2023 20:03:33 -0500 Subject: [PATCH 05/11] update defaults for dandi upload page (so that selecting 1 passes) --- schemas/json/dandi/upload.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schemas/json/dandi/upload.json b/schemas/json/dandi/upload.json index 163d75066..63249619f 100644 --- a/schemas/json/dandi/upload.json +++ b/schemas/json/dandi/upload.json @@ -8,14 +8,14 @@ "type": "integer", "title": "Job Count", "description": "The number of files to upload in parallel. A value of -1 uses all available processes", - "default": 1, + "default": -1, "min": -1 }, "number_of_threads": { "type": "integer", "title": "Threads per Job", "description": "The number of threads to handle each file. A value of -1 uses all available threads per process.", - "default": 1, + "default": -1, "min": -1 }, "cleanup": { From 464245ace198c41993ffe64566387a4d0df5b7cb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 8 Nov 2023 01:04:05 +0000 Subject: [PATCH 06/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/renderer/src/stories/Button.js | 6 +++++- src/renderer/src/stories/Dashboard.js | 5 ++--- src/renderer/src/stories/assets/save.svg | 2 +- src/renderer/src/stories/pages/guided-mode/GuidedHome.js | 1 - src/renderer/src/stories/pages/settings/SettingsPage.js | 5 +---- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/renderer/src/stories/Button.js b/src/renderer/src/stories/Button.js index d2bd0cbfa..066316f26 100644 --- a/src/renderer/src/stories/Button.js +++ b/src/renderer/src/stories/Button.js @@ -104,7 +104,11 @@ export class Button extends LitElement { >${this.icon instanceof HTMLElement ? this.icon : unsafeHTML(this.icon)}` : ""} - ${this.label ? html`${this.label}` : ""} + ${this.label + ? html`${this.label}` + : ""} `; } diff --git a/src/renderer/src/stories/Dashboard.js b/src/renderer/src/stories/Dashboard.js index b89da789d..8c37ab617 100644 --- a/src/renderer/src/stories/Dashboard.js +++ b/src/renderer/src/stories/Dashboard.js @@ -193,9 +193,8 @@ export class Dashboard extends LitElement { // Update Active Page this.#active = page; - // Reset global state if page has no parent - if ( !this.#active.info.parent ) toPass.globalState = {}; + if (!this.#active.info.parent) toPass.globalState = {}; if (isNested) { let parent = info.parent; @@ -203,7 +202,7 @@ export class Dashboard extends LitElement { this.subSidebar.sections = this.#getSections(parent.info.pages, toPass.globalState); // Update sidebar items (if changed) this.subSidebar.active = info.id; // Update active item (if changed) this.sidebar.hide(true); - this.subSidebar.show(); + this.subSidebar.show(); } else { this.sidebar.show(); this.subSidebar.hide(); diff --git a/src/renderer/src/stories/assets/save.svg b/src/renderer/src/stories/assets/save.svg index 9220f74ab..9198ddf73 100644 --- a/src/renderer/src/stories/assets/save.svg +++ b/src/renderer/src/stories/assets/save.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/src/renderer/src/stories/pages/guided-mode/GuidedHome.js b/src/renderer/src/stories/pages/guided-mode/GuidedHome.js index 8cdf45694..64430b163 100644 --- a/src/renderer/src/stories/pages/guided-mode/GuidedHome.js +++ b/src/renderer/src/stories/pages/guided-mode/GuidedHome.js @@ -153,7 +153,6 @@ export class GuidedHomePage extends Page { }; async updated() { - const htmlBase = this.shadowRoot ?? this; // this.content = (this.shadowRoot ?? this).querySelector("#content"); const lottieContainer = htmlBase.querySelector("#new-dataset-lottie-container"); diff --git a/src/renderer/src/stories/pages/settings/SettingsPage.js b/src/renderer/src/stories/pages/settings/SettingsPage.js index 8da39045e..4ea458bf5 100644 --- a/src/renderer/src/stories/pages/settings/SettingsPage.js +++ b/src/renderer/src/stories/pages/settings/SettingsPage.js @@ -20,12 +20,9 @@ import { merge, setUndefinedIfNotDeclared } from "../utils.js"; import { notyf } from "../../../dependencies/globals.js"; -import saveSVG from '../../assets/save.svg?raw' - - +import saveSVG from "../../assets/save.svg?raw"; export class SettingsPage extends Page { - header = { title: "App Settings", subtitle: "This page allows you to set global settings for the GUIDE.", From 722f6b8e37143c093c01e41406fcd1ca3bd47630 Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Tue, 7 Nov 2023 20:14:32 -0500 Subject: [PATCH 07/11] other fixes --- nwb-guide.spec | 4 ++++ package.json | 2 +- pyflask/apis/neuroconv.py | 8 ++++---- pyflask/manageNeuroconv/__init__.py | 2 +- pyflask/manageNeuroconv/manage_neuroconv.py | 14 ++++++-------- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/nwb-guide.spec b/nwb-guide.spec index 8db13407b..fb8ed063d 100644 --- a/nwb-guide.spec +++ b/nwb-guide.spec @@ -13,6 +13,10 @@ hiddenimports = ['scipy._distributor_init', 'scipy._lib.messagestream', 'scipy._ datas += collect_data_files('jsonschema_specifications') tmp_ret = collect_all('dandi') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +tmp_ret = collect_all('keyrings') +datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +tmp_ret = collect_all('unittest') +datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] tmp_ret = collect_all('nwbinspector') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] tmp_ret = collect_all('neuroconv') diff --git a/package.json b/package.json index 13ce918c9..2eee7c2a7 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "build:mac": "npm run build && npm run build:flask && npm run build:electron:mac", "build:linux": "npm run build && npm run build:flask && npm run build:electron:linux", "build:flask": "python -m PyInstaller nwb-guide.spec --log-level DEBUG --clean --noconfirm --distpath ./build/flask", - "build:flask:spec:base": "pyi-makespec --name nwb-guide --onedir --collect-data jsonschema_specifications --collect-all dandi --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ndx_dandi_icephys --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils --hidden-import email_validator ./pyflask/app.py", + "build:flask:spec:base": "pyi-makespec --name nwb-guide --onedir --collect-data jsonschema_specifications --collect-all dandi --collect-all keyrings --collect-all unittest --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ndx_dandi_icephys --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils --hidden-import email_validator ./pyflask/app.py", "build:flask:spec": "npm run build:flask:spec:base && python prepare_pyinstaller_spec.py", "build:electron:win": "electron-builder build --win --publish never", "build:electron:mac": "electron-builder build --mac --publish never", diff --git a/pyflask/apis/neuroconv.py b/pyflask/apis/neuroconv.py index f1eb69ca9..1844dd666 100644 --- a/pyflask/apis/neuroconv.py +++ b/pyflask/apis/neuroconv.py @@ -17,7 +17,7 @@ inspect_nwb_file, inspect_nwb_folder, inspect_multiple_filesystem_objects, - upload_to_dandi, + upload_project_to_dandi, upload_folder_to_dandi, upload_multiple_filesystem_objects_to_dandi, ) @@ -123,11 +123,11 @@ def post(self): @neuroconv_api.route("/upload/project") -class Upload(Resource): +class UploadProject(Resource): @neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): try: - return upload_to_dandi(**neuroconv_api.payload) + return upload_project_to_dandi(**neuroconv_api.payload) except Exception as e: if notBadRequestException(e): @@ -135,7 +135,7 @@ def post(self): @neuroconv_api.route("/upload/folder") -class Upload(Resource): +class UploadFolder(Resource): @neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): try: diff --git a/pyflask/manageNeuroconv/__init__.py b/pyflask/manageNeuroconv/__init__.py index 95cc9bc03..d040232f3 100644 --- a/pyflask/manageNeuroconv/__init__.py +++ b/pyflask/manageNeuroconv/__init__.py @@ -6,7 +6,7 @@ get_metadata_schema, convert_to_nwb, validate_metadata, - upload_to_dandi, + upload_project_to_dandi, upload_folder_to_dandi, upload_multiple_filesystem_objects_to_dandi, listen_to_neuroconv_events, diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 65585c00f..17cfbdf1d 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -169,8 +169,6 @@ def get_all_converter_info() -> dict: return {name: derive_interface_info(converter) for name, converter in module_to_dict(converters).items()} - return output - def get_all_interface_info() -> dict: """Format an information structure to be used for selecting interfaces based on modality and technique.""" @@ -470,7 +468,7 @@ def update_conversion_progress(**kwargs): def upload_multiple_filesystem_objects_to_dandi(**kwargs): - tmp_folder_path = aggregate_symlinks_in_new_directory(kwargs["filesystem_paths"], "upload") + tmp_folder_path = _aggregate_symlinks_in_new_directory(kwargs["filesystem_paths"], "upload") innerKwargs = {**kwargs} del innerKwargs["filesystem_paths"] innerKwargs["nwb_folder_path"] = tmp_folder_path @@ -497,12 +495,12 @@ def upload_folder_to_dandi( nwb_folder_path=Path(nwb_folder_path), staging=staging, cleanup=cleanup, - number_of_jobs=number_of_jobs, - number_of_threads=number_of_threads, + number_of_jobs=1, + number_of_threads=1, ) -def upload_to_dandi( +def upload_project_to_dandi( dandiset_id: str, api_key: str, project: Optional[str] = None, @@ -522,8 +520,8 @@ def upload_to_dandi( nwb_folder_path=CONVERSION_SAVE_FOLDER_PATH / project, # Scope valid DANDI upload paths to GUIDE projects staging=staging, cleanup=cleanup, - number_of_jobs=number_of_jobs, - number_of_threads=number_of_threads, + number_of_jobs=1, + number_of_threads=1, ) From 00aded2cc1433f7e102d904e0f0ba2f876a25329 Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Tue, 7 Nov 2023 20:52:20 -0500 Subject: [PATCH 08/11] restore previous passing --- pyflask/manageNeuroconv/manage_neuroconv.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 17cfbdf1d..ea3accbb4 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -495,8 +495,8 @@ def upload_folder_to_dandi( nwb_folder_path=Path(nwb_folder_path), staging=staging, cleanup=cleanup, - number_of_jobs=1, - number_of_threads=1, + number_of_jobs=number_of_jobs, + number_of_threads=number_of_threads, ) @@ -520,8 +520,8 @@ def upload_project_to_dandi( nwb_folder_path=CONVERSION_SAVE_FOLDER_PATH / project, # Scope valid DANDI upload paths to GUIDE projects staging=staging, cleanup=cleanup, - number_of_jobs=1, - number_of_threads=1, + number_of_jobs=number_of_jobs, + number_of_threads=number_of_threads, ) From c51b0c9e2a0b5602e52ea13a8eecdae77d3611f0 Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Tue, 7 Nov 2023 23:15:09 -0500 Subject: [PATCH 09/11] add zarr too --- nwb-guide.spec | 2 ++ package.json | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/nwb-guide.spec b/nwb-guide.spec index fb8ed063d..27b68c226 100644 --- a/nwb-guide.spec +++ b/nwb-guide.spec @@ -25,6 +25,8 @@ tmp_ret = collect_all('pynwb') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] tmp_ret = collect_all('hdmf') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +tmp_ret = collect_all('hdmf_zarr') +datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] tmp_ret = collect_all('ndx_dandi_icephys') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] tmp_ret = collect_all('ci_info') diff --git a/package.json b/package.json index 2eee7c2a7..d25856f98 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "build:mac": "npm run build && npm run build:flask && npm run build:electron:mac", "build:linux": "npm run build && npm run build:flask && npm run build:electron:linux", "build:flask": "python -m PyInstaller nwb-guide.spec --log-level DEBUG --clean --noconfirm --distpath ./build/flask", - "build:flask:spec:base": "pyi-makespec --name nwb-guide --onedir --collect-data jsonschema_specifications --collect-all dandi --collect-all keyrings --collect-all unittest --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all ndx_dandi_icephys --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils --hidden-import email_validator ./pyflask/app.py", + "build:flask:spec:base": "pyi-makespec --name nwb-guide --onedir --collect-data jsonschema_specifications --collect-all dandi --collect-all keyrings --collect-all unittest --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all hdmf_zarr --collect-all ndx_dandi_icephys --collect-all ci_info --hidden-import scipy._distributor_init --hidden-import scipy._lib.messagestream --hidden-import scipy._lib._ccallback --hidden-import scipy._lib._testutils --hidden-import email_validator ./pyflask/app.py", "build:flask:spec": "npm run build:flask:spec:base && python prepare_pyinstaller_spec.py", "build:electron:win": "electron-builder build --win --publish never", "build:electron:mac": "electron-builder build --mac --publish never", From a59b364c6e6ed1092c279e7a8ffd5a913882fa57 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Wed, 8 Nov 2023 08:13:01 -0800 Subject: [PATCH 10/11] Add note about using 1 --- schemas/json/dandi/upload.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/json/dandi/upload.json b/schemas/json/dandi/upload.json index 63249619f..9667c5168 100644 --- a/schemas/json/dandi/upload.json +++ b/schemas/json/dandi/upload.json @@ -7,7 +7,7 @@ "number_of_jobs": { "type": "integer", "title": "Job Count", - "description": "The number of files to upload in parallel. A value of -1 uses all available processes", + "description": "The number of files to upload in parallel. A value of -1 uses all available processes.
Note: If you encounter an error for any selector value, particularly a Segmentation Fault error, try a value of 1 to run the jobs without parallelization.", "default": -1, "min": -1 }, From f71825e881c4c63becc337036b2d891c834c3e4f Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 8 Nov 2023 11:31:16 -0500 Subject: [PATCH 11/11] Update environments/environment-Windows.yml --- environments/environment-Windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/environment-Windows.yml b/environments/environment-Windows.yml index f9a02d974..77f9390d7 100644 --- a/environments/environment-Windows.yml +++ b/environments/environment-Windows.yml @@ -18,6 +18,6 @@ dependencies: - flask-cors === 3.0.10 - flask_restx == 1.1.0 - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[full] - - dandi @ git+https://github.com/dandi/dandi-cli.git@master + - dandi >= 0.57.0 - pytest == 7.2.2 - pytest-cov == 4.1.0