Auto generate tutorial data

environments/environment-Linux.yml

@@ -21,3 +21,4 @@ dependencies:
       - dandi >= 0.58.1
       - pytest == 7.4.0
       - pytest-cov == 4.1.0
+      - scikit-learn == 1.4.0

environments/environment-MAC-arm64.yml

@@ -25,3 +25,4 @@ dependencies:
       - dandi >= 0.58.1
       - pytest == 7.4.0
       - pytest-cov == 4.1.0
+      - scikit-learn == 1.4.0

environments/environment-MAC.yml

@@ -21,3 +21,4 @@ dependencies:
       - dandi >= 0.58.1
       - pytest == 7.4.0
       - pytest-cov == 4.1.0
+      - scikit-learn == 1.4.0

environments/environment-Windows.yml

@@ -21,3 +21,4 @@ dependencies:
       - dandi >= 0.58.1
       - pytest == 7.2.2
       - pytest-cov == 4.1.0
+      - scikit-learn == 1.4.0

nwb-guide.spec

@@ -33,6 +33,8 @@ tmp_ret = collect_all('hdmf_zarr')
 datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
 tmp_ret = collect_all('ndx_dandi_icephys')
 datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
+tmp_ret = collect_all('sklearn')
+datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
 tmp_ret = collect_all('ci_info')
 datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
 

package.json

@@ -20,14 +20,14 @@
     "build:mac": "npm run build && npm run build:flask && npm run build:electron:mac",
     "build:linux": "npm run build && npm run build:flask && npm run build:electron:linux",
     "build:flask": "python -m PyInstaller nwb-guide.spec --log-level DEBUG --clean --noconfirm --distpath ./build/flask",
-    "build:flask:spec:base": "pyi-makespec --name nwb-guide --onedir --collect-data jsonschema_specifications --collect-all dandi --collect-all keyrings --collect-all unittest --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all hdmf_zarr --collect-all ndx_dandi_icephys --collect-all ci_info ./pyflask/app.py",
+    "build:flask:spec:base": "pyi-makespec --name nwb-guide --onedir --collect-data jsonschema_specifications --collect-all dandi --collect-all keyrings --collect-all unittest --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all hdmf_zarr --collect-all ndx_dandi_icephys --collect-all sklearn --collect-all ci_info ./pyflask/app.py",
     "build:flask:spec": "npm run build:flask:spec:base && python prepare_pyinstaller_spec.py",
     "build:electron:win": "electron-builder build --win --publish never",
     "build:electron:mac": "electron-builder build --mac --publish never",
     "build:electron:linux": "electron-builder build --linux --publish never",
     "test": "npm run test:app && npm run test:server",
     "test:app": "vitest run",
-    "test:server": "pytest pyflask/tests/ -s",
+    "test:server": "pytest pyflask/tests/ -s -vv",
     "wait3s": "node -e \"setTimeout(() => process.exit(0),3000)\"",
     "test:executable": "concurrently -n EXE,TEST --kill-others --success first \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"npm run wait3s && pytest pyflask/tests/ -s --target http://localhost:3434\"",
     "test:coverage": "npm run coverage:app && npm run coverage:server",

paths.config.json

@@ -4,6 +4,6 @@
         "progress": ["pipelines"],
         "conversions": ["conversions"],
         "preview": ["preview"],
-        "tutorial": ["tutorial"]
+        "testdata": ["test-data"]
     }
 }

pyflask/apis/__init__.py

@@ -1,2 +1,3 @@
 from .startup import startup_api
 from .neuroconv import neuroconv_api
+from .data import data_api

pyflask/apis/data.py

@@ -0,0 +1,53 @@
+"""API endpoint definitions for interacting with NeuroConv."""
+
+import traceback
+
+from flask_restx import Namespace, Resource, reqparse
+
+from manageNeuroconv import generate_test_data, generate_dataset
+from errorHandlers import notBadRequestException
+
+data_api = Namespace("data", description="API route for dataset generation in the NWB GUIDE.")
+
+
+@data_api.errorhandler(Exception)
+def exception_handler(error):
+    exceptiondata = traceback.format_exception(type(error), error, error.__traceback__)
+    return {"message": exceptiondata[-1], "traceback": "".join(exceptiondata)}
+
+
+generate_test_data_parser = reqparse.RequestParser()
+generate_test_data_parser.add_argument("output_path", type=str, required=True)
+
+
+@data_api.route("/generate")
+@data_api.expect(generate_test_data_parser)
+class GeneratetestData(Resource):
+    @data_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"})
+    def post(self):
+        try:
+            arguments = generate_test_data_parser.parse_args()
+            generate_test_data(output_path=arguments["output_path"])
+        except Exception as exception:
+            if notBadRequestException(exception):
+                data_api.abort(500, str(exception))
+            raise exception
+
+
+generate_test_dataset_parser = reqparse.RequestParser()
+generate_test_dataset_parser.add_argument("output_path", type=str, required=True)
+generate_test_dataset_parser.add_argument("input_path", type=str, required=True)
+
+
+@data_api.route("/generate/dataset")
+@data_api.expect(generate_test_data_parser)
+class GenerateDataset(Resource):
+    @data_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"})
+    def post(self):
+        try:
+            arguments = generate_test_dataset_parser.parse_args()
+            return generate_dataset(input_path=arguments["input_path"], output_path=arguments["output_path"])
+
+        except Exception as exception:
+            if notBadRequestException(exception):
+                data_api.abort(500, str(exception))

pyflask/apis/neuroconv.py

@@ -264,18 +264,6 @@ def post(self):
                 neuroconv_api.abort(500, str(exception))
 
 
-@neuroconv_api.route("/generate_dataset")
-class GenerateDataset(Resource):
-    @neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"})
-    def post(self):
-        try:
-            return generate_dataset(**neuroconv_api.payload)
-
-        except Exception as exception:
-            if notBadRequestException(exception):
-                neuroconv_api.abort(500, str(exception))
-
-
 # Create an events endpoint
 # announcer.announce('test', 'publish')
 @neuroconv_api.route("/events", methods=["GET"])

pyflask/app.py

@@ -21,7 +21,7 @@
 from flask_cors import CORS
 from flask_restx import Api, Resource
 
-from apis import startup_api, neuroconv_api
+from apis import startup_api, neuroconv_api, data_api
 from manageNeuroconv.info import resource_path, STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH
 
 app = Flask(__name__)
@@ -57,6 +57,7 @@
 )
 api.add_namespace(startup_api)
 api.add_namespace(neuroconv_api)
+api.add_namespace(data_api)
 api.init_app(app)
 
 registered = {}

pyflask/manageNeuroconv/__init__.py

@@ -15,6 +15,7 @@
     inspect_nwb_folder,
     inspect_multiple_filesystem_objects,
     get_interface_alignment,
+    generate_test_data,
 )
 
 

pyflask/manageNeuroconv/info/__init__.py

@@ -3,5 +3,4 @@
     GUIDE_ROOT_FOLDER,
     STUB_SAVE_FOLDER_PATH,
     CONVERSION_SAVE_FOLDER_PATH,
-    TUTORIAL_SAVE_FOLDER_PATH,
 )

pyflask/manageNeuroconv/info/urls.py

@@ -23,11 +23,9 @@ def resource_path(relative_path):
 GUIDE_ROOT_FOLDER = Path(Path.home(), data["root"])
 STUB_SAVE_FOLDER_PATH = Path(Path.home(), data["root"], *data["subfolders"]["preview"])
 CONVERSION_SAVE_FOLDER_PATH = Path(Path.home(), data["root"], *data["subfolders"]["conversions"])
-TUTORIAL_SAVE_FOLDER_PATH = Path(Path.home(), data["root"], *data["subfolders"]["tutorial"])
 
 f.close()
 
 # Create all nested home folders
 STUB_SAVE_FOLDER_PATH.mkdir(exist_ok=True, parents=True)
 CONVERSION_SAVE_FOLDER_PATH.mkdir(exist_ok=True, parents=True)
-TUTORIAL_SAVE_FOLDER_PATH.mkdir(exist_ok=True, parents=True)

pyflask/tests/test_generate_tutorial_data.py

@@ -0,0 +1,11 @@
+from utils import post
+from pathlib import Path
+
+
+def test_generate_test_data(client, tmp_path: Path):
+    # assert client is None
+    result = post(path="/data/generate", json=dict(output_path=str(tmp_path)), client=client)
+    assert result is None
+    assert len(list(Path(tmp_path).iterdir())) != 0
+    assert (Path(tmp_path) / "spikeglx").exists()
+    assert (Path(tmp_path) / "phy").exists()

src/main/main.ts

@@ -472,6 +472,10 @@ ipcMain.on("resize-window", (event, dir) => {
   globals.mainWindow.setSize(x, y);
 });
 
+ipcMain.on('showItemInFolder', function(event, fullPath) {
+  shell.showItemInFolder(fullPath);
+});
+
 // autoUpdater.on("update-available", () => {
 //   onWindowReady(win => send.call(win, "update_available"));
 // });

src/renderer/src/dependencies/simple.js

@@ -20,6 +20,10 @@ export const conversionSaveFolderPath = homeDirectory
     ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.conversions)
     : "";
 
+export const testDataFolderPath = homeDirectory
+    ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.testdata)
+    : "";
+
 // Encryption
 const IV_LENGTH = 16;
 const KEY_LENGTH = 32;

src/renderer/src/pages.js

@@ -18,8 +18,6 @@ import { GuidedInspectorPage } from "./stories/pages/guided-mode/options/GuidedI
 
 import logo from "../assets/img/logo-guide-draft-transparent-tight.png";
 import { GuidedPathExpansionPage } from "./stories/pages/guided-mode/data/GuidedPathExpansion";
-import { TutorialPage } from "./stories/pages/tutorial/Tutorial";
-import tutorialIcon from "./stories/assets/exploration.svg?raw";
 import uploadIcon from "./stories/assets/dandi.svg?raw";
 import inspectIcon from "./stories/assets/inspect.svg?raw";
 import neurosiftIcon from "./stories/assets/neurosift-logo.svg?raw";
@@ -166,11 +164,6 @@ const pages = {
         label: "Upload",
         icon: uploadIcon,
     }),
-    tutorial: new TutorialPage({
-        label: "Tutorial",
-        icon: tutorialIcon,
-        group: resourcesGroup,
-    }),
     docs: new DocumentationPage({
         label: "Documentation",
         icon: documentationIcon,

src/renderer/src/stories/Button.js

@@ -34,7 +34,7 @@ export class Button extends LitElement {
             }
 
             .with-icon {
-                margin-left: 10px;
+                margin-left: 5px;
             }
 
             svg {

src/renderer/src/stories/pages/documentation/Documentation.js

@@ -51,10 +51,10 @@ export class DocumentationPage extends Page {
 
                 <h3 style="padding: 0; margin-top: 25px;">Getting Started</h3>
                 <h4 style="margin-top: 0;">Converting your data</h4>
-                <p>Most users will want to start with the <a @click=${(ev) => {
+                <p>Most users will want to start by generating a <a @click=${(ev) => {
                     ev.preventDefault();
-                    this.to("tutorial");
-                }}>Tutorial</a> to learn how to use the NWB GUIDE.</p>
+                    this.to("settings");
+                }}>test dataset</a> to learn how to use the NWB GUIDE.</p>
                 <p>If you'd like to jump right in, head to the <a @click=${(ev) => {
                     ev.preventDefault();
                     this.to("/");

src/renderer/src/stories/pages/guided-mode/options/GuidedInspectorPage.js

@@ -6,7 +6,7 @@ import folderOpenSVG from "../../../assets/folder_open.svg?raw";
 
 import { electron } from "../../../../electron/index.js";
 import { getSharedPath, removeFilePaths, truncateFilePaths } from "../../../preview/NWBFilePreview.js";
-const { shell } = electron;
+const { ipcRenderer } = electron;
 import { until } from "lit/directives/until.js";
 import { run } from "./utils.js";
 import { InspectorList } from "../../../preview/inspector/InspectorList.js";
@@ -63,12 +63,13 @@ export class GuidedInspectorPage extends Page {
             ...this.headerButtons,
             html`<nwb-button
                 size="small"
-                @click=${() =>
-                    shell
-                        ? shell.showItemInFolder(
-                              getSharedPath(getStubArray(this.info.globalState.preview.stubs).map(({ file }) => file))
-                          )
-                        : ""}
+                @click=${() => {
+                    if (ipcRenderer)
+                        ipcRenderer.send(
+                            "showItemInFolder",
+                            getSharedPath(getStubArray(this.info.globalState.preview.stubs).map(({ file }) => file))
+                        );
+                }}
                 >${unsafeSVG(folderOpenSVG)}</nwb-button
             >`,
         ],

src/renderer/src/stories/pages/guided-mode/options/GuidedStubPreview.js

@@ -6,7 +6,7 @@ import folderOpenSVG from "../../../assets/folder_open.svg?raw";
 
 import { electron } from "../../../../electron/index.js";
 import { NWBFilePreview, getSharedPath } from "../../../preview/NWBFilePreview.js";
-const { shell } = electron;
+const { ipcRenderer } = electron;
 
 export const getStubArray = (stubs) =>
     Object.values(stubs)
@@ -24,12 +24,13 @@ export class GuidedStubPreviewPage extends Page {
         controls: () =>
             html`<nwb-button
                 size="small"
-                @click=${() =>
-                    shell
-                        ? shell.showItemInFolder(
-                              getSharedPath(getStubArray(this.info.globalState.preview.stubs).map((item) => item.file))
-                          )
-                        : ""}
+                @click=${() => {
+                    if (ipcRenderer)
+                        ipcRenderer.send(
+                            "showItemInFolder",
+                            getSharedPath(getStubArray(this.info.globalState.preview.stubs).map((item) => item.file))
+                        );
+                }}
                 >${unsafeSVG(folderOpenSVG)}</nwb-button
             >`,
     };

src/renderer/src/stories/pages/guided-mode/options/utils.js

@@ -45,12 +45,21 @@ export const run = async (url, payload, options = {}) => {
         }));
 
         const element = popup.getHtmlContainer();
+
         const actions = popup.getActions();
         const loader = actions.querySelector(".swal2-loader");
         const container = document.createElement("div");
         container.append(loader);
-        element.innerText = "";
+
+        const notDisplayed = element.style.display === "none";
+
+        Object.assign(element.style, {
+            marginTop: notDisplayed ? "" : "0",
+            display: "unset",
+        });
+
         Object.assign(container.style, {
+            marginTop: notDisplayed ? "" : "25px",
             display: "flex",
             flexDirection: "column",
             alignItems: "center",
@@ -64,6 +73,7 @@ export const run = async (url, payload, options = {}) => {
     }
 
     if (!("base" in options)) options.base = "/neuroconv";
+    if (options.base[0] !== "/") options.base = `/${options.base}`;
 
     // Clear private keys from being passed
     payload = sanitize(structuredClone(payload));