Merge pull request #467 from NeurodataWithoutBorders/add-converters

Allow selection of converters with SpikeGLXConverter Support

guideGlobalMetadata.json

@@ -36,6 +36,7 @@
         "FicTracDataInterface",
         "AudioInterface",
         "MiniscopeBehaviorInterface",
-        "EDFRecordingInterface"
+        "EDFRecordingInterface",
+        "SpikeGLXConverterPipe"
     ]
 }

pyflask/apis/neuroconv.py

@@ -6,6 +6,7 @@
 
 from manageNeuroconv import (
     get_all_interface_info,
+    get_all_converter_info,
     locate_data,
     get_source_schema,
     get_metadata_schema,
@@ -40,7 +41,13 @@ class AllInterfaces(Resource):
     @neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"})
     def get(self):
         try:
-            return get_all_interface_info()
+            # return get_all_interface_info()
+            # return get_all_converter_info()
+
+            return {
+                **get_all_interface_info(),
+                **get_all_converter_info(),
+            }
         except Exception as e:
             if notBadRequestException(e):
                 neuroconv_api.abort(500, str(e))

pyflask/manageNeuroconv/__init__.py

@@ -1,5 +1,6 @@
 from .manage_neuroconv import (
     get_all_interface_info,
+    get_all_converter_info,
     locate_data,
     get_source_schema,
     get_metadata_schema,

pyflask/manageNeuroconv/manage_neuroconv.py

@@ -128,51 +128,67 @@ def locate_data(info: dict) -> dict:
     return organized_output
 
 
+def module_to_dict(my_module):
+    # Create an empty dictionary
+    module_dict = {}
+
+    # Iterate through the module's attributes
+    for attr_name in dir(my_module):
+        if not attr_name.startswith("__"):  # Exclude special attributes
+            attr_value = getattr(my_module, attr_name)
+            module_dict[attr_name] = attr_value
+
+    return module_dict
+
+
+def get_all_converter_info() -> dict:
+    from neuroconv import converters
+
+    return {
+        name: {"keywords": [], "description": f"{converter.__doc__.split('.')[0]}." if converter.__doc__ else ""}
+        for name, converter in module_to_dict(converters).items()
+    }
+
+
 def get_all_interface_info() -> dict:
     """Format an information structure to be used for selecting interfaces based on modality and technique."""
     from neuroconv.datainterfaces import interface_list
 
     exclude_interfaces_from_selection = [
         # Deprecated
-        "SpikeGLXLFP",
+        "SpikeGLXLFPInterface",
         # Aliased
-        "CEDRecording",
-        "OpenEphysBinaryRecording",
-        "OpenEphysLegacyRecording",
+        "CEDRecordingInterface",
+        "OpenEphysBinaryRecordingInterface",
+        "OpenEphysLegacyRecordingInterface",
         # Ignored
-        "AxonaPositionData",
-        "AxonaUnitRecording",
-        "CsvTimeIntervals",
-        "ExcelTimeIntervals",
-        "Hdf5Imaging",
-        "MaxOneRecording",
-        "OpenEphysSorting",
-        "SimaSegmentation",
-    ]  # Should have 'interface' stripped from name
-
-    interfaces_to_load = {interface.__name__.replace("Interface", ""): interface for interface in interface_list}
-    for excluded_interface in exclude_interfaces_from_selection:
-        interfaces_to_load.pop(excluded_interface)
+        "AxonaPositionDataInterface",
+        "AxonaUnitRecordingInterface",
+        "CsvTimeIntervalsInterface",
+        "ExcelTimeIntervalsInterface",
+        "Hdf5ImagingInterface",
+        "MaxOneRecordingInterface",
+        "OpenEphysSortingInterface",
+        "SimaSegmentationInterface",
+    ]
 
     return {
         interface.__name__: {
             "keywords": interface.keywords,
-            # Once we use the raw neuroconv list, we will want to ensure that the interfaces themselves
-            # have a label property
-            "label": format_name
-            # Can also add a description here if we want to provide more information about the interface
+            "description": f"{interface.__doc__.split('.')[0]}." if interface.__doc__ else "",
         }
-        for format_name, interface in interfaces_to_load.items()
+        for interface in interface_list
+        if not interface.__name__ in exclude_interfaces_from_selection
     }
 
 
 # Combine Multiple Interfaces
 def get_custom_converter(interface_class_dict: dict):  # -> NWBConverter:
-    from neuroconv import datainterfaces, NWBConverter
+    from neuroconv import converters, datainterfaces, NWBConverter
 
     class CustomNWBConverter(NWBConverter):
         data_interface_classes = {
-            custom_name: getattr(datainterfaces, interface_name)
+            custom_name: getattr(datainterfaces, interface_name, getattr(converters, interface_name, None))
             for custom_name, interface_name in interface_class_dict.items()
         }
 
@@ -372,7 +388,7 @@ def update_conversion_progress(**kwargs):
     options = (
         {
             interface: {"stub_test": info["stub_test"]}  # , "iter_opts": {"report_hook": update_conversion_progress}}
-            if available_options.get("properties").get(interface).get("properties").get("stub_test")
+            if available_options.get("properties").get(interface).get("properties", {}).get("stub_test")
             else {}
             for interface in info["source_data"]
         }

pyflask/tests/test_neuroconv.py

@@ -13,9 +13,11 @@ def test_get_all_interfaces(client):
                     "type": "object",
                     "properties": {
                         "label": {"type": "string"},
+                        "description": {"type": "string"},
                         "keywords": {"type": "array", "items": {"type": "string"}},
                     },
-                    "required": ["label", "keywords"],
+                    "additionalProperties": False,
+                    "required": ["keywords"],
                 }
             },
         },

src/renderer/src/stories/Search.js

@@ -1,5 +1,6 @@
 import { LitElement, html, css } from "lit";
-import { styleMap } from "lit/directives/style-map.js";
+
+import tippy from "tippy.js";
 
 export class Search extends LitElement {
     constructor({ options, showAllWhenEmpty, disabledLabel } = {}) {
@@ -17,20 +18,18 @@ export class Search extends LitElement {
 
             :host {
                 position: relative;
-                display: block;
+                display: flex;
+                flex-direction: column;
                 background: white;
                 border-radius: 5px;
                 width: 100%;
                 height: 100%;
-                overflow: auto;
+                overflow: hidden;
             }
 
             .header {
                 padding: 25px;
                 background: white;
-                position: sticky;
-                top: 0;
-                z-index: 1;
             }
 
             input {
@@ -45,9 +44,8 @@ export class Search extends LitElement {
                 list-style: none;
                 padding: 0;
                 margin: 0;
-                position: absolute;
-                left: 0;
-                right: 0;
+                position: relative;
+                overflow: auto;
                 background: white;
             }
 
@@ -56,6 +54,15 @@ export class Search extends LitElement {
                 border-top: 1px solid #f2f2f2;
             }
 
+            .category {
+                padding: 10px 25px;
+                background: #f2f2f2;
+                font-weight: bold;
+                position: sticky;
+                top: 0;
+                z-index: 1;
+            }
+
             .option:hover {
                 background: #f2f2f2;
                 cursor: pointer;
@@ -65,7 +72,12 @@ export class Search extends LitElement {
                 margin: 0;
             }
 
-            [disabled] {
+            .label {
+                display: flex;
+                gap: 10px;
+            }
+
+            [disabled]:not([hidden]) {
                 display: flex;
                 justify-content: space-between;
                 align-items: center;
@@ -118,7 +130,11 @@ export class Search extends LitElement {
 
     list = document.createElement("ul");
 
+    categories = {};
+
     render() {
+        this.categories = {};
+
         // Update list
         this.list.remove();
         this.list = document.createElement("ul");
@@ -130,7 +146,14 @@ export class Search extends LitElement {
         this.list.appendChild(slot);
 
         if (this.options) {
-            const unsupported = this.options
+            const options = this.options.map((o) => {
+                return {
+                    label: o.key,
+                    ...o,
+                };
+            });
+
+            const itemEls = options
                 .sort((a, b) => {
                     if (a.label < b.label) return -1;
                     if (a.label > b.label) return 1;
@@ -141,7 +164,7 @@ export class Search extends LitElement {
                     else if (a.disabled) return 1;
                     else if (b.disabled) return -1;
                 }) // Sort with the disabled options at the bottom
-                .filter((option) => {
+                .map((option) => {
                     const li = document.createElement("li");
                     li.classList.add("option");
                     li.setAttribute("hidden", "");
@@ -155,6 +178,20 @@ export class Search extends LitElement {
                     const label = document.createElement("h4");
                     label.classList.add("label");
                     label.innerText = option.label;
+
+                    const info = document.createElement("span");
+
+                    if (option.description) {
+                        info.innerText = "ℹ️";
+                        label.append(info);
+
+                        tippy(info, {
+                            content: `<p>${option.description}</p>`,
+                            allowHTML: true,
+                            placement: "right",
+                        });
+                    }
+
                     container.appendChild(label);
 
                     const keywords = document.createElement("small");
@@ -163,16 +200,41 @@ export class Search extends LitElement {
                     container.appendChild(keywords);
 
                     li.append(container);
-                    this.list.appendChild(li);
 
-                    return option.disabled;
+                    if (option.category) {
+                        let category = this.categories[option.category];
+                        if (!category) {
+                            category = document.createElement("div");
+                            category.innerText = option.category;
+                            category.classList.add("category");
+                            this.categories[option.category] = {
+                                entries: [],
+                                element: category,
+                            };
+                        }
+
+                        this.categories[option.category].entries.push(li);
+                        return;
+                    }
+
+                    return el;
                 })
-                .map((o) => o.value);
+                .filter((el) => el);
 
-            console.warn(`Enabled: ${this.options.length - unsupported.length}/${this.options.length}`);
-            console.warn("Disabled Options:", unsupported);
+            this.list.append(...itemEls);
         }
 
+        // Categories sorted alphabetically
+        const categories = Object.values(this.categories).sort((a, b) => {
+            if (a.element.innerText < b.element.innerText) return -1;
+            if (a.element.innerText > b.element.innerText) return 1;
+            return 0;
+        });
+
+        categories.forEach(({ entries, element }) => {
+            this.list.append(element, ...entries);
+        });
+
         return html`
     <div class="header">
       <input placeholder="Type here to search" @input=${(ev) => {
@@ -204,6 +266,12 @@ export class Search extends LitElement {
                   option.setAttribute("hidden", "");
               }
           });
+
+          categories.forEach(({ entries, element }) => {
+              if (entries.reduce((acc, el) => acc + el.hasAttribute("hidden"), 0) === entries.length)
+                  element.setAttribute("hidden", "");
+              else element.removeAttribute("hidden");
+          });
       }}></input>
     </div>
     ${this.list}

src/renderer/src/stories/pages/guided-mode/data/GuidedStructure.js

@@ -10,6 +10,17 @@ import { List } from "../../../List";
 
 const defaultEmptyMessage = "No interfaces selected";
 
+const categories = [
+    {
+        test: /.*Interface.*/,
+        value: "Single-Stream Interfaces",
+    },
+    {
+        test: /.*Converter.*/,
+        value: "Multi-Stream Converters",
+    },
+];
+
 export class GuidedStructurePage extends Page {
     constructor(...args) {
         super(...args);
@@ -91,10 +102,13 @@ export class GuidedStructurePage extends Page {
             .then((res) => res.json())
             .then((json) =>
                 Object.entries(json).map(([key, value]) => {
+                    const category = categories.find(({ test }) => test.test(key))?.value;
+
                     return {
                         ...value,
-                        key: key.replace("Interface", ""),
+                        key,
                         value: key,
+                        category,
                         disabled: !supportedInterfaces.includes(key),
                     }; // Has label and keywords property already
                 })