Ophys Metadata

guide_testing_suite.yml

@@ -14,6 +14,18 @@ pipelines:
     PhySortingInterface:
       folder_path: ephy_testing_data/phy/phy_example_0
 
-  SpikeGLX-v1-SingleProbe-AP:
+  SpikeGLX_v1_SingleProbe_AP:
     SpikeGLXRecordingInterface:
       file_path: ephy_testing_data/spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.ap.bin
+
+  Suite2P_SinglePlane_SingleChannel:
+    Suite2pSegmentationInterface:
+      folder_path: ophys_testing_data/segmentation_datasets/suite2p
+
+  Caiman_SinglePlane_SingleChannel:
+    CaimanSegmentationInterface:
+      file_path: ophys_testing_data/segmentation_datasets/caiman/caiman_analysis.hdf5
+
+  ScanImage_SinglePlane_SingleChannel:
+    ScanImageImagingInterface:
+      file_path: ophys_testing_data/imaging_datasets/ScanImage/scanimage_20220801_single.tif

pyflask/manageNeuroconv/manage_neuroconv.py

@@ -74,9 +74,17 @@ def replace_none_with_nan(json_object, json_schema):
     def coerce_schema_compliance_recursive(obj, schema):
         if isinstance(obj, dict):
             for key, value in obj.items():
-                if key in schema.get("properties", {}):
+                # Coerce on pattern properties as well
+                pattern_properties = schema.get("patternProperties")
+                if pattern_properties:
+                    for pattern, pattern_schema in pattern_properties.items():
+                        regex = re.compile(pattern)
+                        if regex.match(key):
+                            coerce_schema_compliance_recursive(value, pattern_schema)
+
+                elif key in schema.get("properties", {}):
                     prop_schema = schema["properties"][key]
-                    if prop_schema.get("type") == "number" and value is None:
+                    if prop_schema.get("type") == "number" and (value is None or value == "NaN"):
                         obj[
                             key
                         ] = (

schemas/base-metadata.schema.ts

@@ -1,6 +1,6 @@
 import { serverGlobals, resolve } from '../src/renderer/src/server/globals'
 
-import { header } from '../src/renderer/src/stories/forms/utils'
+import { header, replaceRefsWithValue } from '../src/renderer/src/stories/forms/utils'
 
 import baseMetadataSchema from './json/base_metadata_schema.json' assert { type: "json" }
 
@@ -37,7 +37,9 @@ function getSpeciesInfo(species: any[][] = []) {
 export const preprocessMetadataSchema = (schema: any = baseMetadataSchema, global = false) => {
 
 
-    const copy = structuredClone(schema)
+    const copy = replaceRefsWithValue(structuredClone(schema))
+
+    copy.additionalProperties = false
 
     // Add unit to weight
     const subjectProps = copy.properties.Subject.properties
@@ -82,6 +84,62 @@ export const preprocessMetadataSchema = (schema: any = baseMetadataSchema, globa
     nwbProps.keywords.description = 'Terms to describe your dataset (e.g. Neural circuits, V1, etc.)' // Add description to keywords
 
 
+    const ophys = copy.properties.Ophys
+
+    if (ophys) {
+
+        const getProp = (name: string) => ophys.properties[name]
+
+        if (getProp("TwoPhotonSeries")) {
+            const tpsItemSchema = getProp("TwoPhotonSeries").items
+            tpsItemSchema.order = [
+                "name",
+                "description",
+                "scan_line_rate",
+                "field_of_view"
+            ]
+
+            tpsItemSchema.properties.pmt_gain.title =  'Photomultiplier Gain'
+        }
+
+
+        if (getProp("ImagingPlane")) {
+            const imagingPlaneItems = getProp("ImagingPlane").items
+            imagingPlaneItems.order = [
+                "name",
+                "description",
+                "device",
+                "optical_channel",
+                "excitation_lambda",
+                "indicator",
+                "location",
+                "reference_frame",
+                "imaging_rate",
+                'grid_spacing',
+                "grid_spacing_unit",
+                "origin_coords",
+                'origin_coords_unit'
+            ]
+
+            imagingPlaneItems.properties.optical_channel.items.order = ["name", "description"]
+
+        }
+    }
+
+
+    Object.entries(copy.properties).forEach(([key, value]) => {
+
+        Object.entries(value.properties).forEach(([k, v]) => {
+
+            //  Uniformly grab definitions
+           const ref = v.items ?? v
+           if (!ref.properties) return
+           Object.keys(ref.properties).forEach(k => {
+                const info = ref.properties[k]
+                if (info.description && info.description.includes('DEPRECATED')) delete ref.properties[k] // Remove deprecated properties
+            })
+        })
+    })
 
     // Remove non-global properties
     if (global) {

schemas/json/dandi/global.json

@@ -14,8 +14,10 @@
                     "description": "Your DANDI API key from the <a href='https://gui-staging.dandiarchive.org' target='_blank'>staging server</a>"
                 }
             },
+            "additionalProperties": false,
             "required": ["main_api_key"]
         }
     },
-    "required": ["api_keys"]
+    "required": ["api_keys"],
+    "additionalProperties": false
 }

schemas/json/project/globals.json

@@ -5,5 +5,7 @@
             "format": "directory",
             "description": "Provide a custom output location for your NWB files. Will default to ~/NWB_GUIDE/conversions"
         }
-    }
+    },
+
+    "additionalProperties": false
 }

schemas/json/tutorial.schema.json

@@ -8,5 +8,7 @@
       }
   },
 
+  "additionalProperties": false,
+
   "required": ["test_data_directory_path"]
 }

src/renderer/assets/css/global.css

@@ -50,6 +50,7 @@ body {
     margin: 0;
     height: 100%;
     display: flex;
+    overflow: hidden; /* Dashboard without scroll behavior */
 }
 
 a {

src/renderer/src/progress/index.js

@@ -52,7 +52,7 @@ function decode(text) {
     }
 }
 
-function drill(object, callback) {
+export function drill(object, callback) {
     if (object && typeof object === "object") {
         const copy = Array.isArray(object) ? [...object] : { ...object };
         for (let k in copy) {

src/renderer/src/stories/BasicTable.js

@@ -7,6 +7,8 @@ import { errorHue, warningHue } from "./globals";
 import * as promises from "../promises";
 
 import "./Button";
+import { sortTable } from "./Table";
+import tippy from "tippy.js";
 
 export class BasicTable extends LitElement {
     static get styles() {
@@ -105,6 +107,7 @@ export class BasicTable extends LitElement {
     constructor({
         name,
         schema,
+        ignore,
         data,
         keyColumn,
         maxHeight,
@@ -121,11 +124,27 @@ export class BasicTable extends LitElement {
         this.maxHeight = maxHeight ?? "";
         this.validateEmptyCells = validateEmptyCells ?? true;
 
+        this.ignore = ignore ?? {};
+
         if (validateOnChange) this.validateOnChange = validateOnChange;
         if (onStatusChange) this.onStatusChange = onStatusChange;
         if (onLoaded) this.onLoaded = onLoaded;
     }
 
+    #schema = {};
+    #itemSchema = {};
+    #itemProps = {};
+
+    get schema() {
+        return this.#schema;
+    }
+
+    set schema(schema) {
+        this.#schema = schema;
+        this.#itemSchema = schema.items;
+        this.#itemProps = { ...this.#itemSchema.properties };
+    }
+
     #rendered;
     #updateRendered = (force) =>
         force || this.rendered === true
@@ -134,7 +153,7 @@ export class BasicTable extends LitElement {
     rendered = this.#updateRendered(true);
 
     #renderHeaderContent = (str) => {
-        const required = this.schema.required ? this.schema.required.includes(str) : false;
+        const required = this.#itemSchema.required ? this.#itemSchema.required.includes(str) : false;
         if (required) return html`<div class="relative"><span required>${header(str)}</span></div>`;
         return html`<div class="relative"><span>${header(str)}</span></div>`;
     };
@@ -155,7 +174,7 @@ export class BasicTable extends LitElement {
                 value =
                     (hasRow ? this.data[row][col] : undefined) ??
                     // this.globals[col] ??
-                    this.schema.properties[col].default ??
+                    this.#itemSchema.properties[col].default ??
                     "";
             return value;
         });
@@ -197,7 +216,7 @@ export class BasicTable extends LitElement {
 
         let result;
 
-        const propInfo = this.schema.properties[col] ?? {};
+        const propInfo = this.#itemProps[col] ?? {};
         let thisTypeOf = typeof value;
         let ogType;
         let type = (ogType = propInfo.type || propInfo.data_type);
@@ -218,13 +237,13 @@ export class BasicTable extends LitElement {
         }
 
         // Check if required
-        if (!value && "required" in this.schema && this.schema.required.includes(col))
+        if (!value && "required" in this.#itemSchema.required.includes(col))
             result = [{ message: `${col} is a required property`, type: "error" }];
         // If not required, check matching types for values that are defined
         else if (value !== "" && thisTypeOf !== type)
             result = [{ message: `${col} is expected to be of type ${ogType}, not ${thisTypeOf}`, type: "error" }];
         // Otherwise validate using the specified onChange function
-        else result = this.validateOnChange(col, parent, value);
+        else result = this.validateOnChange([col], parent, value, this.#itemProps[col]);
 
         // Will run synchronously if not a promise result
         return promises.resolve(result, () => {
@@ -239,14 +258,12 @@ export class BasicTable extends LitElement {
 
             if (result === false) errors.push({ message: "Cell is invalid" });
 
-            if (warnings.length) {
-                info.warning = "";
-                info.title = warnings.map((error) => error.message).join("\n");
-            }
-
             if (errors.length) {
                 info.error = "";
                 info.title = errors.map((error) => error.message).join("\n"); // Class switching handled automatically
+            } else if (warnings.length) {
+                info.warning = "";
+                info.title = warnings.map((warning) => warning.message).join("\n");
             }
 
             if (typeof result === "function") result(); // Run if returned value is a function
@@ -265,6 +282,19 @@ export class BasicTable extends LitElement {
                     if (info === true) return;
                     const td = this.shadowRoot.getElementById(`i${i}_j${j}`);
                     if (td) {
+                        const message = info.title;
+                        delete info.title;
+
+                        if (td._tippy) {
+                            td._tippy.destroy();
+                            td.removeAttribute("data-message");
+                        }
+
+                        if (message !== undefined) {
+                            tippy(td, { content: message, allowHTML: true });
+                            td.setAttribute("data-message", message);
+                        }
+
                         for (let key in info) {
                             const value = info[key];
                             if (value === undefined) td.removeAttribute(key);
@@ -326,9 +356,7 @@ export class BasicTable extends LitElement {
         Object.keys(data).forEach((row) => {
             const cols = structuredData[row];
             const latest = (this.data[this.keyColumn ? cols[this.keyColumn] : row] = {});
-            Object.entries(cols).forEach(([key, value]) =>
-                key in this.schema.properties ? (latest[key] = value) : ""
-            ); // Only include data from schema
+            Object.entries(cols).forEach(([key, value]) => (key in this.#itemProps ? (latest[key] = value) : "")); // Only include data from schema
         });
 
         this.onUpdate(null, null, value); // Update the whole table
@@ -338,10 +366,12 @@ export class BasicTable extends LitElement {
     render() {
         this.#updateRendered();
 
-        const entries = { ...this.schema.properties };
+        const entries = this.#itemProps;
+        for (let key in this.ignore) delete entries[key];
+        for (let key in this.ignore["*"] ?? {}) delete entries[key];
 
         // Add existing additional properties to the entries variable if necessary
-        if (this.schema.additionalProperties) {
+        if (this.#itemSchema.additionalProperties) {
             Object.values(this.data).reduce((acc, v) => {
                 Object.keys(v).forEach((k) =>
                     !(k in entries)
@@ -358,13 +388,14 @@ export class BasicTable extends LitElement {
         const keys =
             (this.#keys =
             this.colHeaders =
-                Object.keys(entries).sort((a, b) => {
-                    if (a === this.keyColumn) return -1;
-                    if (b === this.keyColumn) return 1;
-                    if (entries[a].required && !entries[b].required) return -1;
-                    if (!entries[a].required && entries[b].required) return 1;
-                    return 0;
-                }));
+                sortTable(
+                    {
+                        ...this.#itemSchema,
+                        properties: entries,
+                    },
+                    this.keyColumn,
+                    this.#itemSchema.order
+                ));
 
         // Try to guess the key column if unspecified
         if (!Array.isArray(this.data) && !this.keyColumn) {