Add validation tests

.github/workflows/tests.yml

@@ -57,3 +57,42 @@ jobs:
     - name: Test
       working-directory: build
       run: ctest --output-on-failure --no-tests=error -C Release -j 2
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v3
+      with:
+        name: test-files-${{ matrix.os }}
+        path: |
+          build/tests/data/*.nwb
+
+  validate:
+    needs: tests
+    defaults:
+      run:
+        shell: bash
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os }}
+      cancel-in-progress: true
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, ubuntu-latest]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - name: Download test files
+      uses: actions/download-artifact@v3
+      with:
+        name: test-files-${{ matrix.os }}
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - name: Install pynwb and run validation
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install pynwb
+        python -m pynwb.validate *.nwb

.gitignore

@@ -1,6 +1,7 @@
 # folders
 **/.DS_Store
 /tests/data/*
+/resources/schema/*
 
 # IDEs
 .idea/

resources/generate_spec_files.py

@@ -4,17 +4,44 @@
 from ruamel.yaml import YAML
 
 # TODO - setup git submodule or cloning
-schema_dir = Path('./resources/spec/core/2.7.0')
+schema_dir = Path('./resources/schema/')
 
-# create spec strings
-for file in schema_dir.rglob(r"*.yaml"):
+for file in schema_dir.rglob(r"*namespace.yaml"):
     # load file
     yaml = YAML(typ='safe')
     with open(file) as f:
-        spec = yaml.load(file)
+        namespace = yaml.load(file)
 
-    # convert to json
-    outfile = file.with_suffix('.json')
-    print(f'Generating file {outfile}')
-    with open(outfile, 'w') as fo:
-        json.dump(spec, fo, separators=(',', ':'),)
+    # get all the sources
+    for i, ns in enumerate(namespace['namespaces']):
+        spec_dir = Path(f"./resources/spec/{ns['name']}/{ns['version']}")
+        spec_dir.mkdir(parents=True, exist_ok=True)
+
+        # load and convert schema files
+        for s in ns['schema']:
+            if 'source' in s:
+                # load file
+                schema_file = file.parent / s['source']
+                with open(schema_file) as f:
+                    spec = yaml.load(schema_file)
+
+                # convert to json
+                spec_file = (spec_dir / s['source']).with_suffix('.json')
+                print(f'Generating file {spec_file}')
+                with open(spec_file, 'w') as fo:
+                    json.dump(spec, fo, separators=(',', ':'),)    
+
+        # reformat schema sources for namespace file
+        schema = list()
+        for s in ns['schema']:
+            if 'source' in s:
+                s = {'source': s['source'].split('.yaml')[0]}
+            schema.append(s)
+        ns['schema'] = schema
+
+        # convert namespace json
+        ns_file = (spec_dir / file.name).with_suffix('.json')
+        ns_output = {'namespaces': [ns]}
+        print(f'Generating file {ns_file}')
+        with open(ns_file, 'w') as fo:
+            json.dump(ns_output, fo, separators=(',', ':'),)

resources/spec/core/2.7.0/nwb.namespace.json

@@ -1 +1 @@
-{"namespaces":[{"name":"core","doc":"NWB namespace","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter","Keith Godfrey","Jeff Teeters"],"contact":["[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]"],"full_name":"NWB core","schema":[{"namespace":"hdmf-common"},{"doc":"This source module contains base data types used throughout the NWB data format.","source":"nwb.base.yaml","title":"Base data types"},{"doc":"This source module contains neurodata_types for device data.","source":"nwb.device.yaml","title":"Devices"},{"doc":"This source module contains neurodata_types for epoch data.","source":"nwb.epoch.yaml","title":"Epochs"},{"doc":"This source module contains neurodata_types for image data.","source":"nwb.image.yaml","title":"Image data"},{"doc":"Main NWB file specification.","source":"nwb.file.yaml","title":"NWB file"},{"doc":"Miscellaneous types.","source":"nwb.misc.yaml","title":"Miscellaneous neurodata_types."},{"doc":"This source module contains neurodata_types for behavior data.","source":"nwb.behavior.yaml","title":"Behavior"},{"doc":"This source module contains neurodata_types for extracellular electrophysiology data.","source":"nwb.ecephys.yaml","title":"Extracellular electrophysiology"},{"doc":"This source module contains neurodata_types for intracellular electrophysiology data.","source":"nwb.icephys.yaml","title":"Intracellular electrophysiology"},{"doc":"This source module contains neurodata_types for opto-genetics data.","source":"nwb.ogen.yaml","title":"Optogenetics"},{"doc":"This source module contains neurodata_types for optical physiology data.","source":"nwb.ophys.yaml","title":"Optical physiology"},{"doc":"This source module contains neurodata_type for retinotopy data.","source":"nwb.retinotopy.yaml","title":"Retinotopy"}],"version":"2.7.0"}]}
+{"namespaces":[{"name":"core","doc":"NWB namespace","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter","Keith Godfrey","Jeff Teeters"],"contact":["[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]"],"full_name":"NWB core","schema":[{"namespace":"hdmf-common"},{"source":"nwb.base"},{"source":"nwb.device"},{"source":"nwb.epoch"},{"source":"nwb.image"},{"source":"nwb.file"},{"source":"nwb.misc"},{"source":"nwb.behavior"},{"source":"nwb.ecephys"},{"source":"nwb.icephys"},{"source":"nwb.ogen"},{"source":"nwb.ophys"},{"source":"nwb.retinotopy"}],"version":"2.7.0"}]}

resources/spec/hdmf-common/1.8.0/namespace.json

@@ -1 +1 @@
-{"namespaces":[{"name":"hdmf-common","doc":"Common data structures provided by HDMF","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter"],"contact":["[email protected]","[email protected]","[email protected]","[email protected]"],"full_name":"HDMF Common","schema":[{"doc":"base data types","source":"base.yaml","title":"Base data types"},{"doc":"data types for a column-based table","source":"table.yaml","title":"Table data types"},{"doc":"data types for different types of sparse matrices","source":"sparse.yaml","title":"Sparse data types"}],"version":"1.5.0"},{"name":"hdmf-experimental","doc":"Experimental data structures provided by HDMF. These are not guaranteed to be available in the future","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter"],"contact":["[email protected]","[email protected]","[email protected]","[email protected]"],"full_name":"HDMF Experimental","schema":[{"namespace":"hdmf-common"},{"doc":"Experimental data types","source":"experimental.yaml","title":"Experimental data types"},{"doc":"data types for storing references to web accessible resources","source":"resources.yaml","title":"Resource reference data types"}],"version":"0.1.0"}]}
+{"namespaces":[{"name":"hdmf-common","doc":"Common data structures provided by HDMF","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter"],"contact":["[email protected]","[email protected]","[email protected]","[email protected]"],"full_name":"HDMF Common","schema":[{"source":"base"},{"source":"table"},{"source":"sparse"}],"version":"1.8.0"}]}

resources/spec/hdmf-experimental/0.5.0/namespace.json

@@ -0,0 +1 @@
+{"namespaces":[{"name":"hdmf-experimental","doc":"Experimental data structures provided by HDMF. These are not guaranteed to be available in the future.","author":["Andrew Tritt","Oliver Ruebel","Ryan Ly","Ben Dichter","Matthew Avaylon"],"contact":["[email protected]","[email protected]","[email protected]","[email protected]","[email protected]"],"full_name":"HDMF Experimental","schema":[{"namespace":"hdmf-common"},{"source":"experimental"},{"source":"resources"}],"version":"0.5.0"}]}

resources/spec/hdmf-experimental/0.5.0/resources.json

@@ -0,0 +1 @@
+{"groups":[{"data_type_def":"HERD","data_type_inc":"Container","doc":"HDMF External Resources Data Structure. A set of six tables for tracking external resource references in a file or across multiple files.","datasets":[{"data_type_inc":"Data","name":"keys","doc":"A table for storing user terms that are used to refer to external resources.","dtype":[{"name":"key","dtype":"text","doc":"The user term that maps to one or more resources in the `resources` table, e.g., \"human\"."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"files","doc":"A table for storing object ids of files used in external resources.","dtype":[{"name":"file_object_id","dtype":"text","doc":"The object id (UUID) of a file that contains objects that refers to external resources."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"entities","doc":"A table for mapping user terms (i.e., keys) to resource entities.","dtype":[{"name":"entity_id","dtype":"text","doc":"The compact uniform resource identifier (CURIE) of the entity, in the form [prefix]:[unique local identifier], e.g., 'NCBI_TAXON:9606'."},{"name":"entity_uri","dtype":"text","doc":"The URI for the entity this reference applies to. This can be an empty string. e.g., https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=info&id=9606"}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"objects","doc":"A table for identifying which objects in a file contain references to external resources.","dtype":[{"name":"files_idx","dtype":"uint","doc":"The row index to the file in the `files` table containing the object."},{"name":"object_id","dtype":"text","doc":"The object id (UUID) of the object."},{"name":"object_type","dtype":"text","doc":"The data type of the object."},{"name":"relative_path","dtype":"text","doc":"The relative path from the data object with the `object_id` to the dataset or attribute with the value(s) that is associated with an external resource. This can be an empty string if the object is a dataset that contains the value(s) that is associated with an external resource."},{"name":"field","dtype":"text","doc":"The field within the compound data type using an external resource. This is used only if the dataset or attribute is a compound data type; otherwise this should be an empty string."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"object_keys","doc":"A table for identifying which objects use which keys.","dtype":[{"name":"objects_idx","dtype":"uint","doc":"The row index to the object in the `objects` table that holds the key"},{"name":"keys_idx","dtype":"uint","doc":"The row index to the key in the `keys` table."}],"dims":["num_rows"],"shape":[null]},{"data_type_inc":"Data","name":"entity_keys","doc":"A table for identifying which keys use which entity.","dtype":[{"name":"entities_idx","dtype":"uint","doc":"The row index to the entity in the `entities` table."},{"name":"keys_idx","dtype":"uint","doc":"The row index to the key in the `keys` table."}],"dims":["num_rows"],"shape":[null]}]}]}

src/BaseIO.hpp

@@ -217,6 +217,15 @@ class BaseIO
   virtual Status createStringDataSet(const std::string& path,
                                      const std::string& value) = 0;
 
+  /**
+   * @brief Creates a dataset that holds an array of a single string value.
+   * @param path The location in the file of the dataset.
+   * @param text The text value of the dataset.
+   * @return The status of the dataset creation operation.
+   */
+  virtual Status createStringArrayDataSet(const std::string& path,
+                                          const std::string& text) = 0;
+
   /**
    * @brief Creates a dataset that holds an array of references to groups within
    * the file.

src/Utils.hpp

@@ -36,7 +36,18 @@ inline std::string getCurrentTime()
 
   // Format the date and time in ISO 8601 format with the UTC offset
   std::ostringstream oss;
-  oss << std::put_time(&utcTime, "%FT%T%z");
+  oss << std::put_time(&utcTime, "%FT%T");
+
+  // Get the timezone offset
+  auto localTime = std::localtime(&currentTime);
+  auto utcOffset = localTime->tm_hour - utcTime.tm_hour;
+  auto utcOffsetMinutes =
+      (utcOffset < 0 ? -1 : 1) * (localTime->tm_min - utcTime.tm_min);
+
+  // Add the timezone offset to the date and time string
+  oss << (utcOffset < 0 ? "-" : "+") << std::setw(2) << std::setfill('0')
+      << std::abs(utcOffset) << ":" << std::setw(2) << std::setfill('0')
+      << std::abs(utcOffsetMinutes);
 
   return oss.str();
 }

src/hdf5/HDF5IO.cpp

@@ -342,6 +342,19 @@ Status HDF5IO::createStringDataSet(const std::string& path,
   return Status::Success;
 }
 
+Status HDF5IO::createStringArrayDataSet(const std::string& path,
+                                        const std::string& text)
+{
+  std::unique_ptr<BaseRecordingData> dataset;
+  BaseDataType textType = BaseDataType::STR(text.length());
+
+  dataset = std::unique_ptr<BaseRecordingData>(
+      createDataSet(textType, SizeArray {0}, SizeArray {1}, path));
+  dataset->writeDataBlock(1, textType, text.c_str());
+
+  return Status::Success;
+}
+
 AQNWB::BaseRecordingData* HDF5IO::getDataSet(const std::string& path)
 {
   std::unique_ptr<DataSet> data;

src/hdf5/HDF5IO.hpp

@@ -153,6 +153,15 @@ class HDF5IO : public BaseIO
   Status createStringDataSet(const std::string& path,
                              const std::string& value) override;
 
+  /**
+   * @brief Creates a dataset that holds an array of a single string value.
+   * @param path The location in the file of the dataset.
+   * @param text The text value of the dataset.
+   * @return The status of the dataset creation operation.
+   */
+  Status createStringArrayDataSet(const std::string& path,
+                                  const std::string& text) override;
+
   /**
    * @brief Creates a dataset that holds an array of references to groups within
    * the file.

src/nwb/NWBFile.cpp

@@ -59,11 +59,13 @@ Status NWBFile::createFileStructure()
   io->createGroup("general/extracellular_ephys");
 
   io->createGroup("/specifications");
+  io->createReferenceAttribute("/specifications", "/", ".specloc");
   cacheSpecifications("core/", NWBVersion);
   cacheSpecifications("hdmf-common/", HDMFVersion);
+  cacheSpecifications("hdmf-experimental/", HDMFExperimentalVersion);
 
   std::string time = getCurrentTime();
-  io->createStringDataSet("/file_create_date", time);  // TODO - change to array
+  io->createStringArrayDataSet("/file_create_date", time);
   io->createStringDataSet("/session_description", "a recording session");
   io->createStringDataSet("/session_start_time", time);
   io->createStringDataSet("/timestamps_reference_time", time);

src/nwb/NWBFile.hpp

@@ -67,6 +67,11 @@ class NWBFile
    */
   const std::string HDMFVersion = "1.8.0";
 
+  /**
+   * @brief Indicates the HDMF experimental version.
+   */
+  const std::string HDMFExperimentalVersion = "0.5.0";
+
 protected:
   /**
    * @brief Creates the default file structure.

tests/aq-nwb_test.cpp

@@ -76,7 +76,7 @@ TEST_CASE("writeAttributes", "[hdf5io]")
 
 TEST_CASE("saveNWBFile", "[nwb]")
 {
-  std::string filename = getTestFilePath("test_nwb_file.h5");
+  std::string filename = getTestFilePath("test_nwb_file.nwb");
 
   NWB::NWBFile nwbfile("123", std::make_unique<HDF5::HDF5IO>(filename));
   nwbfile.initialize();
@@ -85,7 +85,7 @@ TEST_CASE("saveNWBFile", "[nwb]")
 
 TEST_CASE("startRecording", "[nwb]")
 {
-  std::string filename = getTestFilePath("test_recording.h5");
+  std::string filename = getTestFilePath("test_recording.nwb");
 
   NWB::NWBFile nwbfile("123", std::make_unique<HDF5::HDF5IO>(filename));
   nwbfile.initialize();