Merge pull request #278 from NeurodataWithoutBorders/schema-2.3.0

Schema 2.3.0
NeurodataWithoutBorders · Jun 2, 2021 · 6d47856 · 6d47856
2 parents 8ddf90c + 392f72f
commit 6d47856
Show file tree

Hide file tree

Showing 122 changed files with 9,519 additions and 3,588 deletions.
diff --git a/+tests/+system/DynamicTableTest.m b/+tests/+system/DynamicTableTest.m
@@ -19,8 +19,8 @@ function addContainer(~, file)
                     'id', id(i),...
                     'tablepath', '/intervals/trials');
             end
-            t = table(id(101:200), (101:200) .', (102:201) .', mat2cell(rand(500,1),...
-                repmat(5, 100, 1)), repmat({'TRUE'}, 100, 1),...
+            t = table(id(101:200), (101:200) .', (102:201) .',...
+                mat2cell(rand(500,1), repmat(5, 100, 1)), repmat({'TRUE'}, 100, 1),...
                 'VariableNames', {'id', 'start_time', 'stop_time', 'randomvalues', 'stringdata'});
             file.intervals_trials.addRow(t);
         end

diff --git a/+tests/+system/UnitTimesIOTest.m b/+tests/+system/UnitTimesIOTest.m
@@ -2,23 +2,10 @@
     methods
         function addContainer(~, file)
             vdata = rand(10,1);
-            vd = types.hdmf_common.VectorData('data', vdata, 'description', 'descr');
-
-            spike_loc = '/units/spike_times';
-            vd_ref = [...
-                types.untyped.RegionView(spike_loc, 1),...
-                types.untyped.RegionView(spike_loc, 2:5),...
-                types.untyped.RegionView(spike_loc, 9:10)...
-                ];
-            vi = types.hdmf_common.VectorIndex('data', vd_ref,...
-                'target', types.untyped.ObjectView(spike_loc));
-            ei = types.hdmf_common.ElementIdentifiers('data', 1:3);
-            file.units = types.core.Units(...
-                'colnames', {'spike_times'},...
-                'description', 'test Units',...
-                'spike_times', vd, ...
-                'spike_times_index', vi,...
-                'id', ei);
+            file.units = types.core.Units('description', 'test Units', 'colnames', {'spike_times'});
+            file.units.addRow('spike_times', vdata(1), 'tablepath', '/units');
+            file.units.addRow('spike_times', vdata(2:5));
+            file.units.addRow('spike_times', vdata(6:end));
         end
 
         function c = getContainer(~, file)

diff --git a/+types/+util/+dynamictable/addRawData.m b/+types/+util/+dynamictable/addRawData.m
@@ -26,29 +26,24 @@ function addRawData(DynamicTable, column, data, index)
 if ~isempty(index)
     if isprop(DynamicTable, index)
         VecInd = DynamicTable.(index);
-    else
+    elseif isprop(DynamicTable, 'vectorindex') % Schema < 2.3.0
         VecInd = DynamicTable.vectorindex.get(index);
+    else
+        VecInd = DynamicTable.vectordata.get(index);
     end
 
-    if isa(VecInd.data, 'types.untyped.DataPipe')
-        if 0 == VecInd.data.dims
-            raggedOffset = 0;
-        else
-            raggedOffset = VecInd.data.load(VecInd.data.dims);
-        end
-    else
-        if isempty(VecInd.data)
-            raggedOffset = 0;
-        else
-            raggedOffset = VecInd.data(end);
-        end
+    raggedOffset = 0;
+    if isa(VecInd.data, 'types.untyped.DataPipe') && 0 < VecInd.data.dims
+        raggedOffset = double(VecInd.data.load(VecInd.data.dims));
+    elseif ~isempty(VecInd.data)
+        raggedOffset = double(VecInd.data(end));
     end
 
     raggedValue = raggedOffset + size(data, 1);
     if isa(VecInd.data, 'types.untyped.DataPipe')
         VecInd.data.append(raggedValue);
     else
-        VecInd.data = [VecInd.data; raggedValue];
+        VecInd.data = [double(VecInd.data); raggedValue];
     end
 end
 

diff --git a/+types/+util/+dynamictable/addTableRow.m b/+types/+util/+dynamictable/addTableRow.m
@@ -34,23 +34,25 @@ function addTableRow(DynamicTable, subTable, varargin)
 TypeMap = types.util.dynamictable.getTypeMap(DynamicTable);
 for i = 1:length(rowNames)
     rn = rowNames{i};
-    rv = subTable.(rn);
+    rowColumn = subTable.(rn);
 
     if isKey(TypeMap, rn)
-        validateType(TypeMap(rn), rv);
+        validateType(TypeMap(rn), rowColumn);
     end
 
     % instantiate vector index here because it's dependent on the table
     % fullpath.
     vecIndName = types.util.dynamictable.getIndex(DynamicTable, rn);
-    if isempty(vecIndName) && ~iscellstr(rv) && iscell(rv)
+    if isempty(vecIndName) && (~isempty(p.Results.tablepath) || (~iscellstr(rowColumn) && iscell(rowColumn)))
         vecIndName = types.util.dynamictable.addVecInd(DynamicTable, rn, p.Results.tablepath);
     end
-    if ~iscell(rv) || iscellstr(rv)
-        rv = {rv};
-    end
-    for i = 1:length(rv)
-        types.util.dynamictable.addRawData(DynamicTable, rn, rv{i}, vecIndName);
+    for j = 1:length(rowColumn)
+        if iscell(rowColumn)
+            rv = rowColumn{j};
+        else
+            rv = rowColumn(j);
+        end
+        types.util.dynamictable.addRawData(DynamicTable, rn, rv, vecIndName);
     end
 end
 

diff --git a/+types/+util/+dynamictable/addVarargRow.m b/+types/+util/+dynamictable/addVarargRow.m
@@ -48,7 +48,7 @@ function addVarargRow(DynamicTable, varargin)
     % instantiate vector index here because it's dependent on the table
     % fullpath.
     vecIndName = types.util.dynamictable.getIndex(DynamicTable, rn);
-    if isempty(vecIndName) && size(rv, 1) > 1
+    if isempty(vecIndName) && (~isempty(p.Results.tablepath) || size(rv, 1) > 1)
         vecIndName = types.util.dynamictable.addVecInd(DynamicTable, rn, p.Results.tablepath);
     end
     types.util.dynamictable.addRawData(DynamicTable, rn, rv, vecIndName);
@@ -72,7 +72,7 @@ function addVarargRow(DynamicTable, varargin)
 
 function validateType(TypeStruct, rv)
 if strcmp(TypeStruct.type, 'cellstr')
-    assert(iscellstr(rv) || (ischar(rv) && 1 == size(rv, 1)),...
+    assert(iscellstr(rv) || (ischar(rv) && (isempty(rv) || 1 == size(rv, 1))),...
         'MatNWB:DynamicTable:AddRow:InvalidType',...
         'Type of value must be a cell array of character vectors or a scalar character');
 else

diff --git a/+types/+util/+dynamictable/addVecInd.m b/+types/+util/+dynamictable/addVecInd.m
@@ -19,10 +19,13 @@
     else
         VecData = DynamicTable.vectordata.get(colName);
     end
-    if isa(VecData.data, 'types.untyped.DataPipe')
-        oldDataHeight = VecData.data.offset;
-    else
-        oldDataHeight = size(VecData.data, 1);
+
+    if ~isempty(VecData)
+        if isa(VecData.data, 'types.untyped.DataPipe')
+            oldDataHeight = VecData.data.offset;
+        else
+            oldDataHeight = size(VecData.data, 1);
+        end
     end
 end
 
@@ -31,10 +34,13 @@
 % directly to each row index.
 VecIndex = types.hdmf_common.VectorIndex(...
     'target', vecTarget,...
-    'data', [0:(oldDataHeight-1)] .'); %#ok<NBRAK>
+    'data', [0:(oldDataHeight-1)] .',...
+    'description', sprintf('Index into column %s', colName)); %#ok<NBRAK>
 if isprop(DynamicTable, vecIndName)
     DynamicTable.(vecIndName) = VecIndex;
-else
+elseif isprop(DynamicTable, 'vectorindex')
     DynamicTable.vectorindex.set(vecIndName, VecIndex);
+else
+    DynamicTable.vectordata.set(vecIndName, VecIndex);
 end
 end
diff --git a/+types/+util/+dynamictable/clear.m b/+types/+util/+dynamictable/clear.m
@@ -6,6 +6,8 @@ function clear(DynamicTable)
 
 DynamicTable.vectordata = types.untyped.Set(@(nm, val)types.util.checkConstraint(...
     'vectordata', nm, struct(), {'types.hdmf_common.VectorData'}, val));
-DynamicTable.vectorindex = types.untyped.Set(@(nm, val)types.util.checkConstraint(...
-    'vectorindex', nm, struct(), {'types.hdmf_common.VectorIndex'}, val));
+if isprop(DynamicTable, 'vectorindex') % Schema version <2.3.0
+    DynamicTable.vectorindex = types.untyped.Set(@(nm, val)types.util.checkConstraint(...
+        'vectorindex', nm, struct(), {'types.hdmf_common.VectorIndex'}, val));
+end
 end
diff --git a/+types/+util/+dynamictable/getIndex.m b/+types/+util/+dynamictable/getIndex.m
@@ -10,15 +10,33 @@
     'MatNWB:GetIndex:InvalidColumn',...
     'Column name not found `%s`', column);
 
-vecIndKeys = keys(DynamicTable.vectorindex);
-for i = 1:length(vecIndKeys)
-    vik = vecIndKeys{i};
-    if isVecIndColumn(DynamicTable.vectorindex.get(vik), column)
-        indexName = vik;
+% after Schema version 2.3.0, VectorIndex objects subclass VectorData which
+% meant that vectorindex and vectordata sets could be combined.
+isLegacyDynamicTable = isprop(DynamicTable, 'vectorindex');
+if isLegacyDynamicTable
+    vecKeys = keys(DynamicTable.vectorindex);
+else
+    vecKeys = keys(DynamicTable.vectordata);
+end
+for i = 1:length(vecKeys)
+    vk = vecKeys{i};
+    if isLegacyDynamicTable
+        vecData = DynamicTable.vectorindex.get(vk);
+    else
+        vecData = DynamicTable.vectordata.get(vk);
+    end
+    if ~isa(vecData, 'types.hdmf_common.VectorIndex')
+        continue;
+    end
+    if isVecIndColumn(vecData, column)
+        indexName = vk;
         return;
     end
 end
 
+% check if dynamic table object has extended properties which point to
+% vector indices. These are specifically defined by the schema to be
+% properties.
 DynamicTableProps = properties(DynamicTable);
 isPropVecInd = false(size(DynamicTableProps));
 for i = 1:length(DynamicTableProps)
@@ -27,10 +45,10 @@
 
 DynamicTableProps = DynamicTableProps(isPropVecInd);
 for i = 1:length(DynamicTableProps)
-    vik = DynamicTableProps{i};
-    VecInd = DynamicTable.(vik);
+    vk = DynamicTableProps{i};
+    VecInd = DynamicTable.(vk);
     if isVecIndColumn(VecInd, column)
-        indexName = vik;
+        indexName = vk;
         return;
     end
 end

diff --git a/+types/+util/+dynamictable/getRow.m b/+types/+util/+dynamictable/getRow.m
@@ -66,8 +66,10 @@
 function indMap = getIndexInd(DynamicTable, indexName, matInd)
 if isprop(DynamicTable, indexName)
     VectorIndex = DynamicTable.(indexName);
-else
+elseif isprop(DynamicTable, 'vectorindex') % Schema version < 2.3.0
     VectorIndex = DynamicTable.vectorindex.get(indexName);
+else
+    VectorIndex = DynamicTable.vectordata.get(indexName);
 end
 
 matInd = unique(matInd);

diff --git a/+types/+util/+dynamictable/getTypeMap.m b/+types/+util/+dynamictable/getTypeMap.m
@@ -8,7 +8,7 @@
     return;
 end
 TypeStruct = struct('type', '', 'dims', [0, 0]);
-for i = length(DynamicTable.colnames)
+for i = 1:length(DynamicTable.colnames)
     colnm = DynamicTable.colnames{i};
     if isprop(DynamicTable, colnm)
         colVecData = DynamicTable.(colnm);

diff --git a/+types/+util/checkDtype.m b/+types/+util/checkDtype.m
@@ -23,9 +23,10 @@
                 subv = val(names{i});
             end
             assert(isvector(subv),...
-                ['types.util.checkDtype: struct of arrays as a compound type ',...
-                'cannot have multidimensional data in their fields.  Field data ',...
-                'shape must be scalar or vector to be valid.']);
+                'MatNWB:CheckDType:MultiDimStructArrays',...
+                ['struct of arrays as a compound type ',...
+                'cannot have multidimensional data in their fields. ',...
+                'Field data shape must be scalar or vector to be valid.']);
             sizes(i) = length(subv);
         end
         sizes = unique(sizes);

diff --git a/+types/+util/correctType.m b/+types/+util/correctType.m
@@ -18,11 +18,33 @@
     elseif strcmp(type, 'uint')
         val = uint64(val);
     else
-        val = feval(type, val);
+        val = feval(fitIntType(val, type), val);
     end
 elseif strcmp(type, 'numeric') && ~isnumeric(val)
     val = double(val);
 elseif strcmp(type, 'bool')
     val = logical(val);
 end
+end
+
+function fittingIntType = fitIntType(val, minType)
+intSizeScale = [8 16 32 64];
+typeMatch = regexp(minType, '(u?int)(\d+)', 'once', 'tokens');
+prefix = typeMatch{1};
+minSize = str2double(typeMatch{2});
+
+minVal = min(val(:));
+maxVal = max(val(:));
+minSizeMask = intSizeScale == minSize;
+assert(any(minSizeMask), 'NWB:CorrectType:InvalidIntSize',...
+    'Minimum integer size `%s` not supported.', minType);
+for i = find(minSizeMask, 1):length(intSizeScale)
+    fittingIntType = sprintf('%s%d', prefix, intSizeScale(i));
+    if all(intmin(fittingIntType) <= minVal) && all(intmax(fittingIntType) >= maxVal)
+        return;
+    end
+end
+
+error('NWB:CorrectType:UnfittableInt', 'Could not fit integer into range %d-%d',...
+    minVal, maxVal);
 end