Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cast empty array to correct type before adding row #461

Merged
merged 16 commits into from
Nov 23, 2022
Merged

Cast empty array to correct type before adding row #461

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
2b0977c
Cast empty array to correct type before adding row
lawrence-mbf Aug 17, 2022
2753a5e
Specifically handle logical type concatenation
lawrence-mbf Aug 17, 2022
409225a
Clarify documentation on type conversions
lawrence-mbf Aug 17, 2022
2fcdc3c
revert Dynamic table to use addRow in tests
lawrence-mbf Aug 17, 2022
3b8c7d6
Fix compat with data pipes and dynamic tables
lawrence-mbf Aug 17, 2022
fad8cb3
Fix getRow behavior. Remove redundant tests
lawrence-mbf Aug 17, 2022
845a55e
addrow table support
lawrence-mbf Nov 14, 2022
23ee98c
Fix compatility with old DynamicTable properties
lawrence-mbf Nov 14, 2022
48b0951
Fix special datapipe case
lawrence-mbf Nov 14, 2022
90a4fdb
clarify tutorial test messages
lawrence-mbf Nov 14, 2022
bbefae1
Refactor error ids
lawrence-mbf Nov 14, 2022
0f4c89a
fix dynamic table regression
lawrence-mbf Nov 14, 2022
8cbc56c
support alt size arguments
lawrence-mbf Nov 14, 2022
505e03f
remove generateCore from tests
lawrence-mbf Nov 14, 2022
3775bc8
add `ignorecache` to nwbRead calls
lawrence-mbf Nov 14, 2022
07d929e
remove unused functions
lawrence-mbf Nov 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion +file/mapType.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
return;
end

assert(ischar(dtype), 'MatNWB:MapType:InvalidDtype', ...
assert(ischar(dtype), 'NWB:MapType:InvalidDtype', ...
'schema attribute `dtype` returned in unsupported type `%s`', class(dtype));

switch dtype
Expand Down
4 changes: 2 additions & 2 deletions +io/getRefData.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
try
did = H5D.open(fid, refpaths{i});
catch ME
error('MatNWB:getRefData:InvalidPath',...
error('NWB:getRefData:InvalidPath',...
'Reference path `%s` was invalid', refpaths{i});
end
sid = H5D.get_space(did);
Expand All @@ -29,7 +29,7 @@
try
ref_data(:, i) = H5R.create(fid, ref(i).path, ref(i).reftype, refspace(i));
catch ME
error('MatNWB:getRefData:InvalidPath',...
error('NWB:getRefData:InvalidPath',...
'Reference path `%s` was invalid', ref(i).path);
end
if H5I.is_valid(refspace(i))
Expand Down
2 changes: 1 addition & 1 deletion +io/parseAttributes.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
% MATLAB can't have arbitrary enum values.
attributeValue = strcmp('TRUE', attr.Value);
else
warning('MatNWB:Attribute:UnknownEnum', ...
warning('NWB:Attribute:UnknownEnum', ...
['Encountered unknown enum under field `%s` with %d members. ' ...
'Will be saved as cell array of characters.'], ...
attr.Name, length(attr.Datatype.Type.Member));
Expand Down
2 changes: 1 addition & 1 deletion +io/parseDataset.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
if io.isBool(datatype.Type)
data = strcmp('TRUE', data);
else
warning('MatNWB:Dataset:UnknownEnum', ...
warning('NWB:Dataset:UnknownEnum', ...
['Encountered unknown enum under field `%s` with %d members. ' ...
'Will be saved as cell array of characters.'], ...
info.Name, length(datatype.Type.Member));
Expand Down
215 changes: 34 additions & 181 deletions +tests/+system/DynamicTableTest.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,146 +2,37 @@
methods
function addContainer(~, file)
colnames = {'start_time', 'stop_time', 'randomvalues', ...
'random_multi', 'stringdata', 'compound_data', 'compound_data_struct'};
'random_multi', 'stringdata', 'compound_table', 'compound_struct', 'logical_data'};
%add trailing nulls to columnames
for c =1:length(colnames)
colnames{c} = char([double(colnames{c}) zeros(1,randi(10))]);
end
% until addRow works with multidimensional matrices, define array in one go

nrows = 20;
ids = primes(100)';
start_col = types.hdmf_common.VectorData( ...
'description', 'start_times column', ...
'data', (1:nrows)' ...
);
stop_col = types.hdmf_common.VectorData( ...
'description', 'stop_times column', ...
'data', (1:nrows)'+1 ...
);
rv_col = types.hdmf_common.VectorData( ...
'description', 'randomvalues column', ...
'data', rand(8*nrows,1) ...
);
rv_index = zipArrays(5:8:8*nrows, 8:8:8*nrows);
rv_index_col = types.hdmf_common.VectorIndex( ...
'description', 'index into randomvalues column', ...
'target',types.untyped.ObjectView(rv_col), ...
'data', rv_index ...
);
rv_index_index = 2:2:2*nrows;
rv_index_index_col = types.hdmf_common.VectorIndex( ...
'description', 'index into randomvalues_index column', ...
'target',types.untyped.ObjectView(rv_index_col), ...
'data', rv_index_index ...
);
multi_col = types.hdmf_common.VectorData( ...
'description', 'random_multi column', ...
'data', rand(3,2,nrows) ...
);
str_col = types.hdmf_common.VectorData( ...
'description', 'stringdata column', ...
'data', repmat({'TRUE'}, nrows, 1) ...
);
compound_data = types.hdmf_common.VectorData(...
'description', 'compound data column', ...
'data', table(rand(nrows, 1), rand(nrows, 1), 'VariableNames', {'a', 'b'}));
compound_data_struct = types.hdmf_common.VectorData(...
'description', 'compound data using struct', ...
'data', struct('a', rand(nrows, 1), 'b', rand(nrows, 1)));
file.intervals_trials = types.core.TimeIntervals(...
'description', 'test dynamic table column',...
'colnames', colnames, ...
'start_time', start_col, ...
'stop_time', stop_col, ...
'randomvalues', rv_col, ...
'randomvalues_index', rv_index_col, ...
'randomvalues_index_index', rv_index_index_col, ...
'random_multi', multi_col, ...
'stringdata', str_col, ...
'compound_data', compound_data, ...
'compound_data_struct', compound_data_struct, ...
'id', types.hdmf_common.ElementIdentifiers('data', ids(1:nrows)) ...
);
% check table configuration.
% This can be removed when addRow is added back in
types.util.dynamictable.checkConfig(file.intervals_trials)
end

function addExpandableDynamicTable(~, file, start_array, stop_array, ...
random_array, random_multi_array, ragged_data_array, ...
ragged_index_array, id_array)
% create VectorData objects with DataPipe objects
start_time_exp = types.hdmf_common.VectorData( ...
'description', 'start times', ...
'data', types.untyped.DataPipe( ...
'data', start_array, ...
'maxSize', Inf ...
) ...
);
stop_time_exp = types.hdmf_common.VectorData( ...
'description', 'stop times', ...
'data', types.untyped.DataPipe( ...
'data', stop_array, ...
'maxSize', Inf ...
) ...
);
random_exp = types.hdmf_common.VectorData( ...
'description', 'random data column', ...
'data', types.untyped.DataPipe( ...
'data', random_array', ...
'maxSize', [1, Inf], ...
'axis', 2 ...
)...
);
random_multi_exp = types.hdmf_common.VectorData( ...
'description', 'random data column', ...
'data', types.untyped.DataPipe( ...
'data', random_multi_array, ...
'maxSize', [3 , 2, Inf], ...
'axis', 3 ...
)...
);
ragged_exp = types.hdmf_common.VectorData( ...
'description', 'random data column', ...
'data', types.untyped.DataPipe( ...
'data', ragged_data_array, ...
'maxSize', [Inf, 1], ...
'axis', 1 ...
)...
);
ragged_exp_index = types.hdmf_common.VectorIndex( ...
'description', 'random data column', ...
'target',types.untyped.ObjectView(ragged_exp), ...
'data', types.untyped.DataPipe( ...
'data', ragged_index_array', ...
'maxSize', Inf ...
)...
);
ids_exp = types.hdmf_common.ElementIdentifiers( ...
'data', types.untyped.DataPipe( ...
'data', id_array, ...
'maxSize', Inf ...
) ...
);
% create expandable table
colnames = { ...
'start_time', 'stop_time', ...
'randomvalues', 'random_multi' ,'random_ragged' ...
};
file.intervals_trials = types.core.TimeIntervals( ...
'description', 'test expdandable dynamic table', ...
random_multi_vec = types.hdmf_common.VectorData(...
'description', 'Test multi-dimensional appending using data pipes.', ...
'data', types.untyped.DataPipe(...
'axis', 1, ...
'maxSize', [inf 3 2], ...
'dataType', 'double'));
file.intervals_trials = types.core.TimeIntervals(...
'description', 'test dynamic table column', ...
'colnames', colnames, ...
'start_time', start_time_exp, ...
'stop_time', stop_time_exp, ...
'randomvalues', random_exp, ...
'random_multi', random_multi_exp, ...
'random_ragged', ragged_exp, ...
'random_ragged_index', ragged_exp_index, ...
'id', ids_exp ...
);
% check table configuration.
% This can be removed when addRow is added back in
types.util.dynamictable.checkConfig(file.intervals_trials)
'random_multi', random_multi_vec);
for iRow = 1:nrows
file.intervals_trials.addRow(...
'start_time', iRow, ...
'stop_time', iRow, ...
'randomvalues', {rand(2,3),rand(2,5)}, ...
'random_multi', rand(1,3,2), ...
'stringdata', {'TRUE'}, ...
'id', ids(iRow), ...
'compound_table', table(rand(1), rand(1), 'VariableNames', {'a', 'b'}), ...
'compound_struct', struct('a', rand(), 'b', rand()), ...
'logical_data', true);
end
end

function addContainerUnevenColumns(~, file)
Expand All @@ -164,6 +55,7 @@ function addContainerUnevenColumns(~, file)
) ...
);
end

function addContainerUnmatchedIDs(~, file)
% create and add a container with columns of unmatched length
colnames = {'start_time', 'stop_time', 'randomvalues'};
Expand All @@ -187,6 +79,7 @@ function addContainerUnmatchedIDs(~, file)
) ...
);
end

function addContainerUndefinedIDs(~, file)
% create and add a container with undefined id field
colnames = {'start_time', 'stop_time', 'randomvalues'};
Expand All @@ -207,6 +100,7 @@ function addContainerUndefinedIDs(~, file)
) ...
);
end

function c = getContainer(~, file)
c = file.intervals_trials.vectordata.get('randomvalues');
end
Expand All @@ -231,6 +125,7 @@ function appendContainer(testCase, file)
"NWB:DynamicTable" ...
);
end

function appendRaggedContainer(~, file)
% create synthetic data
data = (100:-1:1);
Expand Down Expand Up @@ -272,7 +167,7 @@ function getRowTest(testCase)
else
startInd = VectorDataInd.data(Indices(iRaggedInd) - 1) + 1;
end
dataIndices{iRaggedInd} = BaseVectorData.data((startInd:endInd) .', :);
dataIndices{iRaggedInd} = BaseVectorData.data(:, (startInd:endInd) .');
end

actualData = Table.getRow(5, 'columns', {'randomvalues'});
Expand Down Expand Up @@ -305,7 +200,7 @@ function getRowRoundtripTest(testCase)

% even if struct is passed in. It is still read back as a
% table. So we cheat a bit here since this is expected a2a.
CompoundStructVector = ExpectedTable.vectordata.get('compound_data_struct');
CompoundStructVector = ExpectedTable.vectordata.get('compound_struct');
ExpectedCompoundStruct = CompoundStructVector.data;
CompoundStructVector.data = struct2table(ExpectedCompoundStruct);

Expand All @@ -315,35 +210,6 @@ function getRowRoundtripTest(testCase)
ActualTable.getRow([13, 19], 'useId', true));
end

function ExpandableTableTest(testCase)
% define data matrices
nrows = 20;
id = 0:nrows-1; % different from row poistion
start_time_array = 1:nrows;
stop_time_array = start_time_array + 1;
rng(1); % to be able replicate random values
random_val_array = rand(nrows, 1);
random_multi_array = rand(3, 2, nrows);
% create expandable table with first half of arrays
testCase.addExpandableDynamicTable(testCase.file, ...
start_time_array(1:10), stop_time_array(1:10), ...
random_val_array(1:10), random_multi_array(:, : ,1:10), ...
rand(nrows*3,1), [sort(randi(nrows*3,9,1)); nrows*3], id(1:10));
% export and read-in expandable table
filename = ['MatNWB.' testCase.className() '.ExpandableTableTest.nwb'];
nwbExport(testCase.file, filename);
% removing addRows test until function is updated
% read in expanded table
readFile = nwbRead(filename, 'ignorecache');
% test getRow
actualData = readFile.intervals_trials.getRow(1:10, ...
'columns', {'randomvalues'});
testCase.verifyEqual( ...
random_val_array(1:10), ...
actualData.randomvalues ...
);
end

function toTableTest(testCase)
% test DynamicTable toTable method.
% 1. For a generic table, the toTable output should be very
Expand Down Expand Up @@ -395,20 +261,21 @@ function toTableTest(testCase)
);
end
end

function DynamicTableCheckTest(testCase)
% Verify that the checkConfig utility function
% throws error when defining an invalid table
%
% 1. Defining a table with columns of unmatched length
testCase.verifyError( ...
@() testCase.addContainerUnevenColumns(testCase.file), ...
'MatNWB:DynamicTable:CheckConfig:InvalidShape' ...
'NWB:DynamicTable:CheckConfig:InvalidShape' ...
)
% 2. Defining a table with length of id's does not match
% the number of columns
testCase.verifyError( ...
@() testCase.addContainerUnmatchedIDs(testCase.file), ...
'MatNWB:DynamicTable:CheckConfig:InvalidId' ...
'NWB:DynamicTable:CheckConfig:InvalidId' ...
)
%3. Defining a table with unspecified IDs
testCase.addContainerUndefinedIDs(testCase.file)
Expand Down Expand Up @@ -436,29 +303,15 @@ function testEmptyTable(testCase)
'description', 'start time column', ...
'data', (1:5)' ...
)), ...
'MatNWB:DynamicTable:CheckConfig:ColumnNamesMismatch');
'NWB:DynamicTable:CheckConfig:ColumnNamesMismatch');

% validate that "vectordata" set checking works.
testCase.verifyError(@()types.core.TimeIntervals(...
'description', 'test error', ...
'randomvalues', types.hdmf_common.VectorData( ...
'description', 'random values', ...
'data', mat2cell(rand(25,2), repmat(5, 5, 1)))), ...
'MatNWB:DynamicTable:CheckConfig:ColumnNamesMismatch');
'NWB:DynamicTable:CheckConfig:ColumnNamesMismatch');
end
end
end
function zipped = zipArrays(A,B)
zipped = zeros((length(A)+length(B)),1);
countA = 1;
countB = 1;
for i = 1:length(A)+length(B)
if mod(i,2)
zipped(i) = A(countA);
countA = countA+1;
else
zipped(i) = B(countB);
countB = countB+1;
end
end
end
7 changes: 6 additions & 1 deletion +tests/+unit/tutorialTest.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,11 @@ function testTutorials(testCase)
if listing.isdir || any(strcmp(skippedTutorials, listing.name))
continue;
end
run(listing.name);
try
run(listing.name);
catch ME
error('NWB:Test:Tutorial', ...
'Error while running test `%s`. Full error message:\n\n%s', listing.name, getReport(ME));
end
end
end
2 changes: 1 addition & 1 deletion +tests/+util/verifyContainerEqual.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ function verifyContainerEqual(testCase, actual, expected, ignoreList)
ignoreList = {};
end
assert(iscellstr(ignoreList),...
'MatNWB:Test:InvalidIgnoreList',...
'NWB:Test:InvalidIgnoreList',...
['Ignore List must be a cell array of character arrays indicating props that should be '...
'ignored.']);
testCase.verifyEqual(class(actual), class(expected));
Expand Down
Loading