Skip to content

Commit

Permalink
Merge pull request #92 from NeurodataWithoutBorders/new_schema4
Browse files Browse the repository at this point in the history 
NWB:N 2.0 release schema
  • Loading branch information
@lawrence-mbf
lawrence-mbf authored Jan 18, 2019
2 parents 78f60d8 + 6c5d439 commit eb37be0
Show file tree
Hide file tree
Showing 23 changed files with 491 additions and 270 deletions.
93 changes: 54 additions & 39 deletions +file/Group.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,9 +145,8 @@
obj.hasAnonData = anonDataCnt > 0;
obj.hasAnonGroups = anonGroupCnt > 0;

obj.elide = obj.scalar && isempty(obj.type) && isempty(obj.attributes)...
&& isempty(obj.links) && ~obj.hasAnonData && ~obj.hasAnonGroups...
&& ~obj.defaultEmpty;
obj.elide = ~isempty(obj.name) && obj.scalar && isempty(obj.type) &&...
isempty(obj.attributes);
end

function props = getProps(obj)
Expand All @@ -159,42 +158,6 @@
error('getProps shouldn''t be called on a constrained set.');
end

%untyped
% parse props and return.

%typed
% containersMap of properties -> types
% parse props and return;

%subgroups
for i=1:length(obj.subgroups)
%if typed, check if constraint
% if constraint, add its type and continue
% otherwise, call getprops and assign to its name.
%if untyped, check if elided
% if elided, add to prefix and check all subgroups, attributes and datasets.
% otherwise, call getprops and assign to its name.
sub = obj.subgroups(i);
if isempty(sub.type)
if sub.elide
subprops = sub.getProps;
epkeys = keys(subprops);
for j=1:length(epkeys)
epk = epkeys{j};
props([sub.name '_' epk]) = subprops(epk);
end
else
props(sub.name) = sub;
end
else
if isempty(sub.name)
props(lower(sub.type)) = sub;
else
props(sub.name) = sub;
end
end
end

%datasets
for i=1:length(obj.datasets)
%if typed, check if constraint
Expand Down Expand Up @@ -230,6 +193,58 @@
props = [props;...
containers.Map({obj.links.name}, num2cell(obj.links))];
end

%untyped
% parse props and return.

%typed
% containersMap of properties -> types
% parse props and return;

%subgroups
for i=1:length(obj.subgroups)
%if typed, check if constraint
% if constraint, add its type and continue
% otherwise, call getprops and assign to its name.
%if untyped, check if elided
% if elided, add to prefix and check all subgroups, attributes and datasets.
% otherwise, call getprops and assign to its name.
sub = obj.subgroups(i);
if isempty(sub.type)
if sub.elide
subprops = sub.getProps;
epkeys = keys(subprops);
for j=1:length(epkeys)
epk = epkeys{j};
epval = subprops(epk);
% hoist constrained sets to the current
% subname.
if (isa(epval, 'file.Group') ||...
isa(epval, 'file.Dataset')) &&...
strcmpi(epk, epval.type) &&...
epval.isConstrainedSet
propname = sub.name;
else
propname = [sub.name '_' epk];
end
if isKey(props, propname)
keyboard;
props(propname) = {props(propname); subprops(epk)};
else
props(propname) = subprops(epk);
end
end
else
props(sub.name) = sub;
end
else
if isempty(sub.name)
props(lower(sub.type)) = sub;
else
props(sub.name) = sub;
end
end
end
end
end
end
22 changes: 12 additions & 10 deletions +file/fillConstructor.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,8 @@
if isempty(names)
return;
end

constrained = false(size(names));
% if there's a root object that is a constrained set, let it be hoistable from dynamic arguments
dynamicConstrained = false(size(names));
anon = false(size(names));
isattr = false(size(names));
typenames = repmat({''}, size(names));
Expand All @@ -123,12 +123,12 @@
prop = props(nm);

if isa(prop, 'file.Group') || isa(prop, 'file.Dataset')
constrained(i) = prop.isConstrainedSet;
dynamicConstrained(i) = prop.isConstrainedSet && strcmpi(nm, prop.type);
anon(i) = ~prop.isConstrainedSet && isempty(prop.name);

if ~isempty(prop.type)
pc_namespace = namespace.getNamespace(prop.type);
varnames{i} = prop.type;
varnames{i} = nm;
if ~isempty(pc_namespace)
typenames{i} = ['types.' pc_namespace.name '.' prop.type];
end
Expand All @@ -144,7 +144,7 @@
'the namespace or class definition for type `%1$s` or fix its schema.'];

invalid = cellfun('isempty', typenames);
invalidWarn = invalid & (constrained | anon) & ~isattr;
invalidWarn = invalid & (dynamicConstrained | anon) & ~isattr;
invalidVars = varnames(invalidWarn);
for i=1:length(invalidVars)
warning(warnmsg, invalidVars{i});
Expand All @@ -155,10 +155,10 @@
deleteFromVars = 'varargin(ivarargin) = [];';
%if constrained/anon sets exist, then check for nonstandard parameters and add as
%container.map
constrainedTypes = typenames(constrained & ~invalid);
constrainedVars = varnames(constrained & ~invalid);
constrainedTypes = typenames(dynamicConstrained & ~invalid);
constrainedVars = varnames(dynamicConstrained & ~invalid);
methodCalls = strcat('[obj.', constrainedVars, ', ivarargin] = ',...
' types.util.parseConstrained(obj,''', pname, ''', ''',...
' types.util.parseConstrained(obj,''', constrainedVars, ''', ''',...
constrainedTypes, ''', varargin{:});');
fullBody = cell(length(methodCalls) * 2,1);
fullBody(1:2:end) = methodCalls;
Expand All @@ -185,11 +185,13 @@
'p.PartialMatching = false;',...
'p.StructExpand = false;'};

names = names(~constrained & ~anon);
names = names(~dynamicConstrained & ~anon);
defaults = cell(size(names));
for i=1:length(names)
prop = props(names{i});
if isa(prop, 'file.Group') && (prop.hasAnonData || prop.hasAnonGroups)
if (isa(prop, 'file.Group') &&...
(prop.hasAnonData || prop.hasAnonGroups || prop.isConstrainedSet)) ||...
(isa(prop, 'file.Dataset') && prop.isConstrainedSet)
defaults{i} = 'types.untyped.Set()';
else
defaults{i} = '[]';
Expand Down
78 changes: 44 additions & 34 deletions +io/parseGroup.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,15 +52,11 @@
parsed(root) = [];
end
else
%elide group properties
propnames = keys(gprops);
typeprops = setdiff(properties(typename), propnames);
elided_typeprops = typeprops(startsWith(typeprops, propnames));
for i=1:length(elided_typeprops)
etp = elided_typeprops{i};
gprops(etp) = elide(etp, gprops);
if gprops.Count > 0
%elide group properties
elided_gprops = elide(gprops, properties(typename));
gprops = [gprops; elided_gprops];
end

%construct as kwargs and instantiate object
kwargs = io.map2kwargs([attrprops; dsprops; gprops; lprops]);
if isempty(root)
Expand All @@ -72,36 +68,50 @@
end
end

function set = elide(propname, elideset)
%given propname and a nested set, elide and return flattened set
set = elideset;
prefix = '';
while ~strcmp(prefix, propname)
ekeys = keys(set);
found = false;
for i=1:length(ekeys)
ek = ekeys{i};
if isempty(prefix)
pek = ek;
else
pek = [prefix '_' ek];
%NOTE: SIDE EFFECTS ALTER THE SET
function elided = elide(set, prop, prefix)
%given raw data representation, match to closest property.
% return a typemap of matching typeprops and their prop values to turn into kwargs
% depth first search through the set to construct a possible type prop
if nargin < 3
prefix = '';
end
elided = containers.Map;
elidekeys = keys(set);
elidevals = values(set);
drop = false(size(elidekeys));
if ~isempty(prefix)
potentials = strcat(prefix, '_', elidekeys);
else
potentials = elidekeys;
end
for i=1:length(potentials)
pvar = potentials{i};
pvalue = elidevals{i};
if isa(pvalue, 'containers.Map') || isa(pvalue, 'types.untyped.Set')
if pvalue.Count == 0
drop(i) = true;
continue; %delete
end
if startsWith(propname, pek)
if isa(set, 'containers.Map')
set = set(ek);
elseif strcmp(propname, pek)
set = set.get(ek);
leads = startsWith(prop, pvar);
if any(leads)
%since set has been edited, we bubble up deletion of the old keys.
subset = elide(pvalue, prop(leads), pvar);
elided = [elided; subset];
if pvalue.Count == 0
drop(i) = true;
elseif any(strcmp(pvar, prop))
elided(pvar) = pvalue;
drop(i) = true;
else
continue;
warning('Unable to match property `%s` under prefix `%s`',...
pvar, prefix);
end
prefix = pek;
found = true;
break;
end
end
if ~found
set = [];
return;
elseif any(strcmp(pvar, prop))
elided(pvar) = pvalue;
drop(i) = true;
end
end
remove(set, elidekeys(drop)); %delete all leftovers that were yielded
end
36 changes: 36 additions & 0 deletions +tests/+system/DynamicTableTest.m
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
classdef DynamicTableTest < tests.system.RoundTripTest
methods
function addContainer(~, file)
start_time = types.core.VectorData(...
'description', 'start_time',...
'data', 1:100);
stop_time = types.core.VectorData(...
'description', 'stop_time',...
'data', 2:200);
colnames = {'start_time', 'stop_time', 'randomvalues'};
id = types.core.ElementIdentifiers(...
'data', 1:100);

randcol = types.core.VectorData(...
'description', 'random data to be indexed into',...
'data', rand(500,1));
randidx = types.core.VectorIndex(...
'target', types.untyped.ObjectView('/intervals/trials/randomvalues'),...
'data', 5:5:500 - 1);

file.intervals_trials = types.core.TimeIntervals(...
'description', 'test dynamic table columns',...
'id', id,...
'start_time', start_time,...
'stop_time', stop_time,...
'colnames', colnames,...
'randomvalues', randcol,...
'randomvalues_index', randidx);
end

function c = getContainer(~, file)
c = file.intervals_trials.vectordata.get('randomvalues');
end
end
end

15 changes: 2 additions & 13 deletions +tests/+system/ElectricalSeriesIOTest.m
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,4 @@
classdef ElectricalSeriesIOTest < tests.system.PyNWBIOTest
methods(Test)
function testOutToPyNWB(testCase)
testCase.assumeFail(['Current schema in MatNWB does not include a ElectrodeTable class used by Python tests. ', ...
'When it does, addContainer in this test will need to be updated to match the Python test']);
end

function testInFromPyNWB(testCase)
testCase.assumeFail(['Current schema in MatNWB does not include a ElectrodeTable class used by Python tests. ', ...
'When it does, addContainer in this test will need to be updated to match the Python test']);
end
end

methods
function addContainer(testCase, file) %#ok<INUSL>
Expand Down Expand Up @@ -45,10 +34,10 @@ function addContainer(testCase, file) %#ok<INUSL>
etColNames, etTblVal)),...
'description', 'electrodes');

file.general_extracellular_ephys.set('electrodes',ettable);
file.general_extracellular_ephys_electrodes = ettable;
file.general_extracellular_ephys.set(egnm, eg);
es = types.core.ElectricalSeries( ...
'data', int32([0:9;10:19]) .', ...
'data', [0:9;10:19] .', ...
'timestamps', (0:9) .', ...
'electrodes', ...
types.core.DynamicTableRegion(...
Expand Down
4 changes: 1 addition & 3 deletions +tests/+system/ImagingPlaneIOTest.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,7 @@ function addContainer(testCase, file) %#ok<INUSL>
'excitation_lambda', 6.28, ...
'imaging_rate', 2.718, ...
'indicator', 'GFP', ...
'location', 'somewhere in the brain',...
'manifold', zeros(3,3,3),...
'reference_frame', 'manifold reference');
'location', 'somewhere in the brain');
file.general_devices.set('imaging_device_1', dev);
file.general_optophysiology.set('imgpln1', ip);
end
Expand Down
6 changes: 2 additions & 4 deletions +tests/+system/PhotonSeriesIOTest.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,10 @@ function addContainer(testCase, file) %#ok<INUSL>
'excitation_lambda', 6.28, ...
'imaging_rate', 2.718, ...
'indicator', 'GFP', ...
'location', 'somewhere in the brain',...
'manifold', zeros(3,1),...
'reference_frame', 'manifold ref');
'location', 'somewhere in the brain');

tps = types.core.TwoPhotonSeries( ...
'data', int32([0:9;10:19]) .', ...
'data', ones(3,3,3), ...
'imaging_plane', types.untyped.SoftLink('/general/optophysiology/imgpln1'), ...
'data_unit', 'image_unit', ...
'format', 'raw', ...
Expand Down
4 changes: 2 additions & 2 deletions +tests/+system/PyNWBIOTest.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ function testInFromPyNWB(testCase)

methods
function [status, cmdout] = runPyTest(testCase, testName)
setenv('PYTHONPATH', fileparts(mfilename('fullpath')));
cmd = sprintf('python -B -m unittest %s.%s.%s', 'PyNWBIOTest', testCase.className(), testName);
%setenv('PYTHONPATH', fileparts(mfilename('fullpath')));
cmd = sprintf('/Users/bendichter/anaconda3/bin/python -B -m unittest %s.%s.%s', 'PyNWBIOTest', testCase.className(), testName);
[status, cmdout] = system(cmd);
end
end
Expand Down
Loading

0 comments on commit eb37be0

Please sign in to comment.