Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Filtered Ephys Reference #539

Merged
merged 10 commits into from
Sep 7, 2023
Merged

Add Filtered Ephys Reference #539

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
bb84253
Fix out-of-order error when adding to dynamictable
lawrence-mbf Sep 6, 2023
6d11420
retab addVarargRow
lawrence-mbf Sep 6, 2023
ab043a0
Fix potential error for dyanmic table
lawrence-mbf Sep 6, 2023
651e18b
retab addRawData
lawrence-mbf Sep 6, 2023
e686b6a
rename variables for addVarargRow
lawrence-mbf Sep 6, 2023
c73ef78
complete ecephys tutorial addition
lawrence-mbf Sep 6, 2023
cab93e3
Fix extraneous nesting when adding Rows
lawrence-mbf Sep 6, 2023
d0f582c
Improve table display for ecephys tutorial
lawrence-mbf Sep 6, 2023
c152bdf
make char rows e2e compatible
lawrence-mbf Sep 6, 2023
f27db70
rebuild ecephys tutorial
lawrence-mbf Sep 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
338 changes: 173 additions & 165 deletions +types/+util/+dynamictable/addRawData.m
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,205 +1,213 @@
function addRawData(DynamicTable, column, data)
%ADDRAWDATA Internal method for adding data to DynamicTable given column
% name and data. Indices are determined based on data format and available
% indices.
validateattributes(column, {'char'}, {'scalartext'});

if (isprop(DynamicTable, column) && isempty(DynamicTable.(column))) ...
|| (~isprop(DynamicTable, column) && ~isKey(DynamicTable.vectordata, column))
% No vecdata found anywhere. Initialize.
initVecData(DynamicTable, column, class(data));
end
%ADDRAWDATA Internal method for adding data to DynamicTable given column
% name and data. Indices are determined based on data format and available
% indices.
validateattributes(column, {'char'}, {'scalartext'});

if (isprop(DynamicTable, column) && isempty(DynamicTable.(column))) ...
|| (~isprop(DynamicTable, column) && ~isKey(DynamicTable.vectordata, column))
% No vecdata found anywhere. Initialize.
initVecData(DynamicTable, column, class(data));
end

if isprop(DynamicTable, column)
Vector = DynamicTable.(column);
elseif isprop(DynamicTable, 'vectorindex') && DynamicTable.vectorindex.isKey(column)
Vector = DynamicTable.vectorindex.get(column);
else
Vector = DynamicTable.vectordata.get(column);
end
if isprop(DynamicTable, column)
Vector = DynamicTable.(column);
elseif isprop(DynamicTable, 'vectorindex') && DynamicTable.vectorindex.isKey(column)
Vector = DynamicTable.vectorindex.get(column);
else
Vector = DynamicTable.vectordata.get(column);
end

% grab all available indices for column.
indexChain = {column};
while true
index = types.util.dynamictable.getIndex(DynamicTable, indexChain{end});
if isempty(index)
break;
% grab all available indices for column.
indexChain = {column};
while true
index = types.util.dynamictable.getIndex(DynamicTable, indexChain{end});
if isempty(index)
break;
end
indexChain{end+1} = index;
end
indexChain{end+1} = index;
end

if ~isa(Vector.data, 'types.untyped.DataPipe')
% validate shape for appending in memory.
checkNestedShape(data);
end
if ~isa(Vector.data, 'types.untyped.DataPipe')
% validate shape for appending in memory.
checkNestedShape(data);
end

% find true nesting depth of column data.
if isa(Vector.data, 'types.untyped.DataPipe')
depth = getNestedDataDepth(data, 'dataPipeDimension', Vector.data.axis);
else
depth = getNestedDataDepth(data);
end
% find true nesting depth of column data.
if isa(Vector.data, 'types.untyped.DataPipe')
depth = getNestedDataDepth(data, 'dataPipeDimension', Vector.data.axis);
else
depth = getNestedDataDepth(data);
end

% add indices until it matches depth.
for iVec = (length(indexChain)+1):depth
indexChain{iVec} = types.util.dynamictable.addVecInd(DynamicTable, indexChain{end});
end
% add indices until it matches depth.
for iVec = (length(indexChain)+1):depth
indexChain{iVec} = types.util.dynamictable.addVecInd(DynamicTable, indexChain{end});
end

% wrap until available vector indices match depth.
for iVec = (depth+1):length(indexChain)
data = {data}; % wrap until the correct number of vector indices are satisfied.
end
% wrap until available vector indices match depth.
for iVec = (depth+1):length(indexChain)
data = {data}; % wrap until the correct number of vector indices are satisfied.
end

if ischar(data)
data = {data};
end

% Now in index->data order.
nestedAdd(DynamicTable, flip(indexChain), data);
% Now in index->data order.
nestedAdd(DynamicTable, flip(indexChain), data);
end

function checkNestedShape(data)
errorId = 'NWB:DynamicTable:AddRow:InvalidShape';
if iscell(data) && ~iscellstr(data)
assert(isvector(data), errorId, ...
'Wrapped cell array data must be a vector for use with ragged arrays.');
for iCell = 1:length(data)
checkNestedShape(data{iCell});
end
else
assert(ismatrix(data), errorId, 'Adding 3D and higher-rank data must use DataPipes.');
end
errorId = 'NWB:DynamicTable:AddRow:InvalidShape';
if iscell(data) && ~iscellstr(data)
assert(isvector(data), errorId, ...
'Wrapped cell array data must be a vector for use with ragged arrays.');
for iCell = 1:length(data)
checkNestedShape(data{iCell});
end
else
assert(ismatrix(data), errorId, 'Adding 3D and higher-rank data must use DataPipes.');
end
end

function initVecData(DynamicTable, column, dataType)
% Don't set the data until after indices are updated.
if 8 == exist('types.hdmf_common.VectorData', 'class')
VecData = types.hdmf_common.VectorData();
else
VecData = types.core.VectorData();
end
% Don't set the data until after indices are updated.
if 8 == exist('types.hdmf_common.VectorData', 'class')
VecData = types.hdmf_common.VectorData();
else
VecData = types.core.VectorData();
end

VecData.description = sprintf('AUTOGENERATED description for column `%s`', column);

if strcmp(dataType, 'logical')
% Logical is the lowest precedent type when concatenating primitive
% types in MATLAB. For more information see:
% https://www.mathworks.com/help/releases/R2022a/matlab/matlab_prog/valid-combinations-of-unlike-classes.html
% That said, we still use doubles by default because character arrays
% will error if concatenated with logical arrays.
VecData.data = logical([]);
else
VecData.data = [];
end
VecData.description = sprintf('AUTOGENERATED description for column `%s`', column);

if isprop(DynamicTable, column)
DynamicTable.(column) = VecData;
else
DynamicTable.vectordata.set(column, VecData);
end
if strcmp(dataType, 'logical')
% Logical is the lowest precedent type when concatenating primitive
% types in MATLAB. For more information see:
% https://www.mathworks.com/help/releases/R2022a/matlab/matlab_prog/valid-combinations-of-unlike-classes.html
% That said, we still use doubles by default because character arrays
% will error if concatenated with logical arrays.
VecData.data = logical([]);
else
VecData.data = [];
end

if isprop(DynamicTable, column)
DynamicTable.(column) = VecData;
else
DynamicTable.vectordata.set(column, VecData);
end
end

function depth = getNestedDataDepth(data, varargin)
p = inputParser;
p.addParameter('dataPipeDimension', [], @(x)isnumeric(x) && (isempty(x) || isscalar(x)));
p.parse(varargin{:});

depth = 1;
subData = data;
while iscell(subData) && ~iscellstr(subData)
depth = depth + 1;
subData = subData{1};
end
p = inputParser;
p.addParameter('dataPipeDimension', [], @(x)isnumeric(x) && (isempty(x) || isscalar(x)));
p.parse(varargin{:});

depth = 1;
subData = data;
while iscell(subData) && ~iscellstr(subData)
depth = depth + 1;
subData = subData{1};
end

% special case where the final data is in fact multiple rows to begin
% with.
if isempty(p.Results.dataPipeDimension)
isMultirow = (ismatrix(subData) && 1 < size(subData, 2)) ...
|| (isvector(subData) && 1 < length(subData));
else
isMultirow = 1 < size(subData, p.Results.dataPipeDimension);
end
if isMultirow
depth = depth + 1;
end
% special case where the final data is in fact multiple rows to begin
% with.
if isempty(p.Results.dataPipeDimension)
if ischar(subData)
isMultiRow = 1 < size(subData, 1);
else
isMultiRow = (ismatrix(subData) && 1 < size(subData, 2)) ...
|| (isvector(subData) && 1 < length(subData));
end
else
isMultiRow = 1 < size(subData, p.Results.dataPipeDimension);
end
if isMultiRow
depth = depth + 1;
end
end

function numRows = nestedAdd(DynamicTable, indChain, data)
name = indChain{1};

if isprop(DynamicTable, name)
Vector = DynamicTable.(name);
elseif isprop(DynamicTable, 'vectorindex') && DynamicTable.vectorindex.isKey(name)
Vector = DynamicTable.vectorindex.get(name);
else
Vector = DynamicTable.vectordata.get(name);
end
name = indChain{1};

if isa(Vector, 'types.hdmf_common.VectorIndex') || isa(Vector, 'types.core.VectorIndex')
if iscell(data) && ~iscellstr(data)
numRows = length(data);
for iEntry = 1:numRows
nestedAdd(DynamicTable, indChain(2:end), data{iEntry});
end
if isprop(DynamicTable, name)
Vector = DynamicTable.(name);
elseif isprop(DynamicTable, 'vectorindex') && DynamicTable.vectorindex.isKey(name)
Vector = DynamicTable.vectorindex.get(name);
else
numRows = nestedAdd(DynamicTable, indChain(2:end), data);
Vector = DynamicTable.vectordata.get(name);
end

add2Index(Vector, numRows);
else
if ischar(data)
data = mat2cell(data, ones(size(data, 1), 1));
end % char matrices converted to cell arrays containing character vectors.
if isa(Vector, 'types.hdmf_common.VectorIndex') || isa(Vector, 'types.core.VectorIndex')
if iscell(data) && ~iscellstr(data)
numRows = length(data);
for iEntry = 1:numRows
nestedAdd(DynamicTable, indChain(2:end), data{iEntry});
end
else
numRows = nestedAdd(DynamicTable, indChain(2:end), data);
end

if isa(Vector.data, 'types.untyped.DataPipe')
Vector.data.append(data);
numRows = size(Vector.data, Vector.data.axis);
add2Index(Vector, numRows);
else
numRows = add2MemData(Vector, data);
if ischar(data)
data = mat2cell(data, ones(size(data, 1), 1));
end % char matrices converted to cell arrays containing character vectors.

if isa(Vector.data, 'types.untyped.DataPipe')
Vector.data.append(data);
numRows = size(Vector.data, Vector.data.axis);
else
numRows = add2MemData(Vector, data);
end
end
end
end

function numRows = add2MemData(VectorData, data)
%ADD2MEMDATA add to in-memory data.

if isempty(VectorData.data) || isscalar(VectorData.data)
appendBasis = data;
else
appendBasis = VectorData.data;
end % determine the basis for finding the concatenation dimension.

if istable(appendBasis)
catDim = 1;
numRows = height(data);
elseif isscalar(appendBasis) || ~isvector(appendBasis) % is scalar or matrix but not vector.
catDim = 2;
assert(2 >= ndims(appendBasis), 'NWB:DynamicTable:AddRow:InvalidShape', ...
['addRow does not support adding to matrices with more than 2 dimensions. ' ...
'For multi-dimensional matrices, use a DataPipe instead.']);
numRows = size(data, 2);
else % vector data
catDim = find(size(appendBasis) > 1);
numRows = length(data);
end
%ADD2MEMDATA add to in-memory data.

VectorData.data = cat(catDim, VectorData.data, data);
if isempty(VectorData.data) || isscalar(VectorData.data)
appendBasis = data;
else
appendBasis = VectorData.data;
end % determine the basis for finding the concatenation dimension.

if istable(appendBasis)
catDim = 1;
numRows = height(data);
elseif isscalar(appendBasis) || ~isvector(appendBasis) % is scalar or matrix but not vector.
catDim = 2;
assert(2 >= ndims(appendBasis), 'NWB:DynamicTable:AddRow:InvalidShape', ...
['addRow does not support adding to matrices with more than 2 dimensions. ' ...
'For multi-dimensional matrices, use a DataPipe instead.']);
numRows = size(data, 2);
else % vector data
catDim = find(size(appendBasis) > 1);
numRows = length(data);
end

VectorData.data = cat(catDim, VectorData.data, data);
end

function add2Index(VectorIndex, numElem)
raggedOffset = 0;
if isa(VectorIndex.data, 'types.untyped.DataPipe')
if isa(VectorIndex.data.internal, 'types.untyped.datapipe.BlueprintPipe')...
&& ~isempty(VectorIndex.data.internal.data)
raggedOffset = VectorIndex.data.internal.data(end);
elseif isa(VectorIndex.data.internal, 'types.untyped.datapipe.BoundPipe')...
&& ~any(VectorIndex.data.internal.stub.dims == 0)
raggedOffset = VectorIndex.data.internal.stub(end);
end
elseif ~isempty(VectorIndex.data)
raggedOffset = VectorIndex.data(end);
end
raggedOffset = 0;
if isa(VectorIndex.data, 'types.untyped.DataPipe')
if isa(VectorIndex.data.internal, 'types.untyped.datapipe.BlueprintPipe')...
&& ~isempty(VectorIndex.data.internal.data)
raggedOffset = VectorIndex.data.internal.data(end);
elseif isa(VectorIndex.data.internal, 'types.untyped.datapipe.BoundPipe')...
&& ~any(VectorIndex.data.internal.stub.dims == 0)
raggedOffset = VectorIndex.data.internal.stub(end);
end
elseif ~isempty(VectorIndex.data)
raggedOffset = VectorIndex.data(end);
end

data = double(raggedOffset) + numElem;
if isa(VectorIndex.data, 'types.untyped.DataPipe')
VectorIndex.data.append(data);
else
VectorIndex.data = [double(VectorIndex.data); data];
end
data = double(raggedOffset) + numElem;
if isa(VectorIndex.data, 'types.untyped.DataPipe')
VectorIndex.data.append(data);
else
VectorIndex.data = [double(VectorIndex.data); data];
end
end
4 changes: 2 additions & 2 deletions +types/+util/+dynamictable/addRow.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,13 @@ function addRow(DynamicTable, varargin)
'of column names before being able to add row data.']);
assert(nargin > 1, 'NWB:DynamicTable:AddRow:NoData', 'Not enough arguments');

types.util.dynamictable.checkConfig(DynamicTable);

assert(~isa(DynamicTable.id.data, 'types.untyped.DataStub'),...
'NWB:DynamicTable:AddRow:Uneditable',...
['Cannot write to on-file Dynamic Tables without enabling data pipes. '...
'If this was produced with pynwb, please enable chunking for this table.']);

types.util.dynamictable.checkConfig(DynamicTable);

if istable(varargin{1})
error("NWB:DynamicTable", ...
['Using MATLAB tables as input to the addRow DynamicTable method has '...
Expand Down
Loading