Support Writing Dynamic Filters with DataPipe

+file/fillProps.m

@@ -11,7 +11,7 @@
 if p.Results.IsRequired
     requiredStr = 'REQUIRED';
 else
-    requiredStr = 'OPTIONAL';
+    requiredStr = '';
 end
 
 proplines = cell(size(names));
@@ -80,13 +80,11 @@
 end
 
 if isa(prop, 'file.Dataset') || isa(prop, 'file.Attribute') || isa(prop, 'file.Group')
-    docStr = prop.doc;
+    propStr = sprintf('(%s) %s', typeStr, prop.doc);
 else
-    docStr = '';
+    propStr = typeStr;
 end
 
-propStr = sprintf('(%s) %s', typeStr, docStr);
-
 if nargin >= 2
     propStr = [propName ' = ' propStr];
 end

+tests/+unit/dataPipeTest.m

@@ -78,6 +78,59 @@ function testAppend(testCase)
 testCase.verifyEqual(readData(1:3), [7, 8, 9] .');
 end
 
+function testExternalFilters(testCase)
+import types.untyped.datapipe.dynamic.Filter;
+import types.untyped.datapipe.properties.DynamicFilter;
+import types.untyped.datapipe.properties.Shuffle;
+
+testCase.assumeTrue(logical(H5Z.filter_avail(uint32(Filter.LZ4))));
+
+filename = 'testExternalWrite.h5';
+
+Pipe = types.untyped.DataPipe(...
+    'maxSize', [10 13 15],...
+    'axis', 3,...
+    'chunkSize', [10 13 1],...
+    'dataType', 'uint8',...
+    'filters', [Shuffle() DynamicFilter(Filter.LZ4)]);
+
+OneDimensionPipe = types.untyped.DataPipe('maxSize', Inf, 'data', [7, 8, 9]);
+
+%% create test file
+fid = H5F.create(filename);
+
+initialData = createData(Pipe.dataType, [10 13 10]);
+Pipe.internal.data = initialData;
+Pipe.export(fid, '/test_data', {}); % bind
+OneDimensionPipe.export(fid, '/test_one_dim_data', {});
+
+H5F.close(fid);
+
+%% append data
+totalLength = 3;
+appendData = zeros([10 13 totalLength], Pipe.dataType);
+for i = 1:totalLength
+    appendData(:,:,i) = createData(Pipe.dataType, Pipe.chunkSize);
+    Pipe.append(appendData(:,:,i));
+end
+
+for i = 1:totalLength
+    OneDimensionPipe.append(rand());
+end
+
+%% verify data
+Pipe = types.untyped.DataPipe('filename', filename, 'path', '/test_data');
+readData = Pipe.load();
+testCase.verifyEqual(readData(:,:,1:10), initialData);
+testCase.verifyEqual(readData(:,:,11:end), appendData);
+
+OneDimensionPipe = types.untyped.DataPipe('filename', filename, 'path', '/test_one_dim_data');
+readData = OneDimensionPipe.load();
+testCase.verifyTrue(isvector(readData));
+testCase.verifyEqual(length(readData), 6);
+testCase.verifyEqual(readData(1:3), [7, 8, 9] .');
+end
+
 function data = createData(dataType, size)
 data = randi(intmax(dataType), size, dataType);
 end

+types/+untyped/+datapipe/+dynamic/Filter.m

@@ -0,0 +1,36 @@
+classdef Filter < uint64
+    %FILTER Compression filter registered to HDF5
+    % as defined by (https://portal.hdfgroup.org/display/support/Filters)
+    % Submit an issue if we're missing one you wish to use!
+
+    enumeration
+        SZ3 (32024)
+        CBF (32006)
+        SZ (32017)
+        BLOSC (32001)
+        BZIP2 (307)
+        JPEG_LS (32012)
+        VBZ (32020)
+        JPEG_XR (32007)
+        CCSDS_123 (32011)
+        FCIDECOMP (32018)
+        BitShuffle (32008)
+        FPZip (32014)
+        B3D (32016)
+        JPEG (32019)
+        LPC_Rice (32010)
+        LZ4 (32004)
+        LZF (32000)
+        LZO (305)
+        MAFISC (32002)
+        ZFP (32013)
+        ZStandard (32015)
+        APAX (32005)
+        Snappy (32003)
+        SPDP (32009)
+        BitGroom (32022)
+        GBR (32023) % Granular BitRound
+        FAPEC (32021)
+    end
+end
+

+types/+untyped/+datapipe/+properties/DynamicFilter.m

@@ -0,0 +1,59 @@
+classdef DynamicFilter < types.untyped.datapipe.Property
+    %DYNAMIC Summary of this class goes here
+    %   Detailed explanation goes here
+
+    properties (SetAccess = private)
+        dynamicFilter;
+    end
+
+    properties
+        parameters;
+    end
+
+    methods
+        function obj = DynamicFilter(filter, parameters)
+            validateattributes(filter, ...
+                {'types.untyped.datapipe.dynamic.Filter'}, ...
+                {'scalar'}, ...
+                'DynamicFilter', 'filter');
+            assert(~verLessThan('matlab', '9.12'), ...
+                ['Your MATLAB version `%s` does not support writing with ' ...
+                'dynamically loaded filters. Please upgrade to version R2022a ' ...
+                'or higher in order to use this feature.'], version);
+            assert(H5Z.filter_avail(uint32(filter)), ...
+                ['Filter `%s` does not appear to be installed on this system. ' ...
+                'Please doublecheck `%s` for more information about writing ' ...
+                'with third-party filters.'], ...
+                filter, ...
+                'https://www.mathworks.com/help/matlab/import_export/read-and-write-hdf5-datasets-using-dynamically-loaded-filters.html');
+
+            obj.dynamicFilter = filter;
+
+            if (1 < nargin)
+                obj.parameters = parameters;
+            else
+                obj.parameters = [];
+            end
+        end
+
+        function tf = isInDcpl(dcpl)
+            tf = false;
+
+            for i = 0:(H5P.get_nfilters(dcpl) - 1)
+                [id, ~, ~, ~, ~] = H5P.get_filter(dcpl, i);
+                if id == uint32(obj.dynamicFilter)
+                    tf = true;
+                    return;
+                end
+            end
+        end
+
+        function addTo(obj, dcpl)
+            H5P.set_filter( ...
+                dcpl, ...
+                uint32(obj.dynamicFilter), ...
+                'H5Z_FLAG_MANDATORY', ...
+                obj.parameters);
+        end
+    end
+end

+types/+untyped/+datapipe/BlueprintPipe.m

@@ -95,12 +95,25 @@ function append(~, ~)
                 'Export the DataPipe to append.']);
         end
 
-         function setPipeProperty(obj, prop)
+        function setPipeProperty(obj, prop)
             assert(isa(prop, 'types.untyped.datapipe.Property'),...
                 'Can only add filters.');
+
+            isDynamicFilter = isa(prop, ...
+                'types.untyped.datapipe.properties.DynamicFilter');
 
+            % dedup pipe properties if it already exists with special case
+            % behavior for dynamic filters.
             for i = 1:length(obj.pipeProperties)
-                if isa(prop, class(obj.pipeProperties{i}))
+                pipeProp = obj.pipeProperties{i};
+
+                isSameClass = isa(prop, class(pipeProp));
+
+                if isSameClass
+                    if isDynamicFilter && prop.dynamicFilter ~= pipeProp.dynamicFilter
+                        continue;
+                    end
+
                     obj.pipeProperties{i} = prop;
                     return;
                 end

+types/+untyped/+datapipe/Property.m

@@ -1,4 +1,4 @@
-classdef Property < handle
+classdef Property < handle & matlab.mixin.Heterogeneous
     %PROPERTY used in datapipe creation
 
     methods (Static, Abstract)

+types/+untyped/DataPipe.m

@@ -44,6 +44,7 @@
 
     properties (SetAccess = private)
         internal;
+        filters;
     end
 
     properties (Dependent)
@@ -69,14 +70,20 @@
             p.addParameter('axis', 1, @(x) isnumeric(x) && isscalar(x) && x > 0);
             p.addParameter('offset', 0, @(x) isnumeric(x) && isscalar(x) && x >= 0);
             p.addParameter('chunkSize', []);
+            % note that compression level is defaulted to ON
+            % This is primarily for legacy support as we move into other
+            % filters.
             p.addParameter('compressionLevel', 3, @(x) isnumeric(x)...
                 && isscalar(x)...
                 && x >= -1);
             p.addParameter('dataType', '');
             p.addParameter('data', []);
             p.addParameter('filename', '');
             p.addParameter('path', '');
-            p.addParameter('hasShuffle', false);
+            p.addParameter('hasShuffle', false, ...
+                @(b) isscalar(b) && (islogical(b) || isnumeric(b)));
+            p.addParameter('filters', DynamicFilter.empty(), ...
+                @(x) isa(x, 'types.untyped.datapipe.Property'));
             p.KeepUnmatched = true;
             p.parse(varargin{:});
 
@@ -149,13 +156,31 @@
             end
             obj.internal.setPipeProperties(Chunking(chunkSize));
 
-            if ~isempty(p.Results.compressionLevel)
-                obj.internal.setPipeProperties(Compression(...
-                    p.Results.compressionLevel));
+            hasFilters = ~isempty(p.Results.filters);
+            usingHasCompressionLevel = ~any(strcmp(p.UsingDefaults, 'compressionLevel'));
+            usingHasShuffle = ~any(strcmp(p.UsingDefaults, 'hasShuffle'));
+            if hasFilters && (usingHasCompressionLevel || usingHasShuffle)
+                warning(['`filters` keyword argument detected. This will ' ...
+                    'override `compressionLevel` and `hasShuffle` keyword ' ...
+                    'arguments. If you wish to use either `compressionLevel` ' ...
+                    'or `hasShuffle`, please add their respective filter ' ...
+                    'properties `types.untyped.datapipe.properties.Compression` ' ...
+                    'and `types.untyped.datapipe.properties.Shuffle` to the ' ...
+                    '`filters` properties array.']);
             end
 
-            if p.Results.hasShuffle
-                obj.internal.setPipeProperties(Shuffle());
+            if hasFilters
+                filterCell = num2cell(p.Results.filters);
+                obj.internal.setPipeProperties(filterCell{:});
+            else
+                if -1 < p.Results.compressionLevel
+                    obj.internal.setPipeProperties(Compression(...
+                        p.Results.compressionLevel));
+                end
+
+                if logical(p.Results.hasShuffle)
+                    obj.internal.setPipeProperties(Shuffle());
+                end
             end
 
             obj.internal.data = p.Results.data;
@@ -197,13 +222,24 @@
         end
 
         function val = get.compressionLevel(obj)
-            val = obj.internal.getPipeProperty(...
-                'types.untyped.datapipe.properties.Compression').level;
+            compressionClass = 'types.untyped.datapipe.properties.Compression';
+            val = -1;
+            if obj.internal.hasPipeProperty(compressionClass)
+                val = obj.internal.getPipeProperty(compressionClass).level;
+            end
         end
 
         function set.compressionLevel(obj, val)
             import types.untyped.datapipe.properties.Compression;
-            obj.internal.setPipeProperty(Compression(val));
+            validateattributes(val, {'numeric'}, {'scalar'}, 1);
+            assert(-1 <= val, 'NWB:SetCompressionLevel:InvalidValue', ...
+                'Compression Level cannot be less than -1.');
+            compressionClass = 'types.untyped.datapipe.properties.Compression';
+            if -1 == val
+                obj.internal.removePipeProperty(compressionClass);
+            else
+                obj.internal.setPipeProperty(Compression(val));
+            end
         end
 
         function tf = get.hasShuffle(obj)

azure-pipelines.yml

@@ -3,31 +3,37 @@ trigger:
 
 pool:
   vmImage: 'ubuntu-latest'
-
+    
 steps:
   - task: InstallMATLAB@0
     # default to latest version of Matlab installed
     #inputs:
-    #  release: R2020a
+    #  release: R2017a
   - checkout: self
 
   - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.8'
+    inputs:       
+      versionSpec: '3.8'    
       architecture: 'x64'
 
-  - script: |
+  - bash: |
       python -m pip install --upgrade pip 
       pip install pynwb
+      pip install hdf5plugin
+      echo "##vso[task.setvariable variable=plugin_path]$(python -c "import hdf5plugin; print(hdf5plugin.PLUGINS_PATH)")"
     displayName: 'Install PyNWB'
 
   - task: RunMATLABCommand@0
     inputs:
       command: "results = assertSuccess(nwbtest); assert(~isempty(results), 'No tests ran');"
+    env:
+      HDF5_PLUGIN_PATH: $(plugin_path)
+
   - task: PublishTestResults@2
     condition: succeededOrFailed()
     inputs:
       testResultsFiles: testResults.xml
+
   - task: PublishCodeCoverageResults@1
     inputs:
       codeCoverageTool: Cobertura