Merge pull request #35 from martindurant/maybe_fletcher

Allow datasets with fletcher32 checksum
fsspec · Jul 6, 2021 · 146bfd1 · 146bfd1
2 parents 97f210f + 02e723e
commit 146bfd1
Showing 1 changed file with 3 additions and 3 deletions.
diff --git a/fsspec_reference_maker/hdf.py b/fsspec_reference_maker/hdf.py
@@ -160,9 +160,6 @@ def _translator(self, name: str, h5obj: Union[h5py.Dataset, h5py.Group]):
             if h5obj.scaleoffset:
                 raise RuntimeError(
                     f'{h5obj.name} uses HDF5 scaleoffset filter - not supported by reference-maker')
-            if h5obj.fletcher32:
-                raise RuntimeError(
-                    f'{h5obj.name} uses fletcher32 checksum - not supported by reference-maker')
             if h5obj.compression in ('szip', 'lzf'):
                 raise RuntimeError(
                     f'{h5obj.name} uses szip or lzf compression - not supported by reference-maker')
@@ -199,6 +196,9 @@ def _translator(self, name: str, h5obj: Union[h5py.Dataset, h5py.Group]):
             # Store chunk location metadata...
             if cinfo:
                 for k, v in cinfo.items():
+                    if h5obj.fletcher32:
+                        logging.info("Discarding fletcher32 checksum")
+                        v['size'] -= 4
                     self.store[za._chunk_key(k)] = [self._uri, v['offset'], v['size']]
 
         elif isinstance(h5obj, h5py.Group):