Merge pull request #2873 from JasonRuonanWang/operator

Simplify operator classes
ornladios · Sep 20, 2021 · 8b2d2ab · 8b2d2ab
2 parents 254b6e8 + 7b4426a
commit 8b2d2ab
Show file tree

Hide file tree

Showing 7 changed files with 28 additions and 55 deletions.
diff --git a/source/adios2/helper/adiosMath.cpp b/source/adios2/helper/adiosMath.cpp
@@ -24,10 +24,10 @@ namespace adios2
 namespace helper
 {
 
-size_t GetTotalSize(const Dims &dimensions) noexcept
+size_t GetTotalSize(const Dims &dimensions, const size_t elementSize) noexcept
 {
-    return std::accumulate(dimensions.begin(), dimensions.end(),
-                           static_cast<size_t>(1), std::multiplies<size_t>());
+    return std::accumulate(dimensions.begin(), dimensions.end(), elementSize,
+                           std::multiplies<size_t>());
 }
 
 bool CheckIndexRange(const int index, const int upperLimit,

diff --git a/source/adios2/helper/adiosMath.h b/source/adios2/helper/adiosMath.h
@@ -29,7 +29,8 @@ namespace helper
  * @param dimensions input containing size on each dimension {Nx, Ny, Nz}
  * @return product of all dimensions Nx * Ny * Nz
  */
-size_t GetTotalSize(const Dims &dimensions) noexcept;
+size_t GetTotalSize(const Dims &dimensions,
+                    const size_t elementSize = 1) noexcept;
 
 /**
  * Populates min and max for a selection region inside

diff --git a/source/adios2/operator/compress/CompressBZIP2.cpp b/source/adios2/operator/compress/CompressBZIP2.cpp
@@ -67,9 +67,8 @@ size_t CompressBZIP2::Compress(const void *dataIn, const Dims &dimensions,
         }
     }
 
-    const size_t sizeIn = std::accumulate(dimensions.begin(), dimensions.end(),
-                                          helper::GetDataTypeSize(type),
-                                          std::multiplies<size_t>());
+    const size_t sizeIn =
+        helper::GetTotalSize(dimensions, helper::GetDataTypeSize(type));
 
     const size_t batches = sizeIn / DefaultMaxFileBatchSize + 1;
     info["batches"] = std::to_string(batches);
@@ -118,9 +117,8 @@ size_t CompressBZIP2::Decompress(const void *bufferIn, const size_t sizeIn,
                                  const Params &parameters, Params &info)
 {
     // TODO: leave defaults at zero?
-    const size_t sizeOut = std::accumulate(blockCount.begin(), blockCount.end(),
-                                           helper::GetDataTypeSize(type),
-                                           std::multiplies<size_t>());
+    const size_t sizeOut =
+        helper::GetTotalSize(blockCount, helper::GetDataTypeSize(type));
     int small = 0;
     int verbosity = 0;
 

diff --git a/source/adios2/operator/compress/CompressBlosc.cpp b/source/adios2/operator/compress/CompressBlosc.cpp
@@ -65,9 +65,8 @@ size_t CompressBlosc::Compress(const void *dataIn, const Dims &dimensions,
                                DataType type, void *bufferOut,
                                const Params &parameters, Params &info)
 {
-    const size_t sizeIn = std::accumulate(dimensions.begin(), dimensions.end(),
-                                          helper::GetDataTypeSize(type),
-                                          std::multiplies<size_t>());
+    const size_t sizeIn =
+        helper::GetTotalSize(dimensions, helper::GetDataTypeSize(type));
 
     bool useMemcpy = false;
     /* input size under this bound would not compressed */
@@ -240,9 +239,8 @@ size_t CompressBlosc::Decompress(const void *bufferIn, const size_t sizeIn,
                                  const Dims &blockStart, const Dims &blockCount,
                                  const Params &parameters, Params &info)
 {
-    size_t sizeOut = std::accumulate(blockCount.begin(), blockCount.end(),
-                                     helper::GetDataTypeSize(type),
-                                     std::multiplies<size_t>());
+    size_t sizeOut =
+        helper::GetTotalSize(blockCount, helper::GetDataTypeSize(type));
 
     assert(sizeIn >= sizeof(DataHeader));
     const bool isChunked =

diff --git a/source/adios2/operator/compress/CompressMGARD.cpp b/source/adios2/operator/compress/CompressMGARD.cpp
@@ -140,15 +140,14 @@ size_t CompressMGARD::Decompress(const void *bufferIn, const size_t sizeIn,
         reinterpret_cast<unsigned char *>(const_cast<void *>(bufferIn)),
         static_cast<int>(sizeIn), r[0], r[1], r[2], 0.0);
 
-    const size_t dataSizeBytes = std::accumulate(
-        blockCount.begin(), blockCount.end(), helper::GetDataTypeSize(type),
-        std::multiplies<size_t>());
-    std::memcpy(dataOut, dataPtr, dataSizeBytes);
+    const size_t sizeOut =
+        helper::GetTotalSize(blockCount, helper::GetDataTypeSize(type));
+    std::memcpy(dataOut, dataPtr, sizeOut);
 
     free(dataPtr);
     dataPtr = nullptr;
 
-    return static_cast<size_t>(dataSizeBytes);
+    return sizeOut;
 }
 
 bool CompressMGARD::IsDataTypeValid(const DataType type) const

diff --git a/source/adios2/operator/compress/CompressPNG.cpp b/source/adios2/operator/compress/CompressPNG.cpp
@@ -175,9 +175,6 @@ size_t CompressPNG::Decompress(const void *bufferIn, const size_t sizeIn,
                                const Dims &blockStart, const Dims &blockCount,
                                const Params &parameters, Params &info)
 {
-    const size_t sizeOut = std::accumulate(blockCount.begin(), blockCount.end(),
-                                           helper::GetDataTypeSize(type),
-                                           std::multiplies<size_t>());
     png_image image;
     std::memset(&image, 0, sizeof(image));
     image.version = PNG_IMAGE_VERSION;
@@ -200,7 +197,7 @@ size_t CompressPNG::Decompress(const void *bufferIn, const size_t sizeIn,
             "to ADIOS2 PNG Decompress\n");
     }
 
-    return sizeOut;
+    return helper::GetTotalSize(blockCount, helper::GetDataTypeSize(type));
 }
 
 bool CompressPNG::IsDataTypeValid(const DataType type) const { return true; }

diff --git a/source/adios2/operator/compress/CompressZFP.cpp b/source/adios2/operator/compress/CompressZFP.cpp
@@ -69,19 +69,6 @@ size_t CompressZFP::Decompress(const void *bufferIn, const size_t sizeIn,
                                const Dims &blockStart, const Dims &blockCount,
                                const Params &parameters, Params &info)
 {
-    auto lf_GetTypeSize = [](const zfp_type zfpType) -> size_t {
-        size_t size = 0;
-        if (zfpType == zfp_type_int32 || zfpType == zfp_type_float)
-        {
-            size = 4;
-        }
-        else if (zfpType == zfp_type_int64 || zfpType == zfp_type_double)
-        {
-            size = 8;
-        }
-        return size;
-    };
-
     Dims convertedDims = ConvertDims(blockCount, type, 3);
 
     zfp_field *field = GetZFPField(dataOut, convertedDims, type);
@@ -105,7 +92,7 @@ size_t CompressZFP::Decompress(const void *bufferIn, const size_t sizeIn,
     zfp_stream_close(stream);
     stream_close(bitstream);
 
-    const size_t typeSizeBytes = lf_GetTypeSize(GetZfpType(type));
+    const size_t typeSizeBytes = helper::GetDataTypeSize(type);
     const size_t dataSizeBytes =
         helper::GetTotalSize(convertedDims) * typeSizeBytes;
 
@@ -169,44 +156,37 @@ zfp_type CompressZFP::GetZfpType(DataType type) const
 zfp_field *CompressZFP::GetZFPField(const void *data, const Dims &dimensions,
                                     DataType type) const
 {
-    auto lf_CheckField = [](const zfp_field *field,
-                            const std::string zfpFieldFunction, DataType type) {
-        if (field == nullptr || field == NULL)
-        {
-            throw std::invalid_argument(
-                "ERROR: " + zfpFieldFunction + " failed for data of type " +
-                ToString(type) +
-                ", data pointer might be corrupted, from "
-                "class CompressZfp Transform\n");
-        }
-    };
-
     zfp_type zfpType = GetZfpType(type);
     zfp_field *field = nullptr;
 
     if (dimensions.size() == 1)
     {
         field = zfp_field_1d(const_cast<void *>(data), zfpType, dimensions[0]);
-        lf_CheckField(field, "zfp_field_1d", type);
     }
     else if (dimensions.size() == 2)
     {
         field = zfp_field_2d(const_cast<void *>(data), zfpType, dimensions[0],
                              dimensions[1]);
-        lf_CheckField(field, "zfp_field_2d", type);
     }
     else if (dimensions.size() == 3)
     {
         field = zfp_field_3d(const_cast<void *>(data), zfpType, dimensions[0],
                              dimensions[1], dimensions[2]);
-        lf_CheckField(field, "zfp_field_3d", type);
     }
     else
     {
         throw std::invalid_argument(
             "ERROR: zfp_field* failed for data of type " + ToString(type) +
             ", only 1D, 2D and 3D dimensions are supported, from "
-            "class CompressZfp Transform\n");
+            "class CompressZfp\n");
+    }
+
+    if (field == nullptr)
+    {
+        throw std::invalid_argument(
+            "ERROR: zfp_field_" + std::to_string(dimensions.size()) +
+            "d failed for data of type " + ToString(type) +
+            ", data might be corrupted, from class CompressZfp\n");
     }
 
     return field;