Merge pull request #2870 from JasonRuonanWang/operator

Simplify operator API in preparation for adding into BP5, Part I
ornladios · Sep 17, 2021 · 0cec587 · 0cec587
2 parents d34cd4a + 349ae32
commit 0cec587
Show file tree

Hide file tree

Showing 27 changed files with 134 additions and 167 deletions.
diff --git a/source/adios2/core/Operator.cpp b/source/adios2/core/Operator.cpp
@@ -82,37 +82,16 @@ size_t Operator::Compress(const void * /*dataIn*/, const Dims & /*dimensions*/,
 }
 
 size_t Operator::Decompress(const void *bufferIn, const size_t sizeIn,
-                            void *dataOut, const size_t sizeOut, Params &info)
+                            void *dataOut, const DataType type,
+                            const Dims &blockStart, const Dims &blockCount,
+                            const Params &parameters, Params &info)
 {
     throw std::invalid_argument(
         "ERROR: signature (const void*, const size_t, void) not supported "
         "by derived class implemented with " +
         m_Type + ", in call to Decompress\n");
 }
 
-size_t Operator::Decompress(const void * /*bufferIn*/, const size_t /*sizeIn*/,
-                            void * /*dataOut*/, const Dims & /*dimensions*/,
-                            const DataType /*type*/,
-                            const Params & /*parameters*/)
-{
-    throw std::invalid_argument("ERROR: signature (const void*, const "
-                                "size_t, void*, const Dims&, const DataType, "
-                                "const Params) not supported "
-                                "by derived class implemented with " +
-                                m_Type + ", in call to Decompress\n");
-}
-
-size_t Operator::Decompress(const void * /*bufferIn*/, const size_t /*sizeIn*/,
-                            void * /*dataOut*/, const Dims & /*start*/,
-                            const Dims & /*count*/, const DataType /*type*/)
-{
-    throw std::invalid_argument("ERROR: signature (const void*, const "
-                                "size_t, void*, const Dims&, const Dims&, "
-                                "const DataType) not supported "
-                                "by derived class implemented with " +
-                                m_Type + ", in call to Decompress\n");
-}
-
 // PROTECTED
 size_t Operator::DoBufferMaxSize(const void *dataIn, const Dims &dimensions,
                                  DataType type, const Params &parameters) const

diff --git a/source/adios2/core/Operator.h b/source/adios2/core/Operator.h
@@ -76,7 +76,6 @@ class Operator
                          const Params &params) const;
 
     /**
-     * BZip2 and Zfp common call
      * @param dataIn
      * @param dimensions
      * @param elementSize
@@ -91,35 +90,9 @@ class Operator
                             Params &info);
 
     virtual size_t Decompress(const void *bufferIn, const size_t sizeIn,
-                              void *dataOut, const size_t sizeOut,
-                              Params &info);
-
-    /**
-     * Zfp signature
-     * @param bufferIn
-     * @param sizeIn
-     * @param dataOut
-     * @param dimensions
-     * @param type
-     * @return
-     */
-    virtual size_t Decompress(const void *bufferIn, const size_t sizeIn,
-                              void *dataOut, const Dims &dimensions,
-                              const DataType type, const Params &parameters);
-
-    /**
-     * Sirius signature
-     * @param bufferIn
-     * @param sizeIn
-     * @param dataOut
-     * @param start
-     * @param count
-     * @param type
-     * @return
-     */
-    virtual size_t Decompress(const void *bufferIn, const size_t sizeIn,
-                              void *dataOut, const Dims &start,
-                              const Dims &count, const DataType type);
+                              void *dataOut, const DataType type,
+                              const Dims &blockStart, const Dims &blockCount,
+                              const Params &parameters, Params &info);
 
     virtual bool IsDataTypeValid(const DataType type) const = 0;
 

diff --git a/source/adios2/operator/compress/CompressBZIP2.cpp b/source/adios2/operator/compress/CompressBZIP2.cpp
@@ -113,10 +113,14 @@ size_t CompressBZIP2::Compress(const void *dataIn, const Dims &dimensions,
 }
 
 size_t CompressBZIP2::Decompress(const void *bufferIn, const size_t sizeIn,
-                                 void *dataOut, const size_t sizeOut,
-                                 Params &info)
+                                 void *dataOut, const DataType type,
+                                 const Dims &blockStart, const Dims &blockCount,
+                                 const Params &parameters, Params &info)
 {
     // TODO: leave defaults at zero?
+    const size_t sizeOut = std::accumulate(blockCount.begin(), blockCount.end(),
+                                           helper::GetDataTypeSize(type),
+                                           std::multiplies<size_t>());
     int small = 0;
     int verbosity = 0;
 

diff --git a/source/adios2/operator/compress/CompressBZIP2.h b/source/adios2/operator/compress/CompressBZIP2.h
@@ -46,7 +46,6 @@ class CompressBZIP2 : public Operator
                     const size_t elementSize, DataType type, void *bufferOut,
                     const Params &parameters, Params &info) final;
 
-    using Operator::Decompress;
     /**
      * Decompression signature for legacy libraries that use void*
      * @param bufferIn
@@ -57,7 +56,9 @@ class CompressBZIP2 : public Operator
      * @return size of decompressed buffer in bytes
      */
     size_t Decompress(const void *bufferIn, const size_t sizeIn, void *dataOut,
-                      const size_t sizeOut, Params &info) final;
+                      const DataType type, const Dims &blockStart,
+                      const Dims &blockCount, const Params &parameters,
+                      Params &info) final;
 
     bool IsDataTypeValid(const DataType type) const final;
 

diff --git a/source/adios2/operator/compress/CompressBlosc.cpp b/source/adios2/operator/compress/CompressBlosc.cpp
@@ -236,9 +236,14 @@ size_t CompressBlosc::Compress(const void *dataIn, const Dims &dimensions,
 }
 
 size_t CompressBlosc::Decompress(const void *bufferIn, const size_t sizeIn,
-                                 void *dataOut, const size_t sizeOut,
-                                 Params &info)
+                                 void *dataOut, const DataType type,
+                                 const Dims &blockStart, const Dims &blockCount,
+                                 const Params &parameters, Params &info)
 {
+    size_t sizeOut = std::accumulate(blockCount.begin(), blockCount.end(),
+                                     helper::GetDataTypeSize(type),
+                                     std::multiplies<size_t>());
+
     assert(sizeIn >= sizeof(DataHeader));
     const bool isChunked =
         reinterpret_cast<const DataHeader *>(bufferIn)->IsChunked();

diff --git a/source/adios2/operator/compress/CompressBlosc.h b/source/adios2/operator/compress/CompressBlosc.h
@@ -59,7 +59,6 @@ class CompressBlosc : public Operator
                     const size_t elementSize, DataType type, void *bufferOut,
                     const Params &parameters, Params &info) final;
 
-    using Operator::Decompress;
     /**
      * Decompression signature for legacy libraries that use void*
      * @param bufferIn
@@ -70,7 +69,9 @@ class CompressBlosc : public Operator
      * @return size of decompressed buffer in bytes
      */
     size_t Decompress(const void *bufferIn, const size_t sizeIn, void *dataOut,
-                      const size_t sizeOut, Params &info) final;
+                      const DataType type, const Dims &blockStart,
+                      const Dims &blockCount, const Params &parameters,
+                      Params &info) final;
 
     bool IsDataTypeValid(const DataType type) const final;
 

diff --git a/source/adios2/operator/compress/CompressLibPressio.cpp b/source/adios2/operator/compress/CompressLibPressio.cpp
@@ -332,20 +332,22 @@ size_t CompressLibPressio::Compress(const void *dataIn, const Dims &dimensions,
 }
 
 size_t CompressLibPressio::Decompress(const void *bufferIn, const size_t sizeIn,
-                                      void *dataOut, const Dims &dimensions,
-                                      DataType varType, const Params &params)
+                                      void *dataOut, const DataType type,
+                                      const Dims &blockStart,
+                                      const Dims &blockCount,
+                                      const Params &parameters, Params &info)
 {
-    std::vector<size_t> dims = adios_to_libpressio_dims(dimensions);
+    std::vector<size_t> dims = adios_to_libpressio_dims(blockCount);
     pressio_data *output_buf = pressio_data_new_owning(
-        adios_to_libpressio_dtype(varType), dims.size(), dims.data());
+        adios_to_libpressio_dtype(type), dims.size(), dims.data());
 
     pressio_data *input_buf = pressio_data_new_nonowning(
         pressio_byte_dtype, const_cast<void *>(bufferIn), 1, &sizeIn);
 
     pressio_compressor *compressor = nullptr;
     try
     {
-        compressor = adios_to_libpressio_compressor(params);
+        compressor = adios_to_libpressio_compressor(parameters);
     }
     catch (std::exception &)
     {

diff --git a/source/adios2/operator/compress/CompressLibPressio.h b/source/adios2/operator/compress/CompressLibPressio.h
@@ -46,8 +46,6 @@ class CompressLibPressio : public Operator
                     const size_t elementSize, DataType type, void *bufferOut,
                     const Params &parameters, Params &info) final;
 
-    using Operator::Decompress;
-
     /**
      * Wrapper around zfp decompression
      * @param bufferIn
@@ -58,8 +56,9 @@ class CompressLibPressio : public Operator
      * @return size of decompressed data in dataOut
      */
     size_t Decompress(const void *bufferIn, const size_t sizeIn, void *dataOut,
-                      const Dims &dimensions, DataType type,
-                      const Params &parameters) final;
+                      const DataType type, const Dims &blockStart,
+                      const Dims &blockCount, const Params &parameters,
+                      Params &info) final;
 
     bool IsDataTypeValid(const DataType type) const final;
 };

diff --git a/source/adios2/operator/compress/CompressMGARD.cpp b/source/adios2/operator/compress/CompressMGARD.cpp
@@ -106,8 +106,9 @@ size_t CompressMGARD::Compress(const void *dataIn, const Dims &dimensions,
 }
 
 size_t CompressMGARD::Decompress(const void *bufferIn, const size_t sizeIn,
-                                 void *dataOut, const Dims &dimensions,
-                                 DataType type, const Params & /*parameters*/)
+                                 void *dataOut, const DataType type,
+                                 const Dims &blockStart, const Dims &blockCount,
+                                 const Params &parameters, Params &info)
 {
     int mgardType = -1;
     size_t elementSize = 0;
@@ -125,22 +126,24 @@ size_t CompressMGARD::Decompress(const void *bufferIn, const size_t sizeIn,
             "MGARD only supports double precision, in call to Get\n");
     }
 
-    const size_t ndims = dimensions.size();
+    const size_t ndims = blockCount.size();
     int r[3];
     r[0] = 1;
     r[1] = 1;
     r[2] = 1;
 
     for (size_t i = 0; i < ndims; i++)
     {
-        r[ndims - i - 1] = static_cast<int>(dimensions[i]);
+        r[ndims - i - 1] = static_cast<int>(blockCount[i]);
     }
 
     void *dataPtr = mgard_decompress(
         reinterpret_cast<unsigned char *>(const_cast<void *>(bufferIn)),
         static_cast<int>(sizeIn), r[0], r[1], r[2], 0.0);
 
-    const size_t dataSizeBytes = helper::GetTotalSize(dimensions) * elementSize;
+    const size_t dataSizeBytes = std::accumulate(
+        blockCount.begin(), blockCount.end(), helper::GetDataTypeSize(type),
+        std::multiplies<size_t>());
     std::memcpy(dataOut, dataPtr, dataSizeBytes);
 
     free(dataPtr);

diff --git a/source/adios2/operator/compress/CompressMGARD.h b/source/adios2/operator/compress/CompressMGARD.h
@@ -44,8 +44,6 @@ class CompressMGARD : public Operator
                     const size_t elementSize, DataType type, void *bufferOut,
                     const Params &parameters, Params &info) final;
 
-    using Operator::Decompress;
-
     /**
      *
      * @param bufferIn
@@ -57,8 +55,9 @@ class CompressMGARD : public Operator
      * @return
      */
     size_t Decompress(const void *bufferIn, const size_t sizeIn, void *dataOut,
-                      const Dims &dimensions, DataType varType,
-                      const Params & /*parameters*/) final;
+                      const DataType type, const Dims &blockStart,
+                      const Dims &blockCount, const Params &parameters,
+                      Params &info) final;
 
     bool IsDataTypeValid(const DataType type) const final;
 };

diff --git a/source/adios2/operator/compress/CompressPNG.cpp b/source/adios2/operator/compress/CompressPNG.cpp
@@ -171,9 +171,13 @@ size_t CompressPNG::Compress(const void *dataIn, const Dims &dimensions,
 }
 
 size_t CompressPNG::Decompress(const void *bufferIn, const size_t sizeIn,
-                               void *dataOut, const size_t sizeOut,
-                               Params &info)
+                               void *dataOut, const DataType type,
+                               const Dims &blockStart, const Dims &blockCount,
+                               const Params &parameters, Params &info)
 {
+    const size_t sizeOut = std::accumulate(blockCount.begin(), blockCount.end(),
+                                           helper::GetDataTypeSize(type),
+                                           std::multiplies<size_t>());
     png_image image;
     std::memset(&image, 0, sizeof(image));
     image.version = PNG_IMAGE_VERSION;

diff --git a/source/adios2/operator/compress/CompressPNG.h b/source/adios2/operator/compress/CompressPNG.h
@@ -46,8 +46,6 @@ class CompressPNG : public Operator
                     const size_t elementSize, DataType type, void *bufferOut,
                     const Params &parameters, Params &info) final;
 
-    using Operator::Decompress;
-
     /**
      * Decompression signature for legacy libraries that use void*
      * @param bufferIn
@@ -58,7 +56,9 @@ class CompressPNG : public Operator
      * @return size of decompressed buffer in bytes
      */
     size_t Decompress(const void *bufferIn, const size_t sizeIn, void *dataOut,
-                      const size_t sizeOut, Params &info) final;
+                      const DataType type, const Dims &blockStart,
+                      const Dims &blockCount, const Params &parameters,
+                      Params &info) final;
 
     bool IsDataTypeValid(const DataType type) const final;
 

diff --git a/source/adios2/operator/compress/CompressSZ.cpp b/source/adios2/operator/compress/CompressSZ.cpp
@@ -270,22 +270,23 @@ size_t CompressSZ::Compress(const void *dataIn, const Dims &dimensions,
 }
 
 size_t CompressSZ::Decompress(const void *bufferIn, const size_t sizeIn,
-                              void *dataOut, const Dims &dimensions,
-                              DataType varType, const Params & /*parameters*/)
+                              void *dataOut, const DataType type,
+                              const Dims &blockStart, const Dims &blockCount,
+                              const Params &parameters, Params &info)
 {
-    Dims convertedDims = ConvertDims(dimensions, varType, 4, true, 1);
+    Dims convertedDims = ConvertDims(blockCount, type, 4, true, 1);
 
     // Get type info
     int dtype = 0;
     size_t typeSizeBytes = 0;
-    if (varType == helper::GetDataType<double>() ||
-        varType == helper::GetDataType<std::complex<double>>())
+    if (type == helper::GetDataType<double>() ||
+        type == helper::GetDataType<std::complex<double>>())
     {
         dtype = SZ_DOUBLE;
         typeSizeBytes = 8;
     }
-    else if (varType == helper::GetDataType<float>() ||
-             varType == helper::GetDataType<std::complex<float>>())
+    else if (type == helper::GetDataType<float>() ||
+             type == helper::GetDataType<std::complex<float>>())
     {
         dtype = SZ_FLOAT;
         typeSizeBytes = 4;

diff --git a/source/adios2/operator/compress/CompressSZ.h b/source/adios2/operator/compress/CompressSZ.h
@@ -46,8 +46,6 @@ class CompressSZ : public Operator
                     const size_t elementSize, DataType type, void *bufferOut,
                     const Params &parameters, Params &info) final;
 
-    using Operator::Decompress;
-
     /**
      * Wrapper around zfp decompression
      * @param bufferIn
@@ -58,8 +56,9 @@ class CompressSZ : public Operator
      * @return size of decompressed data in dataOut
      */
     size_t Decompress(const void *bufferIn, const size_t sizeIn, void *dataOut,
-                      const Dims &dimensions, DataType type,
-                      const Params &parameters) final;
+                      const DataType type, const Dims &blockStart,
+                      const Dims &blockCount, const Params &parameters,
+                      Params &info) final;
 
     bool IsDataTypeValid(const DataType type) const final;
 };

diff --git a/source/adios2/operator/compress/CompressSirius.cpp b/source/adios2/operator/compress/CompressSirius.cpp
@@ -69,15 +69,17 @@ size_t CompressSirius::Compress(const void *dataIn, const Dims &dimensions,
 }
 
 size_t CompressSirius::Decompress(const void *bufferIn, const size_t sizeIn,
-                                  void *dataOut, const Dims &start,
-                                  const Dims &count, DataType type)
+                                  void *dataOut, const DataType type,
+                                  const Dims &blockStart,
+                                  const Dims &blockCount,
+                                  const Params &parameters, Params &info)
 {
-    size_t outputBytes = std::accumulate(count.begin(), count.end(),
-                                         helper::GetDataTypeSize(type),
-                                         std::multiplies<size_t>());
+    const size_t outputBytes = std::accumulate(
+        blockCount.begin(), blockCount.end(), helper::GetDataTypeSize(type),
+        std::multiplies<size_t>());
 
     std::string blockId =
-        helper::DimsToString(start) + helper::DimsToString(count);
+        helper::DimsToString(blockStart) + helper::DimsToString(blockCount);
 
     // decompress data and copy back to m_TierBuffers
     size_t bytesPerTier = outputBytes / m_Tiers;

diff --git a/source/adios2/operator/compress/CompressSirius.h b/source/adios2/operator/compress/CompressSirius.h
@@ -33,11 +33,10 @@ class CompressSirius : public Operator
                     const size_t elementSize, DataType type, void *bufferOut,
                     const Params &params, Params &info) final;
 
-    using Operator::Decompress;
-
     size_t Decompress(const void *bufferIn, const size_t sizeIn, void *dataOut,
-                      const Dims &start, const Dims &count,
-                      DataType type) final;
+                      const DataType type, const Dims &blockStart,
+                      const Dims &blockCount, const Params &parameters,
+                      Params &info) final;
 
     bool IsDataTypeValid(const DataType type) const final;