From e2e2a9c40143e9368b3b6e45df9de07f4cd49b95 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Sun, 19 Sep 2021 19:55:41 -0400 Subject: [PATCH 1/2] removed more meaningless lambda functions in zfp --- .../adios2/operator/compress/CompressZFP.cpp | 40 +++++-------------- 1 file changed, 10 insertions(+), 30 deletions(-) diff --git a/source/adios2/operator/compress/CompressZFP.cpp b/source/adios2/operator/compress/CompressZFP.cpp index aa1f33eab7..af2b3110d5 100644 --- a/source/adios2/operator/compress/CompressZFP.cpp +++ b/source/adios2/operator/compress/CompressZFP.cpp @@ -69,19 +69,6 @@ size_t CompressZFP::Decompress(const void *bufferIn, const size_t sizeIn, const Dims &blockStart, const Dims &blockCount, const Params ¶meters, Params &info) { - auto lf_GetTypeSize = [](const zfp_type zfpType) -> size_t { - size_t size = 0; - if (zfpType == zfp_type_int32 || zfpType == zfp_type_float) - { - size = 4; - } - else if (zfpType == zfp_type_int64 || zfpType == zfp_type_double) - { - size = 8; - } - return size; - }; - Dims convertedDims = ConvertDims(blockCount, type, 3); zfp_field *field = GetZFPField(dataOut, convertedDims, type); @@ -105,7 +92,7 @@ size_t CompressZFP::Decompress(const void *bufferIn, const size_t sizeIn, zfp_stream_close(stream); stream_close(bitstream); - const size_t typeSizeBytes = lf_GetTypeSize(GetZfpType(type)); + const size_t typeSizeBytes = helper::GetDataTypeSize(type); const size_t dataSizeBytes = helper::GetTotalSize(convertedDims) * typeSizeBytes; @@ -169,44 +156,37 @@ zfp_type CompressZFP::GetZfpType(DataType type) const zfp_field *CompressZFP::GetZFPField(const void *data, const Dims &dimensions, DataType type) const { - auto lf_CheckField = [](const zfp_field *field, - const std::string zfpFieldFunction, DataType type) { - if (field == nullptr || field == NULL) - { - throw std::invalid_argument( - "ERROR: " + zfpFieldFunction + " failed for data of type " + - ToString(type) + - ", data pointer might be corrupted, from " - "class CompressZfp Transform\n"); - } - }; - zfp_type zfpType = GetZfpType(type); zfp_field *field = nullptr; if (dimensions.size() == 1) { field = zfp_field_1d(const_cast(data), zfpType, dimensions[0]); - lf_CheckField(field, "zfp_field_1d", type); } else if (dimensions.size() == 2) { field = zfp_field_2d(const_cast(data), zfpType, dimensions[0], dimensions[1]); - lf_CheckField(field, "zfp_field_2d", type); } else if (dimensions.size() == 3) { field = zfp_field_3d(const_cast(data), zfpType, dimensions[0], dimensions[1], dimensions[2]); - lf_CheckField(field, "zfp_field_3d", type); } else { throw std::invalid_argument( "ERROR: zfp_field* failed for data of type " + ToString(type) + ", only 1D, 2D and 3D dimensions are supported, from " - "class CompressZfp Transform\n"); + "class CompressZfp\n"); + } + + if (field == nullptr) + { + throw std::invalid_argument( + "ERROR: zfp_field_" + std::to_string(dimensions.size()) + + "d failed for data of type " + ToString(type) + + ", data might be corrupted, from class CompressZfp\n"); } return field; From 7b4426a64c8de87155f45b41560f6b9b8f10c7e8 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Sun, 19 Sep 2021 20:07:29 -0400 Subject: [PATCH 2/2] add elementSize in helper::GetTotalSize --- source/adios2/helper/adiosMath.cpp | 6 +++--- source/adios2/helper/adiosMath.h | 3 ++- source/adios2/operator/compress/CompressBZIP2.cpp | 10 ++++------ source/adios2/operator/compress/CompressBlosc.cpp | 10 ++++------ source/adios2/operator/compress/CompressMGARD.cpp | 9 ++++----- source/adios2/operator/compress/CompressPNG.cpp | 5 +---- 6 files changed, 18 insertions(+), 25 deletions(-) diff --git a/source/adios2/helper/adiosMath.cpp b/source/adios2/helper/adiosMath.cpp index 5b9ef0280e..426fb4c1f6 100644 --- a/source/adios2/helper/adiosMath.cpp +++ b/source/adios2/helper/adiosMath.cpp @@ -24,10 +24,10 @@ namespace adios2 namespace helper { -size_t GetTotalSize(const Dims &dimensions) noexcept +size_t GetTotalSize(const Dims &dimensions, const size_t elementSize) noexcept { - return std::accumulate(dimensions.begin(), dimensions.end(), - static_cast(1), std::multiplies()); + return std::accumulate(dimensions.begin(), dimensions.end(), elementSize, + std::multiplies()); } bool CheckIndexRange(const int index, const int upperLimit, diff --git a/source/adios2/helper/adiosMath.h b/source/adios2/helper/adiosMath.h index 175fbc28a4..a6a5c6d138 100644 --- a/source/adios2/helper/adiosMath.h +++ b/source/adios2/helper/adiosMath.h @@ -29,7 +29,8 @@ namespace helper * @param dimensions input containing size on each dimension {Nx, Ny, Nz} * @return product of all dimensions Nx * Ny * Nz */ -size_t GetTotalSize(const Dims &dimensions) noexcept; +size_t GetTotalSize(const Dims &dimensions, + const size_t elementSize = 1) noexcept; /** * Populates min and max for a selection region inside diff --git a/source/adios2/operator/compress/CompressBZIP2.cpp b/source/adios2/operator/compress/CompressBZIP2.cpp index aecec0f76c..e7931f0280 100644 --- a/source/adios2/operator/compress/CompressBZIP2.cpp +++ b/source/adios2/operator/compress/CompressBZIP2.cpp @@ -67,9 +67,8 @@ size_t CompressBZIP2::Compress(const void *dataIn, const Dims &dimensions, } } - const size_t sizeIn = std::accumulate(dimensions.begin(), dimensions.end(), - helper::GetDataTypeSize(type), - std::multiplies()); + const size_t sizeIn = + helper::GetTotalSize(dimensions, helper::GetDataTypeSize(type)); const size_t batches = sizeIn / DefaultMaxFileBatchSize + 1; info["batches"] = std::to_string(batches); @@ -118,9 +117,8 @@ size_t CompressBZIP2::Decompress(const void *bufferIn, const size_t sizeIn, const Params ¶meters, Params &info) { // TODO: leave defaults at zero? - const size_t sizeOut = std::accumulate(blockCount.begin(), blockCount.end(), - helper::GetDataTypeSize(type), - std::multiplies()); + const size_t sizeOut = + helper::GetTotalSize(blockCount, helper::GetDataTypeSize(type)); int small = 0; int verbosity = 0; diff --git a/source/adios2/operator/compress/CompressBlosc.cpp b/source/adios2/operator/compress/CompressBlosc.cpp index ffc04caa6c..905e7b67d5 100644 --- a/source/adios2/operator/compress/CompressBlosc.cpp +++ b/source/adios2/operator/compress/CompressBlosc.cpp @@ -65,9 +65,8 @@ size_t CompressBlosc::Compress(const void *dataIn, const Dims &dimensions, DataType type, void *bufferOut, const Params ¶meters, Params &info) { - const size_t sizeIn = std::accumulate(dimensions.begin(), dimensions.end(), - helper::GetDataTypeSize(type), - std::multiplies()); + const size_t sizeIn = + helper::GetTotalSize(dimensions, helper::GetDataTypeSize(type)); bool useMemcpy = false; /* input size under this bound would not compressed */ @@ -240,9 +239,8 @@ size_t CompressBlosc::Decompress(const void *bufferIn, const size_t sizeIn, const Dims &blockStart, const Dims &blockCount, const Params ¶meters, Params &info) { - size_t sizeOut = std::accumulate(blockCount.begin(), blockCount.end(), - helper::GetDataTypeSize(type), - std::multiplies()); + size_t sizeOut = + helper::GetTotalSize(blockCount, helper::GetDataTypeSize(type)); assert(sizeIn >= sizeof(DataHeader)); const bool isChunked = diff --git a/source/adios2/operator/compress/CompressMGARD.cpp b/source/adios2/operator/compress/CompressMGARD.cpp index c81941a181..871df64cfa 100644 --- a/source/adios2/operator/compress/CompressMGARD.cpp +++ b/source/adios2/operator/compress/CompressMGARD.cpp @@ -140,15 +140,14 @@ size_t CompressMGARD::Decompress(const void *bufferIn, const size_t sizeIn, reinterpret_cast(const_cast(bufferIn)), static_cast(sizeIn), r[0], r[1], r[2], 0.0); - const size_t dataSizeBytes = std::accumulate( - blockCount.begin(), blockCount.end(), helper::GetDataTypeSize(type), - std::multiplies()); - std::memcpy(dataOut, dataPtr, dataSizeBytes); + const size_t sizeOut = + helper::GetTotalSize(blockCount, helper::GetDataTypeSize(type)); + std::memcpy(dataOut, dataPtr, sizeOut); free(dataPtr); dataPtr = nullptr; - return static_cast(dataSizeBytes); + return sizeOut; } bool CompressMGARD::IsDataTypeValid(const DataType type) const diff --git a/source/adios2/operator/compress/CompressPNG.cpp b/source/adios2/operator/compress/CompressPNG.cpp index 3ccf97a742..b9b29d628d 100644 --- a/source/adios2/operator/compress/CompressPNG.cpp +++ b/source/adios2/operator/compress/CompressPNG.cpp @@ -175,9 +175,6 @@ size_t CompressPNG::Decompress(const void *bufferIn, const size_t sizeIn, const Dims &blockStart, const Dims &blockCount, const Params ¶meters, Params &info) { - const size_t sizeOut = std::accumulate(blockCount.begin(), blockCount.end(), - helper::GetDataTypeSize(type), - std::multiplies()); png_image image; std::memset(&image, 0, sizeof(image)); image.version = PNG_IMAGE_VERSION; @@ -200,7 +197,7 @@ size_t CompressPNG::Decompress(const void *bufferIn, const size_t sizeIn, "to ADIOS2 PNG Decompress\n"); } - return sizeOut; + return helper::GetTotalSize(blockCount, helper::GetDataTypeSize(type)); } bool CompressPNG::IsDataTypeValid(const DataType type) const { return true; }