diff --git a/source/adios2/core/Operator.cpp b/source/adios2/core/Operator.cpp index f23284832c..6db3804e0e 100644 --- a/source/adios2/core/Operator.cpp +++ b/source/adios2/core/Operator.cpp @@ -21,6 +21,8 @@ Operator::Operator(const std::string type, const Params ¶meters) { } +Operator::Operator(const std::string type) : m_Type(type) {} + void Operator::SetParameter(const std::string key, const std::string value) noexcept { diff --git a/source/adios2/core/Operator.h b/source/adios2/core/Operator.h index 7d65c45057..7b1f6a0ad7 100644 --- a/source/adios2/core/Operator.h +++ b/source/adios2/core/Operator.h @@ -30,12 +30,10 @@ class Operator /** From derived class */ const std::string m_Type; - /** - * Base class constructor - * @param type from derived class object: e.g. bzip2, zfp, callback - */ Operator(const std::string type, const Params ¶meters); + Operator(const std::string type); + virtual ~Operator() = default; void SetParameter(const std::string key, const std::string value) noexcept; diff --git a/source/adios2/operator/compress/CompressBZIP2.cpp b/source/adios2/operator/compress/CompressBZIP2.cpp index f3169a22a2..de8077851c 100644 --- a/source/adios2/operator/compress/CompressBZIP2.cpp +++ b/source/adios2/operator/compress/CompressBZIP2.cpp @@ -32,6 +32,8 @@ CompressBZIP2::CompressBZIP2(const Params ¶meters) { } +CompressBZIP2::CompressBZIP2() : Operator("bzip2") {} + size_t CompressBZIP2::Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, DataType type, char *bufferOut, const Params ¶meters) diff --git a/source/adios2/operator/compress/CompressBZIP2.h b/source/adios2/operator/compress/CompressBZIP2.h index f2039278c5..cfabfb7a39 100644 --- a/source/adios2/operator/compress/CompressBZIP2.h +++ b/source/adios2/operator/compress/CompressBZIP2.h @@ -24,11 +24,10 @@ class CompressBZIP2 : public Operator { public: - /** - * Unique constructor - */ CompressBZIP2(const Params ¶meters); + CompressBZIP2(); + ~CompressBZIP2() = default; /** diff --git a/source/adios2/operator/compress/CompressBlosc.cpp b/source/adios2/operator/compress/CompressBlosc.cpp index 52b27a08a4..a195167e4a 100644 --- a/source/adios2/operator/compress/CompressBlosc.cpp +++ b/source/adios2/operator/compress/CompressBlosc.cpp @@ -42,6 +42,8 @@ CompressBlosc::CompressBlosc(const Params ¶meters) { } +CompressBlosc::CompressBlosc() : Operator("blosc") {} + size_t CompressBlosc::Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType type, char *bufferOut, const Params ¶meters) diff --git a/source/adios2/operator/compress/CompressBlosc.h b/source/adios2/operator/compress/CompressBlosc.h index 7e683c5a9f..52c248c1cd 100644 --- a/source/adios2/operator/compress/CompressBlosc.h +++ b/source/adios2/operator/compress/CompressBlosc.h @@ -36,11 +36,10 @@ class CompressBlosc : public Operator { public: - /** - * Unique constructor - */ CompressBlosc(const Params ¶meters); + CompressBlosc(); + ~CompressBlosc() = default; /** diff --git a/source/adios2/operator/compress/CompressLibPressio.cpp b/source/adios2/operator/compress/CompressLibPressio.cpp index 87b7ef0627..3675bd0710 100644 --- a/source/adios2/operator/compress/CompressLibPressio.cpp +++ b/source/adios2/operator/compress/CompressLibPressio.cpp @@ -27,111 +27,12 @@ namespace core namespace compress { -size_t CompressLibPressio::Operate(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, const DataType type, - char *bufferOut, const Params ¶meters) -{ - const uint8_t bufferVersion = 1; - size_t bufferOutOffset = 0; - - // Universal operator metadata - PutParameter(bufferOut, bufferOutOffset, OperatorType::LIBPRESSIO); - PutParameter(bufferOut, bufferOutOffset, bufferVersion); - PutParameter(bufferOut, bufferOutOffset, static_cast(0)); - // Universal operator metadata end - - const size_t ndims = blockCount.size(); - - // zfp V1 metadata - PutParameter(bufferOut, bufferOutOffset, ndims); - for (const auto &d : blockCount) - { - PutParameter(bufferOut, bufferOutOffset, d); - } - PutParameter(bufferOut, bufferOutOffset, type); - PutParameter(bufferOut, bufferOutOffset, - static_cast(pressio_major_version())); - PutParameter(bufferOut, bufferOutOffset, - static_cast(pressio_minor_version())); - PutParameter(bufferOut, bufferOutOffset, - static_cast(pressio_patch_version())); - PutParameters(bufferOut, bufferOutOffset, parameters); - // zfp V1 metadata end - - auto inputs_dims = adios_to_libpressio_dims(blockCount); - pressio_data *input_buf = pressio_data_new_nonowning( - adios_to_libpressio_dtype(type), const_cast(dataIn), - inputs_dims.size(), inputs_dims.data()); - pressio_data *output_buf = - pressio_data_new_empty(pressio_byte_dtype, 0, nullptr); - pressio_compressor *compressor = nullptr; - try - { - compressor = adios_to_libpressio_compressor(parameters); - } - catch (std::exception &e) - { - pressio_data_free(input_buf); - pressio_data_free(output_buf); - throw; - } - - if (pressio_compressor_compress(compressor, input_buf, output_buf) != 0) - { - pressio_data_free(input_buf); - pressio_data_free(output_buf); - throw std::runtime_error(std::string("pressio_compressor_compress: ") + - pressio_compressor_error_msg(compressor)); - } - - size_t size_in_bytes = 0; - void *bytes = pressio_data_ptr(output_buf, &size_in_bytes); - memcpy(bufferOut + bufferOutOffset, bytes, size_in_bytes); - bufferOutOffset += size_in_bytes; - - pressio_data_free(input_buf); - pressio_data_free(output_buf); - - return bufferOutOffset; -} - -size_t CompressLibPressio::InverseOperate(const char *bufferIn, - const size_t sizeIn, char *dataOut) +CompressLibPressio::CompressLibPressio(const Params ¶meters) +: Operator("libpressio", parameters) { - size_t bufferInOffset = 1; // skip operator type - const uint8_t bufferVersion = - GetParameter(bufferIn, bufferInOffset); - bufferInOffset += 2; // skip two reserved bytes - - if (bufferVersion == 1) - { - return DecompressV1(bufferIn + bufferInOffset, sizeIn - bufferInOffset, - dataOut); - } - else if (bufferVersion == 2) - { - // TODO: if a Version 2 LibPressio buffer is being implemented, put it - // here and keep the DecompressV1 routine for backward compatibility - } - else - { - throw("unknown LibPressio buffer version"); - } - - return 0; } -bool CompressLibPressio::IsDataTypeValid(const DataType type) const -{ -#define declare_type(T) \ - if (helper::GetDataType() == type) \ - { \ - return true; \ - } - ADIOS2_FOREACH_LIBPRESSIO_TYPE_1ARG(declare_type) -#undef declare_type - return false; -} +CompressLibPressio::CompressLibPressio() : Operator("libpressio") {} static pressio_dtype adios_to_libpressio_dtype(DataType var_type) { @@ -386,9 +287,110 @@ static pressio_compressor *adios_to_libpressio_compressor(Params const ¶ms) throw std::runtime_error("missing required \"compressor_id\" setting"); } -CompressLibPressio::CompressLibPressio(const Params ¶meters) -: Operator("libpressio", parameters) +size_t CompressLibPressio::Operate(const char *dataIn, const Dims &blockStart, + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) +{ + const uint8_t bufferVersion = 1; + size_t bufferOutOffset = 0; + + // Universal operator metadata + PutParameter(bufferOut, bufferOutOffset, OperatorType::LIBPRESSIO); + PutParameter(bufferOut, bufferOutOffset, bufferVersion); + PutParameter(bufferOut, bufferOutOffset, static_cast(0)); + // Universal operator metadata end + + const size_t ndims = blockCount.size(); + + // zfp V1 metadata + PutParameter(bufferOut, bufferOutOffset, ndims); + for (const auto &d : blockCount) + { + PutParameter(bufferOut, bufferOutOffset, d); + } + PutParameter(bufferOut, bufferOutOffset, type); + PutParameter(bufferOut, bufferOutOffset, + static_cast(pressio_major_version())); + PutParameter(bufferOut, bufferOutOffset, + static_cast(pressio_minor_version())); + PutParameter(bufferOut, bufferOutOffset, + static_cast(pressio_patch_version())); + PutParameters(bufferOut, bufferOutOffset, parameters); + // zfp V1 metadata end + + auto inputs_dims = adios_to_libpressio_dims(blockCount); + pressio_data *input_buf = pressio_data_new_nonowning( + adios_to_libpressio_dtype(type), const_cast(dataIn), + inputs_dims.size(), inputs_dims.data()); + pressio_data *output_buf = + pressio_data_new_empty(pressio_byte_dtype, 0, nullptr); + pressio_compressor *compressor = nullptr; + try + { + compressor = adios_to_libpressio_compressor(parameters); + } + catch (std::exception &e) + { + pressio_data_free(input_buf); + pressio_data_free(output_buf); + throw; + } + + if (pressio_compressor_compress(compressor, input_buf, output_buf) != 0) + { + pressio_data_free(input_buf); + pressio_data_free(output_buf); + throw std::runtime_error(std::string("pressio_compressor_compress: ") + + pressio_compressor_error_msg(compressor)); + } + + size_t size_in_bytes = 0; + void *bytes = pressio_data_ptr(output_buf, &size_in_bytes); + memcpy(bufferOut + bufferOutOffset, bytes, size_in_bytes); + bufferOutOffset += size_in_bytes; + + pressio_data_free(input_buf); + pressio_data_free(output_buf); + + return bufferOutOffset; +} + +size_t CompressLibPressio::InverseOperate(const char *bufferIn, + const size_t sizeIn, char *dataOut) +{ + size_t bufferInOffset = 1; // skip operator type + const uint8_t bufferVersion = + GetParameter(bufferIn, bufferInOffset); + bufferInOffset += 2; // skip two reserved bytes + + if (bufferVersion == 1) + { + return DecompressV1(bufferIn + bufferInOffset, sizeIn - bufferInOffset, + dataOut); + } + else if (bufferVersion == 2) + { + // TODO: if a Version 2 LibPressio buffer is being implemented, put it + // here and keep the DecompressV1 routine for backward compatibility + } + else + { + throw("unknown LibPressio buffer version"); + } + + return 0; +} + +bool CompressLibPressio::IsDataTypeValid(const DataType type) const { +#define declare_type(T) \ + if (helper::GetDataType() == type) \ + { \ + return true; \ + } + ADIOS2_FOREACH_LIBPRESSIO_TYPE_1ARG(declare_type) +#undef declare_type + return false; } size_t CompressLibPressio::DecompressV1(const char *bufferIn, diff --git a/source/adios2/operator/compress/CompressLibPressio.h b/source/adios2/operator/compress/CompressLibPressio.h index 482c517367..22782649e8 100644 --- a/source/adios2/operator/compress/CompressLibPressio.h +++ b/source/adios2/operator/compress/CompressLibPressio.h @@ -24,11 +24,10 @@ class CompressLibPressio : public Operator { public: - /** - * Unique constructor - */ CompressLibPressio(const Params ¶meters); + CompressLibPressio(); + ~CompressLibPressio() = default; /** diff --git a/source/adios2/operator/compress/CompressMGARD.cpp b/source/adios2/operator/compress/CompressMGARD.cpp index 4befc30f22..6ef34a9bba 100644 --- a/source/adios2/operator/compress/CompressMGARD.cpp +++ b/source/adios2/operator/compress/CompressMGARD.cpp @@ -26,6 +26,8 @@ CompressMGARD::CompressMGARD(const Params ¶meters) { } +CompressMGARD::CompressMGARD() : Operator("mgard") {} + size_t CompressMGARD::Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType type, char *bufferOut, const Params ¶meters) diff --git a/source/adios2/operator/compress/CompressMGARD.h b/source/adios2/operator/compress/CompressMGARD.h index eaa5ba9c33..dc9409b6b1 100644 --- a/source/adios2/operator/compress/CompressMGARD.h +++ b/source/adios2/operator/compress/CompressMGARD.h @@ -24,11 +24,10 @@ class CompressMGARD : public Operator { public: - /** - * Unique constructor - */ CompressMGARD(const Params ¶meters); + CompressMGARD(); + ~CompressMGARD() = default; /** diff --git a/source/adios2/operator/compress/CompressPNG.cpp b/source/adios2/operator/compress/CompressPNG.cpp index 7d9cc60c11..ea0805b8b2 100644 --- a/source/adios2/operator/compress/CompressPNG.cpp +++ b/source/adios2/operator/compress/CompressPNG.cpp @@ -43,10 +43,13 @@ const std::map> CompressPNG::m_BitDepths = { {"PNG_COLOR_TYPE_GA", {8, 16}}}; // PUBLIC + CompressPNG::CompressPNG(const Params ¶meters) : Operator("png", parameters) { } +CompressPNG::CompressPNG() : Operator("png") {} + size_t CompressPNG::Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType type, char *bufferOut, const Params ¶meters) diff --git a/source/adios2/operator/compress/CompressPNG.h b/source/adios2/operator/compress/CompressPNG.h index 0bb5c8a9b6..e691441b80 100644 --- a/source/adios2/operator/compress/CompressPNG.h +++ b/source/adios2/operator/compress/CompressPNG.h @@ -26,11 +26,10 @@ class CompressPNG : public Operator { public: - /** - * Unique constructor - */ CompressPNG(const Params ¶meters); + CompressPNG(); + ~CompressPNG() = default; /** diff --git a/source/adios2/operator/compress/CompressSZ.cpp b/source/adios2/operator/compress/CompressSZ.cpp index 8c139237e8..b7d2820001 100644 --- a/source/adios2/operator/compress/CompressSZ.cpp +++ b/source/adios2/operator/compress/CompressSZ.cpp @@ -29,6 +29,8 @@ namespace compress CompressSZ::CompressSZ(const Params ¶meters) : Operator("sz", parameters) {} +CompressSZ::CompressSZ() : Operator("sz") {} + size_t CompressSZ::Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType varType, char *bufferOut, const Params ¶meters) diff --git a/source/adios2/operator/compress/CompressSZ.h b/source/adios2/operator/compress/CompressSZ.h index 0d0bded72f..a687d04085 100644 --- a/source/adios2/operator/compress/CompressSZ.h +++ b/source/adios2/operator/compress/CompressSZ.h @@ -24,11 +24,10 @@ class CompressSZ : public Operator { public: - /** - * Unique constructor - */ CompressSZ(const Params ¶meters); + CompressSZ(); + ~CompressSZ() = default; /** diff --git a/source/adios2/operator/compress/CompressSirius.cpp b/source/adios2/operator/compress/CompressSirius.cpp index 75ea5a4394..41f649af0b 100644 --- a/source/adios2/operator/compress/CompressSirius.cpp +++ b/source/adios2/operator/compress/CompressSirius.cpp @@ -34,6 +34,13 @@ CompressSirius::CompressSirius(const Params ¶meters) m_TierBuffers.resize(m_Tiers); } +CompressSirius::CompressSirius() : Operator("sirius") +{ + m_Tiers = 4; + m_TierBuffersMap.resize(m_Tiers); + m_TierBuffers.resize(m_Tiers); +} + size_t CompressSirius::Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType varType, char *bufferOut, const Params ¶ms) diff --git a/source/adios2/operator/compress/CompressSirius.h b/source/adios2/operator/compress/CompressSirius.h index a34c99c00b..0f210e8c01 100644 --- a/source/adios2/operator/compress/CompressSirius.h +++ b/source/adios2/operator/compress/CompressSirius.h @@ -27,6 +27,8 @@ class CompressSirius : public Operator public: CompressSirius(const Params ¶meters); + CompressSirius(); + ~CompressSirius() = default; size_t Operate(const char *dataIn, const Dims &blockStart, diff --git a/source/adios2/operator/compress/CompressZFP.cpp b/source/adios2/operator/compress/CompressZFP.cpp index f5ce1bf989..4fd9f7f482 100644 --- a/source/adios2/operator/compress/CompressZFP.cpp +++ b/source/adios2/operator/compress/CompressZFP.cpp @@ -27,6 +27,8 @@ CompressZFP::CompressZFP(const Params ¶meters) : Operator("zfp", parameters) { } +CompressZFP::CompressZFP() : Operator("zfp") {} + size_t CompressZFP::Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType type, char *bufferOut, const Params ¶meters) diff --git a/source/adios2/operator/compress/CompressZFP.h b/source/adios2/operator/compress/CompressZFP.h index 938736a6f4..965366014b 100644 --- a/source/adios2/operator/compress/CompressZFP.h +++ b/source/adios2/operator/compress/CompressZFP.h @@ -29,11 +29,10 @@ class CompressZFP : public Operator { public: - /** - * Unique constructor - */ CompressZFP(const Params ¶meters); + CompressZFP(); + ~CompressZFP() = default; /** diff --git a/source/adios2/operator/compress/CompressorFactory.cpp b/source/adios2/operator/compress/CompressorFactory.cpp index 9d94843bcc..37fcf8776e 100644 --- a/source/adios2/operator/compress/CompressorFactory.cpp +++ b/source/adios2/operator/compress/CompressorFactory.cpp @@ -27,7 +27,7 @@ size_t CompressorFactory::Compress(const char *dataIn, const Dims &blockStart, if (compressorType == "blosc") { #ifdef ADIOS2_HAVE_BLOSC - CompressBlosc c({}); + CompressBlosc c; return c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, parameters); #else @@ -38,7 +38,7 @@ size_t CompressorFactory::Compress(const char *dataIn, const Dims &blockStart, else if (compressorType == "bzip2") { #ifdef ADIOS2_HAVE_BZIP2 - CompressBZIP2 c({}); + CompressBZIP2 c; return c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, parameters); #else @@ -49,7 +49,7 @@ size_t CompressorFactory::Compress(const char *dataIn, const Dims &blockStart, else if (compressorType == "libpressio") { #ifdef ADIOS2_HAVE_LIBPRESSIO - CompressLibPressio c({}); + CompressLibPressio c; return c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, parameters); #else @@ -60,7 +60,7 @@ size_t CompressorFactory::Compress(const char *dataIn, const Dims &blockStart, else if (compressorType == "mgard") { #ifdef ADIOS2_HAVE_MGARD - CompressMGARD c({}); + CompressMGARD c; return c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, parameters); #else @@ -71,29 +71,18 @@ size_t CompressorFactory::Compress(const char *dataIn, const Dims &blockStart, else if (compressorType == "png") { #ifdef ADIOS2_HAVE_PNG - CompressPNG c({}); + CompressPNG c; return c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, parameters); #else throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " "with PNG, can't use compressor\n"); -#endif - } - else if (compressorType == "sirius") - { -#ifdef ADIOS2_HAVE_MHS - CompressSirius c({}); - return c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, - parameters); -#else - throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " - "with MHS, can't use Sirius compressor\n"); #endif } else if (compressorType == "sz") { #ifdef ADIOS2_HAVE_SZ - CompressSZ c({}); + CompressSZ c; return c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, parameters); #else @@ -104,7 +93,7 @@ size_t CompressorFactory::Compress(const char *dataIn, const Dims &blockStart, else if (compressorType == "zfp") { #ifdef ADIOS2_HAVE_ZFP - CompressZFP c({}); + CompressZFP c; return c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, parameters); #else @@ -124,7 +113,7 @@ size_t CompressorFactory::Decompress(const char *bufferIn, const size_t sizeIn, if (compressorType == Operator::OperatorType::BLOSC) { #ifdef ADIOS2_HAVE_BLOSC - compress::CompressBlosc op({}); + compress::CompressBlosc op; return op.InverseOperate(bufferIn, sizeIn, dataOut); #else throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " @@ -134,7 +123,7 @@ size_t CompressorFactory::Decompress(const char *bufferIn, const size_t sizeIn, else if (compressorType == Operator::OperatorType::BZIP2) { #ifdef ADIOS2_HAVE_BZIP2 - compress::CompressBZIP2 op({}); + compress::CompressBZIP2 op; return op.InverseOperate(bufferIn, sizeIn, dataOut); #else throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " @@ -144,7 +133,7 @@ size_t CompressorFactory::Decompress(const char *bufferIn, const size_t sizeIn, else if (compressorType == Operator::OperatorType::LIBPRESSIO) { #ifdef ADIOS2_HAVE_LIBPRESSIO - compress::CompressLibPressio op({}); + compress::CompressLibPressio op; return op.InverseOperate(bufferIn, sizeIn, dataOut); #else throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " @@ -154,7 +143,7 @@ size_t CompressorFactory::Decompress(const char *bufferIn, const size_t sizeIn, else if (compressorType == Operator::OperatorType::MGARD) { #ifdef ADIOS2_HAVE_MGARD - compress::CompressMGARD op({}); + compress::CompressMGARD op; return op.InverseOperate(bufferIn, sizeIn, dataOut); #else throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " @@ -164,7 +153,7 @@ size_t CompressorFactory::Decompress(const char *bufferIn, const size_t sizeIn, else if (compressorType == Operator::OperatorType::PNG) { #ifdef ADIOS2_HAVE_PNG - compress::CompressPNG op({}); + compress::CompressPNG op; return op.InverseOperate(bufferIn, sizeIn, dataOut); #else throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " @@ -174,7 +163,7 @@ size_t CompressorFactory::Decompress(const char *bufferIn, const size_t sizeIn, else if (compressorType == Operator::OperatorType::SIRIUS) { #ifdef ADIOS2_HAVE_MHS - compress::CompressSirius op({}); + compress::CompressSirius op; return op.InverseOperate(bufferIn, sizeIn, dataOut); #else throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " @@ -184,7 +173,7 @@ size_t CompressorFactory::Decompress(const char *bufferIn, const size_t sizeIn, else if (compressorType == Operator::OperatorType::Sz) { #ifdef ADIOS2_HAVE_SZ - compress::CompressSZ op({}); + compress::CompressSZ op; return op.InverseOperate(bufferIn, sizeIn, dataOut); #else throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " @@ -194,7 +183,7 @@ size_t CompressorFactory::Decompress(const char *bufferIn, const size_t sizeIn, else if (compressorType == Operator::OperatorType::ZFP) { #ifdef ADIOS2_HAVE_ZFP - compress::CompressZFP op({}); + compress::CompressZFP op; return op.InverseOperate(bufferIn, sizeIn, dataOut); #else throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp index e06a906fc1..0543e70b6c 100644 --- a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp @@ -301,13 +301,19 @@ BP5Deserializer::ControlInfo *BP5Deserializer::BuildControl(FMFormat Format) return ret; } -void BP5Deserializer::ReverseDimensions(size_t *Dimensions, int count) +void BP5Deserializer::ReverseDimensions(size_t *Dimensions, int count, + int times) { - for (int i = 0; i < count / 2; i++) + int Offset = 0; + for (int j = 0; j < times; j++) { - size_t tmp = Dimensions[i]; - Dimensions[i] = Dimensions[count - i - 1]; - Dimensions[count - i - 1] = tmp; + for (int i = 0; i < count / 2; i++) + { + size_t tmp = Dimensions[Offset + i]; + Dimensions[Offset + i] = Dimensions[Offset + count - i - 1]; + Dimensions[Offset + count - i - 1] = tmp; + } + Offset += count; } } @@ -502,14 +508,18 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen, (ControlFields[i].OrigShapeID == ShapeID::LocalArray)) { MetaArrayRec *meta_base = (MetaArrayRec *)field_data; + size_t BlockCount = + meta_base->Dims ? meta_base->DBCount / meta_base->Dims : 1; if ((meta_base->Dims > 1) && (m_WriterIsRowMajor != m_ReaderIsRowMajor)) { /* if we're getting data from someone of the other array gender, * switcheroo */ - ReverseDimensions(meta_base->Shape, meta_base->Dims); - ReverseDimensions(meta_base->Count, meta_base->Dims); - ReverseDimensions(meta_base->Offsets, meta_base->Dims); + ReverseDimensions(meta_base->Shape, meta_base->Dims, 1); + ReverseDimensions(meta_base->Count, meta_base->Dims, + BlockCount); + ReverseDimensions(meta_base->Offsets, meta_base->Dims, + BlockCount); } if ((WriterRank == 0) || (VarRec->GlobalDims == NULL)) { @@ -533,8 +543,6 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen, } VarRec->DimCount = meta_base->Dims; - size_t BlockCount = - meta_base->Dims ? meta_base->DBCount / meta_base->Dims : 1; if (!m_RandomAccessMode) { if (WriterRank == 0) @@ -921,6 +929,7 @@ bool BP5Deserializer::NeedWriter(BP5ArrayRequest Req, size_t WriterRank, // else Global case for (size_t i = 0; i < writer_meta_base->BlockCount; i++) { + bool NeedThisBlock = true; for (size_t j = 0; j < writer_meta_base->Dims; j++) { size_t SelOffset = Req.Start[j]; @@ -933,17 +942,19 @@ bool BP5Deserializer::NeedWriter(BP5ArrayRequest Req, size_t WriterRank, RankSize = writer_meta_base->Count[i * writer_meta_base->Dims + j]; if ((SelSize == 0) || (RankSize == 0)) { - return false; + NeedThisBlock = false; } if ((RankOffset < SelOffset && (RankOffset + RankSize) <= SelOffset) || (RankOffset >= SelOffset + SelSize)) { - return false; + NeedThisBlock = false; } } + if (NeedThisBlock) + return true; } - return true; + return false; } std::vector @@ -1183,10 +1194,16 @@ void BP5Deserializer::ExtractSelectionFromPartialRM( size_t Left = MAX(PartialOffsets[Dim], SelectionOffsets[Dim]); size_t Right = MIN(PartialOffsets[Dim] + PartialCounts[Dim], SelectionOffsets[Dim] + SelectionCounts[Dim]); + if (Right <= Left) + BlockSize = 0; BlockSize *= (Right - Left); break; } } + if (BlockSize == 0) + { + return; + } if (OperantDims > 0) { SourceBlockStride = PartialCounts[OperantDims - 1] * OperantElementSize; @@ -1203,10 +1220,16 @@ void BP5Deserializer::ExtractSelectionFromPartialRM( SelectionOffsets[Dim] + SelectionCounts[Dim]); if ((OperantDims != 0) && (Dim < OperantDims - 1)) { + if (Right <= Left) + BlockCount = 0; BlockCount *= (Right - Left); } FirstIndex[Dim] = Left; } + if (BlockCount == 0) + { + return; + } size_t *SelectionIndex = (size_t *)malloc(Dims * sizeof(SelectionIndex[0])); MapGlobalToLocalIndex(Dims, FirstIndex, SelectionOffsets, SelectionIndex); DestBlockStartOffset = FindOffset(Dims, SelectionCounts, SelectionIndex); @@ -1271,7 +1294,10 @@ void BP5Deserializer::ExtractSelectionFromPartialCM( int Left = MAX(PartialOffsets[Dim], SelectionOffsets[Dim]); int Right = MIN(PartialOffsets[Dim] + PartialCounts[Dim], SelectionOffsets[Dim] + SelectionCounts[Dim]); + if (Right <= Left) + BlockSize = 0; BlockSize *= (Right - Left); + break; } } @@ -1281,6 +1307,10 @@ void BP5Deserializer::ExtractSelectionFromPartialCM( DestBlockStride = SelectionCounts[0] * OperantElementSize; } + if (BlockSize == 0) + { + return; + } /* calculate first selected element and count */ BlockCount = 1; size_t *FirstIndex = (size_t *)malloc(Dims * sizeof(FirstIndex[0])); @@ -1291,10 +1321,17 @@ void BP5Deserializer::ExtractSelectionFromPartialCM( SelectionOffsets[Dim] + SelectionCounts[Dim]); if (Dim > 0) { + if (Right <= Left) + BlockCount = 0; BlockCount *= (Right - Left); } FirstIndex[Dim] = Left; } + if (BlockCount == 0) + { + free(FirstIndex); + return; + } size_t *SelectionIndex = (size_t *)malloc(Dims * sizeof(SelectionIndex[0])); MapGlobalToLocalIndex(Dims, FirstIndex, SelectionOffsets, SelectionIndex); DestBlockStartOffset = FindOffsetCM(Dims, SelectionCounts, SelectionIndex); @@ -1304,7 +1341,6 @@ void BP5Deserializer::ExtractSelectionFromPartialCM( size_t *PartialIndex = (size_t *)malloc(Dims * sizeof(PartialIndex[0])); MapGlobalToLocalIndex(Dims, FirstIndex, PartialOffsets, PartialIndex); SourceBlockStartOffset = FindOffsetCM(Dims, PartialCounts, PartialIndex); - free(PartialIndex); SourceBlockStartOffset *= OperantElementSize; @@ -1410,6 +1446,27 @@ Engine::MinVarInfo *BP5Deserializer::MinBlocksInfo(const VariableBase &Var, MV->IsReverseDims = ((MV->Dims > 1) && (m_WriterIsRowMajor != m_ReaderIsRowMajor)); + if ((VarRec->OrigShapeID == ShapeID::LocalValue) || + (VarRec->OrigShapeID == ShapeID::GlobalValue)) + { + MV->IsValue = true; + MV->BlocksInfo.reserve(m_WriterCohortSize); + for (size_t WriterRank = 0; WriterRank < m_WriterCohortSize; + WriterRank++) + { + MetaArrayRec *writer_meta_base = + (MetaArrayRec *)GetMetadataBase(VarRec, Step, WriterRank); + if (writer_meta_base) + { + Engine::MinBlockInfo Blk; + Blk.WriterID = WriterRank; + Blk.BlockID = 0; + Blk.BufferP = writer_meta_base; + MV->BlocksInfo.push_back(Blk); + } + } + return MV; + } size_t Id = 0; for (size_t WriterRank = 0; WriterRank < m_WriterCohortSize; WriterRank++) { diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.h b/source/adios2/toolkit/format/bp5/BP5Deserializer.h index c7d7e148a8..c7b80ff088 100644 --- a/source/adios2/toolkit/format/bp5/BP5Deserializer.h +++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.h @@ -150,7 +150,7 @@ class BP5Deserializer : virtual public BP5Base BP5VarRec *LookupVarByKey(void *Key); BP5VarRec *LookupVarByName(const char *Name); BP5VarRec *CreateVarRec(const char *ArrayName); - void ReverseDimensions(size_t *Dimensions, int count); + void ReverseDimensions(size_t *Dimensions, int count, int times); void BreakdownVarName(const char *Name, char **base_name_p, DataType *type_p, int *element_size_p); void BreakdownArrayName(const char *Name, char **base_name_p, diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.h b/source/adios2/toolkit/format/dataman/DataManSerializer.h index 5699604b14..79590fb1a7 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.h +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.h @@ -155,22 +155,6 @@ class DataManSerializer OperatorMap GetOperatorMap(); private: - template - void PutZfp(nlohmann::json &metaj, size_t &datasize, const T *inputData, - const Dims &varCount, const Params ¶ms); - - template - void PutSz(nlohmann::json &metaj, size_t &datasize, const T *inputData, - const Dims &varCount, const Params ¶ms); - - template - void PutBZip2(nlohmann::json &metaj, size_t &datasize, const T *inputData, - const Dims &varCount, const Params ¶ms); - - template - void PutMgard(nlohmann::json &metaj, size_t &datasize, const T *inputData, - const Dims &varCount, const Params ¶ms); - template void PutAttribute(const core::Attribute &attribute); diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.tcc b/source/adios2/toolkit/format/dataman/DataManSerializer.tcc index 80bee2c68c..c5b788d220 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.tcc +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.tcc @@ -145,86 +145,35 @@ void DataManSerializer::PutData( } size_t datasize = 0; - bool compressed = false; std::string compressionMethod; + bool compressed = false; if (not ops.empty()) { compressionMethod = ops[0].Op->m_Type; std::transform(compressionMethod.begin(), compressionMethod.end(), compressionMethod.begin(), ::tolower); - if (compressionMethod == "zfp") + + m_CompressBuffer.reserve(std::accumulate(varCount.begin(), + varCount.end(), sizeof(T), + std::multiplies())); + + if (IsCompressionAvailable(compressionMethod, helper::GetDataType(), + varCount)) { - if (IsCompressionAvailable(compressionMethod, - helper::GetDataType(), varCount)) + try { - try - { - PutZfp(metaj, datasize, inputData, varCount, - ops[0].Parameters); - compressed = true; - } - catch (std::exception &e) - { - std::cout << e.what() << std::endl; - } + core::compress::CompressorFactory c; + datasize = c.Compress( + reinterpret_cast(inputData), varStart, + varCount, helper::GetDataType(), m_CompressBuffer.data(), + ops[0].Parameters, compressionMethod); + compressed = true; } - } - else if (compressionMethod == "sz") - { - if (IsCompressionAvailable(compressionMethod, - helper::GetDataType(), varCount)) + catch (std::exception &e) { - try - { - PutSz(metaj, datasize, inputData, varCount, - ops[0].Parameters); - compressed = true; - } - catch (std::exception &e) - { - std::cout << e.what() << std::endl; - } + std::cout << e.what() << std::endl; } } - else if (compressionMethod == "bzip2") - { - if (IsCompressionAvailable(compressionMethod, - helper::GetDataType(), varCount)) - { - try - { - PutBZip2(metaj, datasize, inputData, varCount, - ops[0].Parameters); - compressed = true; - } - catch (std::exception &e) - { - std::cout << e.what() << std::endl; - } - } - } - else if (compressionMethod == "mgard") - { - if (IsCompressionAvailable(compressionMethod, - helper::GetDataType(), varCount)) - { - try - { - PutMgard(metaj, datasize, inputData, varCount, - ops[0].Parameters); - compressed = true; - } - catch (std::exception &e) - { - std::cout << e.what() << std::endl; - } - } - } - else - { - throw(std::invalid_argument("Compression method " + - compressionMethod + " not supported.")); - } } if (compressed) @@ -371,114 +320,6 @@ int DataManSerializer::GetData(T *outputData, const std::string &varName, return 0; } -template -void DataManSerializer::PutZfp(nlohmann::json &metaj, size_t &datasize, - const T *inputData, const Dims &varCount, - const Params ¶ms) -{ - PERFSTUBS_SCOPED_TIMER_FUNC(); -#ifdef ADIOS2_HAVE_ZFP - core::compress::CompressZFP compressor(params); - m_CompressBuffer.reserve(std::accumulate(varCount.begin(), varCount.end(), - sizeof(T), - std::multiplies())); - try - { - datasize = compressor.Operate(reinterpret_cast(inputData), - {}, varCount, helper::GetDataType(), - m_CompressBuffer.data(), params); - } - catch (std::exception &e) - { - throw(e); - } -#else - throw(std::invalid_argument( - "ZFP compression used but ZFP library is not linked to ADIOS2")); -#endif -} - -template -void DataManSerializer::PutSz(nlohmann::json &metaj, size_t &datasize, - const T *inputData, const Dims &varCount, - const Params ¶ms) -{ - PERFSTUBS_SCOPED_TIMER_FUNC(); -#ifdef ADIOS2_HAVE_SZ - m_CompressBuffer.reserve(std::accumulate(varCount.begin(), varCount.end(), - sizeof(T), - std::multiplies())); - core::compress::CompressSZ compressor(params); - try - { - datasize = compressor.Operate(reinterpret_cast(inputData), - {}, varCount, helper::GetDataType(), - m_CompressBuffer.data(), params); - } - catch (std::exception &e) - { - throw(e); - } -#else - throw(std::invalid_argument( - "SZ compression used but SZ library is not linked to ADIOS2")); -#endif -} - -template -void DataManSerializer::PutBZip2(nlohmann::json &metaj, size_t &datasize, - const T *inputData, const Dims &varCount, - const Params ¶ms) -{ - PERFSTUBS_SCOPED_TIMER_FUNC(); -#ifdef ADIOS2_HAVE_BZIP2 - m_CompressBuffer.reserve(std::accumulate(varCount.begin(), varCount.end(), - sizeof(T), - std::multiplies())); - core::compress::CompressBZIP2 compressor(params); - try - { - datasize = compressor.Operate(reinterpret_cast(inputData), - {}, varCount, helper::GetDataType(), - m_CompressBuffer.data(), params); - } - catch (std::exception &e) - { - throw(e); - } -#else - throw(std::invalid_argument( - "BZip2 compression used but BZip2 library is not linked to ADIOS2")); -#endif -} - -template -void DataManSerializer::PutMgard(nlohmann::json &metaj, size_t &datasize, - const T *inputData, const Dims &varCount, - const Params ¶ms) -{ - PERFSTUBS_SCOPED_TIMER_FUNC(); -#ifdef ADIOS2_HAVE_MGARD - core::compress::CompressMGARD compressor(params); - m_CompressBuffer.reserve(std::accumulate(varCount.begin(), varCount.end(), - sizeof(T), - std::multiplies())); - try - { - datasize = compressor.Operate(reinterpret_cast(inputData), - {}, varCount, helper::GetDataType(), - m_CompressBuffer.data(), params); - } - catch (std::exception &e) - { - throw(e); - } -#else - throw(std::invalid_argument( - "MGARD compression used but MGARD library is not linked to ADIOS2")); -#endif -} - template void DataManSerializer::PutAttribute(const core::Attribute &attribute) {