diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index e8551ff049..976075f3ac 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -77,6 +77,7 @@ add_library(adios2_core toolkit/format/bp/bp4/BP4Base.cpp toolkit/format/bp/bp4/BP4Serializer.cpp toolkit/format/bp/bp4/BP4Serializer.tcc toolkit/format/bp/bp4/BP4Deserializer.cpp toolkit/format/bp/bp4/BP4Deserializer.tcc + toolkit/format/bp/bpBackCompatOperation/compress/BPBackCompatBlosc.cpp toolkit/profiling/iochrono/Timer.cpp toolkit/profiling/iochrono/IOChrono.cpp diff --git a/source/adios2/toolkit/format/bp/BPBase.cpp b/source/adios2/toolkit/format/bp/BPBase.cpp index 8099080e2b..5a8b0d0efc 100644 --- a/source/adios2/toolkit/format/bp/BPBase.cpp +++ b/source/adios2/toolkit/format/bp/BPBase.cpp @@ -13,6 +13,8 @@ #include "adios2/helper/adiosFunctions.h" +#include "adios2/toolkit/format/bp/bpBackCompatOperation/compress/BPBackCompatBlosc.h" + namespace adios2 { namespace format @@ -487,5 +489,16 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation) size_t BPBase::DebugGetDataBufferSize() const { return m_Data.DebugGetSize(); } +std::shared_ptr +BPBase::SetBPBackCompatOperation(const std::string type) const noexcept +{ + std::shared_ptr bpOp; + if (type == "blosc") + { + bpOp = std::make_shared(); + } + return bpOp; +} + } // end namespace format } // end namespace adios2 diff --git a/source/adios2/toolkit/format/bp/BPBase.h b/source/adios2/toolkit/format/bp/BPBase.h index 7daa7cce48..6f4db0984a 100644 --- a/source/adios2/toolkit/format/bp/BPBase.h +++ b/source/adios2/toolkit/format/bp/BPBase.h @@ -21,6 +21,7 @@ #include "adios2/common/ADIOSTypes.h" #include "adios2/helper/adiosFunctions.h" #include "adios2/toolkit/aggregator/mpi/MPIChain.h" +#include "adios2/toolkit/format/bp/bpBackCompatOperation/BPBackCompatOperation.h" #include "adios2/toolkit/format/buffer/Buffer.h" #include "adios2/toolkit/format/buffer/heap/BufferSTL.h" #include "adios2/toolkit/profiling/iochrono/IOChrono.h" @@ -144,6 +145,10 @@ class BPBase uint64_t VarsIndexStart = 0; uint64_t AttributesIndexStart = 0; int8_t Version = -1; + uint8_t ADIOSVersionMajor = 0; + uint8_t ADIOSVersionMinor = 0; + uint8_t ADIOSVersionPatch = 0; + uint32_t ADIOSVersion = 0; // major*1M + minor*1k + patch e.g. 2007001 bool IsLittleEndian = true; bool HasSubFiles = false; @@ -472,6 +477,14 @@ class BPBase TransformTypes TransformTypeEnum(const std::string transformType) const noexcept; + /** + * Returns the proper derived class for BPOperation based on type + * @param type input, must be a supported type under BPOperation + * @return derived class if supported, false pointer if type not supported + */ + std::shared_ptr + SetBPBackCompatOperation(const std::string type) const noexcept; + struct ProcessGroupIndex { uint64_t Offset; diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Base.h b/source/adios2/toolkit/format/bp/bp4/BP4Base.h index 8fce378ffa..239f9de9d9 100644 --- a/source/adios2/toolkit/format/bp/bp4/BP4Base.h +++ b/source/adios2/toolkit/format/bp/bp4/BP4Base.h @@ -61,6 +61,9 @@ class BP4Base : virtual public BPBase static constexpr size_t m_ActiveFlagPosition = 38; static constexpr size_t m_VersionTagPosition = 0; static constexpr size_t m_VersionTagLength = 32; + static constexpr size_t m_VersionMajorPosition = 32; + static constexpr size_t m_VersionMinorPosition = 33; + static constexpr size_t m_VersionPatchPosition = 34; /** * Unique constructor diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.cpp b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.cpp index b3cd684bc9..485bd95959 100644 --- a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.cpp +++ b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.cpp @@ -114,6 +114,23 @@ void BP4Deserializer::ParseMetadataIndex(BufferSTL &bufferSTL, // This has no flag in BP4 header. Always true m_Minifooter.HasSubFiles = true; + // Writer's ADIOS version + position = m_VersionMajorPosition; + uint8_t ascii = helper::ReadValue(buffer, position, + m_Minifooter.IsLittleEndian); + m_Minifooter.ADIOSVersionMajor = ascii - (uint8_t)'0'; + position = m_VersionMinorPosition; + ascii = helper::ReadValue(buffer, position, + m_Minifooter.IsLittleEndian); + m_Minifooter.ADIOSVersionMinor = ascii - (uint8_t)'0'; + position = m_VersionPatchPosition; + ascii = helper::ReadValue(buffer, position, + m_Minifooter.IsLittleEndian); + m_Minifooter.ADIOSVersionPatch = ascii - (uint8_t)'0'; + m_Minifooter.ADIOSVersion = m_Minifooter.ADIOSVersionMajor * 1000000 + + m_Minifooter.ADIOSVersionMinor * 1000 + + m_Minifooter.ADIOSVersionPatch; + // BP version position = m_BPVersionPosition; m_Minifooter.Version = helper::ReadValue( diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.h b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.h index a4bd65c79d..f8812ac487 100644 --- a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.h +++ b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.h @@ -119,6 +119,9 @@ class BP4Deserializer : virtual public BP4Base const bool isRowMajorDestination, const size_t threadID = 0); + void BackCompatDecompress(const helper::SubStreamBoxInfo &subStreamBoxInfo, + const size_t threadID = 0); + /** * Clips and assigns memory to blockInfo.Data from a contiguous memory * input diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc index b09ee68eaf..9668619fd2 100644 --- a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc +++ b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc @@ -158,8 +158,30 @@ void BP4Deserializer::SetVariableBlockInfo( blockOperation.PreSizeOf = sizeof(T); // read metadata from supported type and populate Info - std::memcpy(&blockOperation.PayloadSize, bpOpInfo.Metadata.data() + 8, - 8); + if (m_Minifooter.ADIOSVersion >= 2008000) + { + std::memcpy(&blockOperation.PayloadSize, + bpOpInfo.Metadata.data() + 8, 8); + } + else + { + // Files made before 2.8 have incompatible compression operators + // Add backward compatible fixes here to parse compression metadata + // directly from the BP4 metadata format + std::shared_ptr bpOp = + SetBPBackCompatOperation(bpOpInfo.Type); + if (bpOp) + { + bpOp->GetMetadata(bpOpInfo.Metadata, blockOperation.Info); + blockOperation.PayloadSize = static_cast( + std::stoull(blockOperation.Info.at("OutputSize"))); + } + else + { + std::memcpy(&blockOperation.PayloadSize, + bpOpInfo.Metadata.data() + 8, 8); + } + } subStreamInfo.OperationsInfo.push_back(std::move(blockOperation)); }; @@ -520,34 +542,43 @@ void BP4Deserializer::PostDataRead( { if (subStreamBoxInfo.OperationsInfo.size() > 0) { - const helper::BlockOperationInfo &blockOperationInfo = - InitPostOperatorBlockData(subStreamBoxInfo.OperationsInfo); + if (m_Minifooter.ADIOSVersion >= 2008000) + { + const helper::BlockOperationInfo &blockOperationInfo = + InitPostOperatorBlockData(subStreamBoxInfo.OperationsInfo); - const size_t preOpPayloadSize = - helper::GetTotalSize(blockOperationInfo.PreCount) * - blockOperationInfo.PreSizeOf; - m_ThreadBuffers[threadID][0].resize(preOpPayloadSize); + const size_t preOpPayloadSize = + helper::GetTotalSize(blockOperationInfo.PreCount) * + blockOperationInfo.PreSizeOf; + m_ThreadBuffers[threadID][0].resize(preOpPayloadSize); - // get original block back - char *preOpData = m_ThreadBuffers[threadID][0].data(); - const char *postOpData = m_ThreadBuffers[threadID][1].data(); + // get original block back + char *preOpData = m_ThreadBuffers[threadID][0].data(); + const char *postOpData = m_ThreadBuffers[threadID][1].data(); - std::shared_ptr op = nullptr; - for (auto &o : blockInfo.Operations) - { - if (o->m_Category == "compress" || o->m_Category == "plugin") + std::shared_ptr op = nullptr; + for (auto &o : blockInfo.Operations) { - op = o; - break; + if (o->m_Category == "compress" || o->m_Category == "plugin") + { + op = o; + break; + } } - } - core::Decompress(postOpData, blockOperationInfo.PayloadSize, preOpData, - op); + core::Decompress(postOpData, blockOperationInfo.PayloadSize, + preOpData, op); - // clip block to match selection - helper::ClipVector(m_ThreadBuffers[threadID][0], - subStreamBoxInfo.Seeks.first, - subStreamBoxInfo.Seeks.second); + // clip block to match selection + helper::ClipVector(m_ThreadBuffers[threadID][0], + subStreamBoxInfo.Seeks.first, + subStreamBoxInfo.Seeks.second); + } + else + { + // Files made before 2.8 have incompatible compression operators + // Add backward compatible fixes in the function below + BackCompatDecompress(subStreamBoxInfo, threadID); + } } #ifdef ADIOS2_HAVE_ENDIAN_REVERSE @@ -570,6 +601,48 @@ void BP4Deserializer::PostDataRead( endianReverse, blockInfo.IsGPU); } +void BP4Deserializer::BackCompatDecompress( + const helper::SubStreamBoxInfo &subStreamBoxInfo, const size_t threadID) +{ + // Files made before 2.8 have incompatible compression operators + // Add backward compatible fixes here + const helper::BlockOperationInfo &blockOperationInfo = + InitPostOperatorBlockData(subStreamBoxInfo.OperationsInfo); + + const size_t preOpPayloadSize = + helper::GetTotalSize(blockOperationInfo.PreCount) * + blockOperationInfo.PreSizeOf; + m_ThreadBuffers[threadID][0].resize(preOpPayloadSize); + + std::string opType = blockOperationInfo.Info.at("Type"); + + // get original block back + char *preOpData = m_ThreadBuffers[threadID][0].data(); + const char *postOpData = m_ThreadBuffers[threadID][1].data(); + // get the right bp4Op + std::shared_ptr bp4Op = + SetBPBackCompatOperation(opType); + + if (bp4Op) + { + bp4Op->GetData(postOpData, blockOperationInfo, preOpData); + // clip block to match selection + helper::ClipVector(m_ThreadBuffers[threadID][0], + subStreamBoxInfo.Seeks.first, + subStreamBoxInfo.Seeks.second); + } + else + { + helper::Throw( + "Toolkit", "format::bp::BP4Deserializer", "PostDataRead", + "This file was created by pre-ADIOS 2.8.0 using " + "compression type " + + opType + + ", for which there is no backward compatible reader in this " + "ADIOS version"); + } +} + template std::map::BPInfo>> BP4Deserializer::AllStepsBlocksInfo(const core::Variable &variable) const diff --git a/source/adios2/toolkit/format/bp/bpBackCompatOperation/BPBackCompatOperation.h b/source/adios2/toolkit/format/bp/bpBackCompatOperation/BPBackCompatOperation.h new file mode 100644 index 0000000000..901b55d0d1 --- /dev/null +++ b/source/adios2/toolkit/format/bp/bpBackCompatOperation/BPBackCompatOperation.h @@ -0,0 +1,49 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * BPOperation.h : + * + * Created on: Jul 12, 2018 + * Author: William F Godoy godoywf@ornl.gov + */ + +#ifndef ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_BPOPERATION_H_ +#define ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_BPOPERATION_H_ + +#include +#include + +#include "adios2/common/ADIOSMacros.h" +#include "adios2/core/Variable.h" +#include "adios2/helper/adiosFunctions.h" +#include "adios2/toolkit/format/buffer/heap/BufferSTL.h" + +namespace adios2 +{ +namespace format +{ + +class BPBackCompatOperation +{ +public: + BPBackCompatOperation() = default; + virtual ~BPBackCompatOperation() = default; + + /** + * Deserializes metadata in the form of parameters + * @param buffer contains serialized metadata buffer + * @param info parameters info from metadata buffer + */ + virtual void GetMetadata(const std::vector &buffer, + Params &info) const noexcept = 0; + + virtual void GetData(const char *input, + const helper::BlockOperationInfo &blockOperationInfo, + char *dataOutput) const = 0; +}; + +} // end namespace format +} // end namespace adios2 + +#endif /* ADIOS2_TOOLKIT_FORMAT_BP_OPERATION_BPOPERATION_H_ */ diff --git a/source/adios2/toolkit/format/bp/bpBackCompatOperation/compress/BPBackCompatBlosc.cpp b/source/adios2/toolkit/format/bp/bpBackCompatOperation/compress/BPBackCompatBlosc.cpp new file mode 100644 index 0000000000..3a64ec2100 --- /dev/null +++ b/source/adios2/toolkit/format/bp/bpBackCompatOperation/compress/BPBackCompatBlosc.cpp @@ -0,0 +1,193 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * BPBlosc.cpp + * + * Created on: Jun 21, 2019 + * Author: William F Godoy godoywf@ornl.gov + */ + +#include "BPBackCompatBlosc.h" + +#include "adios2/helper/adiosFunctions.h" + +#ifdef ADIOS2_HAVE_BLOSC +#include "adios2/operator/compress/CompressBlosc.h" +extern "C" { +#include +} +#endif + +#include +#include +#include +#include + +namespace adios2 +{ +namespace format +{ + +void BPBackCompatBlosc::GetMetadata(const std::vector &buffer, + Params &info) const noexcept +{ + size_t position = 0; + info["InputSize"] = + std::to_string(helper::ReadValue(buffer, position)); + info["OutputSize"] = + std::to_string(helper::ReadValue(buffer, position)); +} + +void BPBackCompatBlosc::GetData( + const char *input, const helper::BlockOperationInfo &blockOperationInfo, + char *dataOutput) const +{ +#ifdef ADIOS2_HAVE_BLOSC + core::compress::CompressBlosc op((Params())); + const size_t sizeOut = (sizeof(size_t) == 8) + ? static_cast(helper::StringTo( + blockOperationInfo.Info.at("InputSize"), + "when reading Blosc input size")) + : static_cast(helper::StringTo( + blockOperationInfo.Info.at("InputSize"), + "when reading Blosc input size")); + + Params &info = const_cast(blockOperationInfo.Info); + Decompress(input, blockOperationInfo.PayloadSize, dataOutput, sizeOut, + info); + +#else + throw std::runtime_error( + "ERROR: current ADIOS2 library didn't compile " + "with Blosc, can't read Blosc compressed data, in call " + "to Get\n"); +#endif +} + +#ifdef ADIOS2_HAVE_BLOSC + +size_t BPBackCompatBlosc::Decompress(const void *bufferIn, const size_t sizeIn, + void *dataOut, const size_t sizeOut, + Params &info) const +{ + assert(sizeIn >= sizeof(DataHeader)); + const bool isChunked = + reinterpret_cast(bufferIn)->IsChunked(); + + size_t decompressedSize = 0u; + if (isChunked) + decompressedSize = + DecompressChunkedFormat(bufferIn, sizeIn, dataOut, sizeOut, info); + else + decompressedSize = + DecompressOldFormat(bufferIn, sizeIn, dataOut, sizeOut, info); + + return decompressedSize; +} + +bool BPBackCompatBlosc::IsDataTypeValid(const DataType type) const +{ + return true; +} + +size_t BPBackCompatBlosc::DecompressChunkedFormat(const void *bufferIn, + const size_t sizeIn, + void *dataOut, + const size_t sizeOut, + Params &info) const +{ + const DataHeader *dataPtr = reinterpret_cast(bufferIn); + uint32_t num_chunks = dataPtr->GetNumChunks(); + size_t inputDataSize = sizeIn - sizeof(DataHeader); + + bool isCompressed = true; + if (num_chunks == 0) + isCompressed = false; + + size_t inputOffset = 0u; + size_t currentOutputSize = 0u; + + const uint8_t *inputDataBuff = + reinterpret_cast(bufferIn) + sizeof(DataHeader); + + size_t uncompressedSize = sizeOut; + + if (isCompressed) + { + blosc_init(); + uint8_t *outputBuff = reinterpret_cast(dataOut); + + while (inputOffset < inputDataSize) + { + /* move over the size of the compressed data */ + const uint8_t *in_ptr = inputDataBuff + inputOffset; + + /** read the size of the compress block from the blosc meta data + * + * blosc meta data format (all little endian): + * - 1 byte blosc format version + * - 1 byte blosclz format version + * - 1 byte flags + * - 1 byte typesize + * - 4 byte uncompressed data size + * - 4 byte block size + * - 4 byte compressed data size + * + * we need only the compressed size ( source address + 12 byte) + */ + bloscSize_t max_inputDataSize = + *reinterpret_cast(in_ptr + 12u); + + uint8_t *out_ptr = outputBuff + currentOutputSize; + + size_t outputChunkSize = + std::min(uncompressedSize - currentOutputSize, + static_cast(BLOSC_MAX_BUFFERSIZE)); + bloscSize_t max_output_size = + static_cast(outputChunkSize); + + bloscSize_t decompressdSize = + blosc_decompress(in_ptr, out_ptr, max_output_size); + + if (decompressdSize > 0) + currentOutputSize += static_cast(decompressdSize); + else + { + throw std::runtime_error( + "ERROR: ADIOS2 Blosc Decompress failed. Decompressed chunk " + "results in zero decompressed bytes.\n"); + } + inputOffset += static_cast(max_inputDataSize); + } + blosc_destroy(); + } + else + { + std::memcpy(dataOut, inputDataBuff, inputDataSize); + currentOutputSize = inputDataSize; + inputOffset += inputDataSize; + } + + assert(currentOutputSize == uncompressedSize); + assert(inputOffset == inputDataSize); + + return currentOutputSize; +} + +size_t BPBackCompatBlosc::DecompressOldFormat(const void *bufferIn, + const size_t sizeIn, + void *dataOut, + const size_t sizeOut, + Params &info) const +{ + blosc_init(); + const int decompressedSize = blosc_decompress(bufferIn, dataOut, sizeOut); + blosc_destroy(); + return static_cast(decompressedSize); +} + +#endif // ADIOS2_HAVE_BLOSC + +} // end namespace format +} // end namespace adios2 diff --git a/source/adios2/toolkit/format/bp/bpBackCompatOperation/compress/BPBackCompatBlosc.h b/source/adios2/toolkit/format/bp/bpBackCompatOperation/compress/BPBackCompatBlosc.h new file mode 100644 index 0000000000..eb773a1f74 --- /dev/null +++ b/source/adios2/toolkit/format/bp/bpBackCompatOperation/compress/BPBackCompatBlosc.h @@ -0,0 +1,107 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * BPBlosc.h + * + * Created on: Jun 21, 2019 + * Author: William F Godoy godoywf@ornl.gov + */ + +#ifndef ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPBLOSC_H_ +#define ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPBLOSC_H_ + +#include "adios2/toolkit/format/bp/bpBackCompatOperation/BPBackCompatOperation.h" + +#if defined(_MSC_VER) +#define ADIOS2_CLASS_PACKED(name) __pragma(pack(push, 1)) class name +#define ADIOS2_CLASS_PACKED_SUFFIX __pragma(pack(pop)) +#else +#define ADIOS2_CLASS_PACKED(name) class __attribute__((packed)) name +#define ADIOS2_CLASS_PACKED_SUFFIX +#endif + +namespace adios2 +{ +namespace format +{ + +class BPBackCompatBlosc : public BPBackCompatOperation +{ +public: + BPBackCompatBlosc() = default; + + ~BPBackCompatBlosc() = default; + + void GetMetadata(const std::vector &buffer, Params &info) const + noexcept final; + + void GetData(const char *input, + const helper::BlockOperationInfo &blockOperationInfo, + char *dataOutput) const final; + +private: + /** + * Decompression signature for legacy libraries that use void* + * @param bufferIn + * @param sizeIn + * @param dataOut + * @param dimensions + * @param type + * @return size of decompressed buffer in bytes + */ + size_t Decompress(const void *bufferIn, const size_t sizeIn, void *dataOut, + const size_t sizeOut, Params &info) const; + + bool IsDataTypeValid(const DataType type) const; + + using bloscSize_t = int32_t; + + /** Decompress chunked data */ + size_t DecompressChunkedFormat(const void *bufferIn, const size_t sizeIn, + void *dataOut, const size_t sizeOut, + Params &info) const; + + /** Decompress data written before ADIOS2 supported large variables larger + * 2GiB. */ + size_t DecompressOldFormat(const void *bufferIn, const size_t sizeIn, + void *dataOut, const size_t sizeOut, + Params &info) const; + + ADIOS2_CLASS_PACKED(DataHeader) + { + /** compatible to the first 4 byte of blosc header + * + * blosc meta data format (all little endian): + * - 1 byte blosc format version + * - 1 byte blosclz format version + * - 1 byte flags + * - 1 byte typesize + * + * If zero we writing the new adios blosc format which can handle more + * than 2GiB data chunks. + */ + uint32_t format = 0u; + /** number of blosc chunks within the data blob + * + * If zero the data is not compressed and must be decompressed by using + * 'memcpy' + */ + uint32_t numberOfChunks = 0u; + + public: + void SetNumChunks(const uint32_t numChunks) + { + numberOfChunks = numChunks; + } + uint32_t GetNumChunks() const { return numberOfChunks; } + + bool IsChunked() const { return format == 0; } + } + ADIOS2_CLASS_PACKED_SUFFIX; +}; + +} // end namespace format +} // end namespace adios2 + +#endif /* ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPBLOSC_H_ */ diff --git a/testing/adios2/CMakeLists.txt b/testing/adios2/CMakeLists.txt index 9a840fdd3d..bddb8485f4 100644 --- a/testing/adios2/CMakeLists.txt +++ b/testing/adios2/CMakeLists.txt @@ -12,3 +12,4 @@ add_subdirectory(yaml) add_subdirectory(performance) add_subdirectory(helper) add_subdirectory(hierarchy) +add_subdirectory(backward_compatibility) diff --git a/testing/adios2/backward_compatibility/CMakeLists.txt b/testing/adios2/backward_compatibility/CMakeLists.txt new file mode 100644 index 0000000000..a956aff641 --- /dev/null +++ b/testing/adios2/backward_compatibility/CMakeLists.txt @@ -0,0 +1,9 @@ +#------------------------------------------------------------------------------# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +#------------------------------------------------------------------------------# + +gtest_add_tests_helper(ReadOldCompressed MPI_NONE BP4 Engine.BP. .BP4 + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} EXTRA_ARGS "BP4" +) + diff --git a/testing/adios2/backward_compatibility/TestBP4ReadOldCompressed.cpp b/testing/adios2/backward_compatibility/TestBP4ReadOldCompressed.cpp new file mode 100644 index 0000000000..90bc6dfcf4 --- /dev/null +++ b/testing/adios2/backward_compatibility/TestBP4ReadOldCompressed.cpp @@ -0,0 +1,137 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + */ +#include +#include + +#include +#include //std::iota +#include + +#include + +#include + +std::string engineName; // comes from command line +std::string engineParameters; // comes from command line + +class BP4ReadOldCompressed : public ::testing::Test +{ +public: + BP4ReadOldCompressed() = default; +}; + +class BP4ReadOldCompressedP +: public BP4ReadOldCompressed, + public ::testing::WithParamInterface> +{ +protected: + std::string GetFileName() { return std::get<0>(GetParam()); }; + bool GetSuccess() { return std::get<1>(GetParam()); }; +}; + +// Read an old file (BP4 written with ADIOS 2.7.1) and compressed by an old +// library. Backward compatible reader should work properly if success is +// expected, otherwise a runtime error should be thrown. +TEST_P(BP4ReadOldCompressedP, Read) +{ + const std::string fname = GetFileName(); + const bool expectSuccess = GetSuccess(); + std::cout << "\nRead old file " << fname << " and expect it "; + if (expectSuccess) + { + std::cout << "to succeed\n\n"; + } + else + { + std::cout << "to throw a runtime error\n\n"; + } + + adios2::ADIOS adios; + adios2::IO io = adios.DeclareIO("ReadIO"); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + if (!engineParameters.empty()) + { + io.SetParameters(engineParameters); + } + + adios2::Engine bpReader = io.Open(fname, adios2::Mode::ReadRandomAccess); + + auto var_a = io.InquireVariable("a"); + EXPECT_TRUE(var_a); + ASSERT_EQ(var_a.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_a.Steps(), 1); + ASSERT_EQ(var_a.Shape().size(), 2); + + std::vector R64; + + const adios2::Dims start{0, 0}; + const adios2::Dims count = var_a.Shape(); + const adios2::Box sel(start, count); + var_a.SetSelection(sel); + var_a.SetStepSelection({0, 1}); + // The actual read function fails if there is no backward compatible + // decompression + if (expectSuccess) + { + bpReader.Get(var_a, R64, adios2::Mode::Sync); + ASSERT_EQ(R64.size(), var_a.Shape()[0] * var_a.Shape()[1]); + } + else + { + ASSERT_THROW(bpReader.Get(var_a, R64, adios2::Mode::Sync), + std::runtime_error); + } + bpReader.Close(); +} + +std::vector> GenerateParameters() +{ + std::vector> v; + v.push_back(std::make_tuple("notcompressed_271.bp", true)); +#ifdef ADIOS2_HAVE_BLOSC + v.push_back(std::make_tuple("compressed_blosc_271.bp", true)); +#endif +#ifdef ADIOS2_HAVE_BZIP2 + v.push_back(std::make_tuple("compressed_bzip2_271.bp", false)); +#endif + return v; +} + +INSTANTIATE_TEST_SUITE_P(BP4ReadOldCompressed, BP4ReadOldCompressedP, + ::testing::ValuesIn(GenerateParameters())); + +//****************************************************************************** +// main +//****************************************************************************** + +int main(int argc, char **argv) +{ +#if ADIOS2_USE_MPI + MPI_Init(nullptr, nullptr); +#endif + + int result; + ::testing::InitGoogleTest(&argc, argv); + + if (argc > 1) + { + engineName = std::string(argv[1]); + } + if (argc > 2) + { + engineParameters = std::string(argv[2]); + } + result = RUN_ALL_TESTS(); + +#if ADIOS2_USE_MPI + MPI_Finalize(); +#endif + + return result; +} diff --git a/testing/adios2/backward_compatibility/compressed_blosc_271.bp/data.0 b/testing/adios2/backward_compatibility/compressed_blosc_271.bp/data.0 new file mode 100644 index 0000000000..3aa2913783 Binary files /dev/null and b/testing/adios2/backward_compatibility/compressed_blosc_271.bp/data.0 differ diff --git a/testing/adios2/backward_compatibility/compressed_blosc_271.bp/md.0 b/testing/adios2/backward_compatibility/compressed_blosc_271.bp/md.0 new file mode 100644 index 0000000000..86315efc5b Binary files /dev/null and b/testing/adios2/backward_compatibility/compressed_blosc_271.bp/md.0 differ diff --git a/testing/adios2/backward_compatibility/compressed_blosc_271.bp/md.idx b/testing/adios2/backward_compatibility/compressed_blosc_271.bp/md.idx new file mode 100644 index 0000000000..2a791bcfee Binary files /dev/null and b/testing/adios2/backward_compatibility/compressed_blosc_271.bp/md.idx differ diff --git a/testing/adios2/backward_compatibility/compressed_bzip2_271.bp/data.0 b/testing/adios2/backward_compatibility/compressed_bzip2_271.bp/data.0 new file mode 100644 index 0000000000..59e35133c4 Binary files /dev/null and b/testing/adios2/backward_compatibility/compressed_bzip2_271.bp/data.0 differ diff --git a/testing/adios2/backward_compatibility/compressed_bzip2_271.bp/md.0 b/testing/adios2/backward_compatibility/compressed_bzip2_271.bp/md.0 new file mode 100644 index 0000000000..a9a075cf2f Binary files /dev/null and b/testing/adios2/backward_compatibility/compressed_bzip2_271.bp/md.0 differ diff --git a/testing/adios2/backward_compatibility/compressed_bzip2_271.bp/md.idx b/testing/adios2/backward_compatibility/compressed_bzip2_271.bp/md.idx new file mode 100644 index 0000000000..8cf13840e8 Binary files /dev/null and b/testing/adios2/backward_compatibility/compressed_bzip2_271.bp/md.idx differ diff --git a/testing/adios2/backward_compatibility/create_compressed_file.txt b/testing/adios2/backward_compatibility/create_compressed_file.txt new file mode 100644 index 0000000000..b13da48600 --- /dev/null +++ b/testing/adios2/backward_compatibility/create_compressed_file.txt @@ -0,0 +1,21 @@ +# Build a pre-2.8 version of ADIOS2, e.g. 2.7.1.431 +# Add a compressor operator to create_compressed_file.xml and use it for variable 'a' in io_T1 +# Run +# $ ./bin/adios2_iotest -a 1 -c create_compressed_file.txt -x create_compressed_file.xml -w -t -d 1 +# +# Test read +# $ ./bin/bpls -la blosc.bp/ -d -n 20 +# +# Rename compressed.bp to compressed_2_271.bp +# + +group io_T1 + # item type varname N [dim1 dim2 ... dimN decomp1 decomp2 ... decompN] + array double a 2 100 200 X YZ + +# Task 1 actions +app 1 + steps 1 + write compressed.bp io_T1 + + diff --git a/testing/adios2/backward_compatibility/create_compressed_file.xml b/testing/adios2/backward_compatibility/create_compressed_file.xml new file mode 100644 index 0000000000..af24dcae10 --- /dev/null +++ b/testing/adios2/backward_compatibility/create_compressed_file.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + diff --git a/testing/adios2/backward_compatibility/notcompressed_271.bp/data.0 b/testing/adios2/backward_compatibility/notcompressed_271.bp/data.0 new file mode 100644 index 0000000000..dfa41dd46b Binary files /dev/null and b/testing/adios2/backward_compatibility/notcompressed_271.bp/data.0 differ diff --git a/testing/adios2/backward_compatibility/notcompressed_271.bp/md.0 b/testing/adios2/backward_compatibility/notcompressed_271.bp/md.0 new file mode 100644 index 0000000000..c02c29c039 Binary files /dev/null and b/testing/adios2/backward_compatibility/notcompressed_271.bp/md.0 differ diff --git a/testing/adios2/backward_compatibility/notcompressed_271.bp/md.idx b/testing/adios2/backward_compatibility/notcompressed_271.bp/md.idx new file mode 100644 index 0000000000..7a551459d4 Binary files /dev/null and b/testing/adios2/backward_compatibility/notcompressed_271.bp/md.idx differ