diff --git a/CMakeLists.txt b/CMakeLists.txt index aef4fb581d..8e411b1076 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -151,7 +151,7 @@ adios_option(CUDA "Enable support for Cuda" AUTO) adios_option(DataMan "Enable support for DataMan" AUTO) adios_option(DataSpaces "Enable support for DATASPACES" AUTO) adios_option(SSC "Enable support for SSC" AUTO) -adios_option(Table "Enable support for Table" AUTO) +adios_option(MHS "Enable support for MHS" AUTO) adios_option(SST "Enable support for SST" AUTO) adios_option(BP5 "Enable support for BP5" AUTO) adios_option(ZeroMQ "Enable support for ZeroMQ" AUTO) @@ -175,7 +175,7 @@ if(ADIOS2_HAVE_MPI) endif() set(ADIOS2_CONFIG_OPTS - Blosc BZip2 ZFP SZ MGARD PNG MPI DataMan CUDA DAOS Table SSC SST BP5 DataSpaces ZeroMQ HDF5 HDF5_VOL IME Python Fortran SysVShMem Profiling Endian_Reverse LIBPRESSIO + Blosc BZip2 ZFP SZ MGARD PNG MPI DataMan CUDA DAOS MHS SSC SST BP5 DataSpaces ZeroMQ HDF5 HDF5_VOL IME Python Fortran SysVShMem Profiling Endian_Reverse LIBPRESSIO ) GenerateADIOSHeaderConfig(${ADIOS2_CONFIG_OPTS}) configure_file( diff --git a/ReadMe.md b/ReadMe.md index ce7a713c54..664d2675dd 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -67,7 +67,7 @@ For a `cmake` configuration example see [scripts/runconf/runconf.sh](https://git Once ADIOS2 is installed refer to: -* [Linking ADIO 2](https://adios2.readthedocs.io/en/latest/setting_up/setting_up.html#linking-adios-2) +* [Linking ADIOS2](https://adios2.readthedocs.io/en/latest/setting_up/setting_up.html#linking-adios-2) ## Releases diff --git a/bindings/C/adios2/c/adios2_c_adios.cpp b/bindings/C/adios2/c/adios2_c_adios.cpp index 2f6dd6a615..79086ffdf4 100644 --- a/bindings/C/adios2/c/adios2_c_adios.cpp +++ b/bindings/C/adios2/c/adios2_c_adios.cpp @@ -17,6 +17,30 @@ extern "C" { #endif +adios2::ArrayOrdering adios2_ToArrayOrdering(const adios2_arrayordering Corder) +{ + adios2::ArrayOrdering order = adios2::ArrayOrdering::Auto; + switch (Corder) + { + + case adios2_arrayordering_rowmajor: + order = adios2::ArrayOrdering::RowMajor; + break; + + case adios2_arrayordering_columnmajor: + order = adios2::ArrayOrdering::ColumnMajor; + break; + + case adios2_arrayordering_auto: + order = adios2::ArrayOrdering::Auto; + break; + + default: + break; + } + return order; +} + adios2_adios *adios2_init_config_glue_serial(const char *config_file, const adios2_debug_mode debug_mode, const char *host_language) @@ -48,7 +72,8 @@ adios2_adios *adios2_init_serial() adios2_adios *adios2_init_config_serial(const char *config_file) { - return adios2_init_config_glue_serial("", adios2_debug_mode_off, "C"); + return adios2_init_config_glue_serial(config_file, adios2_debug_mode_off, + "C"); } adios2_io *adios2_declare_io(adios2_adios *adios, const char *name) @@ -68,6 +93,25 @@ adios2_io *adios2_declare_io(adios2_adios *adios, const char *name) return io; } +adios2_io *adios2_declare_io_order(adios2_adios *adios, const char *name, + adios2_arrayordering order) +{ + adios2_io *io = nullptr; + try + { + adios2::helper::CheckForNullptr( + adios, "for adios2_adios, in call to adios2_declare_io"); + io = reinterpret_cast( + &reinterpret_cast(adios)->DeclareIO( + name, adios2_ToArrayOrdering(order))); + } + catch (...) + { + adios2::helper::ExceptionToError("adios2_declare_io"); + } + return io; +} + adios2_io *adios2_at_io(adios2_adios *adios, const char *name) { adios2_io *io = nullptr; diff --git a/bindings/C/adios2/c/adios2_c_adios.h b/bindings/C/adios2/c/adios2_c_adios.h index 3bfa473969..1f135b9130 100644 --- a/bindings/C/adios2/c/adios2_c_adios.h +++ b/bindings/C/adios2/c/adios2_c_adios.h @@ -72,6 +72,16 @@ adios2_adios *adios2_init_config_serial(const char *config_file); */ adios2_io *adios2_declare_io(adios2_adios *adios, const char *name); +/** + * Declares a new io handler with specific array ordering + * @param adios owner the io handler + * @param name unique io identifier within current adios handler + * @param order array ordering + * @return success: handler, failure: NULL + */ +adios2_io *adios2_declare_io_order(adios2_adios *adios, const char *name, + adios2_arrayordering order); + /** * Retrieves a previously declared io handler by name * @param adios owner the io handler diff --git a/bindings/C/adios2/c/adios2_c_attribute.cpp b/bindings/C/adios2/c/adios2_c_attribute.cpp index 1c32816c71..98da6c6da1 100644 --- a/bindings/C/adios2/c/adios2_c_attribute.cpp +++ b/bindings/C/adios2/c/adios2_c_attribute.cpp @@ -162,6 +162,7 @@ adios2_error adios2_attribute_data(void *data, size_t *size, char *dataT = reinterpret_cast(data); attributeCpp->m_DataSingleValue.copy( dataT, attributeCpp->m_DataSingleValue.size()); + dataT[attributeCpp->m_DataSingleValue.size()] = '\0'; *size = 1; } else diff --git a/bindings/C/adios2/c/adios2_c_engine.cpp b/bindings/C/adios2/c/adios2_c_engine.cpp index eb2b7ced39..db3f36347f 100644 --- a/bindings/C/adios2/c/adios2_c_engine.cpp +++ b/bindings/C/adios2/c/adios2_c_engine.cpp @@ -640,6 +640,154 @@ adios2_error adios2_lock_reader_selections(adios2_engine *engine) } } +adios2_varinfo *adios2_inquire_blockinfo(adios2_engine *engine, + adios2_variable *variable, + const size_t step) +{ + auto lf_CopyDims = [](const std::vector &dims) -> size_t * { + size_t *a = nullptr; + size_t ndims = dims.size(); + if (ndims > 0) + { + a = (size_t *)malloc(dims.size() * sizeof(size_t)); + std::memcpy(a, dims.data(), dims.size() * sizeof(size_t)); + } + return a; + }; + + adios2_varinfo *varinfo = NULL; + + try + { + adios2::helper::CheckForNullptr( + engine, "for adios2_engine, in call to adios2_inquire_blockinfo"); + + adios2::core::Engine *engineCpp = + reinterpret_cast(engine); + + if (engineCpp->m_EngineType == "NULL") + { + return NULL; + } + adios2::helper::CheckForNullptr(variable, + "for adios2_variable, in call " + "to adios2_get"); + + adios2::core::VariableBase *variableBase = + reinterpret_cast(variable); + + const adios2::DataType type(variableBase->m_Type); + + const auto minBlocksInfo = + engineCpp->MinBlocksInfo(*variableBase, step); + + if (minBlocksInfo) + { + varinfo = (adios2_varinfo *)malloc(sizeof(adios2_varinfo)); + varinfo->nblocks = minBlocksInfo->BlocksInfo.size(); + varinfo->BlocksInfo = (adios2_blockinfo *)malloc( + varinfo->nblocks * sizeof(adios2_blockinfo)); + auto *b = varinfo->BlocksInfo; + + varinfo->Dims = minBlocksInfo->Dims; + varinfo->Shape = minBlocksInfo->Shape; + varinfo->IsValue = (int)minBlocksInfo->IsValue; + varinfo->IsReverseDims = (int)minBlocksInfo->IsReverseDims; + for (size_t i = 0; i < varinfo->nblocks; ++i) + { + b[i].WriterID = minBlocksInfo->BlocksInfo[i].WriterID; + b[i].BlockID = minBlocksInfo->BlocksInfo[i].BlockID; + b[i].Start = minBlocksInfo->BlocksInfo[i].Start; + b[i].Count = minBlocksInfo->BlocksInfo[i].Count; + if (minBlocksInfo->IsValue) + { + b[i].Value.uint64 = 0; + // = *((T *)minBlocksInfo->BlocksInfo[i].BufferP); + } + else + { + b[i].MinUnion.uint64 = 0; + // = minBlocksInfo->BlocksInfo[i].MinUnion; + b[i].MaxUnion.uint64 = 0; + // = minBlocksInfo->BlocksInfo[i].MaxUnion; + } + } + delete minBlocksInfo; + return varinfo; + } + + /* Call the big gun Engine::BlocksInfo */ + if (type == adios2::DataType::Compound) + { + return varinfo; + } + else if (type == adios2::helper::GetDataType()) + { + const auto blocksInfo = engineCpp->BlocksInfo( + *dynamic_cast *>( + variableBase), + step); + varinfo = (adios2_varinfo *)malloc(sizeof(adios2_varinfo)); + varinfo->nblocks = blocksInfo.size(); + varinfo->BlocksInfo = (adios2_blockinfo *)malloc( + varinfo->nblocks * sizeof(adios2_blockinfo)); + auto *b = varinfo->BlocksInfo; + + varinfo->Dims = static_cast(blocksInfo[0].Shape.size()); + varinfo->Shape = lf_CopyDims(blocksInfo[0].Shape); + varinfo->IsValue = (int)blocksInfo[0].IsValue; + varinfo->IsReverseDims = (int)blocksInfo[0].IsReverseDims; + for (size_t i = 0; i < varinfo->nblocks; ++i) + { + b[i].WriterID = blocksInfo[i].WriterID; + b[i].BlockID = blocksInfo[i].BlockID; + b[i].Start = lf_CopyDims(blocksInfo[i].Start); + b[i].Count = lf_CopyDims(blocksInfo[i].Count); + // minBlocksInfo->BlocksInfo[i].MinUnion; + b[i].MinUnion.uint64 = 0; + // minBlocksInfo->BlocksInfo[i].MaxUnion; + b[i].MaxUnion.uint64 = 0; + b[i].Value.str = (char *)malloc(blocksInfo[i].Value.size() + 1); + std::strcpy(b[i].Value.str, blocksInfo[i].Value.data()); + }; + } +#define declare_template_instantiation(T) \ + else if (type == adios2::helper::GetDataType()) \ + { \ + const auto blocksInfo = engineCpp->BlocksInfo( \ + *dynamic_cast *>(variableBase), step); \ + varinfo = (adios2_varinfo *)malloc(sizeof(adios2_varinfo)); \ + varinfo->nblocks = blocksInfo.size(); \ + varinfo->BlocksInfo = (adios2_blockinfo *)malloc( \ + varinfo->nblocks * sizeof(adios2_blockinfo)); \ + auto *b = varinfo->BlocksInfo; \ + \ + varinfo->Dims = static_cast(blocksInfo[0].Shape.size()); \ + varinfo->Shape = lf_CopyDims(blocksInfo[0].Shape); \ + varinfo->IsValue = (int)blocksInfo[0].IsValue; \ + varinfo->IsReverseDims = (int)blocksInfo[0].IsReverseDims; \ + for (size_t i = 0; i < varinfo->nblocks; ++i) \ + { \ + b[i].WriterID = blocksInfo[i].WriterID; \ + b[i].BlockID = blocksInfo[i].BlockID; \ + b[i].Start = lf_CopyDims(blocksInfo[i].Start); \ + b[i].Count = lf_CopyDims(blocksInfo[i].Count); \ + b[i].MinUnion.uint64 = 0; \ + b[i].MaxUnion.uint64 = 0; \ + b[i].Value.uint64 = 0; \ + }; \ + } + ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_template_instantiation) +#undef declare_template_instantiation + } + catch (...) + { + adios2::helper::ExceptionToError("adios2_inquire_blockinfo"); + return NULL; + } + return varinfo; +} + adios2_error adios2_close_by_index(adios2_engine *engine, const int transport_index) { diff --git a/bindings/C/adios2/c/adios2_c_engine.h b/bindings/C/adios2/c/adios2_c_engine.h index 36465fcd0e..a42a479733 100644 --- a/bindings/C/adios2/c/adios2_c_engine.h +++ b/bindings/C/adios2/c/adios2_c_engine.h @@ -264,6 +264,16 @@ adios2_error adios2_lock_writer_definitions(adios2_engine *engine); */ adios2_error adios2_lock_reader_selections(adios2_engine *engine); +/** + * Get the list of blocks for a variable in a given step. + * In Streaming mode, step is unused, always the current step is processed. + * @return Newly allocated adios2_varinfo structure, nullptr if step does not + * exist. The pointer must be freed by user + */ +adios2_varinfo *adios2_inquire_blockinfo(adios2_engine *engine, + adios2_variable *variable, + const size_t step); + #ifdef __cplusplus } // end extern C #endif diff --git a/bindings/C/adios2/c/adios2_c_io.cpp b/bindings/C/adios2/c/adios2_c_io.cpp index c1a659e26f..bbb33ba43b 100644 --- a/bindings/C/adios2/c/adios2_c_io.cpp +++ b/bindings/C/adios2/c/adios2_c_io.cpp @@ -874,6 +874,60 @@ adios2_error adios2_remove_all_attributes(adios2_io *io) } } +char **adios2_available_variables(adios2_io *io, size_t *size) +{ + try + { + adios2::helper::CheckForNullptr( + io, "for adios2_io, in call to adios2_available_variables"); + std::map> varInfo = + reinterpret_cast(io)->GetAvailableVariables(); + *size = varInfo.size(); + char **names = (char **)malloc(*size * sizeof(char *)); + + size_t cnt = 0; + for (auto var : varInfo) + { + size_t len = var.first.length(); + names[cnt] = (char *)malloc((len + 1) * sizeof(char)); + strcpy(names[cnt], var.first.c_str()); + cnt++; + } + + return names; + } + catch (...) + { + return NULL; + } +} +char **adios2_available_attributes(adios2_io *io, size_t *size) +{ + try + { + adios2::helper::CheckForNullptr( + io, "for adios2_io, in call to adios2_available_attributes"); + std::map> varInfo = + reinterpret_cast(io)->GetAvailableAttributes(); + *size = varInfo.size(); + char **names = (char **)malloc(*size * sizeof(char *)); + + size_t cnt = 0; + for (auto var : varInfo) + { + size_t len = var.first.length(); + names[cnt] = (char *)malloc((len + 1) * sizeof(char)); + strcpy(names[cnt], var.first.c_str()); + cnt++; + } + + return names; + } + catch (...) + { + return NULL; + } +} adios2_engine *adios2_open(adios2_io *io, const char *name, const adios2_mode mode) { diff --git a/bindings/C/adios2/c/adios2_c_io.h b/bindings/C/adios2/c/adios2_c_io.h index e058c9a442..e669661346 100644 --- a/bindings/C/adios2/c/adios2_c_io.h +++ b/bindings/C/adios2/c/adios2_c_io.h @@ -274,6 +274,23 @@ adios2_error adios2_remove_variable(adios2_bool *result, adios2_io *io, * @return adios2_error 0: success, see enum adios2_error for errors */ adios2_error adios2_remove_all_variables(adios2_io *io); +/** + * @brief returns an array or c strings for names of available variables + * Might create dangling pointers + * @param io handler variables io owner + * @param length of array of strings + * @return names of variables as an array of strings + */ +char **adios2_available_variables(adios2_io *io, size_t *size); + +/** + * @brief returns an array or c strings for names of available attributes + * Might create dangling pointers + * @param io handler variables io owner + * @param length of array of strings + * @return names of variables as an array of strings + */ +char **adios2_available_attributes(adios2_io *io, size_t *size); /** * @brief DANGEROUS! Removes an attribute identified by name. Might create diff --git a/bindings/C/adios2/c/adios2_c_types.h b/bindings/C/adios2/c/adios2_c_types.h index 47c3aefe32..8a6d6c9c40 100644 --- a/bindings/C/adios2/c/adios2_c_types.h +++ b/bindings/C/adios2/c/adios2_c_types.h @@ -138,10 +138,54 @@ typedef enum adios2_shapeid_local_array = 4 } adios2_shapeid; +typedef enum +{ + adios2_arrayordering_rowmajor, + adios2_arrayordering_columnmajor, + adios2_arrayordering_auto +} adios2_arrayordering; + static const size_t adios2_string_array_element_max_size = 4096; static const size_t adios2_local_value_dim = SIZE_MAX - 2; +union adios2_PrimitiveStdtypeUnion +{ + int8_t int8; + int16_t int16; + int32_t int32; + int64_t int64; + uint8_t uint8; + uint16_t uint16; + uint32_t uint32; + uint64_t uint64; + float f; + double d; + long double ld; + char *str; +}; + +typedef struct +{ + int WriterID; + size_t BlockID; + size_t *Start; + size_t *Count; + union adios2_PrimitiveStdtypeUnion MinUnion; + union adios2_PrimitiveStdtypeUnion MaxUnion; + union adios2_PrimitiveStdtypeUnion Value; +} adios2_blockinfo; + +typedef struct +{ + int Dims; + size_t *Shape; + int IsValue; + int IsReverseDims; + size_t nblocks; + adios2_blockinfo *BlocksInfo; +} adios2_varinfo; + #ifdef __cplusplus } // end extern C #endif diff --git a/bindings/CXX11/CMakeLists.txt b/bindings/CXX11/CMakeLists.txt index 6ac821bb52..a11452d04d 100644 --- a/bindings/CXX11/CMakeLists.txt +++ b/bindings/CXX11/CMakeLists.txt @@ -42,7 +42,7 @@ if(ADIOS2_HAVE_MPI) ) set_property(TARGET adios2_cxx11_mpi PROPERTY EXPORT_NAME cxx11_mpi) set_property(TARGET adios2_cxx11_mpi PROPERTY OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_cxx11_mpi) - target_link_libraries(adios2_cxx11_mpi PUBLIC adios2_cxx11 PRIVATE adios2_core_mpi PUBLIC MPI::MPI_CXX) + target_link_libraries(adios2_cxx11_mpi PUBLIC adios2_cxx11 PRIVATE adios2_core_mpi PUBLIC MPI::MPI_C) set(maybe_adios2_cxx11_mpi adios2_cxx11_mpi) target_compile_definitions(adios2_cxx11_mpi INTERFACE ADIOS2_USE_MPI) add_library(adios2::cxx11_mpi ALIAS adios2_cxx11_mpi) diff --git a/bindings/CXX11/adios2/cxx11/ADIOS.cpp b/bindings/CXX11/adios2/cxx11/ADIOS.cpp index d8cc46639a..4b159d66b9 100644 --- a/bindings/CXX11/adios2/cxx11/ADIOS.cpp +++ b/bindings/CXX11/adios2/cxx11/ADIOS.cpp @@ -31,10 +31,10 @@ ADIOS::ADIOS(const std::string &configFile, const std::string &hostLanguage, ADIOS::operator bool() const noexcept { return m_ADIOS ? true : false; } -IO ADIOS::DeclareIO(const std::string name) +IO ADIOS::DeclareIO(const std::string name, const ArrayOrdering ArrayOrder) { CheckPointer("for io name " + name + ", in call to ADIOS::DeclareIO"); - return IO(&m_ADIOS->DeclareIO(name)); + return IO(&m_ADIOS->DeclareIO(name, ArrayOrder)); } IO ADIOS::AtIO(const std::string name) diff --git a/bindings/CXX11/adios2/cxx11/ADIOS.h b/bindings/CXX11/adios2/cxx11/ADIOS.h index da8d739b34..fdb147cd82 100644 --- a/bindings/CXX11/adios2/cxx11/ADIOS.h +++ b/bindings/CXX11/adios2/cxx11/ADIOS.h @@ -164,7 +164,8 @@ class ADIOS * @exception std::invalid_argument if IO with unique name is already * declared */ - IO DeclareIO(const std::string name); + IO DeclareIO(const std::string name, + const ArrayOrdering ArrayOrder = ArrayOrdering::Auto); /** * Retrieve an existing IO object previously created with DeclareIO. diff --git a/bindings/CXX11/adios2/cxx11/Engine.cpp b/bindings/CXX11/adios2/cxx11/Engine.cpp index 513f546c8b..eb69869bed 100644 --- a/bindings/CXX11/adios2/cxx11/Engine.cpp +++ b/bindings/CXX11/adios2/cxx11/Engine.cpp @@ -168,11 +168,10 @@ Engine::Engine(core::Engine *engine) : m_Engine(engine) {} #define declare_template_instantiation(T) \ \ - template typename Variable::Span Engine::Put(Variable, const size_t, \ + template typename Variable::Span Engine::Put(Variable, const bool, \ const T &); \ - \ - template void Engine::Get(Variable, T **) const; \ - template typename Variable::Span Engine::Put(Variable); + template typename Variable::Span Engine::Put(Variable); \ + template void Engine::Get(Variable, T **) const; ADIOS2_FOREACH_PRIMITIVE_TYPE_1ARG(declare_template_instantiation) #undef declare_template_instantiation diff --git a/bindings/CXX11/adios2/cxx11/Engine.h b/bindings/CXX11/adios2/cxx11/Engine.h index 14a3ae1f54..76ded5466f 100644 --- a/bindings/CXX11/adios2/cxx11/Engine.h +++ b/bindings/CXX11/adios2/cxx11/Engine.h @@ -100,22 +100,24 @@ class Engine * Put signature that provides access to the internal engine buffer for a * pre-allocated variable including a fill value. Returns a fixed size Span * (based on C++20 std::span) so applications can populate data value after - * this Put. Requires a call to PerformPuts, EndStep, or Close to extract - * the Min/Max bounds. + * this Put and before PerformPuts/EndStep. Requires a call to PerformPuts, + * EndStep, or Close to extract the Min/Max bounds. * @param variable input variable - * @param bufferID if engine has multiple buffers, input 0 when this - * information is not known + * @param initialize bool flag indicating if allocated memory should be + * initialized with the provided value. Some engines (BP3/BP4) may + * initialize the allocated memory anyway to zero if this flag is false. * @param value provide an initial fill value * @return span to variable data in engine internal buffer */ template - typename Variable::Span Put(Variable variable, const size_t bufferID, + typename Variable::Span Put(Variable variable, const bool initialize, const T &value); /** * Put signature that provides access to an internal engine buffer (decided - * by the engine) for a pre-allocated variable with the default fill value - * T(). + * by the engine) for a pre-allocated variable. Allocated buffer may or may + * not be initialized to zero by the engine (e.g. BP3/BP4 does, BP5 does + * not). * @param variable input variable * @return span to variable data in engine internal buffer */ @@ -451,8 +453,7 @@ class Engine #define declare_template_instantiation(T) \ \ extern template typename Variable::Span Engine::Put( \ - Variable, const size_t, const T &); \ - \ + Variable, const bool, const T &); \ extern template typename Variable::Span Engine::Put(Variable); \ extern template void Engine::Get(Variable, T **) const; diff --git a/bindings/CXX11/adios2/cxx11/Engine.tcc b/bindings/CXX11/adios2/cxx11/Engine.tcc index 24bed4ebf0..44033d7584 100644 --- a/bindings/CXX11/adios2/cxx11/Engine.tcc +++ b/bindings/CXX11/adios2/cxx11/Engine.tcc @@ -59,11 +59,61 @@ ToBlocksInfo(const std::vector +static std::vector::Info> +ToBlocksInfo(const core::Engine::MinVarInfo *coreVarInfo) +{ + auto coreBlocksInfo = coreVarInfo->BlocksInfo; + + std::vector::Info> blocksInfo; + blocksInfo.reserve(coreBlocksInfo.size()); + + for (auto &coreBlockInfo : coreBlocksInfo) + { + typename Variable::Info blockInfo; + + if (coreVarInfo->Shape) + { + blockInfo.Start.reserve(coreVarInfo->Dims); + blockInfo.Count.reserve(coreVarInfo->Dims); + for (int i = 0; i < coreVarInfo->Dims; i++) + { + blockInfo.Start.push_back(coreBlockInfo.Start[i]); + blockInfo.Count.push_back(coreBlockInfo.Count[i]); + } + } + else + { + blockInfo.Count.reserve(coreVarInfo->Dims); + for (int i = 0; i < coreVarInfo->Dims; i++) + { + blockInfo.Count.push_back(coreBlockInfo.Count[i]); + } + } + blockInfo.WriterID = coreBlockInfo.WriterID; + + blockInfo.IsValue = coreVarInfo->IsValue; + blockInfo.IsReverseDims = coreVarInfo->IsReverseDims; + if (blockInfo.IsValue) + { + blockInfo.Value = *((T *)coreBlockInfo.BufferP); + } + else + { + blockInfo.Min = *((T *)&coreBlockInfo.MinUnion); + blockInfo.Max = *((T *)&coreBlockInfo.MaxUnion); + } + blockInfo.BlockID = coreBlockInfo.BlockID; + blocksInfo.push_back(blockInfo); + } + + return blocksInfo; +} } // end empty namespace template typename Variable::Span Engine::Put(Variable variable, - const size_t bufferID, const T &value) + const bool initialize, const T &value) { using IOType = typename TypeInfo::IOType; adios2::helper::CheckForNullptr(m_Engine, @@ -79,7 +129,7 @@ typename Variable::Span Engine::Put(Variable variable, typename Variable::Span::CoreSpan *coreSpan = reinterpret_cast::Span::CoreSpan *>( - &m_Engine->Put(*variable.m_Variable, bufferID, + &m_Engine->Put(*variable.m_Variable, initialize, reinterpret_cast(value))); return typename Variable::Span(coreSpan); @@ -88,7 +138,22 @@ typename Variable::Span Engine::Put(Variable variable, template typename Variable::Span Engine::Put(Variable variable) { - return Put(variable, 0, T()); + adios2::helper::CheckForNullptr(m_Engine, + "for Engine in call to Engine::Array"); + + if (m_Engine->m_EngineType == "NULL") + { + return typename Variable::Span(nullptr); + } + + adios2::helper::CheckForNullptr(variable.m_Variable, + "for variable in call to Engine::Array"); + + typename Variable::Span::CoreSpan *coreSpan = + reinterpret_cast::Span::CoreSpan *>( + &m_Engine->Put(*variable.m_Variable, false)); + + return typename Variable::Span(coreSpan); } template @@ -332,6 +397,17 @@ Engine::BlocksInfo(const Variable variable, const size_t step) const adios2::helper::CheckForNullptr( variable.m_Variable, "for variable in call to Engine::BlocksInfo"); + const auto minBlocksInfo = + m_Engine->MinBlocksInfo(*variable.m_Variable, step); + + if (minBlocksInfo) + { + std::vector::Info> Ret = + ToBlocksInfo(minBlocksInfo); + delete minBlocksInfo; + return Ret; + } + const auto blocksInfo = m_Engine->BlocksInfo(*variable.m_Variable, step); return ToBlocksInfo(blocksInfo); diff --git a/bindings/CXX11/adios2/cxx11/IO.cpp b/bindings/CXX11/adios2/cxx11/IO.cpp index 64fcf8204c..f9d409fc60 100644 --- a/bindings/CXX11/adios2/cxx11/IO.cpp +++ b/bindings/CXX11/adios2/cxx11/IO.cpp @@ -152,10 +152,11 @@ std::string IO::AttributeType(const std::string &name) const return ToString(m_IO->InquireAttributeType(name)); } -size_t IO::AddOperation(const Operator op, const Params ¶meters) +void IO::AddOperation(const std::string &variable, + const std::string &operatorType, const Params ¶meters) { helper::CheckForNullptr(m_IO, "in call to IO::AddOperation"); - return m_IO->AddOperation(*op.m_Operator, parameters); + return m_IO->AddOperation(variable, operatorType, parameters); } std::string IO::EngineType() const @@ -182,11 +183,11 @@ ADIOS2_FOREACH_TYPE_1ARG(declare_template_instantiation) #define declare_template_instantiation(T) \ template Attribute IO::DefineAttribute( \ const std::string &, const T *, const size_t, const std::string &, \ - const std::string); \ + const std::string, const bool); \ \ template Attribute IO::DefineAttribute(const std::string &, const T &, \ const std::string &, \ - const std::string); \ + const std::string, const bool); \ \ template Attribute IO::InquireAttribute( \ const std::string &, const std::string &, const std::string); diff --git a/bindings/CXX11/adios2/cxx11/IO.h b/bindings/CXX11/adios2/cxx11/IO.h index 49067e7512..af303de1d2 100644 --- a/bindings/CXX11/adios2/cxx11/IO.h +++ b/bindings/CXX11/adios2/cxx11/IO.h @@ -172,6 +172,7 @@ class IO * @param separator default is "/", hierarchy between variable name and * attribute, e.g. variableName/attribute1, variableName::attribute1. Not * used if variableName is empty. + * @param allowModification true allows redefining existing attribute * @return object reference to internal Attribute in IO * @exception std::invalid_argument if Attribute with unique name (in IO or * Variable) is already defined @@ -180,7 +181,8 @@ class IO Attribute DefineAttribute(const std::string &name, const T *data, const size_t size, const std::string &variableName = "", - const std::string separator = "/"); + const std::string separator = "/", + const bool allowModification = false); /** * @brief Define single value attribute @@ -192,6 +194,7 @@ class IO * @param separator default is "/", hierarchy between variable name and * attribute, e.g. variableName/attribute1, variableName::attribute1. Not * used if variableName is empty. + * @param allowModification true allows redefining existing attribute * @return object reference to internal Attribute in IO * @exception std::invalid_argument if Attribute with unique name (in IO or * Variable) is already defined @@ -199,7 +202,8 @@ class IO template Attribute DefineAttribute(const std::string &name, const T &value, const std::string &variableName = "", - const std::string separator = "/"); + const std::string separator = "/", + const bool allowModification = false); /** * @brief Retrieve an existing attribute @@ -251,7 +255,8 @@ class IO /** * Open an Engine to start heavy-weight input/output operations. * @param name unique engine identifier - * @param mode adios2::Mode::Write, adios2::Mode::Read, or + * @param mode adios2::Mode::Write, adios2::Mode::Read, + * adios2::Mode::ReadStreaming, or * adios2::Mode::Append (BP4 only) * @return engine object */ @@ -338,27 +343,15 @@ class IO std::string AttributeType(const std::string &name) const; /** - * EXPERIMENTAL: carries information about an Operation added with - * AddOperation - */ - struct Operation - { - /** Operator associated with this operation */ - const Operator Op; - /** Parameters settings for this operation */ - const adios2::Params Parameters; - /** Information associated with this operation */ - const adios2::Params Info; - }; - - /** - * EXPERIMENTAL: Adds operation and parameters to current IO object - * @param op operator to be added + * Adds operation and parameters to current IO object + * @param variable variable to add operator to + * @param operatorType operator type to define * @param parameters key/value settings particular to the IO, not to * be confused by op own parameters - * @return operation index handler in Operations() */ - size_t AddOperation(const Operator op, const Params ¶meters = Params()); + void AddOperation(const std::string &variable, + const std::string &operatorType, + const Params ¶meters = Params()); /** * Inspect current engine type from SetEngine @@ -386,11 +379,11 @@ ADIOS2_FOREACH_TYPE_1ARG(declare_template_instantiation) #define declare_template_instantiation(T) \ extern template Attribute IO::DefineAttribute( \ const std::string &, const T *, const size_t, const std::string &, \ - const std::string); \ + const std::string, const bool); \ \ extern template Attribute IO::DefineAttribute( \ const std::string &, const T &, const std::string &, \ - const std::string); \ + const std::string, const bool); \ \ extern template Attribute IO::InquireAttribute( \ const std::string &, const std::string &, const std::string); diff --git a/bindings/CXX11/adios2/cxx11/IO.tcc b/bindings/CXX11/adios2/cxx11/IO.tcc index c83213b09f..2b10b0e4a3 100644 --- a/bindings/CXX11/adios2/cxx11/IO.tcc +++ b/bindings/CXX11/adios2/cxx11/IO.tcc @@ -38,31 +38,32 @@ Variable IO::InquireVariable(const std::string &name) } template -Attribute IO::DefineAttribute(const std::string &name, const T *data, - const size_t size, - const std::string &variableName, - const std::string separator) +Attribute +IO::DefineAttribute(const std::string &name, const T *data, const size_t size, + const std::string &variableName, + const std::string separator, const bool allowModification) { using IOType = typename TypeInfo::IOType; helper::CheckForNullptr(m_IO, "for attribute name " + name + " and variable name " + variableName + ", in call to IO::DefineAttribute"); - return Attribute( - &m_IO->DefineAttribute(name, reinterpret_cast(data), - size, variableName, separator)); + return Attribute(&m_IO->DefineAttribute( + name, reinterpret_cast(data), size, variableName, + separator, allowModification)); } template Attribute IO::DefineAttribute(const std::string &name, const T &value, const std::string &variableName, - const std::string separator) + const std::string separator, + const bool allowModification) { using IOType = typename TypeInfo::IOType; helper::CheckForNullptr(m_IO, "for attribute name " + name + ", in call to IO::DefineAttribute"); return Attribute( &m_IO->DefineAttribute(name, reinterpret_cast(value), - variableName, separator)); + variableName, separator, allowModification)); } template diff --git a/bindings/CXX11/adios2/cxx11/Variable.cpp b/bindings/CXX11/adios2/cxx11/Variable.cpp index e7265226a7..3ddb06416f 100644 --- a/bindings/CXX11/adios2/cxx11/Variable.cpp +++ b/bindings/CXX11/adios2/cxx11/Variable.cpp @@ -189,6 +189,14 @@ namespace adios2 } \ \ template <> \ + void Variable::RemoveOperations() \ + { \ + helper::CheckForNullptr(m_Variable, \ + "in call to Variable::RemoveOperations"); \ + m_Variable->RemoveOperations(); \ + } \ + \ + template <> \ std::pair Variable::MinMax(const size_t step) const \ { \ helper::CheckForNullptr(m_Variable, "in call to Variable::MinMax"); \ diff --git a/bindings/CXX11/adios2/cxx11/Variable.h b/bindings/CXX11/adios2/cxx11/Variable.h index 2b22492b56..df7d217e39 100644 --- a/bindings/CXX11/adios2/cxx11/Variable.h +++ b/bindings/CXX11/adios2/cxx11/Variable.h @@ -303,6 +303,12 @@ class Variable */ std::vector Operations() const; + /** + * Removes all current Operations associated with AddOperation. + * Provides the posibility to apply or not operators on a step basis. + */ + void RemoveOperations(); + /** * Read mode only: return minimum and maximum values for current variable at * a step. For streaming mode (BeginStep/EndStep): use default (leave empty) diff --git a/bindings/Fortran/f2c/adios2_f2c_io.cpp b/bindings/Fortran/f2c/adios2_f2c_io.cpp index e9c3c4523d..74234501c4 100644 --- a/bindings/Fortran/f2c/adios2_f2c_io.cpp +++ b/bindings/Fortran/f2c/adios2_f2c_io.cpp @@ -10,6 +10,7 @@ #include "adios2_f2c_common.h" +#include #include //std::invalid_argument #include //strcpy @@ -220,6 +221,92 @@ void FC_GLOBAL(adios2_define_variable_f2c, ADIOS2_DEFINE_VARIABLE_F2C)( } } +struct cnamelist +{ + char **names; + size_t count; +}; + +void FC_GLOBAL(adios2_available_variables_f2c, + ADIOS2_AVAILABLE_VARIABLES_F2C)(adios2_io **io, + int64_t *namestruct, + int *vars_count, + int *max_var_name_len, int *ierr) +{ + cnamelist *info = new (cnamelist); + info->names = adios2_available_variables(*io, &info->count); + *vars_count = static_cast(info->count); + size_t maxlen = 0; + for (size_t i = 0; i < info->count; ++i) + { + auto l = strlen(info->names[i]); + if (l > maxlen) + { + maxlen = l; + } + } + *max_var_name_len = static_cast(maxlen); + *namestruct = static_cast(reinterpret_cast(info)); + *ierr = 0; +} + +void FC_GLOBAL(adios2_available_attributes_f2c, + ADIOS2_AVAILABLE_ATTRIBUTES_F2C)(adios2_io **io, + int64_t *namestruct, + int *attrs_count, + int *max_attr_name_len, + int *ierr) +{ + cnamelist *info = new (cnamelist); + info->names = adios2_available_attributes(*io, &info->count); + *attrs_count = static_cast(info->count); + + size_t maxlen = 0; + for (size_t i = 0; i < info->count; ++i) + { + auto l = strlen(info->names[i]); + if (l > maxlen) + { + maxlen = l; + } + } + *max_attr_name_len = static_cast(maxlen); + + *namestruct = static_cast(reinterpret_cast(info)); + *ierr = 0; +} + +void FC_GLOBAL(adios2_retrieve_namelist_f2c, + ADIOS2_RETRIEVE_NAMELIST_F2C)(int64_t *namestruct, + void *namelist, int *ierr, + int namelist_len) +{ + cnamelist *info = reinterpret_cast(*namestruct); + if (info != NULL) + { + for (size_t i = 0; i < info->count; i++) + { + char *fs = (char *)namelist + i * namelist_len; + size_t len = strlen(info->names[i]); + if (len > static_cast(namelist_len)) + { + len = static_cast(namelist_len); + } + // copy C string without '\0' + strncpy(fs, info->names[i], len); + // pad with spaces + memset(fs + len, ' ', namelist_len - len); + } + *ierr = 0; + delete (info); + *namestruct = 0; + } + else + { + *ierr = 1; + } +} + void FC_GLOBAL(adios2_inquire_variable_f2c, ADIOS2_INQUIRE_VARIABLE_F2C)(adios2_variable **variable, adios2_io **io, const char *name, diff --git a/bindings/Fortran/modules/adios2_adios_init_mpi_smod.F90 b/bindings/Fortran/modules/adios2_adios_init_mpi_smod.F90 index 2726908404..88495ec4f8 100644 --- a/bindings/Fortran/modules/adios2_adios_init_mpi_smod.F90 +++ b/bindings/Fortran/modules/adios2_adios_init_mpi_smod.F90 @@ -30,6 +30,7 @@ module adios2_adios_init_mpi_mod module procedure adios2_init_config_debug_mpi end interface #endif + external adios2_init_config_mpi_f2c contains diff --git a/bindings/Fortran/modules/adios2_adios_init_serial_smod.F90 b/bindings/Fortran/modules/adios2_adios_init_serial_smod.F90 index 8c95476ed8..217850cb24 100644 --- a/bindings/Fortran/modules/adios2_adios_init_serial_smod.F90 +++ b/bindings/Fortran/modules/adios2_adios_init_serial_smod.F90 @@ -30,6 +30,7 @@ module adios2_adios_init_serial_mod module procedure adios2_init_config_debug_serial end interface #endif + external adios2_init_config_serial_f2c contains diff --git a/bindings/Fortran/modules/adios2_adios_mod.f90 b/bindings/Fortran/modules/adios2_adios_mod.f90 index 556b08bfc2..cc70f031b7 100644 --- a/bindings/Fortran/modules/adios2_adios_mod.f90 +++ b/bindings/Fortran/modules/adios2_adios_mod.f90 @@ -12,6 +12,18 @@ module adios2_adios_mod use adios2_adios_init_mod implicit none + external adios2_declare_io_f2c + external adios2_io_engine_type_length_f2c + external adios2_io_engine_type_f2c + external adios2_at_io_f2c + external adios2_define_operator_f2c + external adios2_inquire_operator_f2c + external adios2_operator_type_length_f2c + external adios2_operator_type_f2c + external adios2_flush_all_f2c + external adios2_remove_io_f2c + external adios2_remove_all_ios_f2c + external adios2_finalize_f2c contains subroutine adios2_declare_io(io, adios, io_name, ierr) diff --git a/bindings/Fortran/modules/adios2_attribute_data_mod.f90 b/bindings/Fortran/modules/adios2_attribute_data_mod.f90 index becafbb89b..ac61408ae1 100644 --- a/bindings/Fortran/modules/adios2_attribute_data_mod.f90 +++ b/bindings/Fortran/modules/adios2_attribute_data_mod.f90 @@ -34,6 +34,8 @@ module adios2_attribute_data_mod end interface + external adios2_attribute_data_f2c + external adios2_attribute_value_f2c contains ! Single value diff --git a/bindings/Fortran/modules/adios2_attribute_mod.f90 b/bindings/Fortran/modules/adios2_attribute_mod.f90 index afa8ab4fce..f68e08cf26 100644 --- a/bindings/Fortran/modules/adios2_attribute_mod.f90 +++ b/bindings/Fortran/modules/adios2_attribute_mod.f90 @@ -18,6 +18,8 @@ subroutine adios2_attribute_name(name, attribute, ierr) character(len=:), allocatable, intent(out) :: name type(adios2_attribute), intent(in) :: attribute integer, intent(out) :: ierr + external adios2_attribute_name_length_f2c + external adios2_attribute_name_f2c !local integer :: length diff --git a/bindings/Fortran/modules/adios2_engine_begin_step_mod.f90 b/bindings/Fortran/modules/adios2_engine_begin_step_mod.f90 index e65953746c..d67cf70d4e 100644 --- a/bindings/Fortran/modules/adios2_engine_begin_step_mod.f90 +++ b/bindings/Fortran/modules/adios2_engine_begin_step_mod.f90 @@ -20,6 +20,7 @@ module adios2_engine_begin_step_mod module procedure adios2_begin_step_default end interface + external adios2_begin_step_f2c contains diff --git a/bindings/Fortran/modules/adios2_engine_get_mod.f90 b/bindings/Fortran/modules/adios2_engine_get_mod.f90 index 9422827a06..1f55a3d344 100644 --- a/bindings/Fortran/modules/adios2_engine_get_mod.f90 +++ b/bindings/Fortran/modules/adios2_engine_get_mod.f90 @@ -303,6 +303,8 @@ module adios2_engine_get_mod end interface + external adios2_get_f2c + external adios2_get_by_name_f2c contains diff --git a/bindings/Fortran/modules/adios2_engine_mod.f90 b/bindings/Fortran/modules/adios2_engine_mod.f90 index 0b9cef1184..0980126959 100644 --- a/bindings/Fortran/modules/adios2_engine_mod.f90 +++ b/bindings/Fortran/modules/adios2_engine_mod.f90 @@ -13,6 +13,15 @@ module adios2_engine_mod use adios2_engine_put_mod use adios2_engine_get_mod implicit none + external adios2_close_f2c + external adios2_current_step_f2c + external adios2_end_step_f2c + external adios2_flush_f2c + external adios2_lock_reader_selections_f2c + external adios2_lock_writer_definitions_f2c + external adios2_perform_gets_f2c + external adios2_perform_puts_f2c + external adios2_steps_f2c contains diff --git a/bindings/Fortran/modules/adios2_engine_put_mod.f90 b/bindings/Fortran/modules/adios2_engine_put_mod.f90 index c8f3419e6b..54966ccba4 100644 --- a/bindings/Fortran/modules/adios2_engine_put_mod.f90 +++ b/bindings/Fortran/modules/adios2_engine_put_mod.f90 @@ -302,6 +302,8 @@ module adios2_engine_put_mod module procedure adios2_put_deferred_by_name_integer8_6d end interface + external adios2_put_f2c + external adios2_put_by_name_f2c contains diff --git a/bindings/Fortran/modules/adios2_io_define_attribute_mod.f90 b/bindings/Fortran/modules/adios2_io_define_attribute_mod.f90 index 0ff982b711..74c427360d 100644 --- a/bindings/Fortran/modules/adios2_io_define_attribute_mod.f90 +++ b/bindings/Fortran/modules/adios2_io_define_attribute_mod.f90 @@ -71,6 +71,8 @@ module adios2_io_define_attribute_mod module procedure adios2_def_var_att_sep_integer8_1d end interface + external adios2_define_vattr_f2c + external adios2_define_vattr_array_f2c contains diff --git a/bindings/Fortran/modules/adios2_io_define_variable_mod.f90 b/bindings/Fortran/modules/adios2_io_define_variable_mod.f90 index 525195dbf7..89a55e6890 100644 --- a/bindings/Fortran/modules/adios2_io_define_variable_mod.f90 +++ b/bindings/Fortran/modules/adios2_io_define_variable_mod.f90 @@ -21,6 +21,9 @@ module adios2_io_define_variable_mod end interface + external adios2_define_global_variable_f2c + external adios2_define_variable_f2c + contains subroutine adios2_define_variable_value(variable, io, name, adios2_type, & diff --git a/bindings/Fortran/modules/adios2_io_mod.f90 b/bindings/Fortran/modules/adios2_io_mod.f90 index 37840517f1..6011df1978 100644 --- a/bindings/Fortran/modules/adios2_io_mod.f90 +++ b/bindings/Fortran/modules/adios2_io_mod.f90 @@ -15,6 +15,32 @@ module adios2_io_mod use adios2_variable_mod implicit none + external adios2_add_transport_f2c + external adios2_attribute_is_value_f2c + external adios2_attribute_length_f2c + external adios2_attribute_type_f2c + external adios2_clear_parameters_f2c + external adios2_flush_all_engines_f2c + external adios2_get_parameter_f2c + external adios2_get_parameter_length_f2c + external adios2_in_config_file_f2c + external adios2_available_variables_f2c + external adios2_available_attributes_f2c + external adios2_retrieve_namelist_f2c + external adios2_inquire_attribute_f2c + external adios2_inquire_variable_attribute_f2c + external adios2_inquire_variable_f2c + external adios2_io_engine_type_f2c + external adios2_io_engine_type_length_f2c + external adios2_remove_all_attributes_f2c + external adios2_remove_all_variables_f2c + external adios2_remove_attribute_f2c + external adios2_remove_variable_f2c + external adios2_set_engine_f2c + external adios2_set_parameter_f2c + external adios2_set_parameters_f2c + external adios2_set_transport_parameter_f2c + contains subroutine adios2_in_config_file(result, io, ierr) @@ -135,6 +161,55 @@ subroutine adios2_set_transport_parameter(io, transport_index, key, value, & ierr) end subroutine + + subroutine adios2_available_variables(io, namestruct, ierr) + type(adios2_io), intent(in) :: io + type(adios2_namestruct), intent(out) :: namestruct + integer, intent(out) :: ierr + + call adios2_available_variables_f2c(io%f2c, namestruct%f2c, & + namestruct%count, namestruct%max_name_len, ierr) + if (ierr == 0) then + namestruct%valid = .true. + endif + end subroutine + + subroutine adios2_retrieve_names(namestruct, namelist, ierr) + type(adios2_namestruct), intent(inout) :: namestruct + character(*), dimension(*), intent(inout) :: namelist + integer, intent(out) :: ierr + + if (namestruct%valid .and. namestruct%f2c > 0_8) then + call adios2_retrieve_namelist_f2c(namestruct%f2c, namelist, ierr) + else + write(*,*) "ADIOS2 Fortran ERROR: invalid namestruct when calling adios2_retrieve_names()" + endif + namestruct%valid = .false. + end subroutine + + ! + ! F2008 implementation that allows for allocating a character array inside + ! + ! subroutine adios2_available_variables(io, nvars, varnamelist, ierr) + ! type(adios2_io), intent(in) :: io + ! integer, intent(out) :: nvars + ! character(len=:), dimension(:), allocatable, intent(out) :: varnamelist + ! integer, intent(out) :: ierr + + ! integer(kind=8):: namestruct + ! integer :: count, max_name_len + + ! call adios2_available_variables_f2c(io%f2c, namestruct, count, & + ! max_name_len, ierr) + ! if (ierr == 0) then + ! allocate(character(len=max_name_len) :: varnamelist(count)) + ! endif + + ! call adios2_retrieve_variable_names_f2c(namestruct, varnamelist, ierr) + ! nvars = count + ! end subroutine + + subroutine adios2_inquire_variable(variable, io, name, ierr) type(adios2_variable), intent(out) :: variable type(adios2_io), intent(in) :: io @@ -189,6 +264,37 @@ subroutine adios2_remove_all_variables(io, ierr) end subroutine + subroutine adios2_available_attributes(io, namestruct, ierr) + type(adios2_io), intent(in) :: io + type(adios2_namestruct), intent(out) :: namestruct + integer, intent(out) :: ierr + + call adios2_available_attributes_f2c(io%f2c, namestruct%f2c, & + namestruct%count, namestruct%max_name_len, ierr) + if (ierr == 0) then + namestruct%valid = .true. + endif + end subroutine + + ! subroutine adios2_available_attributes(io, nattrs, attrnamelist, ierr) + ! type(adios2_io), intent(in) :: io + ! integer, intent(out) :: nattrs + ! character(len=:), dimension(:), allocatable, intent(out) :: attrnamelist + ! integer, intent(out) :: ierr + + ! integer(kind=8):: namestruct + ! integer :: count, max_name_len + + ! call adios2_available_attributes_f2c(io%f2c, namestruct, count, & + ! max_name_len, ierr) + ! if (ierr == 0) then + ! allocate(character(len=max_name_len) :: attrnamelist(count)) + ! endif + + ! call adios2_retrieve_attribute_names_f2c(namestruct, count, & + ! max_name_len, attrnamelist, ierr) + ! nattrs = count + ! end subroutine subroutine adios2_inquire_attribute(attribute, io, name, ierr) type(adios2_attribute), intent(out) :: attribute @@ -224,14 +330,53 @@ subroutine adios2_inquire_attribute(attribute, io, name, ierr) end subroutine + subroutine adios2_inquire_variable_attribute(attribute, io, attribute_name, variable_name, separator, ierr) + type(adios2_attribute), intent(out) :: attribute + type(adios2_io), intent(in) :: io + character*(*), intent(in) :: attribute_name + character*(*), intent(in) :: variable_name + character*(*), intent(in) :: separator + integer, intent(out) :: ierr + !local + integer:: is_valueInt + + call adios2_inquire_variable_attribute_f2c(attribute%f2c, io%f2c, & + TRIM(ADJUSTL(attribute_name))//char(0), & + TRIM(ADJUSTL(variable_name))//char(0), & + TRIM(ADJUSTL(separator))//char(0), & + ierr) + + if(attribute%f2c > 0_8) then + attribute%valid = .true. + attribute%name = TRIM(variable_name)//TRIM(separator)//TRIM(attribute_name) + call adios2_attribute_type_f2c(attribute%type, attribute%f2c, ierr) + call adios2_attribute_length_f2c(attribute%length, attribute%f2c, & + ierr) + call adios2_attribute_is_value_f2c(is_valueInt, attribute%f2c, ierr) + + if(is_valueInt == 0) then + attribute%is_value = .false. + else + attribute%is_value = .true. + end if + + else + attribute%valid = .false. + attribute%name = '' + attribute%type = adios2_type_unknown + attribute%length = 0 + end if + + end subroutine + subroutine adios2_remove_attribute(result, io, name, ierr) type(adios2_io), intent(in) :: io character*(*), intent(in) :: name logical, intent(out) :: result integer, intent(out) :: ierr + ! Local - type(adios2_attribute):: attribute integer :: resultInt call adios2_remove_attribute_f2c(resultInt, io%f2c, & diff --git a/bindings/Fortran/modules/adios2_io_open_mpi_smod.F90 b/bindings/Fortran/modules/adios2_io_open_mpi_smod.F90 index 333a6a5b6d..ed6bb7f61e 100644 --- a/bindings/Fortran/modules/adios2_io_open_mpi_smod.F90 +++ b/bindings/Fortran/modules/adios2_io_open_mpi_smod.F90 @@ -26,6 +26,8 @@ module adios2_io_open_mpi_mod module procedure adios2_open_new_comm end interface #endif + external adios2_open_new_comm_f2c + external adios2_engine_get_type_f2c contains diff --git a/bindings/Fortran/modules/adios2_io_open_serial_smod.F90 b/bindings/Fortran/modules/adios2_io_open_serial_smod.F90 index 5e3f785e51..2501e2a0d3 100644 --- a/bindings/Fortran/modules/adios2_io_open_serial_smod.F90 +++ b/bindings/Fortran/modules/adios2_io_open_serial_smod.F90 @@ -36,6 +36,8 @@ ADIOS2_MODULE_PROCEDURE subroutine adios2_open_old_comm( & character*(*), intent(in) :: name integer, intent(in) :: adios2_mode integer, intent(out) :: ierr + external adios2_open_f2c + external adios2_engine_get_type_f2c call adios2_open_f2c(engine%f2c, io%f2c, TRIM(ADJUSTL(name))//char(0), & adios2_mode, ierr) diff --git a/bindings/Fortran/modules/adios2_operator_mod.f90 b/bindings/Fortran/modules/adios2_operator_mod.f90 index 9eb8b36c7f..d10daf24af 100644 --- a/bindings/Fortran/modules/adios2_operator_mod.f90 +++ b/bindings/Fortran/modules/adios2_operator_mod.f90 @@ -18,6 +18,8 @@ subroutine adios2_operator_type(type, op, ierr) character(len=:), allocatable, intent(out) :: type type(adios2_operator), intent(in) :: op integer, intent(out) :: ierr + external adios2_operator_type_f2c + external adios2_operator_type_length_f2c !local integer :: length diff --git a/bindings/Fortran/modules/adios2_parameters_mod.f90 b/bindings/Fortran/modules/adios2_parameters_mod.f90 index c3376321d7..6be4deb2d9 100644 --- a/bindings/Fortran/modules/adios2_parameters_mod.f90 +++ b/bindings/Fortran/modules/adios2_parameters_mod.f90 @@ -127,4 +127,11 @@ module adios2_parameters_mod character(len=64):: type = '' end type + type adios2_namestruct + integer(kind=8):: f2c = 0_8 + logical :: valid = .false. + integer :: count + integer :: max_name_len + end type + end module diff --git a/bindings/Fortran/modules/adios2_variable_max_mod.f90 b/bindings/Fortran/modules/adios2_variable_max_mod.f90 index ca14b475a7..c1901b4641 100644 --- a/bindings/Fortran/modules/adios2_variable_max_mod.f90 +++ b/bindings/Fortran/modules/adios2_variable_max_mod.f90 @@ -26,6 +26,7 @@ module adios2_variable_max_mod end interface + external adios2_variable_max_f2c contains subroutine adios2_variable_max_real(maximum, variable, ierr) diff --git a/bindings/Fortran/modules/adios2_variable_min_mod.f90 b/bindings/Fortran/modules/adios2_variable_min_mod.f90 index 28530d00a6..fe156e5e42 100644 --- a/bindings/Fortran/modules/adios2_variable_min_mod.f90 +++ b/bindings/Fortran/modules/adios2_variable_min_mod.f90 @@ -25,6 +25,7 @@ module adios2_variable_min_mod module procedure adios2_variable_min_integer8 end interface + external adios2_variable_min_f2c contains diff --git a/bindings/Fortran/modules/adios2_variable_mod.f90 b/bindings/Fortran/modules/adios2_variable_mod.f90 index 2ba0ecfa54..efea67f906 100644 --- a/bindings/Fortran/modules/adios2_variable_mod.f90 +++ b/bindings/Fortran/modules/adios2_variable_mod.f90 @@ -11,7 +11,19 @@ module adios2_variable_mod use adios2_functions_mod implicit none - + external adios2_add_operation_f2c + external adios2_set_block_selection_f2c + external adios2_set_memory_selection_f2c + external adios2_set_operation_parameter_f2c + external adios2_set_selection_f2c + external adios2_set_shape_f2c + external adios2_set_step_selection_f2c + external adios2_variable_name_f2c + external adios2_variable_name_length_f2c + external adios2_variable_ndims_f2c + external adios2_variable_shape_f2c + external adios2_variable_steps_f2c + external adios2_variable_type_f2c contains subroutine adios2_variable_name(name, variable, ierr) diff --git a/bindings/Python/CMakeLists.txt b/bindings/Python/CMakeLists.txt index 9047c017d2..5c6121e8f6 100644 --- a/bindings/Python/CMakeLists.txt +++ b/bindings/Python/CMakeLists.txt @@ -6,6 +6,7 @@ Python_add_library(adios2_py MODULE py11Attribute.cpp py11Engine.cpp py11Operator.cpp + py11Query.cpp py11File.cpp py11File.tcc py11glue.cpp ) diff --git a/bindings/Python/py11Engine.h b/bindings/Python/py11Engine.h index b268e78861..63ea8b84cb 100644 --- a/bindings/Python/py11Engine.h +++ b/bindings/Python/py11Engine.h @@ -30,6 +30,7 @@ class IO; // friend class Engine { friend class IO; + friend class Query; public: struct Info diff --git a/bindings/Python/py11File.cpp b/bindings/Python/py11File.cpp index 096241a3f9..13dc12ae21 100644 --- a/bindings/Python/py11File.cpp +++ b/bindings/Python/py11File.cpp @@ -216,7 +216,7 @@ pybind11::array File::Read(const std::string &name, const Dims &start, m_Stream->Read(name, blockID).front(); pybind11::array_t pyArray(Dims{value.size()}); std::copy(value.begin(), value.end(), pyArray.mutable_data()); - return pyArray; + return std::move(pyArray); } return Read(name, start, count, 0, 0, blockID); @@ -270,7 +270,7 @@ pybind11::array File::ReadAttribute(const std::string &name, pybind11::array_t pyArray(attribute->m_Elements); \ m_Stream->ReadAttribute(name, pyArray.mutable_data(), variableName, \ separator); \ - return pyArray; \ + return std::move(pyArray); \ } ADIOS2_FOREACH_NUMPY_ATTRIBUTE_TYPE_1ARG(declare_type) #undef declare_type diff --git a/bindings/Python/py11File.tcc b/bindings/Python/py11File.tcc index d8fa1b5da8..e102c1bfdf 100644 --- a/bindings/Python/py11File.tcc +++ b/bindings/Python/py11File.tcc @@ -91,7 +91,7 @@ pybind11::array File::DoRead(const std::string &name, const Dims &_start, } m_Stream->m_Engine->Get(variable, pyArray.mutable_data(), Mode::Sync); - return pyArray; + return std::move(pyArray); } } // end namespace py11 diff --git a/bindings/Python/py11Query.cpp b/bindings/Python/py11Query.cpp new file mode 100644 index 0000000000..8d097eeba5 --- /dev/null +++ b/bindings/Python/py11Query.cpp @@ -0,0 +1,45 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * py11Query.h : + * + * Created on: August 5, 2021 + * Author: Junmin Gu (jgu@lbl.gov) + */ + +#include "py11Query.h" + +namespace adios2 +{ +namespace py11 +{ + +Query::Query(adios2::query::Worker *qw) : m_QueryWorker(qw) {} + +Query::Query(std::string queryFile, Engine reader) +{ + adios2::query::Worker *m = + adios2::query::GetWorker(queryFile, reader.m_Engine); + if (m == nullptr) + throw std::invalid_argument("ERROR: unable to construct query. "); + m_QueryWorker = std::make_shared(std::move(*m)); + delete m; +} + +Query::operator bool() const noexcept +{ + return (m_QueryWorker == nullptr) ? false : true; +} + +std::vector> Query::GetResult() +{ + // std::cout<<"Do something"< empty; // look into all data + std::vector> touched_blocks; + m_QueryWorker->GetResultCoverage(empty, touched_blocks); + return touched_blocks; +} + +} // py11 +} // adios2 diff --git a/bindings/Python/py11Query.h b/bindings/Python/py11Query.h new file mode 100644 index 0000000000..68147e17f2 --- /dev/null +++ b/bindings/Python/py11Query.h @@ -0,0 +1,48 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * py11Query.h : + * + * Created on: August 5, 2021 + * Author: Junmin Gu (jgu@lbl.gov) + */ + +#ifndef ADIOS2_BINDINGS_PYTHON_QUERY_H_ +#define ADIOS2_BINDINGS_PYTHON_QUERY_H_ + +#include + +//#include "adios2/toolkit/query/Query.h" +#include "adios2/toolkit/query/Worker.h" +#include "py11Engine.h" + +namespace adios2 +{ +namespace py11 +{ + +class Engine; + +class Query +{ + +public: + Query(std::string queryFile, Engine reader); + ~Query() = default; + + explicit operator bool() const noexcept; + + std::vector> GetResult(); + // const Box< Dims > & refinedSelectionIfAny, + // std::vector< Box< Dims > > &touched_blocks + +private: + Query(adios2::query::Worker *qw); + std::shared_ptr m_QueryWorker; +}; + +} // end namespace py11 +} // end namespace adios2 + +#endif /* BINDINGS_PYTHON_PYQUERY_H_ */ diff --git a/bindings/Python/py11glue.cpp b/bindings/Python/py11glue.cpp index 97d007a555..9f77ab14de 100644 --- a/bindings/Python/py11glue.cpp +++ b/bindings/Python/py11glue.cpp @@ -26,6 +26,7 @@ #include "py11File.h" #include "py11IO.h" #include "py11Operator.h" +#include "py11Query.h" #include "py11Variable.h" #if ADIOS2_USE_MPI @@ -364,6 +365,25 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) .def("RemoveAttribute", &adios2::py11::IO::RemoveAttribute) .def("RemoveAllAttributes", &adios2::py11::IO::RemoveAllAttributes); + pybind11::class_(m, "Query") + .def("__nonzero__", + [](const adios2::py11::Query &query) { + const bool opBool = query ? true : false; + return opBool; + }) + // Python 3 + .def("__bool__", + [](const adios2::py11::Query &query) { + const bool opBool = query ? true : false; + return opBool; + }) + .def( + pybind11::init(), + "adios2 query construction, a xml query File and a read engine", + pybind11::arg("queryFile"), pybind11::arg("reader") = true) + + .def("GetResult", &adios2::py11::Query::GetResult); + pybind11::class_(m, "Variable") // Python 2 .def("__nonzero__", @@ -522,7 +542,6 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) [](const adios2::py11::File &stream) { return stream; }) .def("__exit__", [](adios2::py11::File &stream, pybind11::args) { stream.Close(); }) - .def("__iter__", [](adios2::py11::File &stream) { return stream; }, pybind11::keep_alive<0, 1>()) .def("__next__", diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index 54a25e1196..94ce684e99 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -227,11 +227,11 @@ if(ADIOS2_HAVE_MPI) endif() endif() -# Table -if(ADIOS2_USE_Table STREQUAL AUTO) - set(ADIOS2_HAVE_Table TRUE) -elseif(ADIOS2_USE_Table) - set(ADIOS2_HAVE_Table TRUE) +# MHS +if(ADIOS2_USE_MHS STREQUAL AUTO) + set(ADIOS2_HAVE_MHS TRUE) +elseif(ADIOS2_USE_MHS) + set(ADIOS2_HAVE_MHS TRUE) endif() # DataSpaces diff --git a/cmake/FindPythonModule.cmake b/cmake/FindPythonModule.cmake index 4f9cf29f57..7a71efb78c 100644 --- a/cmake/FindPythonModule.cmake +++ b/cmake/FindPythonModule.cmake @@ -141,7 +141,7 @@ if(PythonModule_${module_NAME}_FOUND AND add_library(${module_NAME} INTERFACE) add_library(Python::${module_NAME} ALIAS ${module_NAME}) foreach(inc_var IN LISTS include_vars) - target_include_directories(${module_NAME} INTERFACE ${${inc_var}}) + target_include_directories(${module_NAME} SYSTEM INTERFACE ${${inc_var}}) endforeach() foreach(lib_var IN LISTS library_vars) target_link_libraries(${module_NAME} INTERFACE ${${lib_var}}) diff --git a/docs/environment.yml b/docs/environment.yml index 1143ed4720..0c0c43fd46 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -9,6 +9,7 @@ dependencies: - adios2-openmpi - sphinx - breathe=4.18.1 + - funcparserlib>=0.3.6 - pip - pip: - blockdiag>=1.5.4 diff --git a/docs/requirements.txt b/docs/requirements.txt index d5f0b0de7d..a3299f4684 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -11,7 +11,7 @@ imagesize==1.2.0 Jinja2==2.11.3 MarkupSafe==1.1.1 packaging==20.1 -Pillow==8.2.0 +Pillow==8.3.2 Pygments==2.7.4 pyparsing==2.4.6 pytz==2019.3 diff --git a/docs/user_guide/source/engines/hdf5.rst b/docs/user_guide/source/engines/hdf5.rst index 6cad70af56..d1dbb85a79 100644 --- a/docs/user_guide/source/engines/hdf5.rst +++ b/docs/user_guide/source/engines/hdf5.rst @@ -13,18 +13,6 @@ or, set it in client code. For example, here is how to create a hdf5 reader: h5IO.SetEngine("HDF5"); adios2::Engine h5Reader = h5IO.Open(filename, adios2::Mode::Read); -In addition, with HDF5 distribution greater or equal to 1.11, one can use the engine ``HDF5Mixer`` -to write files with the VDS (virtual dataset) feature from HDF5. -The corresponding tag in the xml file is: ```` - -and a sample code for VDS writer is: - -.. code-block:: c++ - - adios2::IO h5IO = adios.DeclareIO("SomeName"); - h5IO.SetEngine("HDF5Mixer"); - adios2::Engine h5Writer = h5IO.Open(filename, adios2::Mode::Write); - To read back the h5 files generated with VDS to ADIOS2, one can use the HDF5 engine. Please make sure you are using the HDF5 library that has version greater than or equal to 1.11 in ADIOS2. The h5 file generated by ADIOS2 has two levels of groups: The top Group, ``/`` and its subgroups: ``Step0`` ... ``StepN``, where ``N`` is number of steps. All datasets belong to the subgroups. diff --git a/docs/user_guide/source/setting_up/source/hpc_systems.rst b/docs/user_guide/source/setting_up/source/hpc_systems.rst index 6caa4d3f98..c4dff6aea7 100644 --- a/docs/user_guide/source/setting_up/source/hpc_systems.rst +++ b/docs/user_guide/source/setting_up/source/hpc_systems.rst @@ -13,7 +13,9 @@ Building on HPC Systems Read the system documentation to enable dynamic compilation, usually by setting an environment variable such as ``CRAYPE_LINK_TYPE=dynamic``. Click `here `_ for a fully configurable script example on OLCF systems. -#. **Big Endian and 32-bit systems:** ADIOS2 hasn't been tested on big endian or 32-bit systems. Please be aware before attempting to run. +#. **Big Endian and 32-bit systems:** ADIOS2 hasn't been tested on big + endian and generally will not build on 32-bit systems. Please be + aware before attempting to run. #. **PGI compilers and C++11 support:** Version 15 of the PGI compiler is C++11 compliant. However it relies on the C++ standard library headers supplied by the system version of GCC, which may or may support all the C++11 features used in ADIOS2. @@ -25,3 +27,72 @@ Building on HPC Systems $ module load gcc/6.3.0 $ makelocalrc $(dirname $(which pgc++)) -gcc $(which gcc) -gpp $(which g++) -g77 $(which gfortran) -o -net 1>${HOME}/.mypgirc 2>/dev/null + + + +#. **Enabling RDMA for SST data transfers:** The SST engine in ADIOS2 + is capable of using RDMA networks for transfering data between + writer and reader cohorts, and generally this is the most + performant data transport. However, SST depends upon libfabric to + provide a generic interface to the underlying RDMA capabilities of + the network, and properly configuring libfabric can be a difficult + and error-prone task. HPC computing resources tend to be one-off + custom resources with their own idiosyncracies, so this + documentation cannot offer a definitive guide for every situation, + but we can provide some general guidance and some recommendations + for specific machines. If you are unable to configure ADIOS2 and + libfabric to use RDMA, the best way to get help is to open an issue + on the ADIOS2 github repository. + +Pre-build concerns of note: + + * on some HPC resources, libfabric is available as a loadable + module. That should not be taken as an indication that that + build of libfabric will work with SST, or even that it is + compatible with the system upon which you find it. Your + mileage may vary and you may have to build libfabric + manually. + * libfabric itself depends upon other libraries like + libibverbs and librdmacm. If you build libfabric with a + package manager like spack, spack may build custom versions + of those libraries as well, which may conflict with the + system versions of those libraries. + * MPI on your HPC system may use libfabric itself, and linking + your application with a different version of libfabric (or + its dependent libraries) may result failure, possibly + including opaque error messages from MPI. + * libfabric is structured in such a way that even if it is + found during configuration, ADIOS *cannot* determine at + compile time what providers will be present at run-time, or + what their capabilities are. Therefore even a build that + seems to successfully include libfabric and RDMA may be + rejected at runtime as unable to support SST data transfer. + +Configuration: + ADIOS2 uses the CMake find_package() functionality to locate + libfabric. CMake will automatically search system libraries, + but if you need to specify a libfabric location other than in + a default system location you can add a + "-DLIBFABRIC_ROOT=" argument to direct CMake to + libfabric's location. If CMake finds libfabric, you should + see the line "RDMA Transport for Staging: Available" near the + end of the CMake output. This makes the RDMA DataTransport the + default for SST data movement. (More information about SST + engine parameters like `DataTransport` appears in the SST + engine description.) If instead you see "RDMA Transport for + Staging: Unconfigured", RDMA will not be available to SST. + +Run-time: + Generally, if RDMA is configured and the libfabric provider + has the capabilities that SST needs for RDMA data transfer, + SST will use RDMA without external acknowledgement. However, + if RDMA is configured, but the libfabric provider doesn't have + the capabilities that SST needs, ADIOS will output an error : + 'Warning: Preferred DataPlane "RDMA" not found.' + If you see this warning in a situation where you expect RDMA + to be used, enabling verbose debugging output from SST may + provide more information. The SstVerbose environment + variable can have values from 1 to 5, with 1 being minimal + debugging info (such as confirming which DataTransport is + being used), and 5 being the most detailed debugging + information from all ranks. diff --git a/examples/basics/globalArray/globalArray_write.cpp b/examples/basics/globalArray/globalArray_write.cpp index 2b88a04960..b53d01d9ae 100644 --- a/examples/basics/globalArray/globalArray_write.cpp +++ b/examples/basics/globalArray/globalArray_write.cpp @@ -63,6 +63,8 @@ int main(int argc, char *argv[]) // Get io settings from the config file or // create one with default settings here adios2::IO io = adios.DeclareIO("Output"); + io.SetEngine("BP5"); + io.SetParameter("NumAggregators", "1"); /* * Define global array: type, name, global dimensions diff --git a/examples/basics/values/values_write.cpp b/examples/basics/values/values_write.cpp index 668cffa62c..99f27bb226 100644 --- a/examples/basics/values/values_write.cpp +++ b/examples/basics/values/values_write.cpp @@ -71,6 +71,8 @@ int main(int argc, char *argv[]) */ // 1. Global constant, same value across processes, constant over time adios2::Variable varNproc = io.DefineVariable("Nproc"); + (void)varNproc; // For the sake of the example we create an unused + // variable // 2. Global value, same value across processes, varying value over time adios2::Variable varStep = io.DefineVariable("Step"); diff --git a/examples/heatTransfer/read_fileonly/heatRead_adios2.cpp b/examples/heatTransfer/read_fileonly/heatRead_adios2.cpp index 7e28053682..c12fa78b1e 100644 --- a/examples/heatTransfer/read_fileonly/heatRead_adios2.cpp +++ b/examples/heatTransfer/read_fileonly/heatRead_adios2.cpp @@ -72,6 +72,7 @@ int main(int argc, char *argv[]) adios2::Variable vgndx = bpReaderIO.InquireVariable("gndx"); + (void)vgndx; // gndx = vgndx.GetData()[0]; diff --git a/examples/hello/bpFWriteCRead/FReader.f90 b/examples/hello/bpFWriteCRead/FReader.f90 index 17c7f7374a..67e0f7d461 100644 --- a/examples/hello/bpFWriteCRead/FReader.f90 +++ b/examples/hello/bpFWriteCRead/FReader.f90 @@ -5,7 +5,8 @@ program FReader integer(kind=8), dimension(2) :: sel_start, sel_count real, dimension(:,:), allocatable :: data - integer :: i, j, inx, iny, irank, isize, ierr + integer(kind=8) :: i, j + integer :: irank, isize, ierr ! adios2 handlers type(adios2_adios):: adios diff --git a/examples/hello/bpReader/helloBPReaderHeatMap3D.F90 b/examples/hello/bpReader/helloBPReaderHeatMap3D.F90 index 623cacb53a..a87876f736 100644 --- a/examples/hello/bpReader/helloBPReaderHeatMap3D.F90 +++ b/examples/hello/bpReader/helloBPReaderHeatMap3D.F90 @@ -14,8 +14,9 @@ program helloBPReaderHeatMap3D integer, dimension(:,:,:), allocatable :: temperatures, sel_temperatures integer(kind=8), dimension(3) :: ishape, istart, icount integer(kind=8), dimension(3) :: sel_start, sel_count - integer :: ierr, irank, isize, inx, iny, inz - integer :: i, j, k, iglobal, value, ilinear, icounter + integer(kind=8) :: iglobal, value + integer(kind=8) :: i, j, k + integer(kind=4) :: ierr, irank, isize, inx, iny, inz #if ADIOS2_USE_MPI call MPI_INIT(ierr) @@ -42,7 +43,7 @@ program helloBPReaderHeatMap3D iglobal = istart(1) + i value = (k-1) * ishape(1) * ishape(2) + (j-1) * ishape(1) + & & (iglobal-1) - temperatures(i,j,k) = value + temperatures(i,j,k) = INT(value) end do end do end do diff --git a/examples/hello/bpReader/helloBPReader_nompi.cpp b/examples/hello/bpReader/helloBPReader_nompi.cpp index 776543d8a5..f8161897d0 100644 --- a/examples/hello/bpReader/helloBPReader_nompi.cpp +++ b/examples/hello/bpReader/helloBPReader_nompi.cpp @@ -36,7 +36,7 @@ int main(int argc, char *argv[]) bpIO.AvailableVariables(true); std::cout << "List of variables:"; - for (const auto variablePair : variables) + for (const auto &variablePair : variables) { std::cout << " " << variablePair.first; } diff --git a/examples/hello/bpWriter/helloBPSZ.cpp b/examples/hello/bpWriter/helloBPSZ.cpp index 3b24095e1e..a9943557ed 100644 --- a/examples/hello/bpWriter/helloBPSZ.cpp +++ b/examples/hello/bpWriter/helloBPSZ.cpp @@ -89,6 +89,9 @@ int main(int argc, char *argv[]) adios2::Attribute attribute = bpIO.DefineAttribute("SZ_accuracy", accuracy); + // To avoid compiling warnings + (void)attribute; + /** Engine derived class, spawned to start IO operations */ adios2::Engine bpFileWriter = bpIO.Open("SZexample.bp", adios2::Mode::Write); diff --git a/examples/hello/bpWriter/helloBPSubStreams.cpp b/examples/hello/bpWriter/helloBPSubStreams.cpp index 73e9412775..59ed618eef 100644 --- a/examples/hello/bpWriter/helloBPSubStreams.cpp +++ b/examples/hello/bpWriter/helloBPSubStreams.cpp @@ -67,6 +67,8 @@ int main(int argc, char *argv[]) adios2::Attribute attribute = bpIO.DefineAttribute("attrINT", -1); + (void)attribute; // For the sake of the example we create an unused + // variable /** Engine derived class, spawned to start IO operations */ adios2::Engine bpFileWriter = diff --git a/examples/hello/bpWriter/helloBPWriter.F90 b/examples/hello/bpWriter/helloBPWriter.F90 index 008bd4e372..1a7610f9f2 100644 --- a/examples/hello/bpWriter/helloBPWriter.F90 +++ b/examples/hello/bpWriter/helloBPWriter.F90 @@ -8,7 +8,7 @@ program helloBPWriter integer(kind=8), dimension(1) :: shape_dims, start_dims, count_dims real, dimension(:), allocatable :: myArray - integer :: inx, irank, isize, ierr, i, var1_type + integer :: inx, irank, isize, ierr, i type(adios2_adios) :: adios type(adios2_io) :: io type(adios2_variable) :: var1 diff --git a/examples/hello/bpWriter/helloBPWriter.cpp b/examples/hello/bpWriter/helloBPWriter.cpp index e67c82ba92..6c5714b8e7 100644 --- a/examples/hello/bpWriter/helloBPWriter.cpp +++ b/examples/hello/bpWriter/helloBPWriter.cpp @@ -63,6 +63,8 @@ int main(int argc, char *argv[]) adios2::Variable bpString = bpIO.DefineVariable("bpString"); + (void)bpString; // For the sake of the example we create an unused + // variable std::string filename = "myVector_cpp.bp"; /** Engine derived class, spawned to start IO operations */ diff --git a/examples/hello/hdf5Reader/helloHDF5Reader_nompi.cpp b/examples/hello/hdf5Reader/helloHDF5Reader_nompi.cpp index a6f64151ad..9aab7f8b5f 100644 --- a/examples/hello/hdf5Reader/helloHDF5Reader_nompi.cpp +++ b/examples/hello/hdf5Reader/helloHDF5Reader_nompi.cpp @@ -93,7 +93,7 @@ int main(int argc, char *argv[]) const std::map variables = h5IO.AvailableVariables(); - for (const auto variablePair : variables) + for (const auto &variablePair : variables) { std::cout << "Name: " << variablePair.first; std::cout << std::endl; diff --git a/examples/hello/insituMPI/helloInsituArgs.F90 b/examples/hello/insituMPI/helloInsituArgs.F90 index 59a00c2667..8744780490 100644 --- a/examples/hello/insituMPI/helloInsituArgs.F90 +++ b/examples/hello/insituMPI/helloInsituArgs.F90 @@ -9,13 +9,13 @@ module helloInsituArgs integer(kind=8) :: offx, offy ! Offsets of local array in this process integer :: steps ! number of steps to write integer :: sleeptime ! wait time between steps in seconds - + contains !!*************************** subroutine usage(isWriter) - logical, intent(in) :: isWriter - if (isWriter) then + logical, intent(in) :: isWriter + if (isWriter) then print *, "Usage: helloInsituMPIWriter config N M nx ny steps sleeptime" else print *, "Usage: helloInsituMPIReader config N M" @@ -24,16 +24,16 @@ subroutine usage(isWriter) print *, "config: path of XML config file" print *, "N: number of processes in X dimension" print *, "M: number of processes in Y dimension" - if (isWriter) then + if (isWriter) then print *, "nx: local array size in X dimension per processor" print *, "ny: local array size in Y dimension per processor" - print *, "steps: the total number of steps to output" + print *, "steps: the total number of steps to output" print *, "sleeptime: wait time between steps in seconds" endif -end subroutine usage - +end subroutine usage + !!*************************** -subroutine processArgs(rank, nproc, isWriter) +subroutine processArgs(nproc, isWriter) #if defined(ADIOS2_HAVE_FORTRAN_F03_ARGS) # define ADIOS2_ARGC() command_argument_count() @@ -46,17 +46,16 @@ subroutine processArgs(rank, nproc, isWriter) # define ADIOS2_ARGV(i, v) #endif - integer, intent(in) :: rank integer, intent(in) :: nproc - logical, intent(in) :: isWriter + logical, intent(in) :: isWriter character(len=256) :: npx_str, npy_str, ndx_str, ndy_str character(len=256) :: steps_str,time_str integer :: numargs, expargs !! expected number of arguments - if (isWriter) then + if (isWriter) then expargs = 7 - else + else expargs = 3 endif @@ -70,7 +69,7 @@ subroutine processArgs(rank, nproc, isWriter) ADIOS2_ARGV(1, xmlfile) ADIOS2_ARGV(2, npx_str) ADIOS2_ARGV(3, npy_str) - if (isWriter) then + if (isWriter) then ADIOS2_ARGV(4, ndx_str) ADIOS2_ARGV(5, ndy_str) ADIOS2_ARGV(6, steps_str) @@ -78,7 +77,7 @@ subroutine processArgs(rank, nproc, isWriter) endif read (npx_str,'(i5)') npx read (npy_str,'(i5)') npy - if (isWriter) then + if (isWriter) then read (ndx_str,'(i6)') ndx read (ndy_str,'(i6)') ndy read (steps_str,'(i6)') steps @@ -94,11 +93,10 @@ subroutine processArgs(rank, nproc, isWriter) end subroutine processArgs !!****************************************************** -subroutine DecomposeArray(gndx, gndy, rank, nproc) +subroutine DecomposeArray(gndx, gndy, rank) integer(kind=8), intent(in) :: gndx integer(kind=8), intent(in) :: gndy integer, intent(in) :: rank - integer, intent(in) :: nproc posx = mod(rank, npx); posy = rank / npx; @@ -119,6 +117,5 @@ subroutine DecomposeArray(gndx, gndy, rank, nproc) endif end subroutine DecomposeArray - -end module helloInsituArgs +end module helloInsituArgs diff --git a/examples/hello/insituMPI/helloInsituMPIReader.f90 b/examples/hello/insituMPI/helloInsituMPIReader.f90 index 1c136c8cc7..c5e5139b2f 100644 --- a/examples/hello/insituMPI/helloInsituMPIReader.f90 +++ b/examples/hello/insituMPI/helloInsituMPIReader.f90 @@ -16,7 +16,7 @@ program helloInsituMPIReader integer(kind=8), dimension(:), allocatable :: shape_dims integer(kind=8), dimension(:), allocatable :: sel_start, sel_count integer :: ierr - integer :: i, j, step + integer :: step integer :: comm, color @@ -31,7 +31,7 @@ program helloInsituMPIReader call MPI_Comm_rank(comm, rank, ierr) call MPI_Comm_size(comm, nproc, ierr) - call ProcessArgs(rank, nproc, .false.) + call ProcessArgs(nproc, .false.) ! Start adios2 call adios2_init( adios, xmlfile, comm, adios2_debug_mode_on, ierr ) @@ -55,7 +55,7 @@ program helloInsituMPIReader if (step == 0) then call adios2_variable_shape( shape_dims, ndims, varArray, ierr) ! ndims is assumed to be 2 here - call DecomposeArray( shape_dims(1), shape_dims(2), rank, nproc) + call DecomposeArray( shape_dims(1), shape_dims(2), rank) allocate (sel_start(2), sel_count(2)) sel_start = (/ offx, offy /) sel_count = (/ ndx, ndy /) diff --git a/examples/hello/insituMPI/helloInsituMPIWriter.f90 b/examples/hello/insituMPI/helloInsituMPIWriter.f90 index 37f3fb599d..d99665a949 100644 --- a/examples/hello/insituMPI/helloInsituMPIWriter.f90 +++ b/examples/hello/insituMPI/helloInsituMPIWriter.f90 @@ -8,7 +8,8 @@ program helloInsituMPIWriter integer(kind=8), dimension(2) :: shape_dims, start_dims, count_dims real, dimension(:,:), allocatable :: myArray integer :: wrank, wsize, rank, nproc - integer :: ierr, i, j, step + integer :: ierr, step = 1 + integer(kind=8) :: i, j type(adios2_adios):: adios type(adios2_io):: io type(adios2_variable):: varArray @@ -22,12 +23,12 @@ program helloInsituMPIWriter call MPI_Comm_size(MPI_COMM_WORLD, wsize, ierr) ! Split MPI world into writers and readers. This code is the writers. - color = 1 + color = 1 call MPI_Comm_split(MPI_COMM_WORLD, color, wrank, comm, ierr) call MPI_Comm_rank(comm, rank, ierr) call MPI_Comm_size(comm, nproc, ierr) - call ProcessArgs(rank, nproc, .true.) + call ProcessArgs(nproc, .true.) ! determine offsets posx = mod(rank, npx) ! 1st dim easy: 0, npx, 2npx... are in the same X position @@ -37,10 +38,10 @@ program helloInsituMPIWriter ! Application variables allocate( myArray(ndx,ndy) ) - myArray = step*npx*ndx*npy*ndy + rank*ndx*ndy + myArray = REAL(step*npx*ndx*npy*ndy + rank*ndx*ndy) do j=1,ndy do i=1,ndx - myArray(i,j) = myArray(i,j) + (j-1)*ndx + i-1 + myArray(i,j) = myArray(i,j) + REAL((j-1)*ndx + i-1) end do end do @@ -53,7 +54,7 @@ program helloInsituMPIWriter ! Create adios handler passing the communicator, config file, debug mode and error flag call adios2_init(adios, xmlfile, comm, adios2_debug_mode_on, ierr) - ! Declare an IO process configuration inside adios, + ! Declare an IO process configuration inside adios, ! Engine choice and parameters for 'writer' come from the config file call adios2_declare_io(io, adios, "writer", ierr) diff --git a/scripts/ci/cmake/ci-ascent-gcc.cmake b/scripts/ci/cmake/ci-ascent-gcc.cmake index 9e67411f6e..ebf1fd0743 100644 --- a/scripts/ci/cmake/ci-ascent-gcc.cmake +++ b/scripts/ci/cmake/ci-ascent-gcc.cmake @@ -14,9 +14,6 @@ env_module(load zeromq) set(ENV{CC} gcc) set(ENV{CXX} g++) set(ENV{FC} gfortran) -set(ENV{CFLAGS} "-Werror -Wno-error=builtin-declaration-mismatch") -set(ENV{CXXFLAGS} "-Werror -Wno-error=builtin-declaration-mismatch") -set(ENV{FFLAGS} "-Werror -Wno-error=builtin-declaration-mismatch") set(dashboard_cache " ADIOS2_USE_BZip2:BOOL=OFF @@ -30,6 +27,8 @@ ADIOS2_USE_SST:BOOL=ON ADIOS2_USE_SZ:BOOL=OFF ADIOS2_USE_ZeroMQ:STRING=ON ADIOS2_USE_ZFP:BOOL=OFF + +CMAKE_Fortran_FLAGS:STRING=-Wall ") set(NCPUS 4) diff --git a/scripts/ci/cmake/ci-debian-sid-openmpi.cmake b/scripts/ci/cmake/ci-debian-sid-openmpi.cmake index 889a7ed5a0..c401e70029 100644 --- a/scripts/ci/cmake/ci-debian-sid-openmpi.cmake +++ b/scripts/ci/cmake/ci-debian-sid-openmpi.cmake @@ -12,6 +12,7 @@ set(ENV{CXXFLAGS} "-Wno-deprecated -Wno-deprecated-declarations") set(dashboard_cache " ADIOS2_USE_EXTERNAL_DEPENDENCIES:BOOL=ON ADIOS2_USE_EXTERNAL_EVPATH:BOOL=OFF +ADIOS2_USE_EXTERNAL_NLOHMANN_JSON:BOOL=OFF ADIOS2_USE_BZip2:BOOL=ON ADIOS2_USE_Blosc:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON @@ -28,6 +29,8 @@ CMAKE_INSTALL_PREFIX:STRING=/usr CMAKE_INSTALL_LIBDIR:STRING=lib/${arch} CMAKE_INSTALL_CMAKEDIR:STRING=lib/${arch}/cmake/adios/openmpi +CMAKE_Fortran_FLAGS:STRING=-Wall + MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe MPI_C_COMPILER:FILEPATH=/usr/bin/mpicc.openmpi MPI_CXX_COMPILER:FILEPATH=/usr/bin/mpic++.openmpi diff --git a/scripts/ci/cmake/ci-debian-sid.cmake b/scripts/ci/cmake/ci-debian-sid.cmake index a94eb99b92..2204a0a406 100644 --- a/scripts/ci/cmake/ci-debian-sid.cmake +++ b/scripts/ci/cmake/ci-debian-sid.cmake @@ -12,6 +12,7 @@ set(ENV{CXXFLAGS} "-Wno-deprecated -Wno-deprecated-declarations") set(dashboard_cache " ADIOS2_USE_EXTERNAL_DEPENDENCIES:BOOL=ON ADIOS2_USE_EXTERNAL_EVPATH:BOOL=OFF +ADIOS2_USE_EXTERNAL_NLOHMANN_JSON:BOOL=OFF ADIOS2_USE_BZip2:BOOL=ON ADIOS2_USE_Blosc:BOOL=ON ADIOS2_USE_Fortran:BOOL=ON @@ -28,6 +29,8 @@ CMAKE_INSTALL_PREFIX:STRING=/usr CMAKE_INSTALL_LIBDIR:STRING=lib/${arch} CMAKE_INSTALL_CMAKEDIR:STRING=lib/${arch}/cmake/adios/serial +CMAKE_Fortran_FLAGS:STRING=-Wall + HDF5_C_COMPILER_EXECUTABLE:FILEPATH=/usr/bin/h5cc ") diff --git a/scripts/ci/cmake/ci-el7-gnu8-ohpc.cmake b/scripts/ci/cmake/ci-el7-gnu8-ohpc.cmake index 6bbc905f82..8e142f7225 100644 --- a/scripts/ci/cmake/ci-el7-gnu8-ohpc.cmake +++ b/scripts/ci/cmake/ci-el7-gnu8-ohpc.cmake @@ -10,9 +10,6 @@ env_module(load hdf5) set(ENV{CC} gcc) set(ENV{CXX} g++) set(ENV{FC} gfortran) -set(ENV{CFLAGS} "-Werror -Wno-error=builtin-declaration-mismatch") -set(ENV{CXXFLAGS} "-Werror -Wno-error=builtin-declaration-mismatch") -set(ENV{FFLAGS} "-Werror -Wno-error=builtin-declaration-mismatch") set(dashboard_cache " ADIOS2_USE_BZip2:BOOL=ON @@ -25,6 +22,10 @@ ADIOS2_USE_Python:BOOL=ON ADIOS2_USE_SZ:BOOL=ON ADIOS2_USE_ZeroMQ:STRING=ON ADIOS2_USE_ZFP:BOOL=ON + +CMAKE_C_FLAGS:STRING=-Wall +CMAKE_CXX_FLAGS:STRING=-Wall +CMAKE_Fortran_FLAGS:STRING=-Wall ") set(CTEST_CMAKE_GENERATOR "Unix Makefiles") diff --git a/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc-static.cmake b/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc-static.cmake index 581be161aa..61962398f0 100644 --- a/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc-static.cmake +++ b/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc-static.cmake @@ -16,9 +16,6 @@ env_module(load py3-mpi4py) set(ENV{CC} gcc) set(ENV{CXX} g++) set(ENV{FC} gfortran) -set(ENV{CFLAGS} "-Werror -Wno-error=builtin-declaration-mismatch") -set(ENV{CXXFLAGS} "-Werror -Wno-error=builtin-declaration-mismatch") -set(ENV{FFLAGS} "-Werror -Wno-error=builtin-declaration-mismatch") set(dashboard_cache " BUILD_SHARED_LIBS:BOOL=OFF @@ -36,6 +33,10 @@ ADIOS2_USE_SZ:BOOL=OFF ADIOS2_USE_ZeroMQ:STRING=OFF ADIOS2_USE_ZFP:BOOL=OFF +CMAKE_C_FLAGS:STRING=-Wall +CMAKE_CXX_FLAGS:STRING=-Wall +CMAKE_Fortran_FLAGS:STRING=-Wall + MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} ") diff --git a/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc.cmake b/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc.cmake index 16f5ccf725..2dbec4c744 100644 --- a/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc.cmake +++ b/scripts/ci/cmake/ci-el7-gnu8-openmpi-ohpc.cmake @@ -16,9 +16,6 @@ env_module(load py3-mpi4py) set(ENV{CC} gcc) set(ENV{CXX} g++) set(ENV{FC} gfortran) -set(ENV{CFLAGS} "-Werror -Wno-error=builtin-declaration-mismatch") -set(ENV{CXXFLAGS} "-Werror -Wno-error=builtin-declaration-mismatch") -set(ENV{FFLAGS} "-Werror -Wno-error=builtin-declaration-mismatch") set(dashboard_cache " ADIOS2_USE_BZip2:BOOL=ON @@ -32,6 +29,10 @@ ADIOS2_USE_SZ:BOOL=ON ADIOS2_USE_ZeroMQ:STRING=ON ADIOS2_USE_ZFP:BOOL=ON +CMAKE_C_FLAGS:STRING=-Wall +CMAKE_CXX_FLAGS:STRING=-Wall +CMAKE_Fortran_FLAGS:STRING=-Wall + MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} ") diff --git a/scripts/ci/cmake/ci-el7-intel-ohpc.cmake b/scripts/ci/cmake/ci-el7-intel-ohpc.cmake index 6aec07e815..b3d6504fb1 100644 --- a/scripts/ci/cmake/ci-el7-intel-ohpc.cmake +++ b/scripts/ci/cmake/ci-el7-intel-ohpc.cmake @@ -10,11 +10,10 @@ env_module(load hdf5) set(ENV{CC} icc) set(ENV{CXX} icpc) set(ENV{FC} ifort) -set(ENV{CFLAGS} -Werror) -set(ENV{CXXFLAGS} -Werror) -set(ENV{FFLAGS} "-warn errors") set(dashboard_cache " +CMAKE_C_FLAGS:STRING=-Wall +CMAKE_CXX_FLAGS:STRING=-Wall ADIOS2_USE_BZip2:BOOL=ON ADIOS2_USE_Blosc:BOOL=ON ADIOS2_USE_DataMan:BOOL=ON @@ -25,6 +24,8 @@ ADIOS2_USE_Python:BOOL=ON ADIOS2_USE_SZ:BOOL=ON ADIOS2_USE_ZeroMQ:STRING=ON ADIOS2_USE_ZFP:STRING=ON + +CMAKE_Fortran_FLAGS:STRING=-warn all -stand none ") set(CTEST_CMAKE_GENERATOR "Unix Makefiles") diff --git a/scripts/ci/cmake/ci-el7-intel-openmpi-ohpc.cmake b/scripts/ci/cmake/ci-el7-intel-openmpi-ohpc.cmake index 696ca55aad..6559df5773 100644 --- a/scripts/ci/cmake/ci-el7-intel-openmpi-ohpc.cmake +++ b/scripts/ci/cmake/ci-el7-intel-openmpi-ohpc.cmake @@ -16,9 +16,6 @@ env_module(load py3-mpi4py) set(ENV{CC} icc) set(ENV{CXX} icpc) set(ENV{FC} ifort) -set(ENV{CFLAGS} -Werror) -set(ENV{CXXFLAGS} -Werror) -set(ENV{FFLAGS} "-warn errors") set(dashboard_cache " ADIOS2_USE_BZip2:BOOL=ON @@ -32,6 +29,10 @@ ADIOS2_USE_SZ:BOOL=ON ADIOS2_USE_ZeroMQ:STRING=ON ADIOS2_USE_ZFP:STRING=ON +CMAKE_C_FLAGS:STRING=-Wall +CMAKE_CXX_FLAGS:STRING=-Wall +CMAKE_Fortran_FLAGS:STRING=-warn all -stand none + MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} ") diff --git a/scripts/ci/cmake/ci-el7-spack.cmake b/scripts/ci/cmake/ci-el7-spack.cmake index 6fcd590243..94c1f497c1 100644 --- a/scripts/ci/cmake/ci-el7-spack.cmake +++ b/scripts/ci/cmake/ci-el7-spack.cmake @@ -26,6 +26,10 @@ ADIOS2_USE_EXTERNAL_DEPENDENCIES:BOOL=ON ADIOS2_USE_EXTERNAL_GTEST:BOOL=OFF ADIOS2_USE_EXTERNAL_PUGIXML:BOOL=OFF +CMAKE_C_FLAGS:STRING=-Wall +CMAKE_CXX_FLAGS:STRING=-Wall +CMAKE_Fortran_FLAGS:STRING=-Wall + MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} ") diff --git a/scripts/ci/cmake/ci-el7.cmake b/scripts/ci/cmake/ci-el7.cmake index 4ee0370760..234e20b07a 100644 --- a/scripts/ci/cmake/ci-el7.cmake +++ b/scripts/ci/cmake/ci-el7.cmake @@ -3,9 +3,6 @@ set(ENV{CC} gcc) set(ENV{CXX} g++) set(ENV{FC} gfortran) -set(ENV{CFLAGS} -Werror) -set(ENV{CXXFLAGS} -Werror) -set(ENV{FFLAGS} -Werror) set(dashboard_cache " ADIOS2_USE_BZip2:STRING=ON @@ -16,6 +13,10 @@ ADIOS2_USE_MPI:STRING=OFF ADIOS2_USE_Python:STRING=ON ADIOS2_USE_ZeroMQ:STRING=ON ADIOS2_USE_ZFP:STRING=ON + +CMAKE_C_FLAGS:STRING=-Wall +CMAKE_CXX_FLAGS:STRING=-Wall +CMAKE_Fortran_FLAGS:STRING=-Wall ") set(CTEST_CMAKE_GENERATOR "Unix Makefiles") diff --git a/scripts/ci/cmake/ci-macos1014-xcode103-ninja.cmake b/scripts/ci/cmake/ci-macos1014-xcode103-ninja.cmake index b9f1eba04e..a1ab440d24 100644 --- a/scripts/ci/cmake/ci-macos1014-xcode103-ninja.cmake +++ b/scripts/ci/cmake/ci-macos1014-xcode103-ninja.cmake @@ -7,6 +7,10 @@ set(dashboard_cache " ADIOS2_USE_Fortran:BOOL=ON ADIOS2_USE_MPI:BOOL=OFF ADIOS2_USE_Python:BOOL=ON + +CMAKE_C_FLAGS:STRING=-Wall +CMAKE_CXX_FLAGS:STRING=-Wall +CMAKE_Fortran_FLAGS:STRING=-Wall ") set(ENV{MACOSX_DEPLOYMENT_TARGET} "10.14") diff --git a/scripts/ci/cmake/ci-macos1014-xcode103-openmpi-make.cmake b/scripts/ci/cmake/ci-macos1014-xcode103-openmpi-make.cmake index 51ba0bfc31..f39482a66c 100644 --- a/scripts/ci/cmake/ci-macos1014-xcode103-openmpi-make.cmake +++ b/scripts/ci/cmake/ci-macos1014-xcode103-openmpi-make.cmake @@ -11,6 +11,10 @@ set(dashboard_cache " ADIOS2_USE_Fortran:BOOL=ON ADIOS2_USE_MPI:STRING=ON +CMAKE_C_FLAGS:STRING=-Wall +CMAKE_CXX_FLAGS:STRING=-Wall +CMAKE_Fortran_FLAGS:STRING=-Wall + MPIEXEC_EXTRA_FLAGS:STRING=--oversubscribe MPIEXEC_MAX_NUMPROCS:STRING=4 ") diff --git a/scripts/ci/cmake/ci-macos1015-xcode111-make.cmake b/scripts/ci/cmake/ci-macos1015-xcode111-make.cmake index c422af45a0..5852b90ad3 100644 --- a/scripts/ci/cmake/ci-macos1015-xcode111-make.cmake +++ b/scripts/ci/cmake/ci-macos1015-xcode111-make.cmake @@ -7,6 +7,10 @@ set(dashboard_cache " ADIOS2_USE_Fortran:BOOL=ON ADIOS2_USE_MPI:BOOL=OFF ADISO2_USE_Python:BOOL=ON + +CMAKE_C_FLAGS:STRING=-Wall +CMAKE_CXX_FLAGS:STRING=-Wall +CMAKE_Fortran_FLAGS:STRING=-Wall ") set(ENV{MACOSX_DEPLOYMENT_TARGET} "10.15") diff --git a/scripts/ci/cmake/ci-macos1015-xcode111-openmpi-ninja.cmake b/scripts/ci/cmake/ci-macos1015-xcode111-openmpi-ninja.cmake index af2ffa94c8..c27e122696 100644 --- a/scripts/ci/cmake/ci-macos1015-xcode111-openmpi-ninja.cmake +++ b/scripts/ci/cmake/ci-macos1015-xcode111-openmpi-ninja.cmake @@ -11,6 +11,10 @@ set(dashboard_cache " ADIOS2_USE_Fortran:BOOL=ON ADIOS2_USE_MPI:STRING=ON +CMAKE_C_FLAGS:STRING=-Wall +CMAKE_CXX_FLAGS:STRING=-Wall +CMAKE_Fortran_FLAGS:STRING=-Wall + MPIEXEC_EXTRA_FLAGS:STRING=--oversubscribe MPIEXEC_MAX_NUMPROCS:STRING=4 ") diff --git a/scripts/ci/cmake/ci-ubuntu1804-spack.cmake b/scripts/ci/cmake/ci-ubuntu1804-spack.cmake index 2b833980d4..94c1f497c1 100644 --- a/scripts/ci/cmake/ci-ubuntu1804-spack.cmake +++ b/scripts/ci/cmake/ci-ubuntu1804-spack.cmake @@ -7,9 +7,6 @@ math(EXPR N2CPUS "${NCPUS}*2") set(ENV{CC} gcc) set(ENV{CXX} g++) set(ENV{FC} gfortran) -set(ENV{CFLAGS} "-Werror -Wno-error=builtin-declaration-mismatch") -set(ENV{CXXFLAGS} "-Werror -Wno-error=builtin-declaration-mismatch") -set(ENV{FFLAGS} "-Werror -Wno-error=builtin-declaration-mismatch") set(dashboard_cache " ADIOS2_USE_BZip2:BOOL=ON @@ -29,6 +26,10 @@ ADIOS2_USE_EXTERNAL_DEPENDENCIES:BOOL=ON ADIOS2_USE_EXTERNAL_GTEST:BOOL=OFF ADIOS2_USE_EXTERNAL_PUGIXML:BOOL=OFF +CMAKE_C_FLAGS:STRING=-Wall +CMAKE_CXX_FLAGS:STRING=-Wall +CMAKE_Fortran_FLAGS:STRING=-Wall + MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} ") diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 92f0425cdd..266dafca05 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -5,8 +5,8 @@ add_library(adios2_core common/ADIOSTypes.cpp - - core/Attribute.cpp + + core/Attribute.cpp core/AttributeBase.cpp core/ADIOS.cpp core/Engine.cpp @@ -37,7 +37,7 @@ add_library(adios2_core helper/adiosYAML.cpp helper/adiosCUDA.cu -#engine derived classes +#engine derived classes engine/bp3/BP3Reader.cpp engine/bp3/BP3Reader.tcc engine/bp3/BP3Writer.cpp engine/bp3/BP3Writer.tcc @@ -49,13 +49,15 @@ add_library(adios2_core engine/inline/InlineReader.cpp engine/inline/InlineReader.tcc engine/inline/InlineWriter.cpp engine/inline/InlineWriter.tcc - + engine/null/NullEngine.cpp engine/nullcore/NullCoreWriter.cpp engine/nullcore/NullCoreWriter.tcc #toolkit toolkit/format/buffer/Buffer.cpp toolkit/format/buffer/BufferV.cpp + toolkit/format/buffer/malloc/MallocV.cpp + toolkit/format/buffer/chunk/ChunkV.cpp toolkit/format/buffer/heap/BufferSTL.cpp toolkit/format/bp/BPBase.cpp toolkit/format/bp/BPBase.tcc @@ -68,19 +70,10 @@ add_library(adios2_core toolkit/format/bp/bp4/BP4Base.cpp toolkit/format/bp/bp4/BP4Serializer.cpp toolkit/format/bp/bp4/BP4Serializer.tcc toolkit/format/bp/bp4/BP4Deserializer.cpp toolkit/format/bp/bp4/BP4Deserializer.tcc - - toolkit/format/bp/bpOperation/BPOperation.cpp + + toolkit/format/bp/bpOperation/BPOperation.cpp toolkit/format/bp/bpOperation/BPOperation.tcc - toolkit/format/bp/bpOperation/compress/BPZFP.cpp - toolkit/format/bp/bpOperation/compress/BPZFP.tcc - toolkit/format/bp/bpOperation/compress/BPSZ.cpp - toolkit/format/bp/bpOperation/compress/BPMGARD.cpp - toolkit/format/bp/bpOperation/compress/BPPNG.cpp - toolkit/format/bp/bpOperation/compress/BPBZIP2.cpp - toolkit/format/bp/bpOperation/compress/BPBZIP2.tcc - toolkit/format/bp/bpOperation/compress/BPBlosc.cpp - toolkit/format/bp/bpOperation/compress/BPLIBPRESSIO.cpp - + toolkit/profiling/iochrono/Timer.cpp toolkit/profiling/iochrono/IOChrono.cpp @@ -98,9 +91,12 @@ add_library(adios2_core toolkit/aggregator/mpi/MPIAggregator.cpp toolkit/aggregator/mpi/MPIChain.cpp + toolkit/aggregator/mpi/MPIShmChain.cpp toolkit/burstbuffer/FileDrainer.cpp toolkit/burstbuffer/FileDrainerSingleThread.cpp + + operator/compress/CompressorFactory.cpp ) set_property(TARGET adios2_core PROPERTY EXPORT_NAME core) set_property(TARGET adios2_core PROPERTY OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_core) @@ -127,15 +123,15 @@ if(UNIX) endif() if (ADIOS2_HAVE_BP5) - target_sources(adios2_core PRIVATE + target_sources(adios2_core PRIVATE engine/bp5/BP5Engine.cpp engine/bp5/BP5Reader.cpp engine/bp5/BP5Reader.tcc - engine/bp5/BP5Writer.cpp engine/bp5/BP5Writer.tcc + engine/bp5/BP5Writer.cpp engine/bp5/BP5Writer.tcc engine/bp5/BP5Writer_TwoLevelShm.cpp ) endif() if (ADIOS2_HAVE_BP5 OR ADIOS2_HAVE_SST) - target_sources(adios2_core PRIVATE + target_sources(adios2_core PRIVATE toolkit/format/buffer/ffs/BufferFFS.cpp toolkit/format/bp5/BP5Base.cpp toolkit/format/bp5/BP5Serializer.cpp @@ -213,10 +209,12 @@ if(ADIOS2_HAVE_SSC) target_link_libraries(adios2_core_mpi) endif() -if(ADIOS2_HAVE_Table) +if(ADIOS2_HAVE_MHS) target_sources(adios2_core PRIVATE - engine/table/TableWriter.cpp - engine/table/TableWriter.tcc + engine/mhs/MhsWriter.cpp + engine/mhs/MhsWriter.tcc + engine/mhs/MhsReader.cpp + engine/mhs/MhsReader.tcc ) endif() @@ -275,6 +273,10 @@ if(ADIOS2_HAVE_PNG) target_link_libraries(adios2_core PRIVATE PNG::PNG) endif() +if(ADIOS2_HAVE_MHS) +target_sources(adios2_core PRIVATE operator/compress/CompressSirius.cpp) +endif() + if(ADIOS2_HAVE_HDF5) add_library(adios2_hdf5 OBJECT core/IOHDF5.cpp @@ -282,13 +284,6 @@ if(ADIOS2_HAVE_HDF5) engine/hdf5/HDF5WriterP.cpp toolkit/interop/hdf5/HDF5Common.cpp toolkit/interop/hdf5/HDF5Common.tcc ) - if(NOT HDF5_VERSION VERSION_LESS 1.11) - target_sources(adios2_hdf5 PRIVATE - engine/mixer/HDFMixer.cpp - engine/mixer/HDFMixer.tcc - engine/mixer/HDFMixerWriter.cpp - ) - endif() target_link_libraries(adios2_core PRIVATE adios2_hdf5) set_property(TARGET adios2_hdf5 PROPERTY EXPORT_NAME hdf5) set_property(TARGET adios2_hdf5 PROPERTY OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_hdf5) diff --git a/source/adios2/common/ADIOSMacros.h b/source/adios2/common/ADIOSMacros.h index 5c71827a62..b79fb3e59d 100644 --- a/source/adios2/common/ADIOSMacros.h +++ b/source/adios2/common/ADIOSMacros.h @@ -135,6 +135,8 @@ MACRO(std::complex) \ MACRO(std::complex) +#define ADIOS2_FOREACH_SIRIUS_TYPE_1ARG(MACRO) MACRO(float) + #define ADIOS2_FOREACH_LIBPRESSIO_TYPE_1ARG(MACRO) \ MACRO(uint8_t) \ MACRO(uint16_t) \ diff --git a/source/adios2/common/ADIOSTypes.h b/source/adios2/common/ADIOSTypes.h index a75175fd28..52c007c9ba 100644 --- a/source/adios2/common/ADIOSTypes.h +++ b/source/adios2/common/ADIOSTypes.h @@ -67,6 +67,7 @@ enum class Mode Write, Read, Append, + ReadRandomAccess, // reader random access mode // launch execution modes Sync, Deferred @@ -96,7 +97,7 @@ enum class StepMode { Append, Update, // writer advance mode - Read // reader advance mode + Read, // reader advance mode }; enum class StepStatus @@ -149,6 +150,14 @@ enum class DataType Compound }; +/** Type of ArrayOrdering */ +enum class ArrayOrdering +{ + RowMajor, /// Contiguous elements of a row lie together in memory + ColumnMajor, /// Contiguous elements of a column lie together in memory + Auto /// Default based on language type +}; + // Types using std::size_t; @@ -192,10 +201,22 @@ constexpr uint64_t DefaultMaxBufferSize = MaxSizeT - 1; * for optimizing applications*/ constexpr float DefaultBufferGrowthFactor = 1.05f; +/** default Buffer Chunk Size + * 16Mb */ +constexpr uint64_t DefaultBufferChunkSize = 16 * 1024 * 1024; + +/** default minimum size not copying deferred writes + * 4Mb */ +constexpr size_t DefaultMinDeferredSize = 4 * 1024 * 1024; + /** default size for writing/reading files using POSIX/fstream/stdio write * 2Gb - 100Kb (tolerance)*/ constexpr size_t DefaultMaxFileBatchSize = 2147381248; +/** default maximum shared memory segment size + * 128Mb */ +constexpr uint64_t DefaultMaxShmSize = 128 * 1024 * 1024; + constexpr char PathSeparator = #ifdef _WIN32 '\\'; diff --git a/source/adios2/core/ADIOS.cpp b/source/adios2/core/ADIOS.cpp index 30e9481740..f4fbce5e6a 100644 --- a/source/adios2/core/ADIOS.cpp +++ b/source/adios2/core/ADIOS.cpp @@ -121,7 +121,7 @@ ADIOS::ADIOS(const std::string hostLanguage) ADIOS::~ADIOS() = default; -IO &ADIOS::DeclareIO(const std::string name) +IO &ADIOS::DeclareIO(const std::string name, const ArrayOrdering ArrayOrder) { auto itIO = m_IOs.find(name); @@ -132,6 +132,7 @@ IO &ADIOS::DeclareIO(const std::string name) if (!io.IsDeclared()) // exists from config xml { io.SetDeclared(); + io.SetArrayOrder(ArrayOrder); return io; } else @@ -149,6 +150,7 @@ IO &ADIOS::DeclareIO(const std::string name) std::forward_as_tuple(*this, name, false, m_HostLanguage)); IO &io = ioPair.first->second; io.SetDeclared(); + io.SetArrayOrder(ArrayOrder); return io; } diff --git a/source/adios2/core/ADIOS.h b/source/adios2/core/ADIOS.h index 22ca8b46ba..d7b500d43d 100644 --- a/source/adios2/core/ADIOS.h +++ b/source/adios2/core/ADIOS.h @@ -86,7 +86,8 @@ class ADIOS * @exception std::invalid_argument if IO with unique name is already * declared */ - IO &DeclareIO(const std::string name); + IO &DeclareIO(const std::string name, + const ArrayOrdering ArrayOrder = ArrayOrdering::Auto); /** * Retrieve a reference to an existing IO object created with DeclareIO. diff --git a/source/adios2/core/Attribute.h b/source/adios2/core/Attribute.h index 83fb5fcf1b..84be2c6451 100644 --- a/source/adios2/core/Attribute.h +++ b/source/adios2/core/Attribute.h @@ -38,19 +38,33 @@ class Attribute : public AttributeBase * @param name * @param data * @param elements + * @param allowModifications */ - Attribute(const std::string &name, const T *data, const size_t elements); + Attribute(const std::string &name, const T *data, const size_t elements, + const bool allowModification); /** * Single value constructor * @param name * @param data * @param elements + * @param allowModifications */ - Attribute(const std::string &name, const T &data); + Attribute(const std::string &name, const T &data, + const bool allowModification); ~Attribute() = default; + /** + * Modification of an existing attribute (array) + */ + void Modify(const T *data, const size_t elements); + + /** + * Modification of an existing attribute (single value) + */ + void Modify(const T &data); + private: std::string DoGetInfoValue() const noexcept override; }; diff --git a/source/adios2/core/Attribute.tcc b/source/adios2/core/Attribute.tcc index 3097815e80..8232e8c6f9 100644 --- a/source/adios2/core/Attribute.tcc +++ b/source/adios2/core/Attribute.tcc @@ -54,34 +54,81 @@ struct Pad(arg)[1]); } }; - } template Attribute::Attribute(const Attribute &other) : AttributeBase(other), m_DataArray(other.m_DataArray) { - Pad::Zero(m_DataSingleValue); - m_DataSingleValue = other.m_DataSingleValue; + if (other.m_IsSingleValue) + { + m_DataArray.clear(); + Pad::Zero(m_DataSingleValue); + m_DataSingleValue = other.m_DataSingleValue; + } + else + { + m_DataArray = other.m_DataArray; + Pad::Zero(m_DataSingleValue); + } } template Attribute::Attribute(const std::string &name, const T *array, - const size_t elements) -: AttributeBase(name, helper::GetDataType(), elements) + const size_t elements, const bool allowModification) +: AttributeBase(name, helper::GetDataType(), elements, allowModification) { - Pad::Zero(m_DataSingleValue); m_DataArray = std::vector(array, array + elements); + Pad::Zero(m_DataSingleValue); } template -Attribute::Attribute(const std::string &name, const T &value) -: AttributeBase(name, helper::GetDataType()) +Attribute::Attribute(const std::string &name, const T &value, + const bool allowModification) +: AttributeBase(name, helper::GetDataType(), allowModification) { + m_DataArray.clear(); Pad::Zero(m_DataSingleValue); m_DataSingleValue = value; } +template +void Attribute::Modify(const T *data, const size_t elements) +{ + if (m_AllowModification) + { + m_DataArray = std::vector(data, data + elements); + Pad::Zero(m_DataSingleValue); + this->m_IsSingleValue = false; + this->m_Elements = elements; + } + else + { + throw std::invalid_argument( + "ERROR: Trying to modify attribute " + this->m_Name + + " which has been defined as non-modifiable\n"); + } +} + +template +void Attribute::Modify(const T &data) +{ + if (m_AllowModification) + { + m_DataArray.clear(); + Pad::Zero(m_DataSingleValue); + m_DataSingleValue = data; + this->m_IsSingleValue = true; + this->m_Elements = 1; + } + else + { + throw std::invalid_argument( + "ERROR: Trying to modify attribute " + this->m_Name + + " which has been defined as non-modifiable\n"); + } +} + template std::string Attribute::DoGetInfoValue() const noexcept { diff --git a/source/adios2/core/AttributeBase.cpp b/source/adios2/core/AttributeBase.cpp index f77ccdb4c5..dcc03d60f3 100644 --- a/source/adios2/core/AttributeBase.cpp +++ b/source/adios2/core/AttributeBase.cpp @@ -15,14 +15,18 @@ namespace adios2 namespace core { -AttributeBase::AttributeBase(const std::string &name, const DataType type) -: m_Name(name), m_Type(type), m_Elements(1), m_IsSingleValue(true) +AttributeBase::AttributeBase(const std::string &name, const DataType type, + const bool allowModification) +: m_Name(name), m_Type(type), m_Elements(1), m_IsSingleValue(true), + m_AllowModification(allowModification) { } AttributeBase::AttributeBase(const std::string &name, const DataType type, - const size_t elements) -: m_Name(name), m_Type(type), m_Elements(elements), m_IsSingleValue(false) + const size_t elements, + const bool allowModification) +: m_Name(name), m_Type(type), m_Elements(elements), m_IsSingleValue(false), + m_AllowModification(allowModification) { } @@ -32,6 +36,7 @@ Params AttributeBase::GetInfo() const noexcept info["Type"] = ToString(m_Type); info["Elements"] = std::to_string(m_Elements); info["Value"] = this->DoGetInfoValue(); + info["Modifiable"] = std::to_string(m_AllowModification); return info; } diff --git a/source/adios2/core/AttributeBase.h b/source/adios2/core/AttributeBase.h index 94d2916f5c..d2da95e3e1 100644 --- a/source/adios2/core/AttributeBase.h +++ b/source/adios2/core/AttributeBase.h @@ -29,15 +29,17 @@ class AttributeBase public: const std::string m_Name; const DataType m_Type; - const size_t m_Elements; - const bool m_IsSingleValue; + size_t m_Elements; + bool m_IsSingleValue; + const bool m_AllowModification; /** * Single value constructor used by Attribute derived class * @param name * @param type */ - AttributeBase(const std::string &name, const DataType type); + AttributeBase(const std::string &name, const DataType type, + const bool allowModification); /** * Array constructor used by Attribute derived class @@ -46,7 +48,7 @@ class AttributeBase * @param elements */ AttributeBase(const std::string &name, const DataType type, - const size_t elements); + const size_t elements, const bool allowModification); virtual ~AttributeBase() = default; diff --git a/source/adios2/core/CoreTypes.h b/source/adios2/core/CoreTypes.h new file mode 100644 index 0000000000..3f6eae7bcb --- /dev/null +++ b/source/adios2/core/CoreTypes.h @@ -0,0 +1,47 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * CoreTypes.h : types used only in the core framework, in contrast to + * ADIOSTypes.h, which is a public user-facing header + * + * Created on: Aug 11, 2021 + * Author: Norbert Podhorszki pnorbert@ornl.gov + */ + +#ifndef ADIOS2_CORETYPES_H_ +#define ADIOS2_CORETYPES_H_ + +/// \cond EXCLUDE_FROM_DOXYGEN +#include +#include +#include +/// \endcond + +#include "adios2/common/ADIOSConfig.h" + +namespace adios2 +{ +namespace core +{ + +struct iovec +{ + // Base address of a memory region for input or output. + const void *iov_base; + // The size of the memory pointed to by iov_base. + size_t iov_len; +}; + +typedef std::chrono::duration Seconds; +typedef std::chrono::time_point< + std::chrono::steady_clock, + std::chrono::duration> + TimePoint; + +inline TimePoint Now() { return std::chrono::steady_clock::now(); } + +} // end namespace core +} // end namespace adios2 + +#endif /* ADIOS2_CORETYPES_H_ */ diff --git a/source/adios2/core/Engine.cpp b/source/adios2/core/Engine.cpp index 3aa02bff96..d7da3d8f2e 100644 --- a/source/adios2/core/Engine.cpp +++ b/source/adios2/core/Engine.cpp @@ -100,10 +100,12 @@ void Engine::Init() {} void Engine::InitParameters() {} void Engine::InitTransports() {} +void Engine::NotifyEngineAttribute(std::string name, DataType type) noexcept {} + // DoPut* #define declare_type(T) \ void Engine::DoPut(Variable &, typename Variable::Span &, \ - const size_t, const T &) \ + const bool, const T &) \ { \ ThrowUp("DoPut"); \ } @@ -170,7 +172,8 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type #define declare_type(T, L) \ - T *Engine::DoBufferData_##L(const size_t payloadPosition, \ + T *Engine::DoBufferData_##L(const int bufferIdx, \ + const size_t payloadPosition, \ const size_t bufferID) noexcept \ { \ T *data = nullptr; \ @@ -247,7 +250,7 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation) #define declare_template_instantiation(T) \ template typename Variable::Span &Engine::Put(Variable &, \ - const size_t, const T &); \ + const bool, const T &); \ template void Engine::Get(core::Variable &, T **) const; ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_template_instantiation) diff --git a/source/adios2/core/Engine.h b/source/adios2/core/Engine.h index 76aec1c3bb..1055fbe5f1 100644 --- a/source/adios2/core/Engine.h +++ b/source/adios2/core/Engine.h @@ -113,7 +113,7 @@ class Engine */ template typename Variable::Span & - Put(Variable &variable, const size_t bufferID = 0, const T &value = T{}); + Put(Variable &variable, const bool initialize, const T &value = T{}); /** * @brief Put associates variable and data into adios2 in Engine Write mode. @@ -425,7 +425,7 @@ class Engine std::vector GetAbsoluteSteps(const Variable &variable) const; template - T *BufferData(const size_t payloadOffset, + T *BufferData(const int bufferIdx, const size_t payloadOffset, const size_t bufferID = 0) noexcept; size_t Steps() const; @@ -450,6 +450,56 @@ class Engine /* for adios2 internal testing */ virtual size_t DebugGetDataBufferSize() const; + union PrimitiveStdtypeUnion + { + int8_t int8; + int16_t int16; + int32_t int32; + int64_t int64; + uint8_t uint8; + uint16_t uint16; + uint32_t uint32; + uint64_t uint64; + float f; + double d; + long double ld; + }; + + struct MinBlockInfo + { + int WriterID = 0; + size_t BlockID = 0; + size_t *Start; + size_t *Count; + union PrimitiveStdtypeUnion MinUnion; + union PrimitiveStdtypeUnion MaxUnion; + void *BufferP = NULL; + }; + struct MinVarInfo + { + int Dims; + size_t *Shape; + bool IsValue = false; + bool IsReverseDims = false; + std::vector BlocksInfo; + MinVarInfo(int D, size_t *S) + : Dims(D), Shape(S), IsValue(false), IsReverseDims(false), + BlocksInfo({}) + { + } + }; + + virtual MinVarInfo *MinBlocksInfo(const VariableBase &, + const size_t Step) const + { + return nullptr; + } + + /** Notify the engine when a new attribute is defined. Called from IO.tcc + */ + virtual void NotifyEngineAttribute(std::string name, + DataType type) noexcept; + protected: /** from ADIOS class passed to Engine created with Open * if no communicator is passed */ @@ -479,8 +529,8 @@ class Engine // Put #define declare_type(T) \ virtual void DoPut(Variable &variable, \ - typename Variable::Span &span, const size_t blockID, \ - const T &value); + typename Variable::Span &span, \ + const bool initialize, const T &value); ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type) #undef declare_type @@ -527,7 +577,8 @@ class Engine #undef declare_type #define declare_type(T, L) \ - virtual T *DoBufferData_##L(const size_t payloadPosition, \ + virtual T *DoBufferData_##L(const int bufferIdx, \ + const size_t payloadPosition, \ const size_t bufferID) noexcept; ADIOS2_FOREACH_PRIMITVE_STDTYPE_2ARGS(declare_type) @@ -543,6 +594,10 @@ class Engine */ bool m_ReaderSelectionsLocked = false; + /** true: Currently executing after BeginStep and before EndStep + */ + bool m_BetweenStepPairs = false; + private: /** Throw exception by Engine virtual functions not implemented/supported by * a derived class */ @@ -618,7 +673,7 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation) #define declare_template_instantiation(T) \ extern template typename Variable::Span &Engine::Put( \ - Variable &, const size_t, const T &); \ + Variable &, const bool, const T &); \ extern template void Engine::Get(Variable &, T **) const; ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_template_instantiation) #undef declare_template_instantiation diff --git a/source/adios2/core/Engine.tcc b/source/adios2/core/Engine.tcc index 4de25878a2..6b96929394 100644 --- a/source/adios2/core/Engine.tcc +++ b/source/adios2/core/Engine.tcc @@ -25,7 +25,7 @@ namespace core template typename Variable::Span &Engine::Put(Variable &variable, - const size_t bufferID, const T &value) + const bool initialize, const T &value) { CheckOpenModes({{Mode::Write}}, " for variable " + variable.m_Name + ", in call to Variable::Span Put"); @@ -33,7 +33,7 @@ typename Variable::Span &Engine::Put(Variable &variable, auto itSpan = variable.m_BlocksSpan.emplace( variable.m_BlocksInfo.size(), typename Variable::Span(*this, variable.TotalSize())); - DoPut(variable, itSpan.first->second, bufferID, value); + DoPut(variable, itSpan.first->second, initialize, value); return itSpan.first->second; } @@ -86,7 +86,8 @@ void Engine::Put(const std::string &variableName, const T &datum, template void Engine::Get(Variable &variable, T *data, const Mode launch) { - CommonChecks(variable, data, {{Mode::Read}}, "in call to Get"); + CommonChecks(variable, data, {{Mode::Read}, {Mode::ReadRandomAccess}}, + "in call to Get"); switch (launch) { @@ -224,10 +225,10 @@ std::vector Engine::GetAbsoluteSteps(const Variable &variable) const #define declare_type(T, L) \ template <> \ - T *Engine::BufferData(const size_t payloadPosition, \ + T *Engine::BufferData(const int bufferIdx, const size_t payloadPosition, \ const size_t bufferID) noexcept \ { \ - return DoBufferData_##L(payloadPosition, bufferID); \ + return DoBufferData_##L(bufferIdx, payloadPosition, bufferID); \ } ADIOS2_FOREACH_PRIMITVE_STDTYPE_2ARGS(declare_type) #undef declare_type diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp index c1894c037b..162094bcec 100644 --- a/source/adios2/core/IO.cpp +++ b/source/adios2/core/IO.cpp @@ -28,6 +28,8 @@ #endif #include "adios2/engine/inline/InlineReader.h" #include "adios2/engine/inline/InlineWriter.h" +#include "adios2/engine/mhs/MhsReader.h" +#include "adios2/engine/mhs/MhsWriter.h" #include "adios2/engine/null/NullEngine.h" #include "adios2/engine/nullcore/NullCoreWriter.h" #include "adios2/engine/skeleton/SkeletonReader.h" @@ -48,21 +50,15 @@ #include "adios2/engine/sst/SstWriter.h" #endif -#ifdef ADIOS2_HAVE_TABLE // external dependencies -#include "adios2/engine/table/TableWriter.h" -#endif - namespace adios2 { namespace core { -IO::EngineFactoryEntry IO_MakeEngine_HDFMixer(); IO::EngineFactoryEntry IO_MakeEngine_HDF5(); namespace { - std::unordered_map Factory = { {"bp3", {IO::MakeEngine, IO::MakeEngine}}, @@ -74,14 +70,6 @@ std::unordered_map Factory = { #else IO::NoEngineEntry("ERROR: this version didn't compile with " "BP5 library, can't use BP5 engine\n") -#endif - }, - {"hdfmixer", -#ifdef ADIOS2_HAVE_HDF5 - IO_MakeEngine_HDFMixer() -#else - IO::NoEngineEntry("ERROR: this version didn't compile with " - "HDF5 library, can't use HDF5 engine\n") #endif }, {"dataman", @@ -95,15 +83,12 @@ std::unordered_map Factory = { }, {"ssc", IO::NoEngineEntry("ERROR: this version didn't compile with " "SSC library, can't use SSC engine\n")}, - {"table", -#ifdef ADIOS2_HAVE_TABLE - {IO::NoEngine("ERROR: Table engine only supports Write. It uses other " - "engines as backend. Please use corresponding engines for " - "Read\n"), - IO::MakeEngine} + {"mhs", +#ifdef ADIOS2_HAVE_MHS + {IO::MakeEngine, IO::MakeEngine} #else IO::NoEngineEntry("ERROR: this version didn't compile with " - "Table library, can't use Table engine\n") + "MHS library, can't use MHS engine\n") #endif }, {"sst", @@ -234,7 +219,7 @@ void IO::SetEngine(const std::string engineType) noexcept } else if (engineTypeLC == "filestream") { - finalEngineType = "BP4"; + finalEngineType = "filestream"; lf_InsertParam("OpenTimeoutSecs", "3600"); lf_InsertParam("StreamReader", "true"); } @@ -322,6 +307,19 @@ bool IO::InConfigFile() const noexcept { return m_InConfigFile; } void IO::SetDeclared() noexcept { m_IsDeclared = true; } +void IO::SetArrayOrder(const ArrayOrdering ArrayOrder) noexcept +{ + if (ArrayOrder == ArrayOrdering::Auto) + { + if (helper::IsRowMajor(m_HostLanguage)) + m_ArrayOrder = ArrayOrdering::RowMajor; + else + m_ArrayOrder = ArrayOrdering::ColumnMajor; + } + else + m_ArrayOrder = ArrayOrder; +} + bool IO::IsDeclared() const noexcept { return m_IsDeclared; } bool IO::RemoveVariable(const std::string &name) noexcept @@ -492,12 +490,15 @@ DataType IO::InquireAttributeType(const std::string &name, return itAttribute->second->m_Type; } -size_t IO::AddOperation(Operator &op, const Params ¶meters) noexcept +void IO::AddOperation(const std::string &variable, + const std::string &operatorType, + const Params ¶meters) noexcept { PERFSTUBS_SCOPED_TIMER("IO::other"); - m_Operations.push_back( - Operation{&op, helper::LowerCaseParams(parameters), Params()}); - return m_Operations.size() - 1; + auto params = helper::LowerCaseParams(parameters); + Operator *op = &m_ADIOS.DefineOperator( + m_Name + "_" + variable + "_" + operatorType, operatorType, params); + m_VarOpsPlaceholder[variable].emplace_back(Operation{op, params, Params()}); } Engine &IO::Open(const std::string &name, const Mode mode, helper::Comm comm) @@ -506,6 +507,8 @@ Engine &IO::Open(const std::string &name, const Mode mode, helper::Comm comm) auto itEngineFound = m_Engines.find(name); const bool isEngineFound = (itEngineFound != m_Engines.end()); bool isEngineActive = false; + Mode mode_to_use = mode; + if (isEngineFound) { if (*itEngineFound->second) @@ -542,7 +545,8 @@ Engine &IO::Open(const std::string &name, const Mode mode, helper::Comm comm) } /* Second step in handling virtual engines */ - /* BPFile for read needs to use BP4 or BP3 depending on the file's version + /* BPFile for read needs to use BP5, BP4, or BP3 depending on the file's + * version */ if ((engineTypeLC == "file" || engineTypeLC == "bpfile" || engineTypeLC == "bp" || isDefaultEngine)) @@ -551,11 +555,24 @@ Engine &IO::Open(const std::string &name, const Mode mode, helper::Comm comm) { engineTypeLC = "hdf5"; } - else if (mode == Mode::Read) + else if ((mode_to_use == Mode::Read) || + (mode_to_use == Mode::ReadRandomAccess)) { if (adios2sys::SystemTools::FileIsDirectory(name)) { - engineTypeLC = "bp4"; + char v = helper::BPVersion(name, comm, m_TransportsParameters); + if (v == 'X') + { + // BP4 did not create this file pre 2.8.0 so if not found, + // lets assume bp4 + v = '4'; + } + engineTypeLC = "bp"; + engineTypeLC.push_back(v); + } + else if (adios2sys::SystemTools::FileIsDirectory(name + ".tier0")) + { + engineTypeLC = "mhs"; } else { @@ -581,22 +598,46 @@ Engine &IO::Open(const std::string &name, const Mode mode, helper::Comm comm) } else { + // File default for writing: BP4 engineTypeLC = "bp4"; } } + // filestream is either BP5 or BP4 depending on .bpversion + /* Note: Mismatch between BP4/BP5 writer and FileStream reader is not + handled if writer has not created the directory yet, when FileStream + falls back to default */ + if (engineTypeLC == "filestream") + { + char v = helper::BPVersion(name, comm, m_TransportsParameters); + if (v == 'X') + { + // FileStream default: BP4 + v = '4'; + } + engineTypeLC = "bp"; + engineTypeLC.push_back(v); + // std::cout << "Engine " << engineTypeLC << " selected for FileStream" + // << std::endl; + } + + if ((engineTypeLC != "bp5") && (mode_to_use == Mode::ReadRandomAccess)) + { + // only BP5 special-cases file-reader random access mode + mode_to_use = Mode::Read; + } // For the inline engine, there must be exactly 1 reader, and exactly 1 // writer. if (engineTypeLC == "inline") { - if (mode == Mode::Append) + if (mode_to_use == Mode::Append) { throw std::runtime_error( "Append mode is not supported for the inline engine."); } // See inline.rst:44 - if (mode == Mode::Sync) + if (mode_to_use == Mode::Sync) { throw std::runtime_error( "Sync mode is not supported for the inline engine."); @@ -612,12 +653,12 @@ Engine &IO::Open(const std::string &name, const Mode mode, helper::Comm comm) "added."; throw std::runtime_error(msg); } - // Now protect against declaration of two writers, or declaration of two - // readers: + // Now protect against declaration of two writers, or declaration of + // two readers: if (m_Engines.size() == 1) { auto engine_ptr = m_Engines.begin()->second; - if (engine_ptr->OpenMode() == mode) + if (engine_ptr->OpenMode() == mode_to_use) { std::string msg = "The previously added engine " + engine_ptr->m_Name + @@ -633,13 +674,16 @@ Engine &IO::Open(const std::string &name, const Mode mode, helper::Comm comm) auto f = FactoryLookup(engineTypeLC); if (f != Factory.end()) { - if (mode == Mode::Read) + if ((mode_to_use == Mode::Read) || + (mode_to_use == Mode::ReadRandomAccess)) { - engine = f->second.MakeReader(*this, name, mode, std::move(comm)); + engine = + f->second.MakeReader(*this, name, mode_to_use, std::move(comm)); } else { - engine = f->second.MakeWriter(*this, name, mode, std::move(comm)); + engine = + f->second.MakeWriter(*this, name, mode_to_use, std::move(comm)); } } else @@ -743,8 +787,8 @@ void IO::SetPrefixedNames(const bool isStep) noexcept for (auto itVariable = m_Variables.begin(); itVariable != m_Variables.end(); ++itVariable) { - // if for each step (BP4), check if variable type is not empty (means - // variable exist in that step) + // if for each step (BP4), check if variable type is not empty + // (means variable exist in that step) const DataType type = isStep ? InquireVariableType(itVariable) : itVariable->second->m_Type; @@ -806,10 +850,10 @@ ADIOS2_FOREACH_STDTYPE_1ARG(define_template_instantiation) #define declare_template_instantiation(T) \ template Attribute &IO::DefineAttribute( \ const std::string &, const T *, const size_t, const std::string &, \ - const std::string); \ + const std::string, const bool); \ template Attribute &IO::DefineAttribute( \ const std::string &, const T &, const std::string &, \ - const std::string); \ + const std::string, const bool); \ template Attribute *IO::InquireAttribute( \ const std::string &, const std::string &, const std::string) noexcept; diff --git a/source/adios2/core/IO.h b/source/adios2/core/IO.h index 9d77006057..c09d7dfb76 100644 --- a/source/adios2/core/IO.h +++ b/source/adios2/core/IO.h @@ -59,6 +59,8 @@ class IO /** from ADIOS class passed to Engine created with Open */ const std::string m_HostLanguage = "C++"; + ArrayOrdering m_ArrayOrder; + /** From SetParameter, parameters for a particular engine from m_Type */ Params m_Parameters; @@ -73,9 +75,6 @@ class IO Params Info; }; - /** From AddOperation, contains operators added to this IO */ - std::vector m_Operations; - /** BP3 engine default if unknown */ std::string m_EngineType = "File"; @@ -195,6 +194,8 @@ class IO * @param array pointer to user data * @param elements number of data elements * @param variableName optionally associates the attribute to a Variable + * @param allowModification true allows redefining/modifying existing + * attribute * @return reference to internal Attribute * @exception std::invalid_argument if Attribute with unique name is already * defined @@ -203,12 +204,15 @@ class IO Attribute &DefineAttribute(const std::string &name, const T *array, const size_t elements, const std::string &variableName = "", - const std::string separator = "/"); + const std::string separator = "/", + const bool allowModification = false); /** * @brief Define single value attribute * @param name must be unique for the IO object * @param value single data value + * @param allowModification true allows redefining/modifying existing + * attribute * @return reference to internal Attribute * @exception std::invalid_argument if Attribute with unique name is already * defined @@ -216,7 +220,8 @@ class IO template Attribute &DefineAttribute(const std::string &name, const T &value, const std::string &variableName = "", - const std::string separator = "/"); + const std::string separator = "/", + const bool allowModification = false); /** * @brief Removes an existing Variable in current IO object. @@ -351,6 +356,8 @@ class IO */ void SetDeclared() noexcept; + void SetArrayOrder(const ArrayOrdering ArrayOrder) noexcept; + /** * Check if declared in code * @return true: created with ADIOS DeclareIO, false: dummy from config file @@ -360,12 +367,13 @@ class IO /** * Adds an operator defined by the ADIOS class. Could be a variable set * transform, callback function, etc. - * @param op operator created by the ADIOS class - * @param parameters specific parameters for current IO - * @return operation handler + * @param variable + * @param operatorType + * @param parameters */ - size_t AddOperation(Operator &op, - const Params ¶meters = Params()) noexcept; + void AddOperation(const std::string &variable, + const std::string &operatorType, + const Params ¶meters = Params()) noexcept; /** * @brief Creates a polymorphic object that derives the Engine class, @@ -530,10 +538,10 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation) #define declare_template_instantiation(T) \ extern template Attribute &IO::DefineAttribute( \ const std::string &, const T *, const size_t, const std::string &, \ - const std::string); \ + const std::string, const bool); \ extern template Attribute &IO::DefineAttribute( \ const std::string &, const T &, const std::string &, \ - const std::string); \ + const std::string, const bool); \ extern template Attribute *IO::InquireAttribute( \ const std::string &, const std::string &, const std::string) noexcept; diff --git a/source/adios2/core/IO.tcc b/source/adios2/core/IO.tcc index 6d8adf62c8..63c98dfd32 100644 --- a/source/adios2/core/IO.tcc +++ b/source/adios2/core/IO.tcc @@ -23,6 +23,7 @@ #include "adios2/helper/adiosFunctions.h" #include "adios2/helper/adiosType.h" #include +#include namespace adios2 { @@ -99,7 +100,8 @@ Variable *IO::InquireVariable(const std::string &name) noexcept template Attribute &IO::DefineAttribute(const std::string &name, const T &value, const std::string &variableName, - const std::string separator) + const std::string separator, + const bool allowModification) { PERFSTUBS_SCOPED_TIMER("IO::DefineAttribute"); if (!variableName.empty() && @@ -117,32 +119,51 @@ Attribute &IO::DefineAttribute(const std::string &name, const T &value, auto itExistingAttribute = m_Attributes.find(globalName); if (itExistingAttribute != m_Attributes.end()) { - if (helper::ValueToString(value) == + if (helper::ValueToString(value) != itExistingAttribute->second->GetInfo()["Value"]) { - return static_cast &>(*itExistingAttribute->second); + if (itExistingAttribute->second->m_Type == helper::GetDataType()) + { + Attribute &a = + static_cast &>(*itExistingAttribute->second); + a.Modify(value); + for (auto &e : m_Engines) + { + e.second->NotifyEngineAttribute( + globalName, itExistingAttribute->second->m_Type); + } + } + else + { + throw std::invalid_argument( + "ERROR: modifiable attribute " + globalName + + " has been defined with type " + + ToString(itExistingAttribute->second->m_Type) + + ". Type cannot be changed to " + + ToString(helper::GetDataType()) + "\n"); + } } - else + return static_cast &>(*itExistingAttribute->second); + } + else + { + auto itAttributePair = m_Attributes.emplace( + globalName, std::unique_ptr(new Attribute( + globalName, value, allowModification))); + for (auto &e : m_Engines) { - throw std::invalid_argument( - "ERROR: attribute " + globalName + - " has been defined and its value cannot be changed, in call to " - "DefineAttribute\n"); + e.second->NotifyEngineAttribute( + globalName, itAttributePair.first->second->m_Type); } + return static_cast &>(*itAttributePair.first->second); } - - auto itAttributePair = m_Attributes.emplace( - globalName, - std::unique_ptr(new Attribute(globalName, value))); - - return static_cast &>(*itAttributePair.first->second); } template -Attribute &IO::DefineAttribute(const std::string &name, const T *array, - const size_t elements, - const std::string &variableName, - const std::string separator) +Attribute & +IO::DefineAttribute(const std::string &name, const T *array, + const size_t elements, const std::string &variableName, + const std::string separator, const bool allowModification) { PERFSTUBS_SCOPED_TIMER("IO::DefineAttribute"); if (!variableName.empty() && @@ -165,23 +186,43 @@ Attribute &IO::DefineAttribute(const std::string &name, const T *array, helper::VectorToCSV(std::vector(array, array + elements)) + " }"); - if (itExistingAttribute->second->GetInfo()["Value"] == arrayValues) + if (itExistingAttribute->second->GetInfo()["Value"] != arrayValues) { - return static_cast &>(*itExistingAttribute->second); + if (itExistingAttribute->second->m_Type == helper::GetDataType()) + { + Attribute &a = + static_cast &>(*itExistingAttribute->second); + a.Modify(array, elements); + for (auto &e : m_Engines) + { + e.second->NotifyEngineAttribute( + globalName, itExistingAttribute->second->m_Type); + } + } + else + { + throw std::invalid_argument( + "ERROR: modifiable attribute " + globalName + + " has been defined with type " + + ToString(itExistingAttribute->second->m_Type) + + ". Type cannot be changed to " + + ToString(helper::GetDataType()) + "\n"); + } } - else + return static_cast &>(*itExistingAttribute->second); + } + else + { + auto itAttributePair = m_Attributes.emplace( + globalName, std::unique_ptr(new Attribute( + globalName, array, elements, allowModification))); + for (auto &e : m_Engines) { - throw std::invalid_argument( - "ERROR: attribute " + globalName + - " has been defined and its value cannot be changed, in call to " - "DefineAttribute\n"); + e.second->NotifyEngineAttribute( + globalName, itAttributePair.first->second->m_Type); } + return static_cast &>(*itAttributePair.first->second); } - - auto itAttributePair = m_Attributes.emplace( - globalName, std::unique_ptr( - new Attribute(globalName, array, elements))); - return static_cast &>(*itAttributePair.first->second); } template diff --git a/source/adios2/core/IOHDF5.cpp b/source/adios2/core/IOHDF5.cpp index 92c5896540..1267be6acf 100644 --- a/source/adios2/core/IOHDF5.cpp +++ b/source/adios2/core/IOHDF5.cpp @@ -9,9 +9,6 @@ #include "adios2/engine/hdf5/HDF5ReaderP.h" #include "adios2/engine/hdf5/HDF5WriterP.h" -#if H5_VERSION_GE(1, 11, 0) -#include "adios2/engine/mixer/HDFMixer.h" -#endif namespace adios2 { @@ -37,16 +34,6 @@ std::shared_ptr MakeEngineHDF5(IO &io, const std::string &name, } // end anonymous namespace -IO::EngineFactoryEntry IO_MakeEngine_HDFMixer() -{ -#if H5_VERSION_GE(1, 11, 0) - return IO::EngineFactoryEntry{MakeEngineHDF5, - MakeEngineHDF5}; -#else - return IO::NoEngineEntry("ERROR: update HDF5 >= 1.11 to support VDS."); -#endif -} - IO::EngineFactoryEntry IO_MakeEngine_HDF5() { return IO::EngineFactoryEntry{MakeEngineHDF5, diff --git a/source/adios2/core/Operator.cpp b/source/adios2/core/Operator.cpp index 8c19acbf49..ec5d8605ad 100644 --- a/source/adios2/core/Operator.cpp +++ b/source/adios2/core/Operator.cpp @@ -9,8 +9,6 @@ */ #include "Operator.h" - -#include "adios2/common/ADIOSMacros.h" #include "adios2/helper/adiosFunctions.h" namespace adios2 @@ -51,28 +49,9 @@ void Operator::RunCallback2(void *arg0, const std::string &arg1, CheckCallbackType("Callback2"); } -size_t Operator::BufferMaxSize(const size_t sizeIn) const -{ - throw std::invalid_argument("ERROR: signature (const size_t) not supported " - "by derived class implemented with " + - m_Type + ", in call to BufferMaxSize\n"); -} - -#define declare_type(T) \ - template <> \ - size_t Operator::BufferMaxSize(const T *dataIn, const Dims &dimensions, \ - const Params ¶meters) const \ - { \ - return DoBufferMaxSize(dataIn, dimensions, helper::GetDataType(), \ - parameters); \ - } -ADIOS2_FOREACH_ZFP_TYPE_1ARG(declare_type) -#undef declare_type - -size_t Operator::Compress(const void * /*dataIn*/, const Dims & /*dimensions*/, - const size_t /*elementSize*/, DataType /*type*/, - void * /*bufferOut*/, const Params & /*params*/, - Params & /*info*/) const +size_t Operator::Compress(const char * /*dataIn*/, const Dims & /*blockStart*/, + const Dims & /*blockCount*/, const DataType /*type*/, + char * /*bufferOut*/, const Params & /*params*/) { throw std::invalid_argument("ERROR: signature (const void*, const " "Dims, const size_t, const std::string, " @@ -81,9 +60,8 @@ size_t Operator::Compress(const void * /*dataIn*/, const Dims & /*dimensions*/, m_Type + ", in call to Compress\n"); } -size_t Operator::Decompress(const void *bufferIn, const size_t sizeIn, - void *dataOut, const size_t sizeOut, - Params &info) const +size_t Operator::Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) { throw std::invalid_argument( "ERROR: signature (const void*, const size_t, void) not supported " @@ -91,27 +69,7 @@ size_t Operator::Decompress(const void *bufferIn, const size_t sizeIn, m_Type + ", in call to Decompress\n"); } -size_t Operator::Decompress(const void * /*bufferIn*/, const size_t /*sizeIn*/, - void * /*dataOut*/, const Dims & /*dimensions*/, - DataType /*type*/, - const Params & /*parameters*/) const -{ - throw std::invalid_argument("ERROR: signature (const void*, const " - "size_t, void*, const Dims&, const " - "std::string ) not supported " - "by derived class implemented with " + - m_Type + ", in call to Decompress\n"); -} - // PROTECTED -size_t Operator::DoBufferMaxSize(const void *dataIn, const Dims &dimensions, - DataType type, const Params ¶meters) const -{ - throw std::invalid_argument("ERROR: signature (const void*, const Dims& " - "std::string ) not supported " - "by derived class implemented with " + - m_Type + ", in call to BufferMaxSize\n"); -} Dims Operator::ConvertDims(const Dims &dimensions, const DataType type, const size_t targetDims, const bool enforceDims, diff --git a/source/adios2/core/Operator.h b/source/adios2/core/Operator.h index 8de2756dff..48dc57e8f8 100644 --- a/source/adios2/core/Operator.h +++ b/source/adios2/core/Operator.h @@ -13,14 +13,10 @@ #ifndef ADIOS2_CORE_OPERATOR_H_ #define ADIOS2_CORE_OPERATOR_H_ -/// \cond EXCLUDE_FROM_DOXYGEN -#include -#include -#include -/// \endcond - #include "adios2/common/ADIOSMacros.h" #include "adios2/common/ADIOSTypes.h" +#include +#include namespace adios2 { @@ -57,74 +53,47 @@ class Operator virtual void RunCallback2(void *, const std::string &, const std::string &, const std::string &, const size_t, const Dims &, const Dims &, const Dims &) const; - /** - * Returns a conservative buffer size to hold input data for classes - * @param sizeIn size of input data to be compressed in bytes - * @return recommended allocation for output buffer - */ - virtual size_t BufferMaxSize(const size_t sizeIn) const; /** - * Used by Zfp - * Returns a conservative buffer size to hold input data for classes * @param dataIn - * @param dimensions - * @return recommended allocation for output buffer in bytes - */ - template - size_t BufferMaxSize(const T *dataIn, const Dims &dimensions, - const Params ¶ms) const; - - /** - * BZip2 and Zfp common call - * @param dataIn - * @param dimensions - * @param elementSize + * @param blockStart + * @param blockCount * @param type * @param bufferOut * @param parameters * @return size of compressed buffer */ - virtual size_t Compress(const void *dataIn, const Dims &dimensions, - const size_t elementSize, DataType type, - void *bufferOut, const Params ¶meters, - Params &info) const; - - virtual size_t Decompress(const void *bufferIn, const size_t sizeIn, - void *dataOut, const size_t sizeOut, - Params &info) const; + virtual size_t Compress(const char *dataIn, const Dims &blockStart, + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters); /** - * Zfp signature * @param bufferIn * @param sizeIn * @param dataOut - * @param dimensions - * @param type - * @return + * @return size of decompressed buffer */ - virtual size_t Decompress(const void *bufferIn, const size_t sizeIn, - void *dataOut, const Dims &dimensions, - DataType type, const Params ¶meters) const; + virtual size_t Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut); virtual bool IsDataTypeValid(const DataType type) const = 0; + enum OperatorType : char + { + BLOSC = 0, + BZIP2 = 1, + LIBPRESSIO = 2, + MGARD = 3, + PNG = 4, + SIRIUS = 5, + Sz = 6, + ZFP = 7 + }; + protected: /** Parameters associated with a particular Operator */ Params m_Parameters; - /** - * Used by CompressZfp - * Returns a conservative buffer size to hold input data for classes - * @param dataIn - * @param dimensions - * @param type - * @return conservative buffer size for allocation - */ - virtual size_t DoBufferMaxSize(const void *dataIn, const Dims &dimensions, - DataType type, - const Params ¶meters) const; - /** * Used by lossy compressors with a limitation on complex data types or * dimentions Returns a adios2::Dims object that meets the requirement of a @@ -139,6 +108,63 @@ class Operator const bool enforceDims = false, const size_t defaultDimSize = 1) const; + template + void PutParameter(char *buffer, U &pos, const T ¶meter) + { + std::memcpy(buffer + pos, ¶meter, sizeof(T)); + pos += sizeof(T); + } + + template + T GetParameter(const char *buffer, U &pos) + { + T ret; + std::memcpy(&ret, buffer + pos, sizeof(T)); + pos += sizeof(T); + return ret; + } + + template + void PutParameters(char *buffer, U &pos, const Params ¶meters) + { + uint8_t size = static_cast(parameters.size()); + PutParameter(buffer, pos, size); + for (const auto &p : parameters) + { + size = static_cast(p.first.size()); + PutParameter(buffer, pos, size); + + std::memcpy(buffer + pos, p.first.data(), size); + pos += size; + + size = static_cast(p.second.size()); + PutParameter(buffer, pos, size); + + std::memcpy(buffer + pos, p.second.data(), size); + pos += size; + } + } + + template + Params GetParameters(const char *buffer, U &pos) + { + Params ret; + uint8_t params = GetParameter(buffer, pos); + for (uint8_t i = 0; i < params; ++i) + { + uint8_t size = GetParameter(buffer, pos); + std::string key = + std::string(reinterpret_cast(buffer + pos), size); + pos += size; + size = GetParameter(buffer, pos); + std::string value = + std::string(reinterpret_cast(buffer + pos), size); + pos += size; + ret[key] = value; + } + return ret; + } + private: void CheckCallbackType(const std::string type) const; }; diff --git a/source/adios2/core/Variable.h b/source/adios2/core/Variable.h index 56c6482436..d49f30afb3 100644 --- a/source/adios2/core/Variable.h +++ b/source/adios2/core/Variable.h @@ -35,7 +35,13 @@ class Span public: std::pair m_MinMaxDataPositions; std::pair m_MinMaxMetadataPositions; + + // internal position variables from which the engine + // can return a valid pointer any time + // BP5 needs two levels of reference, BP3/4 uses only one size_t m_PayloadPosition = 0; + int m_BufferIdx = -1; + T m_Value = T{}; Span(Engine &engine, const size_t size); @@ -115,7 +121,7 @@ class Variable : public VariableBase using Span = core::Span; /** Needs a map to preserve iterator as it resizes and the key to match the - * m_BlocksInfo index */ + * m_BlocksInfo index (BP4 ONLY) */ std::map m_BlocksSpan; Variable(const std::string &name, const Dims &shape, const Dims &start, @@ -160,6 +166,8 @@ class Variable : public VariableBase DoAllStepsBlocksInfo() const; void CheckRandomAccess(const size_t step, const std::string hint) const; + + size_t WriterIndex; }; } // end namespace core diff --git a/source/adios2/core/Variable.tcc b/source/adios2/core/Variable.tcc index ccfc5a40b8..206642fe91 100644 --- a/source/adios2/core/Variable.tcc +++ b/source/adios2/core/Variable.tcc @@ -218,7 +218,7 @@ size_t Span::Size() const noexcept template T *Span::Data() const noexcept { - return m_Engine.BufferData(m_PayloadPosition); + return m_Engine.BufferData(m_BufferIdx, m_PayloadPosition); } template @@ -252,15 +252,16 @@ const T &Span::At(const size_t position) const template T &Span::operator[](const size_t position) { - T &data = *m_Engine.BufferData(m_PayloadPosition + position * sizeof(T)); + T &data = *m_Engine.BufferData(m_BufferIdx, + m_PayloadPosition + position * sizeof(T)); return data; } template const T &Span::operator[](const size_t position) const { - const T &data = - *m_Engine.BufferData(m_PayloadPosition + position * sizeof(T)); + const T &data = *m_Engine.BufferData( + m_BufferIdx, m_PayloadPosition + position * sizeof(T)); return data; } diff --git a/source/adios2/core/VariableBase.cpp b/source/adios2/core/VariableBase.cpp index 1652a5be14..cf101fd3c7 100644 --- a/source/adios2/core/VariableBase.cpp +++ b/source/adios2/core/VariableBase.cpp @@ -252,6 +252,8 @@ size_t VariableBase::AddOperation(Operator &op, return m_Operations.size() - 1; } +void VariableBase::RemoveOperations() noexcept { m_Operations.clear(); } + void VariableBase::SetOperationParameter(const size_t operationID, const std::string key, const std::string value) diff --git a/source/adios2/core/VariableBase.h b/source/adios2/core/VariableBase.h index 2c53d13ad1..2f46646cdc 100644 --- a/source/adios2/core/VariableBase.h +++ b/source/adios2/core/VariableBase.h @@ -181,6 +181,12 @@ class VariableBase size_t AddOperation(core::Operator &op, const Params ¶meters = Params()) noexcept; + /** + * Removes all current Operations associated with AddOperation. + * Provides the posibility to apply or not operators on a step basis. + */ + void RemoveOperations() noexcept; + /** * Sets a parameter by key/value in an existing operation from AddOperation * @param operationID returned handler form AddOperation diff --git a/source/adios2/engine/bp3/BP3Reader.cpp b/source/adios2/engine/bp3/BP3Reader.cpp index e0ff9077d7..71f66d6a11 100644 --- a/source/adios2/engine/bp3/BP3Reader.cpp +++ b/source/adios2/engine/bp3/BP3Reader.cpp @@ -50,6 +50,13 @@ StepStatus BP3Reader::BeginStep(StepMode mode, const float timeoutSeconds) "PerformGets() or EndStep()?, in call to BeginStep\n"); } + if (m_BetweenStepPairs) + { + throw std::logic_error("ERROR: BeginStep() is called a second time " + "without an intervening EndStep()"); + } + + m_BetweenStepPairs = true; if (m_FirstStep) { m_FirstStep = false; @@ -78,6 +85,12 @@ size_t BP3Reader::CurrentStep() const { return m_CurrentStep; } void BP3Reader::EndStep() { + if (!m_BetweenStepPairs) + { + throw std::logic_error( + "ERROR: EndStep() is called without a successful BeginStep()"); + } + m_BetweenStepPairs = false; PERFSTUBS_SCOPED_TIMER("BP3Reader::EndStep"); PerformGets(); } @@ -126,6 +139,9 @@ void BP3Reader::Init() " " + m_EndMessage); } + // if IO was involved in reading before this flag may be true now + m_IO.m_ReadStreaming = false; + InitTransports(); InitBuffer(); } diff --git a/source/adios2/engine/bp3/BP3Reader.tcc b/source/adios2/engine/bp3/BP3Reader.tcc index 2da62081a3..6331307f5e 100644 --- a/source/adios2/engine/bp3/BP3Reader.tcc +++ b/source/adios2/engine/bp3/BP3Reader.tcc @@ -105,7 +105,7 @@ void BP3Reader::ReadVariableBlocks(Variable &variable) m_BP3Deserializer.PostDataRead( variable, blockInfo, subStreamBoxInfo, - helper::IsRowMajor(m_IO.m_HostLanguage), 0); + m_IO.m_ArrayOrder == ArrayOrdering::RowMajor, 0); } // substreams loop // advance pointer to next step blockInfo.Data += helper::GetTotalSize(blockInfo.Count); diff --git a/source/adios2/engine/bp3/BP3Writer.cpp b/source/adios2/engine/bp3/BP3Writer.cpp index f25bd0823e..2eaae0522f 100644 --- a/source/adios2/engine/bp3/BP3Writer.cpp +++ b/source/adios2/engine/bp3/BP3Writer.cpp @@ -123,6 +123,7 @@ void BP3Writer::Init() static_cast(m_BP3Serializer.m_SizeMPI)) { m_BP3Serializer.m_Aggregator.Init( + m_BP3Serializer.m_Parameters.NumAggregators, m_BP3Serializer.m_Parameters.NumAggregators, m_Comm); } InitTransports(); @@ -132,10 +133,10 @@ void BP3Writer::Init() #define declare_type(T) \ void BP3Writer::DoPut(Variable &variable, \ typename Variable::Span &span, \ - const size_t bufferID, const T &value) \ + const bool initialize, const T &value) \ { \ PERFSTUBS_SCOPED_TIMER("BP3Writer::Put"); \ - PutCommon(variable, span, bufferID, value); \ + PutCommon(variable, span, 0, value); \ } ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type) @@ -160,6 +161,8 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) void BP3Writer::InitParameters() { m_BP3Serializer.Init(m_IO.m_Parameters, "in call to BP3::Open for writing"); + m_BP3Serializer.ResizeBuffer(m_BP3Serializer.m_Parameters.InitialBufferSize, + "in call to BP3::Open to write"); } void BP3Writer::InitTransports() @@ -175,7 +178,7 @@ void BP3Writer::InitTransports() // only consumers will interact with transport managers std::vector bpSubStreamNames; - if (m_BP3Serializer.m_Aggregator.m_IsConsumer) + if (m_BP3Serializer.m_Aggregator.m_IsAggregator) { // Names passed to IO AddTransport option with key "Name" const std::vector transportsNames = @@ -192,7 +195,7 @@ void BP3Writer::InitTransports() m_BP3Serializer.m_Parameters.NodeLocal); m_BP3Serializer.m_Profiler.Stop("mkdir"); - if (m_BP3Serializer.m_Aggregator.m_IsConsumer) + if (m_BP3Serializer.m_Aggregator.m_IsAggregator) { if (m_BP3Serializer.m_Parameters.AsyncTasks) { @@ -219,7 +222,8 @@ void BP3Writer::InitBPBuffer() else { m_BP3Serializer.PutProcessGroupIndex( - m_IO.m_Name, m_IO.m_HostLanguage, + m_IO.m_Name, + (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) ? "C++" : "Fortran", m_FileDataManager.GetTransportsTypes()); } } @@ -246,7 +250,7 @@ void BP3Writer::DoClose(const int transportIndex) DoFlush(true, transportIndex); - if (m_BP3Serializer.m_Aggregator.m_IsConsumer) + if (m_BP3Serializer.m_Aggregator.m_IsAggregator) { m_FileDataManager.CloseFiles(transportIndex); } @@ -388,15 +392,15 @@ void BP3Writer::AggregateWriteData(const bool isFinal, const int transportIndex) // async? for (int r = 0; r < m_BP3Serializer.m_Aggregator.m_Size; ++r) { - aggregator::MPIAggregator::ExchangeRequests dataRequests = + aggregator::MPIChain::ExchangeRequests dataRequests = m_BP3Serializer.m_Aggregator.IExchange(m_BP3Serializer.m_Data, r); - aggregator::MPIAggregator::ExchangeAbsolutePositionRequests + aggregator::MPIChain::ExchangeAbsolutePositionRequests absolutePositionRequests = m_BP3Serializer.m_Aggregator.IExchangeAbsolutePosition( m_BP3Serializer.m_Data, r); - if (m_BP3Serializer.m_Aggregator.m_IsConsumer) + if (m_BP3Serializer.m_Aggregator.m_IsAggregator) { const format::Buffer &bufferSTL = m_BP3Serializer.m_Aggregator.GetConsumerBuffer( @@ -425,7 +429,7 @@ void BP3Writer::AggregateWriteData(const bool isFinal, const int transportIndex) m_BP3Serializer.AggregateCollectiveMetadata( m_BP3Serializer.m_Aggregator.m_Comm, bufferSTL, false); - if (m_BP3Serializer.m_Aggregator.m_IsConsumer) + if (m_BP3Serializer.m_Aggregator.m_IsAggregator) { m_FileDataManager.WriteFiles(bufferSTL.m_Buffer.data(), bufferSTL.m_Position, transportIndex); @@ -440,10 +444,11 @@ void BP3Writer::AggregateWriteData(const bool isFinal, const int transportIndex) } #define declare_type(T, L) \ - T *BP3Writer::DoBufferData_##L(const size_t payloadPosition, \ + T *BP3Writer::DoBufferData_##L(const int bufferIdx, \ + const size_t payloadPosition, \ const size_t bufferID) noexcept \ { \ - return BufferDataCommon(payloadPosition, bufferID); \ + return BufferDataCommon(bufferIdx, payloadPosition, bufferID); \ } ADIOS2_FOREACH_PRIMITVE_STDTYPE_2ARGS(declare_type) diff --git a/source/adios2/engine/bp3/BP3Writer.h b/source/adios2/engine/bp3/BP3Writer.h index b0c36bd96c..85bd217f38 100644 --- a/source/adios2/engine/bp3/BP3Writer.h +++ b/source/adios2/engine/bp3/BP3Writer.h @@ -69,7 +69,7 @@ class BP3Writer : public core::Engine #define declare_type(T) \ void DoPut(Variable &variable, typename Variable::Span &span, \ - const size_t bufferID, const T &value) final; + const bool initialize, const T &value) final; ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type) #undef declare_type @@ -116,14 +116,14 @@ class BP3Writer : public core::Engine void AggregateWriteData(const bool isFinal, const int transportIndex = -1); #define declare_type(T, L) \ - T *DoBufferData_##L(const size_t payloadPosition, \ + T *DoBufferData_##L(const int bufferIdx, const size_t payloadPosition, \ const size_t bufferID = 0) noexcept final; ADIOS2_FOREACH_PRIMITVE_STDTYPE_2ARGS(declare_type) #undef declare_type template - T *BufferDataCommon(const size_t payloadOffset, + T *BufferDataCommon(const int bufferIdx, const size_t payloadOffset, const size_t bufferID) noexcept; template diff --git a/source/adios2/engine/bp3/BP3Writer.tcc b/source/adios2/engine/bp3/BP3Writer.tcc index 343a3d3933..2197141510 100644 --- a/source/adios2/engine/bp3/BP3Writer.tcc +++ b/source/adios2/engine/bp3/BP3Writer.tcc @@ -40,7 +40,8 @@ void BP3Writer::PutCommon(Variable &variable, if (!m_BP3Serializer.m_MetadataSet.DataPGIsOpen) { m_BP3Serializer.PutProcessGroupIndex( - m_IO.m_Name, m_IO.m_HostLanguage, + m_IO.m_Name, + (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) ? "C++" : "Fortran", m_FileDataManager.GetTransportsTypes()); } @@ -53,7 +54,7 @@ void BP3Writer::PutCommon(Variable &variable, } // WRITE INDEX to data buffer and metadata structure (in memory)// - const bool sourceRowMajor = helper::IsRowMajor(m_IO.m_HostLanguage); + const bool sourceRowMajor = (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor); m_BP3Serializer.PutVariableMetadata(variable, blockInfo, sourceRowMajor, &span); span.m_Value = value; @@ -83,7 +84,8 @@ void BP3Writer::PutSyncCommon(Variable &variable, if (!m_BP3Serializer.m_MetadataSet.DataPGIsOpen) { m_BP3Serializer.PutProcessGroupIndex( - m_IO.m_Name, m_IO.m_HostLanguage, + m_IO.m_Name, + (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) ? "C++" : "Fortran", m_FileDataManager.GetTransportsTypes()); } @@ -94,12 +96,13 @@ void BP3Writer::PutSyncCommon(Variable &variable, // new group index for incoming variable m_BP3Serializer.PutProcessGroupIndex( - m_IO.m_Name, m_IO.m_HostLanguage, + m_IO.m_Name, + (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) ? "C++" : "Fortran", m_FileDataManager.GetTransportsTypes()); } // WRITE INDEX to data buffer and metadata structure (in memory)// - const bool sourceRowMajor = helper::IsRowMajor(m_IO.m_HostLanguage); + const bool sourceRowMajor = (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor); m_BP3Serializer.PutVariableMetadata(variable, blockInfo, sourceRowMajor); m_BP3Serializer.PutVariablePayload(variable, blockInfo, sourceRowMajor); } @@ -123,7 +126,8 @@ void BP3Writer::PutDeferredCommon(Variable &variable, const T *data) } template -T *BP3Writer::BufferDataCommon(const size_t payloadPosition, +T *BP3Writer::BufferDataCommon(const int /*bufferIdx*/, + const size_t payloadPosition, const size_t /*bufferID*/) noexcept { T *data = reinterpret_cast(m_BP3Serializer.m_Data.m_Buffer.data() + diff --git a/source/adios2/engine/bp4/BP4Reader.cpp b/source/adios2/engine/bp4/BP4Reader.cpp index 681da602de..84096267aa 100644 --- a/source/adios2/engine/bp4/BP4Reader.cpp +++ b/source/adios2/engine/bp4/BP4Reader.cpp @@ -44,6 +44,12 @@ StepStatus BP4Reader::BeginStep(StepMode mode, const float timeoutSeconds) "BeginStep\n"); } + if (m_BetweenStepPairs) + { + throw std::logic_error("ERROR: BeginStep() is called a second time " + "without an intervening EndStep()"); + } + if (!m_BP4Deserializer.m_DeferredVariables.empty()) { throw std::invalid_argument( @@ -75,6 +81,7 @@ StepStatus BP4Reader::BeginStep(StepMode mode, const float timeoutSeconds) if (status == StepStatus::OK) { + m_BetweenStepPairs = true; if (m_FirstStep) { m_FirstStep = false; @@ -101,6 +108,12 @@ size_t BP4Reader::CurrentStep() const { return m_CurrentStep; } void BP4Reader::EndStep() { + if (!m_BetweenStepPairs) + { + throw std::logic_error( + "ERROR: EndStep() is called without a successful BeginStep()"); + } + m_BetweenStepPairs = false; PERFSTUBS_SCOPED_TIMER("BP4Reader::EndStep"); PerformGets(); } @@ -148,24 +161,25 @@ void BP4Reader::Init() "supports OpenMode::Read from" + m_Name + " " + m_EndMessage); } + // if IO was involved in reading before this flag may be true now + m_IO.m_ReadStreaming = false; m_BP4Deserializer.Init(m_IO.m_Parameters, "in call to BP4::Open to write"); InitTransports(); /* Do a collective wait for the file(s) to appear within timeout. Make sure every process comes to the same conclusion */ - const Seconds timeoutSeconds = - Seconds(m_BP4Deserializer.m_Parameters.OpenTimeoutSecs); + const Seconds timeoutSeconds( + m_BP4Deserializer.m_Parameters.OpenTimeoutSecs); - Seconds pollSeconds = - Seconds(m_BP4Deserializer.m_Parameters.BeginStepPollingFrequencySecs); + Seconds pollSeconds( + m_BP4Deserializer.m_Parameters.BeginStepPollingFrequencySecs); if (pollSeconds > timeoutSeconds) { pollSeconds = timeoutSeconds; } - TimePoint timeoutInstant = - std::chrono::steady_clock::now() + timeoutSeconds; + TimePoint timeoutInstant = Now() + timeoutSeconds; OpenFiles(timeoutInstant, pollSeconds, timeoutSeconds); if (!m_BP4Deserializer.m_Parameters.StreamReader) @@ -178,7 +192,7 @@ void BP4Reader::Init() bool BP4Reader::SleepOrQuit(const TimePoint &timeoutInstant, const Seconds &pollSeconds) { - auto now = std::chrono::steady_clock::now(); + auto now = Now(); if (now + pollSeconds >= timeoutInstant) { return false; @@ -682,8 +696,7 @@ StepStatus BP4Reader::CheckForNewSteps(Seconds timeoutSeconds) { timeoutSeconds = Seconds(999999999); // max 1 billion seconds wait } - const TimePoint timeoutInstant = - std::chrono::steady_clock::now() + timeoutSeconds; + const TimePoint timeoutInstant = Now() + timeoutSeconds; auto pollSeconds = Seconds(m_BP4Deserializer.m_Parameters.BeginStepPollingFrequencySecs); diff --git a/source/adios2/engine/bp4/BP4Reader.h b/source/adios2/engine/bp4/BP4Reader.h index dcb872b4bc..4161f9d23e 100644 --- a/source/adios2/engine/bp4/BP4Reader.h +++ b/source/adios2/engine/bp4/BP4Reader.h @@ -12,13 +12,12 @@ #define ADIOS2_ENGINE_BP4_BP4READER_H_ #include "adios2/common/ADIOSConfig.h" +#include "adios2/core/CoreTypes.h" #include "adios2/core/Engine.h" #include "adios2/helper/adiosComm.h" #include "adios2/toolkit/format/bp/bp4/BP4Deserializer.h" #include "adios2/toolkit/transportman/TransportMan.h" -#include - namespace adios2 { namespace core @@ -52,12 +51,6 @@ class BP4Reader : public Engine void PerformGets() final; private: - typedef std::chrono::duration Seconds; - typedef std::chrono::time_point< - std::chrono::steady_clock, - std::chrono::duration> - TimePoint; - format::BP4Deserializer m_BP4Deserializer; /* transport manager for metadata file */ transportman::TransportMan m_MDFileManager; diff --git a/source/adios2/engine/bp4/BP4Reader.tcc b/source/adios2/engine/bp4/BP4Reader.tcc index 0a43df821c..863753353b 100644 --- a/source/adios2/engine/bp4/BP4Reader.tcc +++ b/source/adios2/engine/bp4/BP4Reader.tcc @@ -105,7 +105,7 @@ void BP4Reader::ReadVariableBlocks(Variable &variable) m_BP4Deserializer.PostDataRead( variable, blockInfo, subStreamBoxInfo, - helper::IsRowMajor(m_IO.m_HostLanguage), 0); + m_IO.m_ArrayOrder == ArrayOrdering::RowMajor, 0); } // substreams loop // advance pointer to next step blockInfo.Data += helper::GetTotalSize(blockInfo.Count); diff --git a/source/adios2/engine/bp4/BP4Writer.cpp b/source/adios2/engine/bp4/BP4Writer.cpp index d278429a94..169a0d78af 100644 --- a/source/adios2/engine/bp4/BP4Writer.cpp +++ b/source/adios2/engine/bp4/BP4Writer.cpp @@ -127,6 +127,7 @@ void BP4Writer::Init() static_cast(m_BP4Serializer.m_SizeMPI)) { m_BP4Serializer.m_Aggregator.Init( + m_BP4Serializer.m_Parameters.NumAggregators, m_BP4Serializer.m_Parameters.NumAggregators, m_Comm); } InitTransports(); @@ -136,10 +137,10 @@ void BP4Writer::Init() #define declare_type(T) \ void BP4Writer::DoPut(Variable &variable, \ typename Variable::Span &span, \ - const size_t bufferID, const T &value) \ + const bool initialize, const T &value) \ { \ PERFSTUBS_SCOPED_TIMER("BP4Writer::Put"); \ - PutCommon(variable, span, bufferID, value); \ + PutCommon(variable, span, 0, value); \ } ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type) @@ -162,6 +163,8 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) void BP4Writer::InitParameters() { m_BP4Serializer.Init(m_IO.m_Parameters, "in call to BP4::Open to write"); + m_BP4Serializer.ResizeBuffer(m_BP4Serializer.m_Parameters.InitialBufferSize, + "in call to BP4::Open to write"); m_WriteToBB = !(m_BP4Serializer.m_Parameters.BurstBufferPath.empty()); m_DrainBB = m_WriteToBB && m_BP4Serializer.m_Parameters.BurstBufferDrain; } @@ -184,13 +187,13 @@ void BP4Writer::InitTransports() PathSeparator + m_Name; } - if (m_BP4Serializer.m_Aggregator.m_IsConsumer) - { - // Names passed to IO AddTransport option with key "Name" - const std::vector transportsNames = - m_FileDataManager.GetFilesBaseNames(m_BBName, - m_IO.m_TransportsParameters); + // Names passed to IO AddTransport option with key "Name" + const std::vector transportsNames = + m_FileDataManager.GetFilesBaseNames(m_BBName, + m_IO.m_TransportsParameters); + if (m_BP4Serializer.m_Aggregator.m_IsAggregator) + { // /path/name.bp.dir/name.bp.rank m_SubStreamNames = m_BP4Serializer.GetBPSubStreamNames(transportsNames); if (m_DrainBB) @@ -222,7 +225,7 @@ void BP4Writer::InitTransports() } m_BP4Serializer.m_Profiler.Stop("mkdir"); - if (m_BP4Serializer.m_Aggregator.m_IsConsumer) + if (m_BP4Serializer.m_Aggregator.m_IsAggregator) { if (m_BP4Serializer.m_Parameters.AsyncTasks) { @@ -286,6 +289,19 @@ void BP4Writer::InitTransports() } } } + + // last process create .bpversion file with content "4" + if (m_Comm.Rank() == m_Comm.Size() - 1) + { + std::vector versionNames = + m_BP4Serializer.GetBPVersionFileNames(transportsNames); + auto emptyComm = helper::Comm(); + transportman::TransportMan tm(emptyComm); + tm.OpenFiles(versionNames, Mode::Write, m_IO.m_TransportsParameters, + false); + char b[1] = {'4'}; + tm.WriteFiles(b, 1); + } } void BP4Writer::InitBPBuffer() @@ -333,7 +349,7 @@ void BP4Writer::InitBPBuffer() static_cast(lastStep); m_BP4Serializer.m_MetadataSet.CurrentStep += lastStep; - if (m_BP4Serializer.m_Aggregator.m_IsConsumer) + if (m_BP4Serializer.m_Aggregator.m_IsAggregator) { m_BP4Serializer.m_PreDataFileLength = m_FileDataManager.GetFileSize(0); @@ -361,7 +377,7 @@ void BP4Writer::InitBPBuffer() m_BP4Serializer.MakeHeader(m_BP4Serializer.m_MetadataIndex, "Index Table", true); } - if (m_BP4Serializer.m_Aggregator.m_IsConsumer) + if (m_BP4Serializer.m_Aggregator.m_IsAggregator) { m_BP4Serializer.MakeHeader(m_BP4Serializer.m_Data, "Data", false); } @@ -377,7 +393,8 @@ void BP4Writer::InitBPBuffer() } m_BP4Serializer.PutProcessGroupIndex( - m_IO.m_Name, m_IO.m_HostLanguage, + m_IO.m_Name, + (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) ? "C++" : "Fortran", m_FileDataManager.GetTransportsTypes()); } @@ -403,7 +420,7 @@ void BP4Writer::DoClose(const int transportIndex) DoFlush(true, transportIndex); - if (m_BP4Serializer.m_Aggregator.m_IsConsumer) + if (m_BP4Serializer.m_Aggregator.m_IsAggregator) { m_FileDataManager.CloseFiles(transportIndex); // Delete files from temporary storage if draining was on @@ -465,7 +482,7 @@ void BP4Writer::DoClose(const int transportIndex) } } - if (m_BP4Serializer.m_Aggregator.m_IsConsumer && m_DrainBB) + if (m_BP4Serializer.m_Aggregator.m_IsAggregator && m_DrainBB) { /* Signal the BB thread that no more work is coming */ m_FileDrainer.Finish(); @@ -732,15 +749,15 @@ void BP4Writer::AggregateWriteData(const bool isFinal, const int transportIndex) // async? for (int r = 0; r < m_BP4Serializer.m_Aggregator.m_Size; ++r) { - aggregator::MPIAggregator::ExchangeRequests dataRequests = + aggregator::MPIChain::ExchangeRequests dataRequests = m_BP4Serializer.m_Aggregator.IExchange(m_BP4Serializer.m_Data, r); - aggregator::MPIAggregator::ExchangeAbsolutePositionRequests + aggregator::MPIChain::ExchangeAbsolutePositionRequests absolutePositionRequests = m_BP4Serializer.m_Aggregator.IExchangeAbsolutePosition( m_BP4Serializer.m_Data, r); - if (m_BP4Serializer.m_Aggregator.m_IsConsumer) + if (m_BP4Serializer.m_Aggregator.m_IsAggregator) { const format::Buffer &bufferSTL = m_BP4Serializer.m_Aggregator.GetConsumerBuffer( @@ -791,10 +808,11 @@ void BP4Writer::AggregateWriteData(const bool isFinal, const int transportIndex) } #define declare_type(T, L) \ - T *BP4Writer::DoBufferData_##L(const size_t payloadPosition, \ + T *BP4Writer::DoBufferData_##L(const int bufferIdx, \ + const size_t payloadPosition, \ const size_t bufferID) noexcept \ { \ - return BufferDataCommon(payloadPosition, bufferID); \ + return BufferDataCommon(bufferIdx, payloadPosition, bufferID); \ } ADIOS2_FOREACH_PRIMITVE_STDTYPE_2ARGS(declare_type) @@ -805,6 +823,11 @@ size_t BP4Writer::DebugGetDataBufferSize() const return m_BP4Serializer.DebugGetDataBufferSize(); } +void BP4Writer::NotifyEngineAttribute(std::string name, DataType type) noexcept +{ + m_BP4Serializer.m_SerializedAttributes.erase(name); +} + } // end namespace engine } // end namespace core } // end namespace adios2 diff --git a/source/adios2/engine/bp4/BP4Writer.h b/source/adios2/engine/bp4/BP4Writer.h index 6f9f71864e..74002ae2dd 100644 --- a/source/adios2/engine/bp4/BP4Writer.h +++ b/source/adios2/engine/bp4/BP4Writer.h @@ -99,7 +99,7 @@ class BP4Writer : public core::Engine #define declare_type(T) \ void DoPut(Variable &variable, typename Variable::Span &span, \ - const size_t bufferID, const T &value) final; + const bool initialize, const T &value) final; ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type) #undef declare_type @@ -154,18 +154,20 @@ class BP4Writer : public core::Engine void AggregateWriteData(const bool isFinal, const int transportIndex = -1); #define declare_type(T, L) \ - T *DoBufferData_##L(const size_t payloadPosition, \ + T *DoBufferData_##L(const int bufferIdx, const size_t payloadPosition, \ const size_t bufferID = 0) noexcept final; ADIOS2_FOREACH_PRIMITVE_STDTYPE_2ARGS(declare_type) #undef declare_type template - T *BufferDataCommon(const size_t payloadOffset, + T *BufferDataCommon(const int bufferIdx, const size_t payloadOffset, const size_t bufferID) noexcept; template void PerformPutCommon(Variable &variable); + + void NotifyEngineAttribute(std::string name, DataType type) noexcept; }; } // end namespace engine diff --git a/source/adios2/engine/bp4/BP4Writer.tcc b/source/adios2/engine/bp4/BP4Writer.tcc index 5678295acf..fb25f884d6 100644 --- a/source/adios2/engine/bp4/BP4Writer.tcc +++ b/source/adios2/engine/bp4/BP4Writer.tcc @@ -40,7 +40,8 @@ void BP4Writer::PutCommon(Variable &variable, if (!m_BP4Serializer.m_MetadataSet.DataPGIsOpen) { m_BP4Serializer.PutProcessGroupIndex( - m_IO.m_Name, m_IO.m_HostLanguage, + m_IO.m_Name, + (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) ? "C++" : "Fortran", m_FileDataManager.GetTransportsTypes()); } @@ -53,7 +54,7 @@ void BP4Writer::PutCommon(Variable &variable, } // WRITE INDEX to data buffer and metadata structure (in memory)// - const bool sourceRowMajor = helper::IsRowMajor(m_IO.m_HostLanguage); + const bool sourceRowMajor = m_IO.m_ArrayOrder == ArrayOrdering::RowMajor; m_BP4Serializer.PutVariableMetadata(variable, blockInfo, sourceRowMajor, &span); span.m_Value = value; @@ -82,7 +83,8 @@ void BP4Writer::PutSyncCommon(Variable &variable, if (!m_BP4Serializer.m_MetadataSet.DataPGIsOpen) { m_BP4Serializer.PutProcessGroupIndex( - m_IO.m_Name, m_IO.m_HostLanguage, + m_IO.m_Name, + (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) ? "C++" : "Fortran", m_FileDataManager.GetTransportsTypes()); } @@ -93,12 +95,13 @@ void BP4Writer::PutSyncCommon(Variable &variable, // new group index for incoming variable m_BP4Serializer.PutProcessGroupIndex( - m_IO.m_Name, m_IO.m_HostLanguage, + m_IO.m_Name, + (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) ? "C++" : "Fortran", m_FileDataManager.GetTransportsTypes()); } // WRITE INDEX to data buffer and metadata structure (in memory)// - const bool sourceRowMajor = helper::IsRowMajor(m_IO.m_HostLanguage); + const bool sourceRowMajor = m_IO.m_ArrayOrder == ArrayOrdering::RowMajor; m_BP4Serializer.PutVariableMetadata(variable, blockInfo, sourceRowMajor); m_BP4Serializer.PutVariablePayload(variable, blockInfo, sourceRowMajor); } @@ -122,7 +125,8 @@ void BP4Writer::PutDeferredCommon(Variable &variable, const T *data) } template -T *BP4Writer::BufferDataCommon(const size_t payloadPosition, +T *BP4Writer::BufferDataCommon(const int /*bufferIdx*/, + const size_t payloadPosition, const size_t /*bufferID*/) noexcept { T *data = reinterpret_cast(m_BP4Serializer.m_Data.m_Buffer.data() + diff --git a/source/adios2/engine/bp5/BP5Engine.cpp b/source/adios2/engine/bp5/BP5Engine.cpp index 743f3120ad..3be352b204 100644 --- a/source/adios2/engine/bp5/BP5Engine.cpp +++ b/source/adios2/engine/bp5/BP5Engine.cpp @@ -92,6 +92,28 @@ std::string BP5Engine::GetBPMetadataIndexFileName(const std::string &name) const return bpMetaDataIndexRankName; } +std::vector +BP5Engine::GetBPVersionFileNames(const std::vector &names) const + noexcept +{ + std::vector versionFileNames; + versionFileNames.reserve(names.size()); + for (const auto &name : names) + { + versionFileNames.push_back(GetBPVersionFileName(name)); + } + return versionFileNames; +} + +std::string BP5Engine::GetBPVersionFileName(const std::string &name) const + noexcept +{ + const std::string bpName = helper::RemoveTrailingSlash(name); + /* the name of the version file is ".bpversion" */ + const std::string bpVersionFileName(bpName + PathSeparator + ".bpversion"); + return bpVersionFileName; +} + std::string BP5Engine::GetBPSubStreamName(const std::string &name, const size_t id, const bool hasSubFiles, @@ -103,28 +125,21 @@ std::string BP5Engine::GetBPSubStreamName(const std::string &name, } const std::string bpName = helper::RemoveTrailingSlash(name); - - const size_t index = id; - // isReader ? id - // : m_Aggregator.m_IsActive ? m_Aggregator.m_SubStreamIndex : id; - /* the name of a data file starts with "data." */ const std::string bpRankName(bpName + PathSeparator + "data." + - std::to_string(index)); + std::to_string(id)); return bpRankName; } std::vector -BP5Engine::GetBPSubStreamNames(const std::vector &names) const - noexcept +BP5Engine::GetBPSubStreamNames(const std::vector &names, + size_t subFileIndex) const noexcept { std::vector bpNames; bpNames.reserve(names.size()); - for (const auto &name : names) { - bpNames.push_back( - GetBPSubStreamName(name, static_cast(m_RankMPI))); + bpNames.push_back(GetBPSubStreamName(name, subFileIndex)); } return bpNames; } @@ -155,6 +170,31 @@ void BP5Engine::ParseParams(IO &io, struct BP5Params &Params) } } }; + + auto lf_SetFloatParameter = [&](const std::string key, float ¶meter, + float def) { + auto itKey = io.m_Parameters.find(key); + parameter = def; + if (itKey != io.m_Parameters.end()) + { + std::string value = itKey->second; + parameter = + helper::StringTo(value, " in Parameter key=" + key); + } + }; + + auto lf_SetSizeBytesParameter = [&](const std::string key, + size_t ¶meter, size_t def) { + auto itKey = io.m_Parameters.find(key); + parameter = def; + if (itKey != io.m_Parameters.end()) + { + std::string value = itKey->second; + parameter = helper::StringToByteUnits( + value, "for Parameter key=" + key + "in call to Open"); + } + }; + auto lf_SetIntParameter = [&](const std::string key, int ¶meter, int def) { auto itKey = io.m_Parameters.find(key); @@ -167,6 +207,23 @@ void BP5Engine::ParseParams(IO &io, struct BP5Params &Params) return false; }; + auto lf_SetUIntParameter = [&](const std::string key, + unsigned int ¶meter, unsigned int def) { + auto itKey = io.m_Parameters.find(key); + parameter = def; + if (itKey != io.m_Parameters.end()) + { + unsigned long result = std::stoul(itKey->second); + if (result > std::numeric_limits::max()) + { + result = std::numeric_limits::max(); + } + parameter = static_cast(result); + return true; + } + return false; + }; + auto lf_SetStringParameter = [&](const std::string key, std::string ¶meter, const char *def) { auto itKey = io.m_Parameters.find(key); @@ -178,6 +235,63 @@ void BP5Engine::ParseParams(IO &io, struct BP5Params &Params) return false; }; + auto lf_SetBufferVTypeParameter = [&](const std::string key, int ¶meter, + int def) { + auto itKey = io.m_Parameters.find(key); + parameter = def; + if (itKey != io.m_Parameters.end()) + { + std::string value = itKey->second; + std::transform(value.begin(), value.end(), value.begin(), + ::tolower); + if (value == "malloc") + { + parameter = (int)BufferVType::MallocVType; + } + else if (value == "chunk") + { + parameter = (int)BufferVType::ChunkVType; + } + else + { + throw std::invalid_argument( + "ERROR: Unknown BP5 BufferVType parameter \"" + value + + "\" (must be \"malloc\" or \"chunk\""); + } + } + }; + + auto lf_SetAggregationTypeParameter = [&](const std::string key, + int ¶meter, int def) { + auto itKey = io.m_Parameters.find(key); + parameter = def; + if (itKey != io.m_Parameters.end()) + { + std::string value = itKey->second; + std::transform(value.begin(), value.end(), value.begin(), + ::tolower); + if (value == "everyonewrites" || value == "auto") + { + parameter = (int)AggregationType::EveryoneWrites; + } + else if (value == "everyonewritesserial") + { + parameter = (int)AggregationType::EveryoneWritesSerial; + } + else if (value == "twolevelshm") + { + parameter = (int)AggregationType::TwoLevelShm; + } + else + { + throw std::invalid_argument( + "ERROR: Unknown BP5 AggregationType parameter \"" + value + + "\" (must be \"auto\", \"everyonewrites\" or " + "\"twolevelshm\""); + } + } + }; + #define get_params(Param, Type, Typedecl, Default) \ lf_Set##Type##Parameter(#Param, Params.Param, Default); BP5_FOREACH_PARAMETER_TYPE_4ARGS(get_params); diff --git a/source/adios2/engine/bp5/BP5Engine.h b/source/adios2/engine/bp5/BP5Engine.h index e8241f187e..2a1f39deca 100644 --- a/source/adios2/engine/bp5/BP5Engine.h +++ b/source/adios2/engine/bp5/BP5Engine.h @@ -56,7 +56,8 @@ class BP5Engine static constexpr size_t m_VersionTagLength = 32; std::vector - GetBPSubStreamNames(const std::vector &names) const noexcept; + GetBPSubStreamNames(const std::vector &names, + size_t subFileIndex) const noexcept; std::vector GetBPMetadataFileNames(const std::vector &names) const @@ -78,6 +79,28 @@ class BP5Engine const bool hasSubFiles = true, const bool isReader = false) const noexcept; + std::vector + GetBPVersionFileNames(const std::vector &names) const noexcept; + + std::string GetBPVersionFileName(const std::string &name) const noexcept; + + enum class BufferVType + { + MallocVType, + ChunkVType, + Auto + }; + + BufferVType UseBufferV = BufferVType::ChunkVType; + + enum class AggregationType + { + EveryoneWrites, + EveryoneWritesSerial, + TwoLevelShm, + Auto + }; + #define BP5_FOREACH_PARAMETER_TYPE_4ARGS(MACRO) \ MACRO(OpenTimeoutSecs, Int, int, 3600) \ MACRO(BeginStepPollingFrequencySecs, Int, int, 0) \ @@ -87,6 +110,18 @@ class BP5Engine MACRO(NodeLocal, Bool, bool, false) \ MACRO(verbose, Int, int, 0) \ MACRO(CollectiveMetadata, Bool, bool, true) \ + MACRO(NumAggregators, UInt, unsigned int, 0) \ + MACRO(NumSubFiles, UInt, unsigned int, 999999) \ + MACRO(FileSystemPageSize, UInt, unsigned int, 4096) \ + MACRO(AggregationType, AggregationType, int, \ + (int)AggregationType::TwoLevelShm) \ + MACRO(AsyncTasks, Bool, bool, true) \ + MACRO(GrowthFactor, Float, float, DefaultBufferGrowthFactor) \ + MACRO(InitialBufferSize, SizeBytes, size_t, DefaultInitialBufferSize) \ + MACRO(MinDeferredSize, SizeBytes, size_t, DefaultMinDeferredSize) \ + MACRO(BufferChunkSize, SizeBytes, size_t, DefaultBufferChunkSize) \ + MACRO(MaxShmSize, SizeBytes, size_t, DefaultMaxShmSize) \ + MACRO(BufferVType, BufferVType, int, (int)BufferVType::ChunkVType) \ MACRO(ReaderShortCircuitReads, Bool, bool, false) struct BP5Params @@ -113,11 +148,18 @@ class BP5Engine * BP5 header for "Index Table" (64 bytes) * for each Writer, what aggregator writes its data * uint16_t [ WriterCount] - * for each timestep: - * uint64_t 0 : CombinedMetaDataPos - * uint64_t 1 : CombinedMetaDataSize - * for each Writer - * uint64_t DataPos (in the file above) + * for each timestep: (size (WriterCount + 2 ) 64-bit ints + * uint64_t 0 : CombinedMetaDataPos + * uint64_t 1 : CombinedMetaDataSize + * uint64_t 2 : FlushCount + * for each Writer + * for each flush before the last: + * uint64_t DataPos (in the file above) + * uint64_t DataSize + * for the final flush: + * uint64_t DataPos (in the file above) + * So, each timestep takes sizeof(uint64_t)* (3 + ((FlushCount-1)*2 + + *1) * WriterCount) bytes * * MetaMetadata file (mmd.0) contains FFS format information * for each meta metadata item: diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index 739b9ea521..cccebf7b01 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -11,7 +11,6 @@ #include -#include #include namespace adios2 @@ -24,8 +23,8 @@ namespace engine BP5Reader::BP5Reader(IO &io, const std::string &name, const Mode mode, helper::Comm comm) : Engine("BP5Reader", io, name, mode, std::move(comm)), m_MDFileManager(m_Comm), - m_FileMetaMetadataManager(m_Comm), m_DataFileManager(m_Comm), - m_MDIndexFileManager(m_Comm), m_ActiveFlagFileManager(m_Comm) + m_DataFileManager(m_Comm), m_MDIndexFileManager(m_Comm), + m_FileMetaMetadataManager(m_Comm), m_ActiveFlagFileManager(m_Comm) { PERFSTUBS_SCOPED_TIMER("BP5Reader::Open"); Init(); @@ -37,9 +36,55 @@ BP5Reader::~BP5Reader() delete m_BP5Deserializer; } +void BP5Reader::InstallMetadataForTimestep(size_t Step) +{ + size_t pgstart = m_MetadataIndexTable[Step][0]; + size_t Position = pgstart + sizeof(uint64_t); // skip total data size + size_t MDPosition = Position + 2 * sizeof(uint64_t) * m_WriterCount; + for (size_t WriterRank = 0; WriterRank < m_WriterCount; WriterRank++) + { + // variable metadata for timestep + size_t ThisMDSize = helper::ReadValue( + m_Metadata.m_Buffer, Position, m_Minifooter.IsLittleEndian); + char *ThisMD = m_Metadata.m_Buffer.data() + MDPosition; + if (m_OpenMode == Mode::ReadRandomAccess) + { + m_BP5Deserializer->InstallMetaData(ThisMD, ThisMDSize, WriterRank, + Step); + } + else + { + m_BP5Deserializer->InstallMetaData(ThisMD, ThisMDSize, WriterRank); + } + MDPosition += ThisMDSize; + } + for (size_t WriterRank = 0; WriterRank < m_WriterCount; WriterRank++) + { + // attribute metadata for timestep + size_t ThisADSize = helper::ReadValue( + m_Metadata.m_Buffer, Position, m_Minifooter.IsLittleEndian); + char *ThisAD = m_Metadata.m_Buffer.data() + MDPosition; + if (ThisADSize > 0) + m_BP5Deserializer->InstallAttributeData(ThisAD, ThisADSize); + MDPosition += ThisADSize; + } +} + StepStatus BP5Reader::BeginStep(StepMode mode, const float timeoutSeconds) { PERFSTUBS_SCOPED_TIMER("BP5Reader::BeginStep"); + + if (m_OpenMode == Mode::ReadRandomAccess) + { + throw std::logic_error( + "ERROR: BeginStep called in random access mode\n"); + } + if (m_BetweenStepPairs) + { + throw std::logic_error("ERROR: BeginStep() is called a second time " + "without an intervening EndStep()"); + } + if (mode != StepMode::Read) { throw std::invalid_argument("ERROR: mode is not supported yet, " @@ -48,7 +93,6 @@ StepStatus BP5Reader::BeginStep(StepMode mode, const float timeoutSeconds) "BeginStep\n"); } - m_IO.m_ReadStreaming = false; // can't do those checks StepStatus status = StepStatus::OK; if (m_FirstStep) { @@ -67,6 +111,7 @@ StepStatus BP5Reader::BeginStep(StepMode mode, const float timeoutSeconds) } if (status == StepStatus::OK) { + m_BetweenStepPairs = true; if (m_FirstStep) { m_FirstStep = false; @@ -89,31 +134,9 @@ StepStatus BP5Reader::BeginStep(StepMode mode, const float timeoutSeconds) // i++; // } - m_IO.RemoveAllVariables(); m_BP5Deserializer->SetupForTimestep(m_CurrentStep); - size_t pgstart = m_MetadataIndexTable[m_CurrentStep][0]; - size_t Position = pgstart + sizeof(uint64_t); // skip total data size - size_t MDPosition = Position + 2 * sizeof(uint64_t) * m_WriterCount; - for (int i = 0; i < m_WriterCount; i++) - { - // variable metadata for timestep - size_t ThisMDSize = helper::ReadValue( - m_Metadata.m_Buffer, Position, m_Minifooter.IsLittleEndian); - char *ThisMD = m_Metadata.m_Buffer.data() + MDPosition; - m_BP5Deserializer->InstallMetaData(ThisMD, ThisMDSize, i); - MDPosition += ThisMDSize; - } - for (int i = 0; i < m_WriterCount; i++) - { - // attribute metadata for timestep - size_t ThisADSize = helper::ReadValue( - m_Metadata.m_Buffer, Position, m_Minifooter.IsLittleEndian); - char *ThisAD = m_Metadata.m_Buffer.data() + MDPosition; - if (ThisADSize > 0) - m_BP5Deserializer->InstallAttributeData(ThisAD, ThisADSize); - MDPosition += ThisADSize; - } + InstallMetadataForTimestep(m_CurrentStep); m_IO.ResetVariablesStepSelection(false, "in call to BP5 Reader BeginStep"); @@ -130,6 +153,16 @@ size_t BP5Reader::CurrentStep() const { return m_CurrentStep; } void BP5Reader::EndStep() { + if (m_OpenMode == Mode::ReadRandomAccess) + { + throw std::logic_error("ERROR: EndStep called in random access mode\n"); + } + if (!m_BetweenStepPairs) + { + throw std::logic_error( + "ERROR: EndStep() is called without a successful BeginStep()"); + } + m_BetweenStepPairs = false; PERFSTUBS_SCOPED_TIMER("BP5Reader::EndStep"); PerformGets(); } @@ -138,21 +171,49 @@ void BP5Reader::ReadData(const size_t WriterRank, const size_t Timestep, const size_t StartOffset, const size_t Length, char *Destination) { - size_t DataStartPos = m_MetadataIndexTable[Timestep][2]; - DataStartPos += WriterRank * sizeof(uint64_t); - size_t DataStart = helper::ReadValue( - m_MetadataIndex.m_Buffer, DataStartPos, m_Minifooter.IsLittleEndian); + size_t FlushCount = m_MetadataIndexTable[Timestep][2]; + size_t DataPosPos = m_MetadataIndexTable[Timestep][3]; + size_t SubfileNum = m_WriterToFileMap[WriterRank]; + // check if subfile is already opened - if (m_DataFileManager.m_Transports.count(WriterRank) == 0) + if (m_DataFileManager.m_Transports.count(SubfileNum) == 0) { const std::string subFileName = GetBPSubStreamName( - m_Name, WriterRank, m_Minifooter.HasSubFiles, true); + m_Name, SubfileNum, m_Minifooter.HasSubFiles, true); - m_DataFileManager.OpenFileID(subFileName, WriterRank, Mode::Read, + m_DataFileManager.OpenFileID(subFileName, SubfileNum, Mode::Read, {{"transport", "File"}}, false); } - m_DataFileManager.ReadFile(Destination, Length, DataStart + StartOffset, - WriterRank); + + size_t InfoStartPos = + DataPosPos + (WriterRank * (2 * FlushCount + 1) * sizeof(uint64_t)); + size_t ThisFlushInfo = InfoStartPos; + size_t RemainingLength = Length; + size_t ThisDataPos; + size_t Offset = StartOffset; + for (size_t flush = 0; flush < FlushCount; flush++) + { + + ThisDataPos = + helper::ReadValue(m_MetadataIndex.m_Buffer, ThisFlushInfo, + m_Minifooter.IsLittleEndian); + size_t ThisDataSize = + helper::ReadValue(m_MetadataIndex.m_Buffer, ThisFlushInfo, + m_Minifooter.IsLittleEndian); + if (ThisDataSize > RemainingLength) + ThisDataSize = RemainingLength; + m_DataFileManager.ReadFile(Destination, ThisDataSize, + ThisDataPos + Offset, SubfileNum); + Destination += ThisDataSize; + RemainingLength -= ThisDataSize; + Offset = 0; + if (RemainingLength == 0) + return; + } + ThisDataPos = helper::ReadValue( + m_MetadataIndex.m_Buffer, ThisFlushInfo, m_Minifooter.IsLittleEndian); + m_DataFileManager.ReadFile(Destination, RemainingLength, ThisDataPos, + SubfileNum); } void BP5Reader::PerformGets() @@ -172,15 +233,19 @@ void BP5Reader::PerformGets() // PRIVATE void BP5Reader::Init() { - if (m_OpenMode != Mode::Read) + if ((m_OpenMode != Mode::Read) && (m_OpenMode != Mode::ReadRandomAccess)) { - throw std::invalid_argument("ERROR: BPFileReader only " - "supports OpenMode::Read from" + - m_Name + " " + m_EndMessage); + throw std::invalid_argument( + "ERROR: BPFileReader only " + "supports OpenMode::Read or OpenMode::ReadRandomAccess from" + + m_Name + " " + m_EndMessage); } + // if IO was involved in reading before this flag may be true now + m_IO.m_ReadStreaming = false; + ParseParams(m_IO, m_Parameters); - m_ReaderIsRowMajor = helper::IsRowMajor(m_IO.m_HostLanguage); + m_ReaderIsRowMajor = (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor); InitTransports(); /* Do a collective wait for the file(s) to appear within timeout. @@ -193,21 +258,18 @@ void BP5Reader::Init() pollSeconds = timeoutSeconds; } - TimePoint timeoutInstant = - std::chrono::steady_clock::now() + timeoutSeconds; + TimePoint timeoutInstant = Now() + timeoutSeconds; OpenFiles(timeoutInstant, pollSeconds, timeoutSeconds); - if (!m_Parameters.StreamReader) - { - /* non-stream reader gets as much steps as available now */ - InitBuffer(timeoutInstant, pollSeconds / 10, timeoutSeconds); - } + + /* non-stream reader gets as much steps as available now */ + InitBuffer(timeoutInstant, pollSeconds / 10, timeoutSeconds); } bool BP5Reader::SleepOrQuit(const TimePoint &timeoutInstant, const Seconds &pollSeconds) { - auto now = std::chrono::steady_clock::now(); + auto now = Now(); if (now + pollSeconds >= timeoutInstant) { return false; @@ -359,6 +421,12 @@ void BP5Reader::OpenFiles(TimePoint &timeoutInstant, const Seconds &pollSeconds, */ } +Engine::MinVarInfo *BP5Reader::MinBlocksInfo(const VariableBase &Var, + const size_t Step) const +{ + return m_BP5Deserializer->MinBlocksInfo(Var, Step); +} + void BP5Reader::InitTransports() { if (m_IO.m_TransportsParameters.empty()) @@ -498,13 +566,21 @@ void BP5Reader::InitBuffer(const TimePoint &timeoutInstant, // done m_BP5Deserializer = new format::BP5Deserializer( - m_WriterCount, m_WriterIsRowMajor, m_ReaderIsRowMajor); + m_WriterCount, m_WriterIsRowMajor, m_ReaderIsRowMajor, + (m_OpenMode == Mode::ReadRandomAccess)); m_BP5Deserializer->m_Engine = this; InstallMetaMetaData(m_MetaMetadata); m_IdxHeaderParsed = true; + if (m_OpenMode == Mode::ReadRandomAccess) + { + for (size_t Step = 0; Step < m_MetadataIndexTable.size(); Step++) + { + InstallMetadataForTimestep(Step); + } + } // fills IO with Variables and Attributes // m_MDFileProcessedSize = ParseMetadata( // m_Metadata, *this, true); @@ -599,19 +675,34 @@ void BP5Reader::ParseMetadataIndex(format::BufferSTL &bufferSTL, buffer, position, m_Minifooter.IsLittleEndian); const uint64_t MetadataSize = helper::ReadValue( buffer, position, m_Minifooter.IsLittleEndian); + const uint64_t FlushCount = helper::ReadValue( + buffer, position, m_Minifooter.IsLittleEndian); ptrs.push_back(MetadataPos); ptrs.push_back(MetadataSize); + ptrs.push_back(FlushCount); ptrs.push_back(position); m_MetadataIndexTable[currentStep] = ptrs; +#ifdef DUMPDATALOCINFO for (uint64_t i = 0; i < m_WriterCount; i++) { - size_t DataPosPos = ptrs[2] + sizeof(uint64_t) * i; + size_t DataPosPos = ptrs[3]; + std::cout << "Writer " << i << " data at "; + for (uint64_t j = 0; j < FlushCount; j++) + { + const uint64_t DataPos = helper::ReadValue( + buffer, DataPosPos, m_Minifooter.IsLittleEndian); + const uint64_t DataSize = helper::ReadValue( + buffer, DataPosPos, m_Minifooter.IsLittleEndian); + std::cout << "loc:" << DataPos << " siz:" << DataSize << "; "; + } const uint64_t DataPos = helper::ReadValue( buffer, DataPosPos, m_Minifooter.IsLittleEndian); + std::cout << "loc:" << DataPos << std::endl; } +#endif - position += sizeof(uint64_t) * m_WriterCount; + position += sizeof(uint64_t) * m_WriterCount * ((2 * FlushCount) + 1); m_StepsCount++; currentStep++; } while (!oneStepOnly && position < buffer.size()); @@ -638,12 +729,12 @@ void BP5Reader::DoClose(const int transportIndex) m_MDFileManager.CloseFiles(); } +// DoBlocksInfo will not be called because MinBlocksInfo is operative #define declare_type(T) \ std::vector::BPInfo> BP5Reader::DoBlocksInfo( \ const Variable &variable, const size_t step) const \ { \ - PERFSTUBS_SCOPED_TIMER("BP5Reader::BlocksInfo"); \ - return m_BP5Deserializer->BlocksInfo(variable, step); \ + return std::vector::BPInfo>(); \ } ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) diff --git a/source/adios2/engine/bp5/BP5Reader.h b/source/adios2/engine/bp5/BP5Reader.h index 6cd73b6310..7de7f8bb71 100644 --- a/source/adios2/engine/bp5/BP5Reader.h +++ b/source/adios2/engine/bp5/BP5Reader.h @@ -12,6 +12,7 @@ #define ADIOS2_ENGINE_BP5_BP5READER_H_ #include "adios2/common/ADIOSConfig.h" +#include "adios2/core/CoreTypes.h" #include "adios2/core/Engine.h" #include "adios2/engine/bp5/BP5Engine.h" #include "adios2/helper/adiosComm.h" @@ -52,13 +53,9 @@ class BP5Reader : public BP5Engine, public Engine void PerformGets() final; -private: - typedef std::chrono::duration Seconds; - typedef std::chrono::time_point< - std::chrono::steady_clock, - std::chrono::duration> - TimePoint; + MinVarInfo *MinBlocksInfo(const VariableBase &, const size_t Step) const; +private: format::BP5Deserializer *m_BP5Deserializer = nullptr; /* transport manager for metadata file */ transportman::TransportMan m_MDFileManager; @@ -197,6 +194,7 @@ class BP5Reader : public BP5Engine, public Engine uint64_t MetadataExpectedMinFileSize(const std::string &IdxFileName, bool hasHeader); void InstallMetaMetaData(format::BufferSTL MetaMetadata); + void InstallMetadataForTimestep(size_t Step); void ReadData(const size_t WriterRank, const size_t Timestep, const size_t StartOffset, const size_t Length, char *Destination); diff --git a/source/adios2/engine/bp5/BP5Writer.cpp b/source/adios2/engine/bp5/BP5Writer.cpp index c7223d089e..4af91baa05 100644 --- a/source/adios2/engine/bp5/BP5Writer.cpp +++ b/source/adios2/engine/bp5/BP5Writer.cpp @@ -12,6 +12,8 @@ #include "adios2/common/ADIOSMacros.h" #include "adios2/core/IO.h" #include "adios2/helper/adiosFunctions.h" //CheckIndexRange +#include "adios2/toolkit/format/buffer/chunk/ChunkV.h" +#include "adios2/toolkit/format/buffer/malloc/MallocV.h" #include "adios2/toolkit/transport/file/FileFStream.h" #include @@ -31,7 +33,8 @@ BP5Writer::BP5Writer(IO &io, const std::string &name, const Mode mode, helper::Comm comm) : Engine("BP5Writer", io, name, mode, std::move(comm)), m_BP5Serializer(), m_FileDataManager(m_Comm), m_FileMetadataManager(m_Comm), - m_FileMetaMetadataManager(m_Comm), m_FileMetadataIndexManager(m_Comm) + m_FileMetadataIndexManager(m_Comm), m_FileMetaMetadataManager(m_Comm), + m_Profiler(m_Comm) { PERFSTUBS_SCOPED_TIMER("BP5Writer::Open"); m_IO.m_ReadStreaming = false; @@ -42,7 +45,27 @@ BP5Writer::BP5Writer(IO &io, const std::string &name, const Mode mode, StepStatus BP5Writer::BeginStep(StepMode mode, const float timeoutSeconds) { - m_WriterStep++; + if (m_BetweenStepPairs) + { + throw std::logic_error("ERROR: BeginStep() is called a second time " + "without an intervening EndStep()"); + } + + m_BetweenStepPairs = true; + if (m_Parameters.BufferVType == (int)BufferVType::MallocVType) + { + m_BP5Serializer.InitStep(new MallocV("BP5Writer", false, + m_Parameters.InitialBufferSize, + m_Parameters.GrowthFactor)); + } + else + { + m_BP5Serializer.InitStep(new ChunkV("BP5Writer", + false /* always copy */, + m_Parameters.BufferChunkSize)); + } + m_ThisTimestepDataSize = 0; + return StepStatus::OK; } @@ -51,6 +74,9 @@ size_t BP5Writer::CurrentStep() const { return m_WriterStep; } void BP5Writer::PerformPuts() { PERFSTUBS_SCOPED_TIMER("BP5Writer::PerformPuts"); + m_Profiler.Start("PP"); + m_BP5Serializer.PerformPuts(); + m_Profiler.Stop("PP"); return; } @@ -70,9 +96,9 @@ void BP5Writer::WriteMetaMetadata( } } -uint64_t BP5Writer::WriteMetadata( - const std::vector MetaDataBlocks, - const std::vector AttributeBlocks) +uint64_t +BP5Writer::WriteMetadata(const std::vector &MetaDataBlocks, + const std::vector &AttributeBlocks) { uint64_t MDataTotalSize = 0; uint64_t MetaDataSize = 0; @@ -120,37 +146,135 @@ uint64_t BP5Writer::WriteMetadata( void BP5Writer::WriteData(format::BufferV *Data) { - format::BufferV::BufferV_iovec DataVec = Data->DataVec(); - size_t DataSize = 0; - int i = 0; - while (DataVec[i].iov_base != NULL) - { - m_FileDataManager.WriteFiles((char *)DataVec[i].iov_base, - DataVec[i].iov_len); - DataSize += DataVec[i].iov_len; - i++; - } - m_DataPos += DataSize; - delete[] DataVec; + switch (m_Parameters.AggregationType) + { + case (int)AggregationType::EveryoneWrites: + WriteData_EveryoneWrites(Data, false); + break; + case (int)AggregationType::EveryoneWritesSerial: + WriteData_EveryoneWrites(Data, true); + break; + case (int)AggregationType::TwoLevelShm: + WriteData_TwoLevelShm(Data); + break; + default: + throw std::invalid_argument( + "Aggregation method " + + std::to_string(m_Parameters.AggregationType) + + "is not supported in BP5"); + } +} + +void BP5Writer::WriteData_EveryoneWrites(format::BufferV *Data, + bool SerializedWriters) +{ + const aggregator::MPIChain *a = + dynamic_cast(m_Aggregator); + + std::vector DataVec = Data->DataVec(); + + // new step writing starts at offset m_DataPos on aggregator + // others will wait for the position to arrive from the rank below + + if (a->m_Comm.Rank() > 0) + { + a->m_Comm.Recv(&m_DataPos, 1, a->m_Comm.Rank() - 1, 0, + "Chain token in BP5Writer::WriteData"); + } + + // align to PAGE_SIZE + m_DataPos += helper::PaddingToAlignOffset(m_DataPos, + m_Parameters.FileSystemPageSize); + m_StartDataPos = m_DataPos; + + if (!SerializedWriters && a->m_Comm.Rank() < a->m_Comm.Size() - 1) + { + /* Send the token before writing so everyone can start writing asap */ + uint64_t nextWriterPos = m_DataPos + Data->Size(); + a->m_Comm.Isend(&nextWriterPos, 1, a->m_Comm.Rank() + 1, 0, + "Chain token in BP5Writer::WriteData"); + } + + m_FileDataManager.WriteFileAt(DataVec.data(), DataVec.size(), + m_StartDataPos); + m_DataPos += Data->Size(); + + if (SerializedWriters && a->m_Comm.Rank() < a->m_Comm.Size() - 1) + { + /* send token now, effectively serializing the writers in the chain */ + uint64_t nextWriterPos = m_DataPos; + a->m_Comm.Isend(&nextWriterPos, 1, a->m_Comm.Rank() + 1, 0, + "Chain token in BP5Writer::WriteData"); + } + + if (a->m_Comm.Size() > 1) + { + // at the end, last rank sends back the final data pos to first rank + // so it can update its data pos + if (a->m_Comm.Rank() == a->m_Comm.Size() - 1) + { + a->m_Comm.Isend(&m_DataPos, 1, 0, 0, + "Final chain token in BP5Writer::WriteData"); + } + if (a->m_Comm.Rank() == 0) + { + a->m_Comm.Recv(&m_DataPos, 1, a->m_Comm.Size() - 1, 0, + "Chain token in BP5Writer::WriteData"); + } + } } void BP5Writer::WriteMetadataFileIndex(uint64_t MetaDataPos, - uint64_t MetaDataSize, - std::vector DataSizes) + uint64_t MetaDataSize) { m_FileMetadataManager.FlushFiles(); - uint64_t buf[2]; + std::vector buf; + buf.resize(3 + ((FlushPosSizeInfo.size() * 2) + 1) * m_Comm.Size()); buf[0] = MetaDataPos; buf[1] = MetaDataSize; - m_FileMetadataIndexManager.WriteFiles((char *)buf, sizeof(buf)); - m_FileMetadataIndexManager.WriteFiles((char *)m_WriterDataPos.data(), - DataSizes.size() * sizeof(uint64_t)); - for (int i = 0; i < DataSizes.size(); i++) + buf[2] = FlushPosSizeInfo.size(); + + uint64_t pos = 3; + + for (int writer = 0; writer < m_Comm.Size(); writer++) + { + for (size_t flushNum = 0; flushNum < FlushPosSizeInfo.size(); + flushNum++) + { + buf[pos + (flushNum * 2)] = FlushPosSizeInfo[flushNum][2 * writer]; + buf[pos + (flushNum * 2) + 1] = + FlushPosSizeInfo[flushNum][2 * writer + 1]; + } + buf[pos + FlushPosSizeInfo.size() * 2] = m_WriterDataPos[writer]; + pos += (FlushPosSizeInfo.size() * 2) + 1; + } + + m_FileMetadataIndexManager.WriteFiles((char *)buf.data(), + buf.size() * sizeof(uint64_t)); + +#ifdef DUMPDATALOCINFO + std::cout << "Flush count is :" << FlushPosSizeInfo.size() << std::endl; + std::cout << "Write Index positions = {" << std::endl; + + for (size_t i = 0; i < m_Comm.Size(); ++i) { - m_WriterDataPos[i] += DataSizes[i]; + std::cout << "Writer " << i << " has data at: " << std::endl; + uint64_t eachWriterSize = FlushPosSizeInfo.size() * 2 + 1; + for (size_t j = 0; j < FlushPosSizeInfo.size(); ++j) + { + std::cout << "loc:" << buf[3 + eachWriterSize * i + j * 2] + << " siz:" << buf[3 + eachWriterSize * i + j * 2 + 1] + << std::endl; + } + std::cout << "loc:" << buf[3 + eachWriterSize * (i + 1) - 1] + << std::endl; } + std::cout << "}" << std::endl; +#endif + /* reset for next timestep */ + FlushPosSizeInfo.clear(); } void BP5Writer::MarshalAttributes() @@ -169,7 +293,14 @@ void BP5Writer::MarshalAttributes() for (const auto &attributePair : attributes) { const std::string name(attributePair.first); - const DataType type(attributePair.second->m_Type); + auto baseAttr = &attributePair.second; + const DataType type((*baseAttr)->m_Type); + int element_count = -1; + + if (!attributePair.second->m_IsSingleValue) + { + element_count = (*baseAttr)->m_Elements; + } if (type == DataType::None) { @@ -178,11 +309,23 @@ void BP5Writer::MarshalAttributes() { core::Attribute &attribute = *m_IO.InquireAttribute(name); - int element_count = -1; - const char *data_addr = attribute.m_DataSingleValue.c_str(); - if (!attribute.m_IsSingleValue) + void *data_addr; + if (attribute.m_IsSingleValue) { - // + data_addr = (void *)attribute.m_DataSingleValue.c_str(); + } + else + { + const char **tmp = + (const char **)malloc(sizeof(char *) * element_count); + for (int i = 0; i < element_count; i++) + { + auto str = &attribute.m_DataArray[i]; + printf("Marshalling attr array string %s\n", str->c_str()); + tmp[i] = str->c_str(); + } + // tmp will be free'd after final attribute marshalling + data_addr = (void *)tmp; } m_BP5Serializer.MarshalAttribute(name.c_str(), type, sizeof(char *), @@ -203,7 +346,7 @@ void BP5Writer::MarshalAttributes() sizeof(T), element_count, data_addr); \ } - ADIOS2_FOREACH_ATTRIBUTE_PRIMITIVE_STDTYPE_1ARG(declare_type) + ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type) #undef declare_type } m_MarshaledAttributesCount = attributesCount; @@ -211,8 +354,9 @@ void BP5Writer::MarshalAttributes() void BP5Writer::EndStep() { + m_BetweenStepPairs = false; PERFSTUBS_SCOPED_TIMER("BP5Writer::EndStep"); - + m_Profiler.Start("endstep"); MarshalAttributes(); // true: advances step @@ -222,9 +366,15 @@ void BP5Writer::EndStep() * AttributeEncodeBuffer and the data encode Vector */ /* the first */ + m_Profiler.Start("AWD"); + WriteData(TSInfo.DataBuffer); + m_Profiler.Stop("AWD"); + + m_ThisTimestepDataSize += TSInfo.DataBuffer->Size(); + std::vector MetaBuffer = m_BP5Serializer.CopyMetadataToContiguous( TSInfo.NewMetaMetaBlocks, TSInfo.MetaEncodeBuffer, - TSInfo.AttributeEncodeBuffer, TSInfo.DataBuffer->Size()); + TSInfo.AttributeEncodeBuffer, m_ThisTimestepDataSize, m_StartDataPos); size_t LocalSize = MetaBuffer.size(); std::vector RecvCounts = m_Comm.GatherValues(LocalSize, 0); @@ -237,24 +387,44 @@ void BP5Writer::EndStep() TotalSize += n; RecvBuffer->resize(TotalSize); } + + m_Profiler.Start("meta_gather"); m_Comm.GathervArrays(MetaBuffer.data(), LocalSize, RecvCounts.data(), RecvCounts.size(), RecvBuffer->data(), 0); + m_Profiler.Stop("meta_gather"); if (m_Comm.Rank() == 0) { std::vector UniqueMetaMetaBlocks; std::vector DataSizes; - std::vector AttributeBlocks; + std::vector AttributeBlocks; auto Metadata = m_BP5Serializer.BreakoutContiguousMetadata( RecvBuffer, RecvCounts, UniqueMetaMetaBlocks, AttributeBlocks, - DataSizes); + DataSizes, m_WriterDataPos); + if (m_MetaDataPos == 0) + { + // First time, write the headers + format::BufferSTL b; + MakeHeader(b, "Metadata", false); + m_FileMetadataManager.WriteFiles(b.m_Buffer.data(), b.m_Position); + m_MetaDataPos = b.m_Position; + format::BufferSTL bi; + MakeHeader(bi, "Index Table", true); + m_FileMetadataIndexManager.WriteFiles(bi.m_Buffer.data(), + bi.m_Position); + // where each rank's data will end up + m_FileMetadataIndexManager.WriteFiles((char *)m_Assignment.data(), + sizeof(m_Assignment[0]) * + m_Assignment.size()); + } WriteMetaMetadata(UniqueMetaMetaBlocks); uint64_t ThisMetaDataPos = m_MetaDataPos; uint64_t ThisMetaDataSize = WriteMetadata(Metadata, AttributeBlocks); - WriteMetadataFileIndex(ThisMetaDataPos, ThisMetaDataSize, DataSizes); + WriteMetadataFileIndex(ThisMetaDataPos, ThisMetaDataSize); } delete RecvBuffer; - WriteData(TSInfo.DataBuffer); + m_Profiler.Stop("endstep"); + m_WriterStep++; } // PRIVATE @@ -263,10 +433,23 @@ void BP5Writer::Init() m_BP5Serializer.m_Engine = this; m_RankMPI = m_Comm.Rank(); InitParameters(); + InitAggregator(); InitTransports(); InitBPBuffer(); } +#define declare_type(T) \ + void BP5Writer::DoPut(Variable &variable, \ + typename Variable::Span &span, \ + const bool initialize, const T &value) \ + { \ + PERFSTUBS_SCOPED_TIMER("BP5Writer::Put"); \ + PutCommonSpan(variable, span, initialize, value); \ + } + +ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type) +#undef declare_type + #define declare_type(T) \ void BP5Writer::DoPutSync(Variable &variable, const T *data) \ { \ @@ -280,16 +463,93 @@ void BP5Writer::Init() ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type +#define declare_type(T, L) \ + T *BP5Writer::DoBufferData_##L(const int bufferIdx, \ + const size_t payloadPosition, \ + const size_t bufferID) noexcept \ + { \ + return reinterpret_cast( \ + m_BP5Serializer.GetPtr(bufferIdx, payloadPosition)); \ + } + +ADIOS2_FOREACH_PRIMITVE_STDTYPE_2ARGS(declare_type) +#undef declare_type + void BP5Writer::InitParameters() { ParseParams(m_IO, m_Parameters); m_WriteToBB = !(m_Parameters.BurstBufferPath.empty()); m_DrainBB = m_WriteToBB && m_Parameters.BurstBufferDrain; + + if (m_Parameters.NumAggregators > static_cast(m_Comm.Size())) + { + m_Parameters.NumAggregators = static_cast(m_Comm.Size()); + } + + if (m_Parameters.NumSubFiles > m_Parameters.NumAggregators) + { + m_Parameters.NumSubFiles = m_Parameters.NumAggregators; + } + + if (m_Parameters.FileSystemPageSize == 0) + { + m_Parameters.FileSystemPageSize = 4096; + } + if (m_Parameters.FileSystemPageSize > 67108864) + { + // Limiting to max 64MB page size + m_Parameters.FileSystemPageSize = 67108864; + } +} + +void BP5Writer::InitAggregator() +{ + // in BP5, aggregation is "always on", but processes may be alone, so + // m_Aggregator.m_IsActive is always true + // m_Aggregator.m_Comm.Rank() will always succeed (not abort) + // m_Aggregator.m_SubFileIndex is always set + if (m_Parameters.AggregationType == (int)AggregationType::EveryoneWrites || + m_Parameters.AggregationType == + (int)AggregationType::EveryoneWritesSerial) + { + m_Parameters.NumSubFiles = m_Parameters.NumAggregators; + m_AggregatorEveroneWrites.Init(m_Parameters.NumAggregators, + m_Parameters.NumSubFiles, m_Comm); + m_IAmDraining = m_AggregatorEveroneWrites.m_IsAggregator; + m_IAmWritingDataHeader = m_AggregatorEveroneWrites.m_IsAggregator; + m_IAmWritingData = true; + DataWritingComm = &m_AggregatorEveroneWrites.m_Comm; + m_Aggregator = static_cast( + &m_AggregatorEveroneWrites); + } + else + { + size_t numNodes = m_AggregatorTwoLevelShm.PreInit(m_Comm); + (void)numNodes; + m_AggregatorTwoLevelShm.Init(m_Parameters.NumAggregators, + m_Parameters.NumSubFiles, m_Comm); + + /*std::cout << "Rank " << m_RankMPI << " aggr? " + << m_AggregatorTwoLevelShm.m_IsAggregator << " master? " + << m_AggregatorTwoLevelShm.m_IsMasterAggregator + << " aggr size = " << m_AggregatorTwoLevelShm.m_Size + << " rank = " << m_AggregatorTwoLevelShm.m_Rank + << " subfile = " << m_AggregatorTwoLevelShm.m_SubStreamIndex + << " type = " << m_Parameters.AggregationType + + << std::endl;*/ + + m_IAmDraining = m_AggregatorTwoLevelShm.m_IsMasterAggregator; + m_IAmWritingData = m_AggregatorTwoLevelShm.m_IsAggregator; + m_IAmWritingDataHeader = m_AggregatorTwoLevelShm.m_IsMasterAggregator; + DataWritingComm = &m_AggregatorTwoLevelShm.m_AggregatorChainComm; + m_Aggregator = + static_cast(&m_AggregatorTwoLevelShm); + } } void BP5Writer::InitTransports() { - // TODO need to add support for aggregators here later if (m_IO.m_TransportsParameters.empty()) { Params defaultTransportParameters; @@ -297,28 +557,39 @@ void BP5Writer::InitTransports() m_IO.m_TransportsParameters.push_back(defaultTransportParameters); } - // only consumers will interact with transport managers - m_BBName = m_Name; if (m_WriteToBB) { m_BBName = m_Parameters.BurstBufferPath + PathSeparator + m_Name; } - - if (m_Aggregator.m_IsConsumer) + else { - // Names passed to IO AddTransport option with key "Name" - const std::vector transportsNames = - m_FileDataManager.GetFilesBaseNames(m_BBName, - m_IO.m_TransportsParameters); - - // /path/name.bp.dir/name.bp.rank - m_SubStreamNames = GetBPSubStreamNames(transportsNames); + m_BBName = m_Name; + } + /* From this point, engine writes to m_BBName, which points to either + the BB file system if BB is turned on, or to the target file system. + m_Name always points to the target file system, to which the drainer + should write if BB is turned on + */ + + // Names passed to IO AddTransport option with key "Name" + const std::vector transportsNames = + m_FileDataManager.GetFilesBaseNames(m_BBName, + m_IO.m_TransportsParameters); + + // /path/name.bp.dir/name.bp.rank + m_SubStreamNames = + GetBPSubStreamNames(transportsNames, m_Aggregator->m_SubStreamIndex); + + if (m_IAmDraining) + { + // Only (master)aggregators will run draining processes if (m_DrainBB) { const std::vector drainTransportNames = m_FileDataManager.GetFilesBaseNames( m_Name, m_IO.m_TransportsParameters); - m_DrainSubStreamNames = GetBPSubStreamNames(drainTransportNames); + m_DrainSubStreamNames = GetBPSubStreamNames( + drainTransportNames, m_Aggregator->m_SubStreamIndex); /* start up BB thread */ // m_FileDrainer.SetVerbose( // m_Parameters.BurstBufferVerbose, @@ -332,10 +603,6 @@ void BP5Writer::InitTransports() if (m_Comm.Rank() == 0) { - const std::vector transportsNames = - m_FileMetadataManager.GetFilesBaseNames( - m_Name, m_IO.m_TransportsParameters); - m_MetadataFileNames = GetBPMetadataFileNames(transportsNames); m_MetaMetadataFileNames = GetBPMetaMetadataFileNames(transportsNames); m_MetadataIndexFileNames = GetBPMetadataIndexFileNames(transportsNames); @@ -351,20 +618,27 @@ void BP5Writer::InitTransports() m_Parameters.NodeLocal); } - if (m_Aggregator.m_IsConsumer) + /* Everyone opens its data file. Each aggregation chain opens + one data file and does so in chain, not everyone at once */ + if (m_Parameters.AsyncTasks) { -#ifdef NOTDEF - if (m_Parameters.AsyncTasks) + for (size_t i = 0; i < m_IO.m_TransportsParameters.size(); ++i) { - for (size_t i = 0; i < m_IO.m_TransportsParameters.size(); ++i) - { - m_IO.m_TransportsParameters[i]["asynctasks"] = "true"; - } + m_IO.m_TransportsParameters[i]["asynctasks"] = "true"; } -#endif + } + + bool useProfiler = true; + + if (m_IAmWritingData) + { m_FileDataManager.OpenFiles(m_SubStreamNames, m_OpenMode, - m_IO.m_TransportsParameters, false); + m_IO.m_TransportsParameters, useProfiler, + *DataWritingComm); + } + if (m_IAmDraining) + { if (m_DrainBB) { for (const auto &name : m_DrainSubStreamNames) @@ -377,15 +651,16 @@ void BP5Writer::InitTransports() if (m_Comm.Rank() == 0) { m_FileMetaMetadataManager.OpenFiles(m_MetaMetadataFileNames, m_OpenMode, - m_IO.m_TransportsParameters, false); + m_IO.m_TransportsParameters, + useProfiler); m_FileMetadataManager.OpenFiles(m_MetadataFileNames, m_OpenMode, - m_IO.m_TransportsParameters, false); + m_IO.m_TransportsParameters, + useProfiler); - uint64_t WriterCount = m_Comm.Size(); m_FileMetadataIndexManager.OpenFiles( m_MetadataIndexFileNames, m_OpenMode, m_IO.m_TransportsParameters, - false); + useProfiler); if (m_DrainBB) { @@ -407,6 +682,19 @@ void BP5Writer::InitTransports() } } } + + // last process create .bpversion file with content "5" + if (m_Comm.Rank() == m_Comm.Size() - 1) + { + std::vector versionNames = + GetBPVersionFileNames(transportsNames); + auto emptyComm = helper::Comm(); + transportman::TransportMan tm(emptyComm); + tm.OpenFiles(versionNames, Mode::Write, m_IO.m_TransportsParameters, + useProfiler); + char b[1] = {'5'}; + tm.WriteFiles(b, 1); + } } /*generate the header for the metadata index file*/ @@ -522,10 +810,10 @@ void BP5Writer::MakeHeader(format::BufferSTL &b, const std::string fileType, // byte 48 columnMajor // write if data is column major in metadata and data const uint8_t columnMajor = - (helper::IsRowMajor(m_IO.m_HostLanguage) == false) ? 'y' : 'n'; + (m_IO.m_ArrayOrder == ArrayOrdering::ColumnMajor) ? 'y' : 'n'; helper::CopyToBuffer(buffer, position, &columnMajor); - // byte 45-63: unused + // byte 49-63: unused position += 15; absolutePosition = position; } @@ -536,6 +824,10 @@ void BP5Writer::InitBPBuffer() * Make headers in data buffer and metadata buffer (but do not write * them yet so that Open() can stay free of writing to disk) */ + + const uint64_t a = static_cast(m_Aggregator->m_SubStreamIndex); + std::vector Assignment = m_Comm.GatherValues(a, 0); + if (m_Comm.Rank() == 0) { format::BufferSTL b; @@ -546,58 +838,84 @@ void BP5Writer::InitBPBuffer() MakeHeader(bi, "Index Table", true); m_FileMetadataIndexManager.WriteFiles(bi.m_Buffer.data(), bi.m_Position); - std::vector Assignment(m_Comm.Size()); - for (uint64_t i = 0; i < m_Comm.Size(); i++) - { - Assignment[i] = i; // Change when we do aggregation - } // where each rank's data will end up m_FileMetadataIndexManager.WriteFiles((char *)Assignment.data(), sizeof(Assignment[0]) * Assignment.size()); } - if (m_Aggregator.m_IsConsumer) + + if (m_Aggregator->m_IsAggregator) + { + m_DataPos = 0; + } + + if (m_Comm.Rank() == 0) { - format::BufferSTL d; - MakeHeader(d, "Data", false); - m_FileDataManager.WriteFiles(d.m_Buffer.data(), d.m_Position); - m_DataPos = d.m_Position; m_WriterDataPos.resize(m_Comm.Size()); - for (auto &DataPos : m_WriterDataPos) - { - DataPos = m_DataPos; - } } } -void BP5Writer::DoFlush(const bool isFinal, const int transportIndex) +void BP5Writer::NotifyEngineAttribute(std::string name, DataType type) noexcept { - m_FileMetadataManager.FlushFiles(); - m_FileMetaMetadataManager.FlushFiles(); - m_FileDataManager.FlushFiles(); - // m_BP4Serializer.ResetBuffer(m_BP4Serializer.m_Data, false, false); - - // if (m_Parameters.CollectiveMetadata) - // { - // WriteCollectiveMetadataFile(); - // } - // if (m_BP4Serializer.m_Aggregator.m_IsActive) - // { - // AggregateWriteData(isFinal, transportIndex); - // } - // else - // { - // WriteData(isFinal, transportIndex); - // } + m_MarshaledAttributesCount = 0; } +void BP5Writer::FlushData(const bool isFinal) +{ + BufferV *DataBuf; + if (m_Parameters.BufferVType == (int)BufferVType::MallocVType) + { + DataBuf = m_BP5Serializer.ReinitStepData( + new MallocV("BP5Writer", false, m_Parameters.InitialBufferSize, + m_Parameters.GrowthFactor)); + } + else + { + DataBuf = m_BP5Serializer.ReinitStepData( + new ChunkV("BP5Writer", false /* always copy */, + m_Parameters.BufferChunkSize)); + } + + WriteData(DataBuf); + + m_ThisTimestepDataSize += DataBuf->Size(); + + if (!isFinal) + { + size_t tmp[2]; + // aggregate start pos and data size to rank 0 + tmp[0] = m_StartDataPos; + tmp[1] = DataBuf->Size(); + + std::vector RecvBuffer; + if (m_Comm.Rank() == 0) + { + RecvBuffer.resize(m_Comm.Size() * 2); + } + m_Comm.GatherArrays(tmp, 2, RecvBuffer.data(), 0); + if (m_Comm.Rank() == 0) + { + FlushPosSizeInfo.push_back(RecvBuffer); + } + } + delete DataBuf; +} + +void BP5Writer::Flush(const int transportIndex) { FlushData(false); } + void BP5Writer::DoClose(const int transportIndex) { PERFSTUBS_SCOPED_TIMER("BP5Writer::Close"); - PerformPuts(); - - DoFlush(true, transportIndex); + if ((m_WriterStep == 0) && !m_BetweenStepPairs) + { + /* never did begin step, do one now */ + BeginStep(StepMode::Update); + } + if (m_BetweenStepPairs) + { + EndStep(); + } m_FileDataManager.CloseFiles(transportIndex); // Delete files from temporary storage if draining was on @@ -612,6 +930,85 @@ void BP5Writer::DoClose(const int transportIndex) // close metadata index file m_FileMetadataIndexManager.CloseFiles(); } + + FlushProfiler(); +} + +void BP5Writer::FlushProfiler() +{ + auto transportTypes = m_FileDataManager.GetTransportsTypes(); + + // find first File type output, where we can write the profile + int fileTransportIdx = -1; + for (size_t i = 0; i < transportTypes.size(); ++i) + { + if (transportTypes[i].compare(0, 4, "File") == 0) + { + fileTransportIdx = static_cast(i); + } + } + + auto transportProfilers = m_FileDataManager.GetTransportsProfilers(); + + auto transportTypesMD = m_FileMetadataManager.GetTransportsTypes(); + auto transportProfilersMD = m_FileMetadataManager.GetTransportsProfilers(); + + transportTypes.insert(transportTypes.end(), transportTypesMD.begin(), + transportTypesMD.end()); + + transportProfilers.insert(transportProfilers.end(), + transportProfilersMD.begin(), + transportProfilersMD.end()); + + // m_Profiler.WriteOut(transportTypes, transportProfilers); + + const std::string lineJSON( + m_Profiler.GetRankProfilingJSON(transportTypes, transportProfilers) + + ",\n"); + + const std::vector profilingJSON( + m_Profiler.AggregateProfilingJSON(lineJSON)); + + if (m_RankMPI == 0) + { + // std::cout << "write profiling file!" << std::endl; + std::string profileFileName; + if (m_DrainBB) + { + // auto bpTargetNames = m_BP4Serializer.GetBPBaseNames({m_Name}); + std::vector bpTargetNames = {m_Name}; + if (fileTransportIdx > -1) + { + profileFileName = + bpTargetNames[fileTransportIdx] + "/profiling.json"; + } + else + { + profileFileName = bpTargetNames[0] + "_profiling.json"; + } + m_FileDrainer.AddOperationWrite( + profileFileName, profilingJSON.size(), profilingJSON.data()); + } + else + { + transport::FileFStream profilingJSONStream(m_Comm); + // auto bpBaseNames = m_BP4Serializer.GetBPBaseNames({m_BBName}); + std::vector bpBaseNames = {m_Name}; + if (fileTransportIdx > -1) + { + profileFileName = + bpBaseNames[fileTransportIdx] + "/profiling.json"; + } + else + { + profileFileName = bpBaseNames[0] + "_profiling.json"; + } + profilingJSONStream.Open(profileFileName, Mode::Write); + profilingJSONStream.Write(profilingJSON.data(), + profilingJSON.size()); + profilingJSONStream.Close(); + } + } } /*write the content of metadata index file*/ diff --git a/source/adios2/engine/bp5/BP5Writer.h b/source/adios2/engine/bp5/BP5Writer.h index bcab5389f3..4c257763b6 100644 --- a/source/adios2/engine/bp5/BP5Writer.h +++ b/source/adios2/engine/bp5/BP5Writer.h @@ -10,10 +10,13 @@ #define ADIOS2_ENGINE_BP5_BP5WRITER_H_ #include "adios2/common/ADIOSConfig.h" +#include "adios2/core/CoreTypes.h" #include "adios2/core/Engine.h" #include "adios2/engine/bp5/BP5Engine.h" #include "adios2/helper/adiosComm.h" +#include "adios2/helper/adiosMemory.h" // PaddingToAlignOffset #include "adios2/toolkit/aggregator/mpi/MPIChain.h" +#include "adios2/toolkit/aggregator/mpi/MPIShmChain.h" #include "adios2/toolkit/burstbuffer/FileDrainerSingleThread.h" #include "adios2/toolkit/format/bp5/BP5Serializer.h" #include "adios2/toolkit/format/buffer/BufferV.h" @@ -46,6 +49,7 @@ class BP5Writer : public BP5Engine, public core::Engine size_t CurrentStep() const final; void PerformPuts() final; void EndStep() final; + void Flush(const int transportIndex = -1) final; private: /** Single object controlling BP buffering */ @@ -62,7 +66,7 @@ class BP5Writer : public BP5Engine, public core::Engine transportman::TransportMan m_FileMetaMetadataManager; - int64_t m_WriterStep = -1; + int64_t m_WriterStep = 0; /* * Burst buffer variables */ @@ -90,14 +94,30 @@ class BP5Writer : public BP5Engine, public core::Engine std::vector m_DrainMetadataIndexFileNames; std::vector m_ActiveFlagFileNames; + bool m_BetweenStepPairs = false; + void Init() final; /** Parses parameters from IO SetParameters */ void InitParameters() final; + /** Set up the aggregator */ + void InitAggregator(); /** Parses transports and parameters from IO AddTransport */ void InitTransports() final; /** Allocates memory and starts a PG group */ void InitBPBuffer(); + void NotifyEngineAttribute(std::string name, DataType type) noexcept; + +#define declare_type(T) \ + void DoPut(Variable &variable, typename Variable::Span &span, \ + const bool initialize, const T &value) final; + + ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type) +#undef declare_type + + template + void PutCommonSpan(Variable &variable, typename Variable::Span &span, + const bool initialize, const T &value); #define declare_type(T) \ void DoPutSync(Variable &, const T *) final; \ @@ -109,7 +129,14 @@ class BP5Writer : public BP5Engine, public core::Engine template void PutCommon(Variable &variable, const T *data, bool sync); - void DoFlush(const bool isFinal = false, const int transportIndex = -1); +#define declare_type(T, L) \ + T *DoBufferData_##L(const int bufferIdx, const size_t payloadPosition, \ + const size_t bufferID = 0) noexcept final; + + ADIOS2_FOREACH_PRIMITVE_STDTYPE_2ARGS(declare_type) +#undef declare_type + + void FlushData(const bool isFinal = false); void DoClose(const int transportIndex = -1) final; @@ -120,14 +147,16 @@ class BP5Writer : public BP5Engine, public core::Engine void WriteMetaMetadata( const std::vector MetaMetaBlocks); - void WriteMetadataFileIndex(uint64_t MetaDataPos, uint64_t MetaDataSize, - std::vector DataSizes); + void WriteMetadataFileIndex(uint64_t MetaDataPos, uint64_t MetaDataSize); - uint64_t - WriteMetadata(const std::vector MetaDataBlocks, - const std::vector AttributeBlocks); + uint64_t WriteMetadata(const std::vector &MetaDataBlocks, + const std::vector &AttributeBlocks); + /** Write Data to disk, in an aggregator chain */ void WriteData(format::BufferV *Data); + void WriteData_EveryoneWrites(format::BufferV *Data, + bool SerializedWriters); + void WriteData_TwoLevelShm(format::BufferV *Data); void PopulateMetadataIndexFileContent( format::BufferSTL &buffer, const uint64_t currentStep, @@ -141,35 +170,60 @@ class BP5Writer : public BP5Engine, public core::Engine void MarshalAttributes(); - /** - * N-to-N data buffers writes, including metadata file - * @param transportIndex - */ - // void WriteData(const bool isFinal, const int transportIndex = -1); - - /** - * N-to-M (aggregation) data buffers writes, including metadata file - * @param transportIndex - */ - void AggregateWriteData(const bool isFinal, const int transportIndex = -1); - - template - T *BufferDataCommon(const size_t payloadOffset, - const size_t bufferID) noexcept; + /* Two-level-shm aggregator functions */ + void WriteMyOwnData(format::BufferV *Data); + void SendDataToAggregator(format::BufferV *Data); + void WriteOthersData(const size_t TotalSize); template void PerformPutCommon(Variable &variable); + void FlushProfiler(); + /** manages all communication tasks in aggregation */ - aggregator::MPIChain m_Aggregator; + aggregator::MPIAggregator *m_Aggregator; // points to one of these below + aggregator::MPIShmChain m_AggregatorTwoLevelShm; + aggregator::MPIChain m_AggregatorEveroneWrites; + bool m_IAmDraining = false; + bool m_IAmWritingData = false; + helper::Comm *DataWritingComm; // processes that write the same data file + bool m_IAmWritingDataHeader = false; + + adios2::profiling::JSONProfiler m_Profiler; private: - uint64_t m_MetaDataPos = 0; // updated during WriteMetaData - uint64_t m_DataPos = 0; // updated during WriteData + // updated during WriteMetaData + uint64_t m_MetaDataPos = 0; + + /** On every process, at the end of writing, this holds the offset + * where they started writing (needed for global metadata) + */ + uint64_t m_StartDataPos = 0; + /** On aggregators, at the end of writing, this holds the starting offset + * to the next step's writing; otherwise used as temporary offset variable + * during writing on every process and points to the end of the process' + * data block in the file (not used for anything) + */ + uint64_t m_DataPos = 0; + + /* + * Total data written this timestep + */ + uint64_t m_ThisTimestepDataSize = 0; + + /** rank 0 collects m_StartDataPos in this vector for writing it + * to the index file + */ + std::vector m_WriterDataPos; + uint32_t m_MarshaledAttributesCount = 0; // updated during EndStep/MarshalAttributes - std::vector m_WriterDataPos; + // where each writer rank writes its data, init in InitBPBuffer; + std::vector m_Assignment; + + std::vector> FlushPosSizeInfo; + void MakeHeader(format::BufferSTL &b, const std::string fileType, const bool isActive); }; diff --git a/source/adios2/engine/bp5/BP5Writer.tcc b/source/adios2/engine/bp5/BP5Writer.tcc index 75785bfaa5..af19b388d8 100644 --- a/source/adios2/engine/bp5/BP5Writer.tcc +++ b/source/adios2/engine/bp5/BP5Writer.tcc @@ -4,13 +4,12 @@ * * BP5Writer.tcc implementation of template functions with known type * - * Created on: Aug 1, 2018 - * Author: Lipeng Wan wanl@ornl.gov */ #ifndef ADIOS2_ENGINE_BP5_BP5WRITER_TCC_ #define ADIOS2_ENGINE_BP5_BP5WRITER_TCC_ #include "BP5Writer.h" +#include "adios2/helper/adiosMath.h" namespace adios2 { @@ -22,13 +21,92 @@ namespace engine template void BP5Writer::PutCommon(Variable &variable, const T *values, bool sync) { + if (!m_BetweenStepPairs) + { + BeginStep(StepMode::Update); + } variable.SetData(values); + size_t *Shape = NULL; + size_t *Start = NULL; + size_t *Count = NULL; + size_t DimCount = variable.m_Count.size(); + + if (variable.m_ShapeID == ShapeID::GlobalArray) + { + Shape = variable.m_Shape.data(); + Count = variable.m_Count.data(); + Start = variable.m_Start.data(); + } + else if (variable.m_ShapeID == ShapeID::LocalArray) + { + Count = variable.m_Count.data(); + } + + if (!sync) + { + /* If arrays is small, force copying to internal buffer to aggregate + * small writes */ + size_t n = helper::GetTotalSize(variable.m_Count) * sizeof(T); + if (n < m_Parameters.MinDeferredSize) + { + sync = true; + } + } + + if (!variable.m_MemoryCount.empty()) + { + // get a temporary span then fill with memselection now + format::BufferV::BufferPos bp5span(0, 0, 0); + m_BP5Serializer.Marshal((void *)&variable, variable.m_Name.c_str(), + variable.m_Type, variable.m_ElementSize, + DimCount, Shape, Count, Start, nullptr, false, + &bp5span); + T *ptr = reinterpret_cast( + m_BP5Serializer.GetPtr(bp5span.bufferIdx, bp5span.posInBuffer)); + + const bool sourceRowMajor = helper::IsRowMajor(m_IO.m_HostLanguage); + + helper::CopyMemoryBlock( + ptr, variable.m_Start, variable.m_Count, sourceRowMajor, values, + variable.m_Start, variable.m_Count, sourceRowMajor, false, Dims(), + Dims(), variable.m_MemoryStart, variable.m_MemoryCount); + } + else + { + if (std::is_same::value) + { + std::string &source = *(std::string *)values; + void *p = &(source[0]); + m_BP5Serializer.Marshal((void *)&variable, variable.m_Name.c_str(), + variable.m_Type, variable.m_ElementSize, + DimCount, Shape, Count, Start, &p, sync, + nullptr); + } + else + m_BP5Serializer.Marshal((void *)&variable, variable.m_Name.c_str(), + variable.m_Type, variable.m_ElementSize, + DimCount, Shape, Count, Start, values, sync, + nullptr); + } +} + +template +void BP5Writer::PutCommonSpan(Variable &variable, + typename Variable::Span &span, + const bool initialize, const T &value) +{ + format::BufferV::BufferPos bp5span(0, 0, 0); + size_t *Shape = NULL; size_t *Start = NULL; size_t *Count = NULL; size_t DimCount = 0; + if (!m_BetweenStepPairs) + { + BeginStep(StepMode::Update); + } if (variable.m_ShapeID == ShapeID::GlobalArray) { DimCount = variable.m_Shape.size(); @@ -41,9 +119,42 @@ void BP5Writer::PutCommon(Variable &variable, const T *values, bool sync) DimCount = variable.m_Count.size(); Count = variable.m_Count.data(); } - m_BP5Serializer.Marshal((void *)&variable, variable.m_Name.c_str(), - variable.m_Type, variable.m_ElementSize, DimCount, - Shape, Count, Start, values, sync); + + if (std::is_same::value) + { + m_BP5Serializer.Marshal((void *)&variable, variable.m_Name.c_str(), + variable.m_Type, variable.m_ElementSize, + DimCount, Shape, Count, Start, nullptr, false, + &bp5span); + } + else + m_BP5Serializer.Marshal((void *)&variable, variable.m_Name.c_str(), + variable.m_Type, variable.m_ElementSize, + DimCount, Shape, Count, Start, nullptr, false, + &bp5span); + + span.m_PayloadPosition = bp5span.posInBuffer; + span.m_BufferIdx = bp5span.bufferIdx; + span.m_Value = value; + + /* initialize buffer if needed */ + if (initialize) + { + const size_t ElemCount = m_BP5Serializer.CalcSize(DimCount, Count); + T *itBegin = reinterpret_cast( + m_BP5Serializer.GetPtr(span.m_BufferIdx, span.m_PayloadPosition)); + + // TODO from BP4: does std::fill_n have a bug in gcc or due to + // optimizations this is impossible due to memory alignment? This seg + // faults in Release mode only . Even RelWithDebInfo works, replacing + // with explicit loop below using access operator [] + // std::fill_n(itBegin, blockSize, span->m_Value); + + for (size_t i = 0; i < ElemCount; ++i) + { + itBegin[i] = value; + } + } } } // end namespace engine diff --git a/source/adios2/engine/bp5/BP5Writer_TwoLevelShm.cpp b/source/adios2/engine/bp5/BP5Writer_TwoLevelShm.cpp new file mode 100644 index 0000000000..5518861754 --- /dev/null +++ b/source/adios2/engine/bp5/BP5Writer_TwoLevelShm.cpp @@ -0,0 +1,313 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * BP5Writer.cpp + * + */ + +#include "BP5Writer.h" + +#include "adios2/common/ADIOSMacros.h" +#include "adios2/core/IO.h" +#include "adios2/helper/adiosFunctions.h" //CheckIndexRange, PaddingToAlignOffset +#include "adios2/toolkit/format/buffer/chunk/ChunkV.h" +#include "adios2/toolkit/format/buffer/malloc/MallocV.h" +#include "adios2/toolkit/transport/file/FileFStream.h" +#include + +#include +#include +#include + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +using namespace adios2::format; + +void BP5Writer::WriteData_TwoLevelShm(format::BufferV *Data) +{ + aggregator::MPIShmChain *a = + dynamic_cast(m_Aggregator); + + // new step writing starts at offset m_DataPos on master aggregator + // other aggregators to the same file will need to wait for the position + // to arrive from the rank below + + // align to PAGE_SIZE (only valid on master aggregator at this point) + m_DataPos += helper::PaddingToAlignOffset(m_DataPos, + m_Parameters.FileSystemPageSize); + + /* + // Each aggregator needs to know the total size they write + // including alignment to page size + // This calculation is valid on aggregators only + std::vector mySizes = a->m_Comm.GatherValues(Data->Size()); + uint64_t myTotalSize = 0; + uint64_t pos = m_DataPos; + for (auto s : mySizes) + { + uint64_t alignment = + helper::PaddingToAlignOffset(pos, m_Parameters.FileSystemPageSize); + myTotalSize += alignment + s; + pos += alignment + s; + } + */ + + // Each aggregator needs to know the total size they write + // This calculation is valid on aggregators only + std::vector mySizes = a->m_Comm.GatherValues(Data->Size()); + uint64_t myTotalSize = 0; + uint64_t maxSize = 0; + for (auto s : mySizes) + { + myTotalSize += s; + if (s > maxSize) + { + maxSize = s; + } + } + + if (a->m_Comm.Size() > 1) + { + a->CreateShm(static_cast(maxSize), m_Parameters.MaxShmSize); + } + + if (a->m_IsAggregator) + { + // In each aggregator chain, send from master down the line + // these total sizes, so every aggregator knows where to start + if (a->m_AggregatorChainComm.Rank() > 0) + { + a->m_AggregatorChainComm.Recv( + &m_DataPos, 1, a->m_AggregatorChainComm.Rank() - 1, 0, + "AggregatorChain token in BP5Writer::WriteData_TwoLevelShm"); + // align to PAGE_SIZE + m_DataPos += helper::PaddingToAlignOffset( + m_DataPos, m_Parameters.FileSystemPageSize); + } + m_StartDataPos = m_DataPos; // metadata needs this info + if (a->m_AggregatorChainComm.Rank() < + a->m_AggregatorChainComm.Size() - 1) + { + uint64_t nextWriterPos = m_DataPos + myTotalSize; + a->m_AggregatorChainComm.Isend( + &nextWriterPos, 1, a->m_AggregatorChainComm.Rank() + 1, 0, + "Chain token in BP5Writer::WriteData"); + } + else if (a->m_AggregatorChainComm.Size() > 1) + { + // send back final position from last aggregator in file to master + // aggregator + uint64_t nextWriterPos = m_DataPos + myTotalSize; + a->m_AggregatorChainComm.Isend( + &nextWriterPos, 1, 0, 0, "Chain token in BP5Writer::WriteData"); + } + + /*std::cout << "Rank " << m_Comm.Rank() + << " aggregator start writing step " << m_WriterStep + << " to subfile " << a->m_SubStreamIndex << " at pos " + << m_DataPos << " totalsize " << myTotalSize << std::endl;*/ + + // Send token to first non-aggregator to start filling shm + // Also informs next process its starting offset (for correct metadata) + if (a->m_Comm.Size() > 1) + { + uint64_t nextWriterPos = m_DataPos + Data->Size(); + a->m_Comm.Isend(&nextWriterPos, 1, a->m_Comm.Rank() + 1, 0, + "Shm token in BP5Writer::WriteData_TwoLevelShm"); + } + + WriteMyOwnData(Data); + + /* Write from shm until every non-aggr sent all data */ + if (a->m_Comm.Size() > 1) + { + WriteOthersData(myTotalSize - Data->Size()); + } + + // Master aggregator needs to know where the last writing ended by the + // last aggregator in the chain, so that it can start from the correct + // position at the next output step + if (a->m_AggregatorChainComm.Size() > 1 && + !a->m_AggregatorChainComm.Rank()) + { + a->m_AggregatorChainComm.Recv( + &m_DataPos, 1, a->m_AggregatorChainComm.Size() - 1, 0, + "Chain token in BP5Writer::WriteData"); + } + } + else + { + // non-aggregators fill shared buffer in marching order + // they also receive their starting offset this way + a->m_Comm.Recv(&m_StartDataPos, 1, a->m_Comm.Rank() - 1, 0, + "Shm token in BP5Writer::WriteData_TwoLevelShm"); + + /*std::cout << "Rank " << m_Comm.Rank() + << " non-aggregator recv token to fill shm = " + << m_StartDataPos << std::endl;*/ + + SendDataToAggregator(Data); + + if (a->m_Comm.Rank() < a->m_Comm.Size() - 1) + { + uint64_t nextWriterPos = m_StartDataPos + Data->Size(); + a->m_Comm.Isend(&nextWriterPos, 1, a->m_Comm.Rank() + 1, 0, + "Shm token in BP5Writer::WriteData_TwoLevelShm"); + } + } + + if (a->m_Comm.Size() > 1) + { + a->DestroyShm(); + } +} + +void BP5Writer::WriteMyOwnData(format::BufferV *Data) +{ + std::vector DataVec = Data->DataVec(); + m_StartDataPos = m_DataPos; + m_FileDataManager.WriteFileAt(DataVec.data(), DataVec.size(), + m_StartDataPos); + m_DataPos += Data->Size(); +} + +/*std::string DoubleBufferToString(const double *b, int n) +{ + std::ostringstream out; + out.precision(1); + out << std::fixed << "["; + char s[32]; + + for (int i = 0; i < n; ++i) + { + snprintf(s, sizeof(s), "%g", b[i]); + out << s; + if (i < n - 1) + { + out << ", "; + } + } + out << "]"; + return out.str(); +}*/ + +void BP5Writer::SendDataToAggregator(format::BufferV *Data) +{ + /* Only one process is running this function at once + See shmFillerToken in the caller function + + In a loop, copy the local data into the shared memory, alternating + between the two segments. + */ + + aggregator::MPIShmChain *a = + dynamic_cast(m_Aggregator); + + std::vector DataVec = Data->DataVec(); + size_t nBlocks = DataVec.size(); + + size_t sent = 0; + size_t block = 0; + size_t temp_offset = 0; + while (block < nBlocks) + { + // potentially blocking call waiting on Aggregator + aggregator::MPIShmChain::ShmDataBuffer *b = a->LockProducerBuffer(); + // b->max_size: how much we can copy + // b->actual_size: how much we actually copy + b->actual_size = 0; + while (true) + { + /* Copy n bytes from the current block, current offset to shm + making sure to use up to shm_size bytes + */ + size_t n = DataVec[block].iov_len - temp_offset; + if (n > (b->max_size - b->actual_size)) + { + n = b->max_size - b->actual_size; + } + std::memcpy(&b->buf[b->actual_size], + (const char *)DataVec[block].iov_base + temp_offset, n); + b->actual_size += n; + + /* Have we processed the entire block or staying with it? */ + if (n + temp_offset < DataVec[block].iov_len) + { + temp_offset += n; + } + else + { + temp_offset = 0; + ++block; + } + + /* Have we reached the max allowed shm size ?*/ + if (b->actual_size >= b->max_size) + { + break; + } + if (block >= nBlocks) + { + break; + } + } + sent += b->actual_size; + + /*if (m_RankMPI >= 42) + { + std::cout << "Rank " << m_Comm.Rank() + << " filled shm, data_size = " << b->actual_size + << " block = " << block + << " temp offset = " << temp_offset << " sent = " << sent + << " buf = " << static_cast(b->buf) << " = " + << DoubleBufferToString((double *)b->buf, + b->actual_size / sizeof(double)) + << std::endl; + }*/ + + a->UnlockProducerBuffer(); + } +} +void BP5Writer::WriteOthersData(size_t TotalSize) +{ + /* Only an Aggregator calls this function */ + aggregator::MPIShmChain *a = + dynamic_cast(m_Aggregator); + + size_t wrote = 0; + while (wrote < TotalSize) + { + // potentially blocking call waiting on some non-aggr process + aggregator::MPIShmChain::ShmDataBuffer *b = a->LockConsumerBuffer(); + + /*std::cout << "Rank " << m_Comm.Rank() + << " write from shm, data_size = " << b->actual_size + << " total so far = " << wrote + << " buf = " << static_cast(b->buf) << " = " + << DoubleBufferToString((double *)b->buf, + b->actual_size / sizeof(double)) + << std::endl;*/ + /*<< " buf = " << static_cast(b->buf) << " = [" + << (int)b->buf[0] << (int)b->buf[1] << "..." + << (int)b->buf[b->actual_size - 2] + << (int)b->buf[b->actual_size - 1] << "]" << std::endl;*/ + + // b->actual_size: how much we need to write + m_FileDataManager.WriteFiles(b->buf, b->actual_size); + + wrote += b->actual_size; + + a->UnlockConsumerBuffer(); + } + m_DataPos += TotalSize; +} + +} // end namespace engine +} // end namespace core +} // end namespace adios2 diff --git a/source/adios2/engine/dataman/DataManMonitor.cpp b/source/adios2/engine/dataman/DataManMonitor.cpp index 434c327831..cbcc178bd8 100644 --- a/source/adios2/engine/dataman/DataManMonitor.cpp +++ b/source/adios2/engine/dataman/DataManMonitor.cpp @@ -12,7 +12,7 @@ #include #include #include -#include +#include namespace adios2 { @@ -230,7 +230,8 @@ void DataManMonitor::OutputCsv(const std::string &filename) } file << floor(log2(m_TotalRate)) << ", "; file << floor(log2(m_AccumulatedLatency / - static_cast(m_CurrentStep + 1))) + static_cast(m_CurrentStep + 1))) - + 8 << ", "; if (m_RequiredAccuracy == 0) { @@ -245,7 +246,7 @@ void DataManMonitor::OutputCsv(const std::string &filename) file << round(log10(m_RequiredAccuracy)) + 6 << ", "; } file << ceil(log2(m_DropRate * 100 + 1)) << ", "; - file << floor(log(m_StepBytes) / log(5)) - 4 << ", "; + file << floor(log2(m_StepBytes)) - 10 << ", "; file << ceil(log2(m_CombiningSteps)) << ", "; if (m_CompressionAccuracy == 0) { diff --git a/source/adios2/engine/dataman/DataManMonitor.h b/source/adios2/engine/dataman/DataManMonitor.h index e8d732e8a6..9a95cf496d 100644 --- a/source/adios2/engine/dataman/DataManMonitor.h +++ b/source/adios2/engine/dataman/DataManMonitor.h @@ -76,7 +76,6 @@ class DataManMonitor bool m_WriterThreading = false; bool m_Verbose = true; - bool m_JsonOutput = true; }; } // end namespace engine diff --git a/source/adios2/engine/dataman/DataManReader.cpp b/source/adios2/engine/dataman/DataManReader.cpp index e205c4a72a..a804d1cdc0 100644 --- a/source/adios2/engine/dataman/DataManReader.cpp +++ b/source/adios2/engine/dataman/DataManReader.cpp @@ -22,7 +22,7 @@ DataManReader::DataManReader(IO &io, const std::string &name, const Mode openMode, helper::Comm comm) : Engine("DataManReader", io, name, openMode, std::move(comm)), m_FinalStep(std::numeric_limits::max()), - m_Serializer(m_Comm, helper::IsRowMajor(io.m_HostLanguage)), + m_Serializer(m_Comm, (io.m_ArrayOrder == ArrayOrdering::RowMajor)), m_RequesterThreadActive(true), m_SubscriberThreadActive(true) { m_MpiRank = m_Comm.Rank(); @@ -169,6 +169,8 @@ StepStatus DataManReader::BeginStep(StepMode stepMode, m_Serializer.GetAttributes(m_IO); + m_IO.RemoveAllVariables(); + for (const auto &i : *m_CurrentStepMetadata) { if (i.step == static_cast(m_CurrentStep)) diff --git a/source/adios2/engine/dataman/DataManReader.tcc b/source/adios2/engine/dataman/DataManReader.tcc index 0e80df21df..cf176209da 100644 --- a/source/adios2/engine/dataman/DataManReader.tcc +++ b/source/adios2/engine/dataman/DataManReader.tcc @@ -31,7 +31,7 @@ void DataManReader::GetSyncCommon(Variable &variable, T *data) template void DataManReader::GetDeferredCommon(Variable &variable, T *data) { - if (helper::IsRowMajor(m_IO.m_HostLanguage)) + if (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) { while (true) { diff --git a/source/adios2/engine/dataman/DataManWriter.cpp b/source/adios2/engine/dataman/DataManWriter.cpp index e270311594..587f1b9c46 100644 --- a/source/adios2/engine/dataman/DataManWriter.cpp +++ b/source/adios2/engine/dataman/DataManWriter.cpp @@ -20,7 +20,7 @@ namespace engine DataManWriter::DataManWriter(IO &io, const std::string &name, const Mode openMode, helper::Comm comm) : Engine("DataManWriter", io, name, openMode, std::move(comm)), m_SentSteps(0), - m_Serializer(m_Comm, helper::IsRowMajor(io.m_HostLanguage)), + m_Serializer(m_Comm, (io.m_ArrayOrder == ArrayOrdering::RowMajor)), m_ReplyThreadActive(true), m_PublishThreadActive(true) { @@ -211,13 +211,20 @@ void DataManWriter::DoClose(const int transportIndex) m_SerializerBufferSize = buffer->size(); } - if (m_Threading || m_TransportMode == "reliable") + if (m_TransportMode == "reliable") { PushBufferQueue(buffer); } - else + else if (m_TransportMode == "fast") { - m_Publisher.Send(buffer); + if (m_Threading) + { + PushBufferQueue(buffer); + } + else + { + m_Publisher.Send(buffer); + } } } @@ -227,16 +234,33 @@ void DataManWriter::DoClose(const int transportIndex) auto cvp = std::make_shared>(s.size()); std::memcpy(cvp->data(), s.c_str(), s.size()); - if (m_Threading || m_TransportMode == "reliable") + if (m_TransportMode == "reliable") { PushBufferQueue(cvp); } - else + else if (m_TransportMode == "fast") { - m_Publisher.Send(cvp); + if (m_Threading) + { + while (!IsBufferQueueEmpty()) + { + } + for (int i = 0; i < 3; ++i) + { + PushBufferQueue(cvp); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + } + else + { + for (int i = 0; i < 3; ++i) + { + m_Publisher.Send(cvp); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + } } - m_PublishThreadActive = false; if (m_ReplyThreadActive) { while (m_SentSteps < static_cast(m_CurrentStep + 2)) @@ -244,12 +268,12 @@ void DataManWriter::DoClose(const int transportIndex) } m_ReplyThreadActive = false; } - if (m_ReplyThread.joinable()) { m_ReplyThread.join(); } + m_PublishThreadActive = false; if (m_PublishThread.joinable()) { m_PublishThread.join(); @@ -263,6 +287,12 @@ void DataManWriter::DoClose(const int transportIndex) } } +bool DataManWriter::IsBufferQueueEmpty() +{ + std::lock_guard l(m_BufferQueueMutex); + return m_BufferQueue.empty(); +} + void DataManWriter::PushBufferQueue(std::shared_ptr> buffer) { std::lock_guard l(m_BufferQueueMutex); diff --git a/source/adios2/engine/dataman/DataManWriter.h b/source/adios2/engine/dataman/DataManWriter.h index 53cd997dc0..5638a81806 100644 --- a/source/adios2/engine/dataman/DataManWriter.h +++ b/source/adios2/engine/dataman/DataManWriter.h @@ -77,6 +77,7 @@ class DataManWriter : public Engine void PushBufferQueue(std::shared_ptr> buffer); std::shared_ptr> PopBufferQueue(); + bool IsBufferQueueEmpty(); void Handshake(); void ReplyThread(); diff --git a/source/adios2/engine/dataman/DataManWriter.tcc b/source/adios2/engine/dataman/DataManWriter.tcc index ea1f4fc846..02c1e39aea 100644 --- a/source/adios2/engine/dataman/DataManWriter.tcc +++ b/source/adios2/engine/dataman/DataManWriter.tcc @@ -32,7 +32,7 @@ template void DataManWriter::PutDeferredCommon(Variable &variable, const T *values) { variable.SetData(values); - if (helper::IsRowMajor(m_IO.m_HostLanguage)) + if (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) { m_Serializer.PutData(variable, m_Name, CurrentStep(), m_MpiRank, ""); } diff --git a/source/adios2/engine/dataspaces/DataSpacesReader.cpp b/source/adios2/engine/dataspaces/DataSpacesReader.cpp index f2c066f10b..c95b2c0aaa 100644 --- a/source/adios2/engine/dataspaces/DataSpacesReader.cpp +++ b/source/adios2/engine/dataspaces/DataSpacesReader.cpp @@ -179,7 +179,7 @@ StepStatus DataSpacesReader::BeginStep(StepMode mode, const float timeout_sec) shape.resize(var_dim_size); if (var_dim_size > 0) { - bool isOrderC = helper::IsRowMajor(m_IO.m_HostLanguage); + bool isOrderC = io.m_ArrayOrder == RowMajor; for (int i = 0; i < var_dim_size; i++) { if (isOrderC) diff --git a/source/adios2/engine/dataspaces/DataSpacesReader.tcc b/source/adios2/engine/dataspaces/DataSpacesReader.tcc index 6c589dde2a..5621d81fdc 100644 --- a/source/adios2/engine/dataspaces/DataSpacesReader.tcc +++ b/source/adios2/engine/dataspaces/DataSpacesReader.tcc @@ -60,7 +60,7 @@ void DataSpacesReader::ReadDsData(Variable &variable, T *data, int version) { uint64_t lb_in[MAX_DS_NDIM], ub_in[MAX_DS_NDIM], gdims_in[MAX_DS_NDIM]; int ndims = std::max(variable.m_Shape.size(), variable.m_Count.size()); - bool isOrderC = helper::IsRowMajor(m_IO.m_HostLanguage); + bool isOrderC = io.m_ArrayOrder == RowMajor; /* Order of dimensions: in DataSpaces: fast --> slow --> slowest For example: Fortran: i,j,k --> i, j, k = lb[0], lb[1], lb[2] diff --git a/source/adios2/engine/dataspaces/DataSpacesWriter.tcc b/source/adios2/engine/dataspaces/DataSpacesWriter.tcc index b705122a93..e02da5d826 100644 --- a/source/adios2/engine/dataspaces/DataSpacesWriter.tcc +++ b/source/adios2/engine/dataspaces/DataSpacesWriter.tcc @@ -42,7 +42,7 @@ void DataSpacesWriter::DoPutSyncCommon(Variable &variable, const T *values) unsigned int version; version = m_CurrentStep; int ndims = std::max(variable.m_Shape.size(), variable.m_Count.size()); - bool isOrderC = helper::IsRowMajor(m_IO.m_HostLanguage); + bool isOrderC = io.m_ArrayOrder == RowMajor; /* Order of dimensions: in DataSpaces: fast --> slow --> slowest For example: Fortran: i,j,k --> i, j, k = lb[0], lb[1], lb[2] diff --git a/source/adios2/engine/hdf5/HDF5ReaderP.cpp b/source/adios2/engine/hdf5/HDF5ReaderP.cpp index 25bafada34..a953e9f5b4 100644 --- a/source/adios2/engine/hdf5/HDF5ReaderP.cpp +++ b/source/adios2/engine/hdf5/HDF5ReaderP.cpp @@ -159,7 +159,7 @@ size_t HDF5ReaderP::ReadDataset(hid_t dataSetId, hid_t h5Type, else { std::vector start(ndims), count(ndims), stride(ndims); - bool isOrderC = helper::IsRowMajor(m_IO.m_HostLanguage); + bool isOrderC = (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor); for (size_t i = 0u; i < ndims; i++) { @@ -202,8 +202,8 @@ size_t HDF5ReaderP::ReadDataset(hid_t dataSetId, hid_t h5Type, size_t typesize = H5Tget_size(h5Type); char *val = (char *)(calloc(typesize, sizeof(char))); - hid_t ret2 = H5Dread(dataSetId, h5Type, memDataSpace, fileSpace, - H5P_DEFAULT, val); + H5Dread(dataSetId, h5Type, memDataSpace, fileSpace, H5P_DEFAULT, + val); ((std::string *)values)->assign(val, typesize); free(val); diff --git a/source/adios2/engine/insitumpi/InSituMPIReader.cpp b/source/adios2/engine/insitumpi/InSituMPIReader.cpp index 690500579c..8fe2b3f0cf 100644 --- a/source/adios2/engine/insitumpi/InSituMPIReader.cpp +++ b/source/adios2/engine/insitumpi/InSituMPIReader.cpp @@ -343,8 +343,7 @@ void InSituMPIReader::PerformGets() // bool writer_IsRowMajor = m_BP3Deserializer.m_IsRowMajor; // recalculate seek offsets to payload offset 0 (beginning of blocks) int nRequests = insitumpi::FixSeeksToZeroOffset( - m_ReadScheduleMap, helper::IsRowMajor(m_IO.m_HostLanguage)); - + m_ReadScheduleMap, (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor)); if (m_CurrentStep == 0 || !m_ReaderSelectionsLocked) { // Send schedule to writers diff --git a/source/adios2/engine/insitumpi/InSituMPIWriter.cpp b/source/adios2/engine/insitumpi/InSituMPIWriter.cpp index 6d0f08844b..56459e0c23 100644 --- a/source/adios2/engine/insitumpi/InSituMPIWriter.cpp +++ b/source/adios2/engine/insitumpi/InSituMPIWriter.cpp @@ -38,6 +38,9 @@ InSituMPIWriter::InSituMPIWriter(IO &io, const std::string &name, m_EndMessage = " in call to InSituMPIWriter " + m_Name + " Open\n"; Init(); m_BP3Serializer.Init(m_IO.m_Parameters, "in call to InSituMPI::Open write"); + m_BP3Serializer.ResizeBuffer( + m_BP3Serializer.m_Parameters.InitialBufferSize, + "in call to BP3::Open for writing by InSituMPI engine"); m_RankAllPeers = insitumpi::FindPeers(CommAsMPI(m_Comm), m_Name, true, m_CommWorld); diff --git a/source/adios2/engine/mhs/MhsReader.cpp b/source/adios2/engine/mhs/MhsReader.cpp new file mode 100644 index 0000000000..fc4030b9ca --- /dev/null +++ b/source/adios2/engine/mhs/MhsReader.cpp @@ -0,0 +1,110 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * MhsReader.cpp + * + * Created on: Aug 04, 2021 + * Author: Jason Wang jason.ruonan.wang@gmail.com + */ + +#include "MhsReader.tcc" +#include "adios2/helper/adiosFunctions.h" + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +MhsReader::MhsReader(IO &io, const std::string &name, const Mode mode, + helper::Comm comm) +: Engine("MhsReader", io, name, mode, std::move(comm)) +{ + helper::GetParameter(io.m_Parameters, "Tiers", m_Tiers); + Params params = {{"Tiers", std::to_string(m_Tiers)}}; + m_SiriusCompressor = std::make_shared(params); + io.SetEngine(""); + m_SubIOs.emplace_back(&io); + m_SubEngines.emplace_back(&io.Open(m_Name + ".tier0", adios2::Mode::Read)); + + for (int i = 1; i < m_Tiers; ++i) + { + m_SubIOs.emplace_back( + &io.m_ADIOS.DeclareIO("SubIO" + std::to_string(i))); + m_SubEngines.emplace_back(&m_SubIOs.back()->Open( + m_Name + ".tier" + std::to_string(i), adios2::Mode::Read)); + } +} + +MhsReader::~MhsReader() +{ + for (int i = 1; i < m_Tiers; ++i) + { + m_IO.m_ADIOS.RemoveIO("SubIO" + std::to_string(i)); + } +} + +StepStatus MhsReader::BeginStep(const StepMode mode, const float timeoutSeconds) +{ + bool endOfStream = false; + for (auto &e : m_SubEngines) + { + auto status = e->BeginStep(mode, timeoutSeconds); + if (status == StepStatus::EndOfStream) + { + endOfStream = true; + } + } + if (endOfStream) + { + return StepStatus::EndOfStream; + } + return StepStatus::OK; +} + +size_t MhsReader::CurrentStep() const { return m_SubEngines[0]->CurrentStep(); } + +void MhsReader::PerformGets() +{ + for (auto &e : m_SubEngines) + { + e->PerformGets(); + } +} + +void MhsReader::EndStep() +{ + for (auto &e : m_SubEngines) + { + e->EndStep(); + } +} + +// PRIVATE + +#define declare_type(T) \ + void MhsReader::DoGetSync(Variable &variable, T *data) \ + { \ + GetSyncCommon(variable, data); \ + } \ + void MhsReader::DoGetDeferred(Variable &variable, T *data) \ + { \ + GetDeferredCommon(variable, data); \ + } + +ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + +void MhsReader::DoClose(const int transportIndex) +{ + for (auto &e : m_SubEngines) + { + e->Close(); + } +} + +} // end namespace engine +} // end namespace core +} // end namespace adios2 diff --git a/source/adios2/engine/mhs/MhsReader.h b/source/adios2/engine/mhs/MhsReader.h new file mode 100644 index 0000000000..39244ab3b9 --- /dev/null +++ b/source/adios2/engine/mhs/MhsReader.h @@ -0,0 +1,62 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * MhsReader.h + * + * Created on: Aug 04, 2021 + * Author: Jason Wang jason.ruonan.wang@gmail.com + */ + +#ifndef ADIOS2_ENGINE_MHSREADER_H_ +#define ADIOS2_ENGINE_MHSREADER_H_ + +#include "adios2/core/Engine.h" +#include "adios2/operator/compress/CompressSirius.h" + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +class MhsReader : public Engine +{ +public: + MhsReader(IO &adios, const std::string &name, const Mode mode, + helper::Comm comm); + virtual ~MhsReader(); + + StepStatus BeginStep(StepMode mode = StepMode::Read, + const float timeoutSeconds = -1.0) final; + size_t CurrentStep() const final; + void PerformGets() final; + void EndStep() final; + +private: + std::vector m_SubIOs; + std::vector m_SubEngines; + std::shared_ptr m_SiriusCompressor; + int m_Tiers; + +#define declare_type(T) \ + void DoGetSync(Variable &, T *) final; \ + void DoGetDeferred(Variable &, T *) final; + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + + void DoClose(const int transportIndex = -1); + + template + void GetSyncCommon(Variable &variable, T *data); + + template + void GetDeferredCommon(Variable &variable, T *data); +}; + +} // end namespace engine +} // end namespace core +} // end namespace adios2 + +#endif /* ADIOS2_ENGINE_MHSREADER_H_ */ diff --git a/source/adios2/engine/mhs/MhsReader.tcc b/source/adios2/engine/mhs/MhsReader.tcc new file mode 100644 index 0000000000..4c3928bfc7 --- /dev/null +++ b/source/adios2/engine/mhs/MhsReader.tcc @@ -0,0 +1,60 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * MhsReader.tcc + * + * Created on: Aug 04, 2021 + * Author: Jason Wang jason.ruonan.wang@gmail.com + */ + +#ifndef ADIOS2_ENGINE_MHSREADER_TCC_ +#define ADIOS2_ENGINE_MHSREADER_TCC_ + +#include "MhsReader.h" + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +template +inline void MhsReader::GetSyncCommon(Variable &variable, T *data) +{ + GetDeferredCommon(variable, data); + PerformGets(); +} + +template <> +inline void MhsReader::GetDeferredCommon(Variable &variable, + std::string *data) +{ + m_SubEngines[0]->Get(variable, data, Mode::Sync); +} + +template +void MhsReader::GetDeferredCommon(Variable &variable, T *data) +{ + for (int i = 0; i < m_Tiers; ++i) + { + auto var = m_SubIOs[i]->InquireVariable(variable.m_Name); + if (!var) + { + break; + } + var->SetSelection({variable.m_Start, variable.m_Count}); + m_SubEngines[i]->Get(*var, data, Mode::Sync); + if (m_SiriusCompressor->m_CurrentReadFinished) + { + break; + } + } +} + +} // end namespace engine +} // end namespace core +} // end namespace adios2 + +#endif // ADIOS2_ENGINE_MHSREADER_TCC_ diff --git a/source/adios2/engine/mhs/MhsWriter.cpp b/source/adios2/engine/mhs/MhsWriter.cpp new file mode 100644 index 0000000000..f3b1808ada --- /dev/null +++ b/source/adios2/engine/mhs/MhsWriter.cpp @@ -0,0 +1,127 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * MhsWriter.cpp + * + * Created on: Apr 6, 2019 + * Author: Jason Wang w4g@ornl.gov + */ + +#include "MhsWriter.tcc" +#include "adios2/helper/adiosFunctions.h" +#include "adios2/operator/compress/CompressSirius.h" + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +MhsWriter::MhsWriter(IO &io, const std::string &name, const Mode mode, + helper::Comm comm) +: Engine("MhsWriter", io, name, mode, std::move(comm)) +{ + helper::GetParameter(io.m_Parameters, "Tiers", m_Tiers); + for (const auto &transportParams : io.m_TransportsParameters) + { + auto itVar = transportParams.find("variable"); + if (itVar == transportParams.end()) + { + continue; + } + auto itTransport = transportParams.find("transport"); + if (itTransport == transportParams.end()) + { + continue; + } + + if (itTransport->second == "sirius") + { + m_TransportMap.emplace( + itVar->second, + std::make_shared(io.m_Parameters)); + } + else + { + throw("invalid operator"); + } + } + for (int i = 0; i < m_Tiers; ++i) + { + m_SubIOs.emplace_back( + &io.m_ADIOS.DeclareIO("SubIO" + std::to_string(i))); + m_SubEngines.emplace_back(&m_SubIOs.back()->Open( + m_Name + ".tier" + std::to_string(i), adios2::Mode::Write)); + } +} + +MhsWriter::~MhsWriter() +{ + for (int i = 0; i < m_Tiers; ++i) + { + m_IO.m_ADIOS.RemoveIO("SubIO" + std::to_string(i)); + } +} + +StepStatus MhsWriter::BeginStep(StepMode mode, const float timeoutSeconds) +{ + for (auto &e : m_SubEngines) + { + e->BeginStep(mode, timeoutSeconds); + } + return StepStatus::OK; +} + +size_t MhsWriter::CurrentStep() const { return m_SubEngines[0]->CurrentStep(); } + +void MhsWriter::PerformPuts() +{ + for (auto &e : m_SubEngines) + { + e->PerformPuts(); + } +} + +void MhsWriter::EndStep() +{ + for (auto &e : m_SubEngines) + { + e->EndStep(); + } +} + +void MhsWriter::Flush(const int transportIndex) +{ + for (auto &e : m_SubEngines) + { + e->Flush(transportIndex); + } +} + +// PRIVATE + +#define declare_type(T) \ + void MhsWriter::DoPutSync(Variable &variable, const T *data) \ + { \ + PutSyncCommon(variable, data); \ + } \ + void MhsWriter::DoPutDeferred(Variable &variable, const T *data) \ + { \ + PutDeferredCommon(variable, data); \ + } +ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + +void MhsWriter::DoClose(const int transportIndex) +{ + for (auto &e : m_SubEngines) + { + e->Close(); + } +} + +} // end namespace engine +} // end namespace core +} // end namespace adios2 diff --git a/source/adios2/engine/table/TableWriter.h b/source/adios2/engine/mhs/MhsWriter.h similarity index 64% rename from source/adios2/engine/table/TableWriter.h rename to source/adios2/engine/mhs/MhsWriter.h index 2520ce2b85..107ea659dc 100644 --- a/source/adios2/engine/table/TableWriter.h +++ b/source/adios2/engine/mhs/MhsWriter.h @@ -2,19 +2,16 @@ * Distributed under the OSI-approved Apache License, Version 2.0. See * accompanying file Copyright.txt for details. * - * TableWriter.h + * MhsWriter.h * * Created on: Apr 6, 2019 * Author: Jason Wang w4g@ornl.gov */ -#ifndef ADIOS2_ENGINE_TABLEWRITER_H_ -#define ADIOS2_ENGINE_TABLEWRITER_H_ +#ifndef ADIOS2_ENGINE_MHSWRITER_H_ +#define ADIOS2_ENGINE_MHSWRITER_H_ -#include "adios2/core/ADIOS.h" #include "adios2/core/Engine.h" -#include "adios2/core/IO.h" -#include "adios2/core/Variable.h" namespace adios2 { @@ -23,14 +20,13 @@ namespace core namespace engine { -class TableWriter : public Engine +class MhsWriter : public Engine { public: - TableWriter(IO &adios, const std::string &name, const Mode mode, - helper::Comm comm); - - virtual ~TableWriter(); + MhsWriter(IO &adios, const std::string &name, const Mode mode, + helper::Comm comm); + virtual ~MhsWriter(); StepStatus BeginStep(StepMode mode, const float timeoutSeconds = -1.0) final; @@ -40,15 +36,10 @@ class TableWriter : public Engine void Flush(const int transportIndex = -1) final; private: - int m_MpiRank; - int m_MpiSize; - ADIOS m_SubAdios; - IO &m_SubIO; - Engine *m_SubEngine = nullptr; - Operator *m_Compressor = nullptr; - std::unordered_map m_IndexerMap; - std::string m_UseCompressor; - std::string m_UseAccuracy; + std::vector m_SubIOs; + std::vector m_SubEngines; + std::unordered_map> m_TransportMap; + int m_Tiers = 1; void PutSubEngine(bool finalPut = false); @@ -71,4 +62,4 @@ class TableWriter : public Engine } // end namespace core } // end namespace adios2 -#endif // ADIOS2_ENGINE_TABLEWRITER_H_ +#endif // ADIOS2_ENGINE_MHSWRITER_H_ diff --git a/source/adios2/engine/mhs/MhsWriter.tcc b/source/adios2/engine/mhs/MhsWriter.tcc new file mode 100644 index 0000000000..e37e3050ff --- /dev/null +++ b/source/adios2/engine/mhs/MhsWriter.tcc @@ -0,0 +1,92 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * MhsWriter.tcc + * + * Created on: Apr 6, 2019 + * Author: Jason Wang w4g@ornl.gov + */ + +#ifndef ADIOS2_ENGINE_MHSWRITER_TCC_ +#define ADIOS2_ENGINE_MHSWRITER_TCC_ + +#include "MhsWriter.h" + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +template <> +void MhsWriter::PutDeferredCommon(Variable &variable, + const std::string *data) +{ + auto var = m_SubIOs[0]->InquireVariable(variable.m_Name); + if (!var) + { + var = &m_SubIOs[0]->DefineVariable(variable.m_Name, + {LocalValueDim}); + } + m_SubEngines[0]->Put(variable, data, Mode::Sync); +} + +template +void MhsWriter::PutSyncCommon(Variable &variable, const T *data) +{ + PutDeferredCommon(variable, data); + PerformPuts(); +} + +template +void MhsWriter::PutDeferredCommon(Variable &variable, const T *data) +{ + bool putToAll = false; + auto itVar = m_TransportMap.find(variable.m_Name); + if (itVar != m_TransportMap.end()) + { + if (itVar->second->m_Type == "sirius") + { + putToAll = true; + } + } + + auto var0 = m_SubIOs[0]->InquireVariable(variable.m_Name); + if (!var0) + { + var0 = + &m_SubIOs[0]->DefineVariable(variable.m_Name, variable.m_Shape); + itVar = m_TransportMap.find(variable.m_Name); + if (itVar != m_TransportMap.end()) + { + var0->AddOperation(*itVar->second, {}); + } + } + + var0->SetSelection({variable.m_Start, variable.m_Count}); + m_SubEngines[0]->Put(*var0, data, Mode::Sync); + + if (putToAll) + { + for (size_t i = 1; i < m_SubEngines.size(); ++i) + { + auto var = m_SubIOs[i]->InquireVariable(variable.m_Name); + if (!var) + { + var = &m_SubIOs[i]->DefineVariable(variable.m_Name, + variable.m_Shape); + var->AddOperation(*itVar->second, {}); + } + var->SetSelection({variable.m_Start, variable.m_Count}); + m_SubEngines[i]->Put(*var, data, Mode::Sync); + } + } +} + +} // end namespace engine +} // end namespace core +} // end namespace adios2 + +#endif /* ADIOS2_ENGINE_MHSWRITER_TCC_ */ diff --git a/source/adios2/engine/mixer/HDFMixer.cpp b/source/adios2/engine/mixer/HDFMixer.cpp deleted file mode 100644 index be6128f229..0000000000 --- a/source/adios2/engine/mixer/HDFMixer.cpp +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * HDFMixer.cpp - * - * Created on: Dec 19, 2016 - * Author: Junmin GU - */ - -#include "HDFMixer.h" -#include "HDFMixer.tcc" - -#include "adios2/core/IO.h" -#include "adios2/helper/adiosFunctions.h" //CheckIndexRange -#include "adios2/toolkit/transport/file/FileFStream.h" - -namespace adios2 -{ -namespace core -{ -namespace engine -{ - -HDFMixer::HDFMixer(IO &io, const std::string &name, const Mode openMode, - helper::Comm comm) -: Engine("HDFMixer", io, name, openMode, std::move(comm)), - m_HDFVDSWriter(m_Comm), m_HDFSerialWriter(helper::Comm()), - m_TransportsManager(m_Comm) -{ - m_EndMessage = " in call to IO Open HDFMixer " + m_Name + "\n"; - Init(); -} - -HDFMixer::~HDFMixer() = default; - -void HDFMixer::Init() -{ - InitParameters(); - InitTransports(); - InitBuffer(); -} - -#define declare_type(T) \ - void HDFMixer::DoPutSync(Variable &variable, const T *values) \ - { \ - DoPutSyncCommon(variable, values); \ - } \ - void HDFMixer::DoPutDeferred(Variable &variable, const T *values) \ - { \ - DoPutSyncCommon(variable, values); \ - } -ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) -#undef declare_type - -StepStatus HDFMixer::BeginStep(StepMode mode, const float timeout_sec) -{ - return StepStatus::OK; -} - -void HDFMixer::PerformPuts() {} - -// void HDFMixer::Advance(const float /*timeout_sec*/) -// void HDFMixer::EndStep(const float /*timeout_sec*/) -void HDFMixer::EndStep() -{ - m_HDFSerialWriter.Advance(); - m_HDFVDSWriter.Advance(); -} - -// PRIVATE FUNCTIONS -void HDFMixer::InitParameters() -{ - // no need for hdf5 - // m_HDFSerialWriter.InitParameters(m_IO.m_Parameters); -} - -void HDFMixer::InitTransports() -{ - if (m_IO.m_TransportsParameters.empty()) - { - Params defaultTransportParameters; - defaultTransportParameters["transport"] = "File"; - m_IO.m_TransportsParameters.push_back(defaultTransportParameters); - } - -#ifdef NEVER -/* - // TODO need to add support for aggregators here later - - // Names are std::vector - auto transportsNames = m_TransportsManager.GetFilesBaseNames( - m_Name, m_IO.m_TransportsParameters); - auto bpBaseNames = m_HDFSerialWriter.GetBPBaseNames(transportsNames); - auto bpNames = m_HDFSerialWriter.GetBPNames(transportsNames); - - m_TransportsManager.OpenFiles(bpBaseNames, bpNames, m_OpenMode, - m_IO.m_TransportsParameters, - m_HDFSerialWriter.m_Profiler.IsActive); -*/ -#else - - m_HDFSerialWriter.Init(m_Name, m_Comm.Rank()); - m_HDFVDSWriter.Init(m_Name); -/* -auto transportsNames = m_TransportsManager.GetFilesBaseNames( - m_Name, -m_IO.m_TransportsParameters); auto baseNames = -m_HDFSerialWriter.GetBaseNames(transportsNames); - -auto h5name = m_HDFSerialWriter.GetLocalFileNames(baseNames, m_Name); // e.g. -base=/my/path/xy_1.h5 m_TransportsManager.OpenFiles(baseNames, h5name, -m_OpenMode, m_IO.m_TransportsParameters, m_HDFSerialWriter.m_Profiler.IsActive); -*/ -#endif -} - -void HDFMixer::InitBuffer() -{ - /* - if (m_OpenMode == OpenMode::Append) - { - throw std::invalid_argument( - "ADIOS2: OpenMode Append hasn't been implemented, yet"); - // TODO: Get last pg timestep and update timestep counter in - } - else - { - m_HDFSerialWriter.WriteProcessGroupIndex( - m_IO.m_HostLanguage, - m_TransportsManager.GetTransportsTypes()); - } - */ -} - -void HDFMixer::WriteProfilingJSONFile() {} - -void HDFMixer::DoClose(const int transportIndex) -{ - m_HDFSerialWriter.Close(); - m_HDFVDSWriter.Close(); -} - -} // end namespace engine -} // end namespace core -} // end namespace adios2 diff --git a/source/adios2/engine/mixer/HDFMixer.h b/source/adios2/engine/mixer/HDFMixer.h deleted file mode 100644 index de61fe2b4d..0000000000 --- a/source/adios2/engine/mixer/HDFMixer.h +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * HDFMixer.h - * - * Created on: Aug 16 2017 - * Author: Junmin GU - */ - -#ifndef ADIOS2_ENGINE_H5_HDFMIXER_H_ -#define ADIOS2_ENGINE_H5_HDFMIXER_H_ - -#include "adios2/common/ADIOSConfig.h" -#include "adios2/core/Engine.h" -//#include "adios2/toolkit/format/bp1/BP1Writer.h" //format::BP1Writer - -/// \cond EXCLUDE_FROM_DOXYGEN -#include //std::count, std::copy, std::for_each -#include //std::ceil -#include //std::memcpy -/// \endcond - -#include "HDFMixerWriter.h" -#include "adios2/common/ADIOSConfig.h" -#include "adios2/common/ADIOSMacros.h" -#include "adios2/common/ADIOSTypes.h" -#include "adios2/core/Variable.h" -#include "adios2/helper/adiosComm.h" -#include "adios2/helper/adiosFunctions.h" -//#include "adios2/toolkit/capsule/heap/STLVector.h" -#include "adios2/toolkit/transportman/TransportMan.h" //transport::TransportsMan - -#include - -namespace adios2 -{ -namespace core -{ -namespace engine -{ - -class HDFMixer : public Engine -{ - -public: - /** - * Constructor for file Writer in H5 format - * @param name unique name given to the engine - * @param openMode w (supported), r, a from OpenMode in ADIOSTypes.h - * @param comm multi-process communicator - */ - HDFMixer(IO &io, const std::string &name, const Mode openMode, - helper::Comm comm); - - ~HDFMixer(); - - // void Advance(const float timeoutSeconds = - // std::numeric_limits::max()) final; - StepStatus BeginStep(StepMode mode, const float timeout_sec); - // void EndStep(const float /*timeout_sec*/); - void EndStep() final; - - void PerformPuts() final; - - void CreateName(std::string &pathName, std::string &rootName, - std::string &fullH5Name, int rank); - -private: - /** Single object controlling H5 buffering */ - // format::H51Writer m_H51Writer; - HDFSerialWriter m_HDFSerialWriter; - HDFVDSWriter m_HDFVDSWriter; - - /** single object controlling a vector of Transports from IO AddTransport */ - transportman::TransportMan m_TransportsManager; - - /** true: due to buffer overflow, move to transports manager */ - bool m_DoTransportFlush = false; - - void Init() final; - - /** Parses parameters from IO SetParameters */ - void InitParameters() final; - /** Parses transports and parameters from IO AddTransport */ - void InitTransports() final; - - void InitBuffer(); - -#define declare_type(T) \ - void DoPutSync(Variable &variable, const T *values) /*final */; \ - void DoPutDeferred(Variable &variable, const T *values) /*final */; - ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) -#undef declare_type - - /** - * Closes a single transport or all transports - * @param transportIndex, if -1 (default) closes all transports, - * otherwise it - * closes a transport in m_Transport[transportIndex]. transportIndex is - * bounds-checked. - */ - void DoClose(const int transportIndex = -1) final; - - /** - * Common function for primitive (including std::complex) writes - * @param variable - * @param values - */ - template - void DoPutSyncCommon(Variable &variable, const T *values); - - /** Write a profiling.json file from m_H51Writer and m_TransportsManager - * profilers*/ - void WriteProfilingJSONFile(); -}; - -} // end namespace engine -} // end namespace core -} // end namespace adios2 - -#endif /* ADIOS2_ENGINE_H5_HDFMIXER_H_ */ diff --git a/source/adios2/engine/mixer/HDFMixer.tcc b/source/adios2/engine/mixer/HDFMixer.tcc deleted file mode 100644 index d1acfd002a..0000000000 --- a/source/adios2/engine/mixer/HDFMixer.tcc +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * HDFMixer.tcc implementation of template functions with known type - * - * Created on: Aug 16 2017 - * Author: Junmin GU - */ - -#include "HDFMixer.h" - -namespace adios2 -{ -namespace core -{ -namespace engine -{ - -template -void HDFMixer::DoPutSyncCommon(Variable &variable, const T *values) -{ - // set values - variable.SetData(values); - // m_WrittenVariables.insert(variable.m_Name); - Variable local(variable.m_Name, {}, {}, variable.m_Count, - variable.IsConstantDims()); - - // m_HDFSerialWriter.m_H5File.Write(variable, values); - // writes only the m_Count() part - int nDims = std::max(variable.m_Shape.size(), variable.m_Count.size()); - if (nDims == 0) - { - // this is scalar - if (m_HDFVDSWriter.m_Rank == 0) - { - m_HDFVDSWriter.m_VDSFile.Write(local, values); - } - } - else - { - m_HDFSerialWriter.m_H5File.Write(local, values); - // std::cout<<" ==> "<< variable.m_Name<()); - m_HDFVDSWriter.AddVar(variable, - m_HDFSerialWriter.m_H5File.GetHDF5Type()); - } -} - -} // end namespace engine -} // end namespace core -} // namespace adios2 diff --git a/source/adios2/engine/mixer/HDFMixerWriter.cpp b/source/adios2/engine/mixer/HDFMixerWriter.cpp deleted file mode 100644 index 8ece096a93..0000000000 --- a/source/adios2/engine/mixer/HDFMixerWriter.cpp +++ /dev/null @@ -1,451 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * HDFMixer.h - * - * Created on: Aug 16 2017 - * Author: Junmin GU - */ - -#include - -#include "HDFMixerWriter.h" -#include "adios2/helper/adiosFunctions.h" - -// -// class HDFSerialWriter -// -namespace adios2 -{ -namespace core -{ -namespace engine -{ - -HDFVDSWriter::HDFVDSWriter(helper::Comm const &comm) -: m_VDSFile(), m_Rank(-1), m_SubfileComm(comm) -{ - m_NumSubFiles = m_SubfileComm.Size(); - m_Rank = m_SubfileComm.Rank(); -} - -void HDFVDSWriter::Init(const std::string &name) -{ - if (m_Rank > 0) - { - return; - } - - // - // VDS can only operate on one process. So let rank = 0 handle it - // - std::string h5Name = adios2::helper::AddExtension(name, ".h5"); - m_VDSFile.Init(h5Name, helper::Comm(), true); - // m_FileName = h5Name; - m_FileName = name; -} - -void HDFVDSWriter::GetVarInfo(const VariableBase &var, - std::vector &dimsf, int nDims, - std::vector &start, - std::vector &count, - std::vector &one) -{ // interop::HDF5Common summaryFile(true); - // std::vector dimsf, start, one, count; - // int nDims = std::max(var.m_Shape.size(), var.m_Count.size()); - - for (int i = 0; i < nDims; i++) - { - if (var.m_Shape.size() > 0) - { - dimsf.push_back(var.m_Shape[i]); - } - else - { - dimsf.push_back(var.m_Count[i]); - } - if (var.m_Start.size() > 0) - { - start.push_back(var.m_Start[i]); - } - else - { - start.push_back(0); - } - if (var.m_Count.size() > 0) - { - count.push_back(var.m_Count[i]); - } - else if (var.m_Shape.size() > 0) - { - count.push_back(var.m_Shape[i]); - } - else - { - count.push_back(0); - } - one.push_back(1); - } -} - -void HDFVDSWriter::AddVar(const VariableBase &var, hid_t h5Type) -{ - hid_t space; - /* Create VDS dataspace. */ - int nDims = std::max(var.m_Shape.size(), var.m_Count.size()); - - if (nDims == 0) - { - if (m_Rank == 0) - { - /* - std::cout<<" will deal with scalar later?"< dimsf, start, one, count; - GetVarInfo(var, dimsf, nDims, start, count, one); - // - - m_SubfileComm.Gather(start.data(), nDims, all_starts[0], nDims, 0); - m_SubfileComm.Gather(count.data(), nDims, all_counts[0], nDims, 0); - - herr_t status; - if (m_Rank == 0) - { - m_VDSFile.CheckWriteGroup(); - /* Set VDS creation property. */ - hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE); - // status = H5Pset_fill_value(dcpl, ADIOS2_MPI_SIZE_T, 0); - - space = H5Screate_simple(nDims, dimsf.data(), NULL); - // summaryFile.Init(fileName.c_str(), MPI_COMM_SELF, true); - - hsize_t currCount[nDims], currStart[nDims]; - // std::string subfileVarName="TimeStep0/"+var.m_Name; // need full - // path? NEED TO GET the RIGHT SUBFILE VAR NAME RELATED to TIMESTEP!! - std::string subfileVarName; - interop::HDF5Common::StaticGetAdiosStepString( - subfileVarName, m_VDSFile.m_CurrentAdiosStep); - subfileVarName += "/" + var.m_Name; - - for (int i = 0; i < m_NumSubFiles; i++) - { - for (int j = 0; j < nDims; j++) - { - currCount[j] = all_counts[i][j]; - currStart[j] = all_starts[i][j]; - // std::cout< 0) - { - return; - } - - m_VDSFile.Advance(); -} - -void HDFVDSWriter::Close(const int transportIndex) -{ - if (m_Rank > 0) - { - return; - } - - m_VDSFile.Close(); -} - -// -// class HDFSerialWriter -// -HDFSerialWriter::HDFSerialWriter(helper::Comm const &comm) -: m_H5File(), m_LocalComm(comm) -{ -} - -void HDFSerialWriter::Advance(const float timeoutSeconds) -{ - m_H5File.Advance(); -} -void HDFSerialWriter::Close(const int transportIndex) { m_H5File.Close(); }; - -void HDFSerialWriter::StaticCreateName(std::string &pathName, - std::string &rootName, - std::string &fullH5Name, - const std::string &input, int rank) -{ - - auto lf_GetBaseName = [](const std::string &name) -> std::string { - const std::string baseName(adios2::helper::AddExtension(name, ".h5") + - ".dir"); - return baseName; - }; - - auto lf_GetRootTag = [](const std::string &userTag) -> std::string { - std::string h5RootName = userTag; - const auto lastPathSeparator(userTag.find_last_of(PathSeparator)); - if (lastPathSeparator != std::string::npos) - { - h5RootName = userTag.substr(lastPathSeparator); - } - return h5RootName; - }; - - pathName = lf_GetBaseName(input); - rootName = lf_GetRootTag(input); - - fullH5Name = - (pathName + "/" + rootName + "_" + std::to_string(rank) + ".h5"); -} - -void HDFSerialWriter::Init(const std::string &name, int rank) -{ - /* - auto lf_GetBaseName = [](const std::string &name) -> std::string { - const std::string baseName(AddExtension(name, ".h5") + ".dir"); - return baseName; - }; - - auto lf_GetRootTag = [] (const std::string &userTag) -> std::string { - std::string h5RootName = userTag; - const auto lastPathSeparator(userTag.find_last_of(PathSeparator)); - if (lastPathSeparator != std::string::npos) - { - h5RootName = userTag.substr(lastPathSeparator); - } - return h5RootName; - }; - - std::string baseName=lf_GetBaseName(name); - - auto rootTag = lf_GetRootTag(name); - const std::string h5Name(baseName + "/" + - rootTag+"_"+std::to_string(rank)+".h5"); - - */ - std::string baseName, rootTag, h5Name; - StaticCreateName(baseName, rootTag, h5Name, name, rank); - // std::cout<<"rank="< baseNames; - baseNames.reserve(names.size()); - - for (const auto &name : names) - { - baseNames.push_back(lf_GetBaseName(name)); - } - return baseNames; - } - - - std::vector - GetLocalFileNames(const std::vector &baseNames, - const std::string &userTag) const noexcept - { - // e.g. /some/where/xy.h5.dir - // e.g. xy - - auto lf_GetH5Name = [](const std::string &baseName, - const std::string &userTag, - const int rank) -> std::string { -#ifdef NEVER - const std::string h5BaseName = AddExtension(baseName, ".h5"); - - std::string h5RootName = h5BaseName; - const auto lastPathSeparator(h5BaseName.find_last_of(PathSeparator)); - - if (lastPathSeparator != std::string::npos) - { - h5RootName = h5BaseName.substr(lastPathSeparator); - } - const std::string h5Name(h5BaseName + ".dir/" + h5RootName + "." + - std::to_string(rank)); -#else - const std::string h5Name(baseName + "/" + -userTag+"_"+std::to_string(rank)+".h5"); #endif return h5Name; - }; - - - auto lf_GetRootTag = [] (const std::string &userTag) -> std::string { - std::string h5RootName = userTag; - const auto lastPathSeparator(userTag.find_last_of(PathSeparator)); - if (lastPathSeparator != std::string::npos) - { - h5RootName = userTag.substr(lastPathSeparator); - } - return h5RootName; - }; - - std::vector h5Names; - h5Names.reserve(baseNames.size()); - - auto rootTag = lf_GetRootTag(userTag); - for (const auto &baseName : baseNames) - { - h5Names.push_back(lf_GetH5Name(baseName, rootTag, m_RankMPI)); - } - return h5Names; - - } - - - enum class ResizeResult - { - Failure, //!< FAILURE, caught a std::bad_alloc - Unchanged, //!< UNCHANGED, no need to resize (sufficient capacity) - Success, //!< SUCCESS, resize was successful - Flush //!< FLUSH, need to flush to transports for current variable - }; - - - template - ResizeResult ResizeBuffer(const Variable &variable) - { std::cout<<"ResizeBuffer() Forcing Flush for now."< -&transportsTypes, const std::vector &transportsProfilers) -noexcept - { - std::cout<<"GetRankProfilingJSON() returns empty string now "< &transportsTypes) -noexcept - { - std::cout<<"WriteProcessGroupIndex() to hdf5"< - void WriteVariableMetadata(const Variable &variable) noexcept - { - std::cout<<"WriteVariableMetadata() to hdf5"< - void WriteVariablePayload(const Variable &variable) noexcept - { - std::cout<<"WriteVariablePayload() to hdf5"<::max()); - void Close(const int transportIndex = -1); - - interop::HDF5Common m_VDSFile; - int m_Rank; - -private: - void GetVarInfo(const VariableBase &var, std::vector &dimsf, - int nDim, std::vector &start, - std::vector &count, std::vector &one); - - int m_NumSubFiles; - std::string m_FileName; - helper::Comm const - &m_SubfileComm; // only rank 0 in this comm can build VDS; -}; - -class HDFSerialWriter -{ -public: - HDFSerialWriter(helper::Comm const &comm); - void - Advance(const float timeoutSeconds = std::numeric_limits::max()); - void Close(const int transportIndex = -1); - void Init(const std::string &name, int rank); - - static void StaticCreateName(std::string &pathName, std::string &rootName, - std::string &fullH5Name, - const std::string &input, int rank); - /** contains data buffer and position */ - // capsule::STLVector m_HeapBuffer; - - // int m_Rank; - interop::HDF5Common m_H5File; - std::string m_FileName; - -private: - helper::Comm const - &m_LocalComm; // all ranks in this comm write to the same file - int m_Rank; -}; - -} // end namespace engine -} // end namespace core -} // end namespace adios2 - -#endif // ADIOS2_ENGINE_BP_HDFSerialWriter diff --git a/source/adios2/engine/nullcore/NullCoreWriter.h b/source/adios2/engine/nullcore/NullCoreWriter.h index 3fff8b7969..2b08889099 100644 --- a/source/adios2/engine/nullcore/NullCoreWriter.h +++ b/source/adios2/engine/nullcore/NullCoreWriter.h @@ -44,7 +44,7 @@ class NullCoreWriter : public core::Engine // Put #define declare_type(T) \ void DoPut(Variable &variable, typename Variable::Span &span, \ - const size_t blockID, const T &value) override; + const bool initialize, const T &value) override; ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type) #undef declare_type diff --git a/source/adios2/engine/nullcore/NullCoreWriter.tcc b/source/adios2/engine/nullcore/NullCoreWriter.tcc index 8dfc23eb85..081a0872a0 100644 --- a/source/adios2/engine/nullcore/NullCoreWriter.tcc +++ b/source/adios2/engine/nullcore/NullCoreWriter.tcc @@ -22,7 +22,7 @@ namespace engine #define instantiate_type(T) \ void NullCoreWriter::DoPut(Variable &variable, \ typename Variable::Span &span, \ - const size_t blockID, const T &value) \ + const bool initialize, const T &value) \ { \ } diff --git a/source/adios2/engine/ssc/SscHelper.cpp b/source/adios2/engine/ssc/SscHelper.cpp index b5cf30a550..2e3c9c7189 100644 --- a/source/adios2/engine/ssc/SscHelper.cpp +++ b/source/adios2/engine/ssc/SscHelper.cpp @@ -375,7 +375,7 @@ void Deserialize(const Buffer &input, BlockVecVec &output, IO &io, { \ Dims vStart = b.start; \ Dims vShape = b.shape; \ - if (!helper::IsRowMajor(io.m_HostLanguage)) \ + if (io.m_ArrayOrder != ArrayOrdering::RowMajor) \ { \ std::reverse(vStart.begin(), vStart.end()); \ std::reverse(vShape.begin(), vShape.end()); \ @@ -421,7 +421,7 @@ void AggregateMetadata(const Buffer &localBuffer, Buffer &globalBuffer, globalBuffer.resize(globalSize + 10); std::vector displs(mpiSize); - for (size_t i = 1; i < mpiSize; ++i) + for (size_t i = 1; i < static_cast(mpiSize); ++i) { displs[i] = displs[i - 1] + localSizes[i - 1]; } @@ -438,7 +438,7 @@ void BroadcastMetadata(Buffer &globalBuffer, const int root, MPI_Comm comm) { int globalBufferSize = static_cast(globalBuffer.size()); MPI_Bcast(&globalBufferSize, 1, MPI_INT, root, comm); - if (globalBuffer.size() < globalBufferSize) + if (globalBuffer.size() < static_cast(globalBufferSize)) { globalBuffer.resize(globalBufferSize); } diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp index 7701d2d21b..f82fb325e9 100644 --- a/source/adios2/engine/ssc/SscReader.cpp +++ b/source/adios2/engine/ssc/SscReader.cpp @@ -140,9 +140,12 @@ StepStatus SscReader::BeginStep(const StepMode stepMode, auto variable = m_IO.InquireVariable(v.name); \ if (variable) \ { \ - std::memcpy(&variable->m_Min, value.data(), value.size()); \ - std::memcpy(&variable->m_Max, value.data(), value.size()); \ - std::memcpy(&variable->m_Value, value.data(), value.size()); \ + std::memcpy(reinterpret_cast(&variable->m_Min), \ + value.data(), value.size()); \ + std::memcpy(reinterpret_cast(&variable->m_Max), \ + value.data(), value.size()); \ + std::memcpy(reinterpret_cast(&variable->m_Value), \ + value.data(), value.size()); \ } \ } ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) diff --git a/source/adios2/engine/ssc/SscReader.tcc b/source/adios2/engine/ssc/SscReader.tcc index d787a4d763..48d7c463be 100644 --- a/source/adios2/engine/ssc/SscReader.tcc +++ b/source/adios2/engine/ssc/SscReader.tcc @@ -33,7 +33,7 @@ void SscReader::GetDeferredDeltaCommon(Variable &variable, T *data) Dims vCount = variable.m_Count; Dims vShape = variable.m_Shape; - if (!helper::IsRowMajor(m_IO.m_HostLanguage)) + if (m_IO.m_ArrayOrder != ArrayOrdering::RowMajor) { std::reverse(vStart.begin(), vStart.end()); std::reverse(vCount.begin(), vCount.end()); @@ -101,7 +101,7 @@ void SscReader::GetDeferredCommon(Variable &variable, T *data) Dims vCount = variable.m_Count; Dims vShape = variable.m_Shape; - if (!helper::IsRowMajor(m_IO.m_HostLanguage)) + if (m_IO.m_ArrayOrder != ArrayOrdering::RowMajor) { std::reverse(vStart.begin(), vStart.end()); std::reverse(vCount.begin(), vCount.end()); @@ -183,7 +183,7 @@ SscReader::BlocksInfoCommon(const Variable &variable, b.Step = m_CurrentStep; b.StepsStart = m_CurrentStep; b.StepsCount = 1; - if (!helper::IsRowMajor(m_IO.m_HostLanguage)) + if (m_IO.m_ArrayOrder != ArrayOrdering::RowMajor) { std::reverse(b.Start.begin(), b.Start.end()); std::reverse(b.Count.begin(), b.Count.end()); @@ -197,11 +197,13 @@ SscReader::BlocksInfoCommon(const Variable &variable, m_WriterDefinitionsLocked == false || m_ReaderSelectionsLocked == false) { - std::memcpy(&b.Value, v.value.data(), v.value.size()); + std::memcpy(reinterpret_cast(&b.Value), + v.value.data(), v.value.size()); } else { - std::memcpy(&b.Value, m_Buffer.data() + v.bufferStart, + std::memcpy(reinterpret_cast(&b.Value), + m_Buffer.data() + v.bufferStart, v.bufferCount); } } diff --git a/source/adios2/engine/ssc/SscWriter.tcc b/source/adios2/engine/ssc/SscWriter.tcc index 77ba801fe8..42ebd5eb24 100644 --- a/source/adios2/engine/ssc/SscWriter.tcc +++ b/source/adios2/engine/ssc/SscWriter.tcc @@ -94,7 +94,7 @@ void SscWriter::PutDeferredCommon(Variable &variable, const T *data) Dims vCount = variable.m_Count; Dims vShape = variable.m_Shape; - if (!helper::IsRowMajor(m_IO.m_HostLanguage)) + if (m_IO.m_ArrayOrder != ArrayOrdering::RowMajor) { std::reverse(vStart.begin(), vStart.end()); std::reverse(vCount.begin(), vCount.end()); diff --git a/source/adios2/engine/sst/SstParamParser.cpp b/source/adios2/engine/sst/SstParamParser.cpp index f31938ccaa..5c928bdde3 100644 --- a/source/adios2/engine/sst/SstParamParser.cpp +++ b/source/adios2/engine/sst/SstParamParser.cpp @@ -117,7 +117,7 @@ void SstParamParser::ParseParams(IO &io, struct _SstParams &Params) // not really a parameter, but a convenient way to pass this around auto lf_SetIsRowMajorParameter = [&](const std::string key, int ¶meter) { - parameter = adios2::helper::IsRowMajor(io.m_HostLanguage); + parameter = (io.m_ArrayOrder == adios2::ArrayOrdering::RowMajor); return true; }; diff --git a/source/adios2/engine/sst/SstReader.cpp b/source/adios2/engine/sst/SstReader.cpp index a5aae390e6..6dce8aeaed 100644 --- a/source/adios2/engine/sst/SstReader.cpp +++ b/source/adios2/engine/sst/SstReader.cpp @@ -179,6 +179,7 @@ SstReader::SstReader(IO &io, const std::string &name, const Mode mode, class SstReader::SstReader *Reader = reinterpret_cast(reader); size_t currentStep = SstCurrentStep(Reader->m_Input); + struct MinBlockInfo BI; /* * setup shape of array variable as global (I.E. Count == Shape, * Start == 0) @@ -226,8 +227,45 @@ SstReader::SstReader(IO &io, const std::string &name, const Mode mode, return; }; - SstReaderInitFFSCallback(m_Input, this, varFFSCallback, arrayFFSCallback, - attrFFSCallback, arrayBlocksInfoCallback); + auto MinArraySetupUpcall = [](void *reader, int DimCount, size_t *Shape) { + MinVarInfo *MV = new MinVarInfo(DimCount, Shape); + return (void *)MV; + }; + auto arrayMinBlocksInfoCallback = + [](void *reader, void *MV, const int type, int WriterRank, int DimCount, + size_t *Shape, size_t *Start, size_t *Count) { + struct MinBlockInfo MBI; + struct MinVarInfo *MinVar = (struct MinVarInfo *)MV; + + MBI.WriterID = WriterRank; + MBI.BlockID = 0; + MBI.Start = Start; + MBI.Count = Count; + + MinVar->BlocksInfo.push_back(MBI); + return; + }; + + static int UseMin = 1; + if (UseMin == -1) + { + if (getenv("OldBlocksInfo") == NULL) + { + UseMin = 0; + } + else + { + UseMin = 1; + } + } + if (UseMin) + SstReaderInitFFSCallback(m_Input, this, varFFSCallback, + arrayFFSCallback, MinArraySetupUpcall, + attrFFSCallback, arrayMinBlocksInfoCallback); + else + SstReaderInitFFSCallback(m_Input, this, varFFSCallback, + arrayFFSCallback, NULL, attrFFSCallback, + arrayBlocksInfoCallback); delete[] cstr; } @@ -458,6 +496,7 @@ void SstReader::EndStep() // unknown marshaling method, shouldn't happen } SstReleaseStep(m_Input); + m_InfoMap.clear(); } void SstReader::Flush(const int transportIndex) {} @@ -486,7 +525,7 @@ void SstReader::Init() size_t *Start = NULL; \ size_t *Count = NULL; \ size_t DimCount = 0; \ - int NeedSync; \ + int NeedSync = 0; \ \ if (variable.m_SelectionType == \ adios2::SelectionType::BoundingBox) \ @@ -586,7 +625,7 @@ void SstReader::Init() } \ if (m_WriterMarshalMethod == SstMarshalBP5) \ { \ - bool need_sync = m_BP5Deserializer->QueueGet(variable, data); \ + m_BP5Deserializer->QueueGet(variable, data); \ } \ } ADIOS2_FOREACH_STDTYPE_1ARG(declare_gets) @@ -701,6 +740,26 @@ void SstReader::PerformGets() void SstReader::DoClose(const int transportIndex) { SstReaderClose(m_Input); } +Engine::MinVarInfo *SstReader::MinBlocksInfo(const VariableBase &Var, + const size_t Step) const +{ + if (m_WriterMarshalMethod == SstMarshalBP) + { + return nullptr; + } + else if (m_WriterMarshalMethod == SstMarshalFFS) + { + return (Engine::MinVarInfo *)SstFFSGetBlocksInfo(m_Input, (void *)&Var); + } + else if (m_WriterMarshalMethod == SstMarshalBP5) + { + return (Engine::MinVarInfo *)m_BP5Deserializer->MinBlocksInfo(Var, + Step); + } + throw std::invalid_argument( + "ERROR: Unknown marshal mechanism in MinBlocksInfo\n"); +} + #define declare_type(T) \ std::map::BPInfo>> \ SstReader::DoAllStepsBlocksInfo(const Variable &variable) const \ @@ -731,7 +790,8 @@ void SstReader::DoClose(const int transportIndex) { SstReaderClose(m_Input); } } \ else if (m_WriterMarshalMethod == SstMarshalBP5) \ { \ - return m_BP5Deserializer->BlocksInfo(variable, 0); \ + std::vector::BPInfo> tmp; \ + return tmp; \ } \ throw std::invalid_argument( \ "ERROR: Unknown marshal mechanism in DoBlocksInfo\n"); \ diff --git a/source/adios2/engine/sst/SstReader.h b/source/adios2/engine/sst/SstReader.h index 45359688b8..9c876995e4 100644 --- a/source/adios2/engine/sst/SstReader.h +++ b/source/adios2/engine/sst/SstReader.h @@ -50,6 +50,7 @@ class SstReader : public Engine void EndStep(); void PerformGets(); void Flush(const int transportIndex = -1) final; + MinVarInfo *MinBlocksInfo(const VariableBase &, const size_t Step) const; private: template @@ -70,7 +71,6 @@ class SstReader : public Engine SstMarshalMethod m_WriterMarshalMethod; int m_WriterIsRowMajor; bool m_DefinitionsNotified = false; - bool m_BetweenStepPairs = false; /* --- Used only with BP marshaling --- */ SstFullMetadata m_CurrentStepMetaData = NULL; @@ -80,6 +80,8 @@ class SstReader : public Engine struct _SstParams Params; + std::unordered_map> + m_InfoMap; #define declare_type(T) \ void DoGetSync(Variable &, T *) final; \ void DoGetDeferred(Variable &, T *) final; \ diff --git a/source/adios2/engine/sst/SstReader.tcc b/source/adios2/engine/sst/SstReader.tcc index c573dcfbb6..7f0bde51b8 100644 --- a/source/adios2/engine/sst/SstReader.tcc +++ b/source/adios2/engine/sst/SstReader.tcc @@ -172,7 +172,8 @@ void SstReader::ReadVariableBlocksFill(Variable &variable, m_BP3Deserializer->PostDataRead( variable, blockInfo, subStreamInfo, - helper::IsRowMajor(m_IO.m_HostLanguage), threadID); + (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor), + threadID); ++iter; } // if remote data buffer is not compressed diff --git a/source/adios2/engine/sst/SstWriter.cpp b/source/adios2/engine/sst/SstWriter.cpp index 118a9f9ad7..4bb50ded96 100644 --- a/source/adios2/engine/sst/SstWriter.cpp +++ b/source/adios2/engine/sst/SstWriter.cpp @@ -14,6 +14,7 @@ #include "SstParamParser.h" #include "SstWriter.h" #include "SstWriter.tcc" +#include "adios2/toolkit/format/buffer/malloc/MallocV.h" #include namespace adios2 @@ -144,13 +145,20 @@ StepStatus SstWriter::BeginStep(StepMode mode, const float timeout_sec) new format::BP3Serializer(m_Comm)); m_BP3Serializer->Init(m_IO.m_Parameters, "in call to BP3::Open for writing", "sst"); + m_BP3Serializer->ResizeBuffer( + m_BP3Serializer->m_Parameters.InitialBufferSize, + "in call to BP3::Open for writing by SST engine"); m_BP3Serializer->m_MetadataSet.TimeStep = 1; m_BP3Serializer->m_MetadataSet.CurrentStep = m_WriterStep; } else if (Params.MarshalMethod == SstMarshalBP5) { - m_BP5Serializer = - std::unique_ptr(new format::BP5Serializer()); + if (!m_BP5Serializer) + { + m_BP5Serializer = std::unique_ptr( + new format::BP5Serializer()); + } + m_BP5Serializer->InitStep(new format::MallocV("SstWriter", true)); m_BP5Serializer->m_Engine = this; // m_BP5Serializer->Init(m_IO.m_Parameters, // "in call to BP5::Open for writing", @@ -285,11 +293,18 @@ void SstWriter::EndStep() newblock->MetaMetaBlocks = MetaMetaBlocks; newblock->metadata.DataSize = TSInfo->MetaEncodeBuffer->m_FixedSize; newblock->metadata.block = TSInfo->MetaEncodeBuffer->Data(); - format::BufferV::BufferV_iovec iovec = TSInfo->DataBuffer->DataVec(); - newblock->data.DataSize = iovec[0].iov_len; - newblock->data.block = (char *)iovec[0].iov_base; + std::vector iovec = TSInfo->DataBuffer->DataVec(); + if (!iovec.empty()) + { + newblock->data.DataSize = iovec[0].iov_len; + newblock->data.block = (char *)iovec[0].iov_base; + } + else + { + newblock->data.DataSize = 0; + newblock->data.block = nullptr; + } newblock->TSInfo = TSInfo; - delete[] iovec; if (TSInfo->AttributeEncodeBuffer) { newblock->attribute_data.DataSize = diff --git a/source/adios2/engine/sst/SstWriter.h b/source/adios2/engine/sst/SstWriter.h index 076df32b05..a8b979339e 100644 --- a/source/adios2/engine/sst/SstWriter.h +++ b/source/adios2/engine/sst/SstWriter.h @@ -78,7 +78,6 @@ class SstWriter : public Engine SstStream m_Output; long m_WriterStep = -1; - bool m_BetweenStepPairs = false; bool m_DefinitionsNotified = false; size_t m_MarshaledAttributesCount = 0; struct _SstParams Params; diff --git a/source/adios2/engine/sst/SstWriter.tcc b/source/adios2/engine/sst/SstWriter.tcc index c7a66f7021..841c76d390 100644 --- a/source/adios2/engine/sst/SstWriter.tcc +++ b/source/adios2/engine/sst/SstWriter.tcc @@ -67,7 +67,7 @@ void SstWriter::PutSyncCommon(Variable &variable, const T *values) m_BP5Serializer->Marshal((void *)&variable, variable.m_Name.c_str(), variable.m_Type, variable.m_ElementSize, DimCount, Shape, Count, Start, values, - true); + true, nullptr); } } else if (Params.MarshalMethod == SstMarshalBP) @@ -77,8 +77,11 @@ void SstWriter::PutSyncCommon(Variable &variable, const T *values) if (!m_BP3Serializer->m_MetadataSet.DataPGIsOpen) { - m_BP3Serializer->PutProcessGroupIndex(m_IO.m_Name, - m_IO.m_HostLanguage, {"SST"}); + m_BP3Serializer->PutProcessGroupIndex( + m_IO.m_Name, + (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) ? "C++" + : "Fortran", + {"SST"}); } const size_t dataSize = helper::PayloadSize(blockInfo.Data, blockInfo.Count) + @@ -93,7 +96,8 @@ void SstWriter::PutSyncCommon(Variable &variable, const T *values) throw std::runtime_error("Failed to resize BP3 serializer buffer"); } - const bool sourceRowMajor = helper::IsRowMajor(m_IO.m_HostLanguage); + const bool sourceRowMajor = + (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor); m_BP3Serializer->PutVariableMetadata(variable, blockInfo, sourceRowMajor); m_BP3Serializer->PutVariablePayload(variable, blockInfo, diff --git a/source/adios2/engine/table/TableWriter.cpp b/source/adios2/engine/table/TableWriter.cpp deleted file mode 100644 index 07574a8609..0000000000 --- a/source/adios2/engine/table/TableWriter.cpp +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * TableWriter.cpp - * - * Created on: Apr 6, 2019 - * Author: Jason Wang w4g@ornl.gov - */ - -#include "TableWriter.tcc" -#include "adios2/helper/adiosString.h" - -namespace adios2 -{ -namespace core -{ -namespace engine -{ - -TableWriter::TableWriter(IO &io, const std::string &name, const Mode mode, - helper::Comm comm) -: Engine("TableWriter", io, name, mode, std::move(comm)), - m_SubAdios(m_Comm.Duplicate(), io.m_HostLanguage), - m_SubIO(m_SubAdios.DeclareIO("SubIO")) -{ - m_MpiRank = m_Comm.Rank(); - m_MpiSize = m_Comm.Size(); - helper::GetParameter(m_IO.m_Parameters, "Compressor", m_UseCompressor); - helper::GetParameter(m_IO.m_Parameters, "Accuracy", m_UseAccuracy); - m_SubEngine = &m_SubIO.Open(m_Name, adios2::Mode::Write); -} - -TableWriter::~TableWriter() -{ - if (m_Compressor) - { - delete m_Compressor; - } - m_Compressor = nullptr; -} - -StepStatus TableWriter::BeginStep(StepMode mode, const float timeoutSeconds) -{ - return m_SubEngine->BeginStep(mode, timeoutSeconds); -} - -size_t TableWriter::CurrentStep() const { return 0; } - -void TableWriter::PerformPuts() { m_SubEngine->PerformPuts(); } - -void TableWriter::EndStep() { m_SubEngine->EndStep(); } - -void TableWriter::Flush(const int transportIndex) -{ - m_SubEngine->Flush(transportIndex); -} - -// PRIVATE - -#define declare_type(T) \ - void TableWriter::DoPutSync(Variable &variable, const T *data) \ - { \ - PutSyncCommon(variable, data); \ - } \ - void TableWriter::DoPutDeferred(Variable &variable, const T *data) \ - { \ - PutDeferredCommon(variable, data); \ - } -ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) -#undef declare_type - -void TableWriter::DoClose(const int transportIndex) -{ - m_SubEngine->Close(); - m_SubEngine = nullptr; -} - -} // end namespace engine -} // end namespace core -} // end namespace adios2 diff --git a/source/adios2/engine/table/TableWriter.tcc b/source/adios2/engine/table/TableWriter.tcc deleted file mode 100644 index a1c91f21e3..0000000000 --- a/source/adios2/engine/table/TableWriter.tcc +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * TableWriter.tcc implementation of template functions with known type - * - * Created on: Apr 6, 2019 - * Author: Jason Wang w4g@ornl.gov - */ - -#ifndef ADIOS2_ENGINE_TABLEWRITER_TCC_ -#define ADIOS2_ENGINE_TABLEWRITER_TCC_ - -#include "TableWriter.h" - -#ifdef ADIOS2_HAVE_BLOSC -#include "adios2/operator/compress/CompressBlosc.h" -#endif - -#ifdef ADIOS2_HAVE_BZIP2 -#include "adios2/operator/compress/CompressBZIP2.h" -#endif - -#ifdef ADIOS2_HAVE_ZFP -#include "adios2/operator/compress/CompressZFP.h" -#endif - -#ifdef ADIOS2_HAVE_SZ -#include "adios2/operator/compress/CompressSZ.h" -#endif - -namespace adios2 -{ -namespace core -{ -namespace engine -{ - -template <> -void TableWriter::PutSyncCommon(Variable &variable, - const std::string *data) -{ - auto var = m_SubIO.InquireVariable(variable.m_Name); - if (!var) - { - var = &m_SubIO.DefineVariable(variable.m_Name, - {LocalValueDim}); - } - m_SubEngine->Put(*var, data, Mode::Sync); -} - -template <> -void TableWriter::PutDeferredCommon( - Variable &variable, const std::string *data) -{ - auto var = m_SubIO.InquireVariable(variable.m_Name); - if (!var) - { - var = &m_SubIO.DefineVariable(variable.m_Name, - {LocalValueDim}); - } - m_SubEngine->Put(*var, data, Mode::Deferred); -} - -template -void TableWriter::PutSyncCommon(Variable &variable, const T *data) -{ - PutDeferredCommon(variable, data); - PerformPuts(); -} - -template -void TableWriter::PutDeferredCommon(Variable &variable, const T *data) -{ - auto var = m_SubIO.InquireVariable(variable.m_Name); - if (!var) - { - var = &m_SubIO.DefineVariable(variable.m_Name, variable.m_Shape); - if (m_UseCompressor == "blosc") - { -#ifdef ADIOS2_HAVE_BLOSC - m_Compressor = new compress::CompressBlosc({}); - var->AddOperation(*m_Compressor, {}); -#else - std::cerr - << "ADIOS2 is not compiled with c-blosc " - "(https://github.com/Blosc/c-blosc), compressor not added" - << std::endl; -#endif - } - else if (m_UseCompressor == "bzip2") - { -#ifdef ADIOS2_HAVE_BZIP2 - m_Compressor = new compress::CompressBZIP2({}); - var->AddOperation(*m_Compressor, {}); -#else - std::cerr << "ADIOS2 is not compiled with Bzip2 " - "(https://gitlab.com/federicomenaquintero/bzip2), " - "compressor not added" - << std::endl; -#endif - } - else if (m_UseCompressor == "zfp") - { -#ifdef ADIOS2_HAVE_ZFP - if (var->m_Type == helper::GetDataType() || - var->m_Type == helper::GetDataType() || - var->m_Type == helper::GetDataType>() || - var->m_Type == helper::GetDataType>()) - { - if (m_UseAccuracy.empty()) - { - std::cerr << "Parameter accuracy for lossy compression is " - "not specified, compressor not added" - << std::endl; - } - else - { - m_Compressor = new compress::CompressZFP({}); - var->AddOperation(*m_Compressor, {{ops::zfp::key::accuracy, - m_UseAccuracy}}); - } - } -#else - std::cerr << "ADIOS2 is not compiled with ZFP " - "(https://github.com/LLNL/zfp), " - "compressor not added" - << std::endl; -#endif - } - else if (m_UseCompressor == "sz") - { -#ifdef ADIOS2_HAVE_SZ - if (var->m_Type == helper::GetDataType() || - var->m_Type == helper::GetDataType() || - var->m_Type == helper::GetDataType>() || - var->m_Type == helper::GetDataType>()) - { - if (m_UseAccuracy.empty()) - { - std::cerr << "Parameter accuracy for lossy compression is " - "not specified, compressor not added" - << std::endl; - } - else - { - m_Compressor = new compress::CompressSZ({}); - var->AddOperation(*m_Compressor, {{ops::sz::key::accuracy, - m_UseAccuracy}}); - } - } -#else - std::cerr << "ADIOS2 is not compiled with SZ " - "(https://github.com/szcompressor/SZ), " - "compressor not added" - << std::endl; -#endif - } - } - - if (m_IndexerMap[variable.m_Name]) - { - // TODO: implement indexing - } - else - { - var->SetSelection({variable.m_Start, variable.m_Count}); - m_SubEngine->Put(*var, data, Mode::Deferred); - } -} - -} // end namespace engine -} // end namespace core -} // end namespace adios2 - -#endif /* ADIOS2_ENGINE_TABLEWRITER_TCC_ */ diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp index 42ce4d7fce..4c119f65cc 100644 --- a/source/adios2/helper/adiosComm.cpp +++ b/source/adios2/helper/adiosComm.cpp @@ -114,6 +114,31 @@ std::vector Comm::GetGathervDisplacements(const size_t *counts, return displacements; } +Comm::Win Comm::Win_allocate_shared(size_t size, int disp_unit, void *baseptr, + const std::string &hint) +{ + return m_Impl->Win_allocate_shared(size, disp_unit, baseptr, hint); +} +int Comm::Win_shared_query(Comm::Win &win, int rank, size_t *size, + int *disp_unit, void *baseptr, + const std::string &hint) +{ + return m_Impl->Win_shared_query(win, rank, size, disp_unit, baseptr, hint); +} +int Comm::Win_free(Win &win, const std::string &hint) +{ + return m_Impl->Win_free(win, hint); +} +int Comm::Win_Lock(LockType lock_type, int rank, int assert, Win &win, + const std::string &hint) +{ + return m_Impl->Win_Lock(lock_type, rank, assert, win, hint); +} +int Comm::Win_Unlock(int rank, Win &win, const std::string &hint) +{ + return m_Impl->Win_Unlock(rank, win, hint); +} + Comm::Req::Req() = default; Comm::Req::Req(std::unique_ptr impl) : m_Impl(std::move(impl)) {} @@ -135,6 +160,27 @@ Comm::Status Comm::Req::Wait(const std::string &hint) return status; } +Comm::Win::Win() = default; + +Comm::Win::Win(std::unique_ptr impl) : m_Impl(std::move(impl)) {} + +Comm::Win::~Win() = default; + +Comm::Win::Win(Win &&win) = default; + +Comm::Win &Comm::Win::operator=(Win &&win) = default; + +int Comm::Win::Free(const std::string &hint) +{ + int status = 0; + if (m_Impl) + { + status = m_Impl->Free(hint); + m_Impl.reset(); + } + return status; +} + CommImpl::~CommImpl() = default; size_t CommImpl::SizeOf(Datatype datatype) { return ToSize(datatype); } @@ -149,9 +195,18 @@ Comm::Req CommImpl::MakeReq(std::unique_ptr impl) return Comm::Req(std::move(impl)); } +Comm::Win CommImpl::MakeWin(std::unique_ptr impl) +{ + return Comm::Win(std::move(impl)); +} + CommImpl *CommImpl::Get(Comm const &comm) { return comm.m_Impl.get(); } +CommWinImpl *CommWinImpl::Get(Comm::Win const &win) { return win.m_Impl.get(); } + CommReqImpl::~CommReqImpl() = default; +CommWinImpl::~CommWinImpl() = default; + } // end namespace helper } // end namespace adios2 diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h index f8e2c7cf9e..4f4ed2fc54 100644 --- a/source/adios2/helper/adiosComm.h +++ b/source/adios2/helper/adiosComm.h @@ -19,6 +19,7 @@ namespace helper class CommImpl; class CommReqImpl; +class CommWinImpl; /** @brief Encapsulation for communication in a multi-process environment. */ class Comm @@ -26,6 +27,7 @@ class Comm public: class Req; class Status; + class Win; /** * @brief Enumeration of element-wise accumulation operations. @@ -49,6 +51,15 @@ class Comm None, }; + /** + * @brief Enumeration of locking operations. + */ + enum class LockType + { + Exclusive, + Shared + }; + /** * @brief Default constructor. Produces an empty communicator. * @@ -254,6 +265,16 @@ class Comm Req Irecv(T *buffer, const size_t count, int source, int tag, const std::string &hint = std::string()) const; + Win Win_allocate_shared(size_t size, int disp_unit, void *baseptr, + const std::string &hint = std::string()); + int Win_shared_query(Win &win, int rank, size_t *size, int *disp_unit, + void *baseptr, + const std::string &hint = std::string()); + int Win_free(Win &win, const std::string &hint = std::string()); + int Win_Lock(LockType lock_type, int rank, int assert, Win &win, + const std::string &hint = std::string()); + int Win_Unlock(int rank, Win &win, const std::string &hint = std::string()); + private: friend class CommImpl; @@ -340,6 +361,59 @@ class Comm::Status bool Cancelled = false; }; +class Comm::Win +{ +public: + /** + * @brief Default constructor. Produces an empty Win. + * + * An empty Win may not be used. + */ + Win(); + + /** + * @brief Move constructor. Moves Win state from that given. + * + * The moved-from Win is left empty and may not be used. + */ + Win(Win &&); + + /** + * @brief Deleted copy constructor. A Win may not be copied. + */ + Win(Win const &) = delete; + + ~Win(); + + /** + * @brief Move assignment. Moves Win state from that given. + * + * The moved-from Win is left empty and may not be used. + */ + Win &operator=(Win &&); + + /** + * @brief Deleted copy assignment. A Win may not be copied. + */ + Win &operator=(Win const &) = delete; + + /** + * @brief Free the Win object. + * + * On return, the Win is empty. + * For an MPI Win object this is equivalent to the call MPI_Win_free() + */ + int Free(const std::string &hint = std::string()); + +private: + friend class CommImpl; + friend class CommWinImpl; + + explicit Win(std::unique_ptr impl); + + std::unique_ptr m_Impl; +}; + class CommImpl { public: @@ -437,10 +511,23 @@ class CommImpl int source, int tag, const std::string &hint) const = 0; + virtual Comm::Win Win_allocate_shared(size_t size, int disp_unit, + void *baseptr, + const std::string &hint) const = 0; + virtual int Win_shared_query(Comm::Win &win, int rank, size_t *size, + int *disp_unit, void *baseptr, + const std::string &hint) const = 0; + virtual int Win_free(Comm::Win &win, const std::string &hint) const = 0; + virtual int Win_Lock(Comm::LockType lock_type, int rank, int assert, + Comm::Win &win, const std::string &hint) const = 0; + virtual int Win_Unlock(int rank, Comm::Win &win, + const std::string &hint) const = 0; + static size_t SizeOf(Datatype datatype); static Comm MakeComm(std::unique_ptr impl); static Comm::Req MakeReq(std::unique_ptr impl); + static Comm::Win MakeWin(std::unique_ptr impl); static CommImpl *Get(Comm const &comm); }; @@ -451,6 +538,14 @@ class CommReqImpl virtual Comm::Status Wait(const std::string &hint) = 0; }; +class CommWinImpl +{ +public: + virtual ~CommWinImpl() = 0; + virtual int Free(const std::string &hint) = 0; + static CommWinImpl *Get(Comm::Win const &win); +}; + } // end namespace helper } // end namespace adios2 diff --git a/source/adios2/helper/adiosCommDummy.cpp b/source/adios2/helper/adiosCommDummy.cpp index eddf9827d5..7c4ae8fd00 100644 --- a/source/adios2/helper/adiosCommDummy.cpp +++ b/source/adios2/helper/adiosCommDummy.cpp @@ -38,6 +38,17 @@ class CommReqImplDummy : public CommReqImpl CommReqImplDummy::~CommReqImplDummy() = default; +class CommWinImplDummy : public CommWinImpl +{ +public: + CommWinImplDummy() {} + ~CommWinImplDummy() override; + + int Free(const std::string &hint) override; +}; + +CommWinImplDummy::~CommWinImplDummy() = default; + class CommImplDummy : public CommImpl { public: @@ -104,6 +115,16 @@ class CommImplDummy : public CommImpl Comm::Req Irecv(void *buffer, size_t count, Datatype datatype, int source, int tag, const std::string &hint) const override; + + Comm::Win Win_allocate_shared(size_t size, int disp_unit, void *baseptr, + const std::string &hint) const override; + int Win_shared_query(Comm::Win &win, int rank, size_t *size, int *disp_unit, + void *baseptr, const std::string &hint) const override; + int Win_free(Comm::Win &win, const std::string &hint) const override; + int Win_Lock(Comm::LockType lock_type, int rank, int assert, Comm::Win &win, + const std::string &hint) const override; + int Win_Unlock(int rank, Comm::Win &win, + const std::string &hint) const override; }; CommImplDummy::~CommImplDummy() = default; @@ -285,12 +306,50 @@ Comm::Req CommImplDummy::Irecv(void *, size_t, Datatype, int, int, return MakeReq(std::move(req)); } +Comm::Win CommImplDummy::Win_allocate_shared(size_t size, int disp_unit, + void *baseptr, + const std::string &) const +{ + auto win = std::unique_ptr(new CommWinImplDummy()); + // TODO: How do you set the out pointer to NULL? baseptr = nullptr; + return MakeWin(std::move(win)); +} + +int CommImplDummy::Win_shared_query(Comm::Win &win, int rank, size_t *size, + int *disp_unit, void *baseptr, + const std::string &) const +{ + *size = 0; + *disp_unit = 1; + // TODO: How do you set the out pointer to NULL? baseptr = nullptr; + return 0; +} + +int CommImplDummy::Win_free(Comm::Win &win, const std::string &) const +{ + win.Free(); + return 0; +} + +int CommImplDummy::Win_Lock(Comm::LockType lock_type, int rank, int assert, + Comm::Win &win, const std::string &) const +{ + return 0; +} +int CommImplDummy::Win_Unlock(int rank, Comm::Win &win, + const std::string &) const +{ + return 0; +} + Comm::Status CommReqImplDummy::Wait(const std::string &hint) { Comm::Status status; return status; } +int CommWinImplDummy::Free(const std::string &hint) { return 0; } + Comm CommDummy() { auto comm = std::unique_ptr(new CommImplDummy()); diff --git a/source/adios2/helper/adiosCommMPI.cpp b/source/adios2/helper/adiosCommMPI.cpp index eebc4e7251..a4ae9e0893 100644 --- a/source/adios2/helper/adiosCommMPI.cpp +++ b/source/adios2/helper/adiosCommMPI.cpp @@ -43,6 +43,10 @@ const MPI_Op OpToMPI[] = { MPI_Op ToMPI(Comm::Op op) { return OpToMPI[int(op)]; } +const int LockTypeToMPI[] = {MPI_LOCK_EXCLUSIVE, MPI_LOCK_SHARED}; + +int ToMPI(Comm::LockType lock_type) { return LockTypeToMPI[int(lock_type)]; } + const MPI_Datatype DatatypeToMPI[] = { MPI_SIGNED_CHAR, MPI_CHAR, @@ -74,15 +78,16 @@ void CheckMPIReturn(const int value, const std::string &hint) } std::string error; - switch (value) + if (value == MPI_ERR_COMM) { - case MPI_ERR_COMM: error = "MPI_ERR_COMM"; - break; - case MPI_ERR_INTERN: + } + else if (value == MPI_ERR_INTERN) + { error = "MPI_ERR_INTERN"; - break; - default: + } + else + { error = "MPI_ERR number: " + std::to_string(value); } @@ -108,6 +113,19 @@ class CommReqImplMPI : public CommReqImpl CommReqImplMPI::~CommReqImplMPI() = default; +class CommWinImplMPI : public CommWinImpl +{ +public: + CommWinImplMPI() {} + ~CommWinImplMPI() override; + + int Free(const std::string &hint) override; + + MPI_Win m_Win; +}; + +CommWinImplMPI::~CommWinImplMPI() = default; + class CommImplMPI : public CommImpl { public: @@ -177,6 +195,16 @@ class CommImplMPI : public CommImpl Comm::Req Irecv(void *buffer, size_t count, Datatype datatype, int source, int tag, const std::string &hint) const override; + + Comm::Win Win_allocate_shared(size_t size, int disp_unit, void *baseptr, + const std::string &hint) const override; + int Win_shared_query(Comm::Win &win, int rank, size_t *size, int *disp_unit, + void *baseptr, const std::string &hint) const override; + int Win_free(Comm::Win &win, const std::string &hint) const override; + int Win_Lock(Comm::LockType lock_type, int rank, int assert, Comm::Win &win, + const std::string &hint) const override; + int Win_Unlock(int rank, Comm::Win &win, + const std::string &hint) const override; }; CommImplMPI::~CommImplMPI() @@ -519,6 +547,56 @@ Comm::Req CommImplMPI::Irecv(void *buffer, size_t count, Datatype datatype, return MakeReq(std::move(req)); } +Comm::Win CommImplMPI::Win_allocate_shared(size_t size, int disp_unit, + void *baseptr, + const std::string &hint) const +{ + auto w = std::unique_ptr(new CommWinImplMPI()); + MPI_Aint asize = static_cast(size); + CheckMPIReturn(MPI_Win_allocate_shared(asize, disp_unit, MPI_INFO_NULL, + m_MPIComm, baseptr, &w->m_Win), + "in call to Win_allocate_shared " + hint + "\n"); + return MakeWin(std::move(w)); +} + +int CommImplMPI::Win_shared_query(Comm::Win &win, int rank, size_t *size, + int *disp_unit, void *baseptr, + const std::string &hint) const +{ + CommWinImplMPI *w = dynamic_cast(CommWinImpl::Get(win)); + MPI_Aint asize; + int ret = MPI_Win_shared_query(w->m_Win, rank, &asize, disp_unit, baseptr); + CheckMPIReturn(ret, "in call to Win_shared_query " + hint + "\n"); + *size = static_cast(asize); + return ret; +} + +int CommImplMPI::Win_free(Comm::Win &win, const std::string &hint) const +{ + CommWinImplMPI *w = dynamic_cast(CommWinImpl::Get(win)); + int ret = MPI_Win_free(&w->m_Win); + CheckMPIReturn(ret, "in call to Win_free " + hint + "\n"); + return ret; +} + +int CommImplMPI::Win_Lock(Comm::LockType lock_type, int rank, int assert, + Comm::Win &win, const std::string &hint) const +{ + CommWinImplMPI *w = dynamic_cast(CommWinImpl::Get(win)); + int mpi_lock_type = ToMPI(lock_type); + int ret = MPI_Win_lock(mpi_lock_type, rank, assert, w->m_Win); + CheckMPIReturn(ret, "in call to Win_Lock " + hint + "\n"); + return ret; +} +int CommImplMPI::Win_Unlock(int rank, Comm::Win &win, + const std::string &hint) const +{ + CommWinImplMPI *w = dynamic_cast(CommWinImpl::Get(win)); + int ret = MPI_Win_unlock(rank, w->m_Win); + CheckMPIReturn(ret, "in call to Win_Lock " + hint + "\n"); + return ret; +} + Comm::Status CommReqImplMPI::Wait(const std::string &hint) { Comm::Status status; @@ -580,6 +658,11 @@ Comm::Status CommReqImplMPI::Wait(const std::string &hint) return status; } +int CommWinImplMPI::Free(const std::string &hint) +{ + return MPI_Win_free(&m_Win); +} + Comm CommWithMPI(MPI_Comm mpiComm) { static InitMPI const initMPI; diff --git a/source/adios2/helper/adiosJSONcomplex.h b/source/adios2/helper/adiosJSONcomplex.h index e3a9e3ae92..496336ffb5 100644 --- a/source/adios2/helper/adiosJSONcomplex.h +++ b/source/adios2/helper/adiosJSONcomplex.h @@ -11,8 +11,8 @@ #ifndef ADIOS2_HELPER_ADIOSJSONCOMPLEX_H_ #define ADIOS2_HELPER_ADIOSJSONCOMPLEX_H_ -#include "nlohmann/json.hpp" #include +#include // JSON std::complex handling namespace std diff --git a/source/adios2/helper/adiosMath.cpp b/source/adios2/helper/adiosMath.cpp index 4922f9b999..426fb4c1f6 100644 --- a/source/adios2/helper/adiosMath.cpp +++ b/source/adios2/helper/adiosMath.cpp @@ -24,10 +24,10 @@ namespace adios2 namespace helper { -size_t GetTotalSize(const Dims &dimensions) noexcept +size_t GetTotalSize(const Dims &dimensions, const size_t elementSize) noexcept { - return std::accumulate(dimensions.begin(), dimensions.end(), - static_cast(1), std::multiplies()); + return std::accumulate(dimensions.begin(), dimensions.end(), elementSize, + std::multiplies()); } bool CheckIndexRange(const int index, const int upperLimit, @@ -44,6 +44,10 @@ bool CheckIndexRange(const int index, const int upperLimit, size_t NextExponentialSize(const size_t requiredSize, const size_t currentSize, const float growthFactor) noexcept { + if (currentSize == 0) + { + return requiredSize; + } if (currentSize >= requiredSize) { return currentSize; diff --git a/source/adios2/helper/adiosMath.h b/source/adios2/helper/adiosMath.h index 175fbc28a4..a6a5c6d138 100644 --- a/source/adios2/helper/adiosMath.h +++ b/source/adios2/helper/adiosMath.h @@ -29,7 +29,8 @@ namespace helper * @param dimensions input containing size on each dimension {Nx, Ny, Nz} * @return product of all dimensions Nx * Ny * Nz */ -size_t GetTotalSize(const Dims &dimensions) noexcept; +size_t GetTotalSize(const Dims &dimensions, + const size_t elementSize = 1) noexcept; /** * Populates min and max for a selection region inside diff --git a/source/adios2/helper/adiosMemory.cpp b/source/adios2/helper/adiosMemory.cpp index 3bf8c89adc..549c2c3af4 100644 --- a/source/adios2/helper/adiosMemory.cpp +++ b/source/adios2/helper/adiosMemory.cpp @@ -297,5 +297,15 @@ size_t PaddingToAlignPointer(const void *ptr) return padSize; } +uint64_t PaddingToAlignOffset(uint64_t offset, uint64_t alignment_size) +{ + uint64_t padSize = alignment_size - (offset % alignment_size); + if (padSize == alignment_size) + { + padSize = 0; + } + return padSize; +} + } // end namespace helper } // end namespace adios2 diff --git a/source/adios2/helper/adiosMemory.h b/source/adios2/helper/adiosMemory.h index 89a17dfe85..9a7fa96ada 100644 --- a/source/adios2/helper/adiosMemory.h +++ b/source/adios2/helper/adiosMemory.h @@ -252,6 +252,10 @@ size_t PayloadSize(const T *data, const Dims &count) noexcept; */ size_t PaddingToAlignPointer(const void *ptr); +/** Calculate padding to an arbitrary offset to be aligned to + * the size alignment_size */ +uint64_t PaddingToAlignOffset(uint64_t offset, uint64_t alignment_size); + } // end namespace helper } // end namespace adios2 diff --git a/source/adios2/helper/adiosMemory.inl b/source/adios2/helper/adiosMemory.inl index de3fecea96..724ed50e3d 100644 --- a/source/adios2/helper/adiosMemory.inl +++ b/source/adios2/helper/adiosMemory.inl @@ -639,10 +639,11 @@ void Resize(std::vector &vec, const size_t dataSize, const std::string hint, // functions) and copies to the output buffer in blocks. the memory address // calculation complexity for copying each block is minimized to O(1), which is // independent of the number of dimensions. -static void NdCopyRecurDFSeqPadding(size_t curDim, const char *&inOvlpBase, - char *&outOvlpBase, Dims &inOvlpGapSize, - Dims &outOvlpGapSize, Dims &ovlpCount, - size_t &minContDim, size_t &blockSize) +static inline void +NdCopyRecurDFSeqPadding(size_t curDim, const char *&inOvlpBase, + char *&outOvlpBase, Dims &inOvlpGapSize, + Dims &outOvlpGapSize, Dims &ovlpCount, + size_t &minContDim, size_t &blockSize) { // note: all elements in and below this node are contiguous on input and // output @@ -682,7 +683,7 @@ static void NdCopyRecurDFSeqPadding(size_t curDim, const char *&inOvlpBase, // each element is minimized to average O(1), which is independent of // the number of dimensions. -static void +static inline void NdCopyRecurDFSeqPaddingRevEndian(size_t curDim, const char *&inOvlpBase, char *&outOvlpBase, Dims &inOvlpGapSize, Dims &outOvlpGapSize, Dims &ovlpCount, @@ -723,11 +724,12 @@ NdCopyRecurDFSeqPaddingRevEndian(size_t curDim, const char *&inOvlpBase, // used for buffer of Column major // the memory address calculation complexity for copying each element is // minimized to average O(1), which is independent of the number of dimensions. -static void NdCopyRecurDFNonSeqDynamic(size_t curDim, const char *inBase, - char *outBase, Dims &inRltvOvlpSPos, - Dims &outRltvOvlpSPos, Dims &inStride, - Dims &outStride, Dims &ovlpCount, - size_t elmSize) +static inline void NdCopyRecurDFNonSeqDynamic(size_t curDim, const char *inBase, + char *outBase, + Dims &inRltvOvlpSPos, + Dims &outRltvOvlpSPos, + Dims &inStride, Dims &outStride, + Dims &ovlpCount, size_t elmSize) { if (curDim == inStride.size()) { @@ -753,7 +755,7 @@ static void NdCopyRecurDFNonSeqDynamic(size_t curDim, const char *inBase, // The memory address calculation complexity for copying each element is // minimized to average O(1), which is independent of the number of dimensions. -static void NdCopyRecurDFNonSeqDynamicRevEndian( +static inline void NdCopyRecurDFNonSeqDynamicRevEndian( size_t curDim, const char *inBase, char *outBase, Dims &inRltvOvlpSPos, Dims &outRltvOvlpSPos, Dims &inStride, Dims &outStride, Dims &ovlpCount, size_t elmSize) @@ -779,10 +781,11 @@ static void NdCopyRecurDFNonSeqDynamicRevEndian( } } -static void NdCopyIterDFSeqPadding(const char *&inOvlpBase, char *&outOvlpBase, - Dims &inOvlpGapSize, Dims &outOvlpGapSize, - Dims &ovlpCount, size_t minContDim, - size_t blockSize) +static inline void NdCopyIterDFSeqPadding(const char *&inOvlpBase, + char *&outOvlpBase, + Dims &inOvlpGapSize, + Dims &outOvlpGapSize, Dims &ovlpCount, + size_t minContDim, size_t blockSize) { Dims pos(ovlpCount.size(), 0); size_t curDim = 0; @@ -810,7 +813,7 @@ static void NdCopyIterDFSeqPadding(const char *&inOvlpBase, char *&outOvlpBase, } } -static void NdCopyIterDFSeqPaddingRevEndian( +static inline void NdCopyIterDFSeqPaddingRevEndian( const char *&inOvlpBase, char *&outOvlpBase, Dims &inOvlpGapSize, Dims &outOvlpGapSize, Dims &ovlpCount, size_t minContDim, size_t blockSize, size_t elmSize, size_t numElmsPerBlock) @@ -846,10 +849,11 @@ static void NdCopyIterDFSeqPaddingRevEndian( } while (pos[curDim] == ovlpCount[curDim]); } } -static void NdCopyIterDFDynamic(const char *inBase, char *outBase, - Dims &inRltvOvlpSPos, Dims &outRltvOvlpSPos, - Dims &inStride, Dims &outStride, - Dims &ovlpCount, size_t elmSize) +static inline void NdCopyIterDFDynamic(const char *inBase, char *outBase, + Dims &inRltvOvlpSPos, + Dims &outRltvOvlpSPos, Dims &inStride, + Dims &outStride, Dims &ovlpCount, + size_t elmSize) { size_t curDim = 0; Dims pos(ovlpCount.size() + 1, 0); @@ -883,11 +887,12 @@ static void NdCopyIterDFDynamic(const char *inBase, char *outBase, } } -static void NdCopyIterDFDynamicRevEndian(const char *inBase, char *outBase, - Dims &inRltvOvlpSPos, - Dims &outRltvOvlpSPos, Dims &inStride, - Dims &outStride, Dims &ovlpCount, - size_t elmSize) +static inline void NdCopyIterDFDynamicRevEndian(const char *inBase, + char *outBase, + Dims &inRltvOvlpSPos, + Dims &outRltvOvlpSPos, + Dims &inStride, Dims &outStride, + Dims &ovlpCount, size_t elmSize) { size_t curDim = 0; Dims pos(ovlpCount.size() + 1, 0); diff --git a/source/adios2/helper/adiosNetwork.cpp b/source/adios2/helper/adiosNetwork.cpp index d966a10e6f..b35a19c0ed 100644 --- a/source/adios2/helper/adiosNetwork.cpp +++ b/source/adios2/helper/adiosNetwork.cpp @@ -25,7 +25,7 @@ #include //AvailableIpAddresses() ioctl #include //AvailableIpAddresses() close -#include +#include namespace adios2 { diff --git a/source/adios2/helper/adiosSystem.cpp b/source/adios2/helper/adiosSystem.cpp index 5acf33e091..9edaa360df 100644 --- a/source/adios2/helper/adiosSystem.cpp +++ b/source/adios2/helper/adiosSystem.cpp @@ -161,5 +161,40 @@ bool IsHDF5File(const std::string &name, helper::Comm &comm, return (flag == 1); } +char BPVersion(const std::string &name, helper::Comm &comm, + const std::vector &transportsParameters) noexcept +{ + char version[] = {'X'}; + std::string versionFileName = name + PathSeparator + ".bpversion"; + if (!comm.Rank()) + { + try + { + transportman::TransportMan tm(comm); + if (transportsParameters.empty()) + { + std::vector defaultTransportParameters(1); + defaultTransportParameters[0]["transport"] = "File"; + tm.OpenFiles({versionFileName}, adios2::Mode::Read, + defaultTransportParameters, false); + } + else + { + tm.OpenFiles({versionFileName}, adios2::Mode::Read, + transportsParameters, false); + } + if (tm.GetFileSize(0) > 0) + { + tm.ReadFile(version, 1, 0); + } + tm.CloseFiles(); + } + catch (std::ios_base::failure &) + { + } + } + return version[0]; +} + } // end namespace helper } // end namespace adios2 diff --git a/source/adios2/helper/adiosSystem.h b/source/adios2/helper/adiosSystem.h index 56f32aab32..5c9c8e68f3 100644 --- a/source/adios2/helper/adiosSystem.h +++ b/source/adios2/helper/adiosSystem.h @@ -77,7 +77,8 @@ int ExceptionToError(const std::string &function); bool IsHDF5File(const std::string &name, helper::Comm &comm, const std::vector &transportsParameters) noexcept; - +char BPVersion(const std::string &name, helper::Comm &comm, + const std::vector &transportsParameters) noexcept; } // end namespace helper } // end namespace adios2 diff --git a/source/adios2/helper/adiosType.cpp b/source/adios2/helper/adiosType.cpp index 8cdc0c24c0..b84a8d94cb 100644 --- a/source/adios2/helper/adiosType.cpp +++ b/source/adios2/helper/adiosType.cpp @@ -90,6 +90,18 @@ DataType GetDataTypeFromString(std::string const &type) noexcept return DataType::None; } +size_t GetDataTypeSize(DataType type) +{ +#define declare_type(T) \ + if (type == helper::GetDataType()) \ + { \ + return sizeof(T); \ + } + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + throw(std::runtime_error("unknown data type")); +} + std::string DimsToCSV(const Dims &dimensions) noexcept { std::string dimsCSV; diff --git a/source/adios2/helper/adiosType.h b/source/adios2/helper/adiosType.h index fe1193c51c..01ab9ebb13 100644 --- a/source/adios2/helper/adiosType.h +++ b/source/adios2/helper/adiosType.h @@ -109,6 +109,8 @@ DataType GetDataType() noexcept; */ DataType GetDataTypeFromString(std::string const &) noexcept; +size_t GetDataTypeSize(DataType type); + /** * Converts a vector of dimensions to a CSV string * @param dims vector of dimensions diff --git a/source/adios2/helper/adiosType.inl b/source/adios2/helper/adiosType.inl index 23eb7d02f8..280255230c 100644 --- a/source/adios2/helper/adiosType.inl +++ b/source/adios2/helper/adiosType.inl @@ -179,7 +179,7 @@ inline std::string VectorToCSV(const std::vector &input) noexcept } std::ostringstream valueSS; - for (const auto value : input) + for (const auto &value : input) { valueSS << "\"" << value << "\", "; } @@ -200,7 +200,7 @@ inline std::string VectorToCSV(const std::vector &input) noexcept } \ \ std::ostringstream valueSS; \ - for (const auto value : input) \ + for (const auto &value : input) \ { \ const int valueInt = static_cast(value); \ valueSS << valueInt << ", "; \ @@ -223,7 +223,7 @@ inline std::string VectorToCSV(const std::vector &input) noexcept } std::ostringstream valueSS; - for (const auto value : input) + for (const auto &value : input) { valueSS << value << ", "; } diff --git a/source/adios2/helper/adiosXMLUtil.cpp b/source/adios2/helper/adiosXMLUtil.cpp index 74e0a9b990..61f49600e0 100644 --- a/source/adios2/helper/adiosXMLUtil.cpp +++ b/source/adios2/helper/adiosXMLUtil.cpp @@ -125,7 +125,7 @@ adios2::Params XMLGetParameters(const pugi::xml_node &node, ", " + hint); Params parameters; - for (const pugi::xml_node paramNode : node.children("parameter")) + for (const pugi::xml_node ¶mNode : node.children("parameter")) { const std::unique_ptr key = XMLAttribute("key", paramNode, errorMessage); diff --git a/source/adios2/operator/compress/CompressBZIP2.cpp b/source/adios2/operator/compress/CompressBZIP2.cpp index caef10aeda..5c85719b1a 100644 --- a/source/adios2/operator/compress/CompressBZIP2.cpp +++ b/source/adios2/operator/compress/CompressBZIP2.cpp @@ -32,21 +32,32 @@ CompressBZIP2::CompressBZIP2(const Params ¶meters) { } -size_t CompressBZIP2::BufferMaxSize(const size_t sizeIn) const +size_t CompressBZIP2::Compress(const char *dataIn, const Dims &blockStart, + const Dims &blockCount, DataType type, + char *bufferOut, const Params ¶meters) { - return static_cast(std::ceil(1.1 * sizeIn) + 600); -} -size_t CompressBZIP2::Compress(const void *dataIn, const Dims &dimensions, - const size_t elementSize, DataType type, - void *bufferOut, const Params ¶meters, - Params &info) const -{ - // defaults + const uint8_t bufferVersion = 1; + unsigned int destOffset = 0; + + // Universal operator metadata + PutParameter(bufferOut, destOffset, OperatorType::BZIP2); + PutParameter(bufferOut, destOffset, bufferVersion); + PutParameter(bufferOut, destOffset, static_cast(0)); + // Universal operator metadata end + + const size_t sizeIn = + helper::GetTotalSize(blockCount, helper::GetDataTypeSize(type)); + const size_t batches = sizeIn / DefaultMaxFileBatchSize + 1; + + // bzip2 V1 metadata + PutParameter(bufferOut, destOffset, sizeIn); + PutParameter(bufferOut, destOffset, batches); + // bzip2 V1 metadata end + int blockSize100k = 1; int verbosity = 0; int workFactor = 0; - if (!parameters.empty()) { const std::string hint(" in call to CompressBZIP2 Compress " + @@ -56,7 +67,6 @@ size_t CompressBZIP2::Compress(const void *dataIn, const Dims &dimensions, helper::SetParameterValueInt("verbosity", parameters, verbosity, hint); helper::SetParameterValueInt("workFactor", parameters, workFactor, hint); - if (blockSize100k < 1 || blockSize100k > 9) { throw std::invalid_argument( @@ -68,42 +78,35 @@ size_t CompressBZIP2::Compress(const void *dataIn, const Dims &dimensions, } } - const size_t sizeIn = - static_cast(helper::GetTotalSize(dimensions) * elementSize); - - const size_t batches = sizeIn / DefaultMaxFileBatchSize + 1; - info["batches"] = std::to_string(batches); - - unsigned int destOffset = 0; unsigned int sourceOffset = 0; + unsigned int batchInfoOffset = destOffset; + destOffset += batches * 4 * sizeof(unsigned int); for (size_t b = 0; b < batches; ++b) { - char *source = - const_cast(reinterpret_cast(dataIn)) + - sourceOffset; + char *source = const_cast(dataIn) + sourceOffset; const size_t batchSize = (b == batches - 1) ? sizeIn % DefaultMaxFileBatchSize : DefaultMaxFileBatchSize; unsigned int sourceLen = static_cast(batchSize); - char *dest = reinterpret_cast(bufferOut) + destOffset; + char *dest = bufferOut + destOffset; unsigned int destLen = sourceLen; int status = BZ2_bzBuffToBuffCompress(dest, &destLen, source, sourceLen, blockSize100k, verbosity, workFactor); - const std::string bStr = std::to_string(b); + CheckStatus(status, "in call to ADIOS2 BZIP2 Compress batch " + + std::to_string(b) + "\n"); - CheckStatus(status, "in call to ADIOS2 BZIP2 Compress batch " + bStr); - - // record metadata info - info["OriginalOffset_" + bStr] = std::to_string(sourceOffset); - info["OriginalSize_" + bStr] = std::to_string(sourceLen); - info["CompressedOffset_" + bStr] = std::to_string(destOffset); - info["CompressedSize_" + bStr] = std::to_string(destLen); + // bzip2 V1 metadata + PutParameter(bufferOut, batchInfoOffset, sourceOffset); + PutParameter(bufferOut, batchInfoOffset, sourceLen); + PutParameter(bufferOut, batchInfoOffset, destOffset); + PutParameter(bufferOut, batchInfoOffset, destLen); + // bzip2 V1 metadata end sourceOffset += sourceLen; destOffset += destLen; @@ -112,63 +115,88 @@ size_t CompressBZIP2::Compress(const void *dataIn, const Dims &dimensions, return destOffset; } -size_t CompressBZIP2::Decompress(const void *bufferIn, const size_t sizeIn, - void *dataOut, const size_t sizeOut, - Params &info) const +size_t CompressBZIP2::DecompressV1(const char *bufferIn, const size_t sizeIn, + char *dataOut) { - // TODO: leave defaults at zero? + // Do NOT remove even if the buffer version is updated. Data might be still + // in lagacy formats. This function must be kept for backward compatibility. + // If a newer buffer format is implemented, create another function, e.g. + // DecompressV2 and keep this function for decompressing lagacy data. + + size_t bufferInOffset = 4; // skip the first four bytes + + size_t sizeOut = GetParameter(bufferIn, bufferInOffset); + size_t batches = GetParameter(bufferIn, bufferInOffset); + int small = 0; int verbosity = 0; - auto itBatches = info.find("batches"); - const size_t batches = - (itBatches == info.end()) - ? 1 - : static_cast(helper::StringTo( - itBatches->second, - "when extracting batches in ADIOS2 BZIP2 Decompress")); - size_t expectedSizeOut = 0; for (size_t b = 0; b < batches; ++b) { - const std::string bStr = std::to_string(b); + unsigned int destOffset = + GetParameter(bufferIn, bufferInOffset); - const size_t destOffset = - static_cast(helper::StringTo( - info.at("OriginalOffset_" + bStr), - "when extracting batches in ADIOS2 BZIP2 Decompress")); + bufferInOffset += sizeof(unsigned int); - char *dest = reinterpret_cast(dataOut) + destOffset; + char *dest = dataOut + destOffset; const size_t batchSize = (b == batches - 1) ? sizeOut % DefaultMaxFileBatchSize : DefaultMaxFileBatchSize; + unsigned int destLen = static_cast(batchSize); - const size_t sourceOffset = - static_cast(helper::StringTo( - info.at("CompressedOffset_" + bStr), - "when extracting batches in ADIOS2 BZIP2 Decompress")); - char *source = - const_cast(reinterpret_cast(bufferIn)) + - sourceOffset; + unsigned int sourceOffset = + GetParameter(bufferIn, bufferInOffset); + + char *source = const_cast(bufferIn) + sourceOffset; unsigned int sourceLen = - static_cast(helper::StringTo( - info.at("CompressedSize_" + bStr), - "when extracting batches in ADIOS2 BZIP2 Decompress")); + GetParameter(bufferIn, bufferInOffset); int status = BZ2_bzBuffToBuffDecompress(dest, &destLen, source, sourceLen, small, verbosity); - CheckStatus(status, "in call to ADIOS2 BZIP2 Decompress\n"); - // TODO verify size of decompressed buffer + CheckStatus(status, "in call to ADIOS2 BZIP2 Decompress batch " + + std::to_string(b) + "\n"); expectedSizeOut += static_cast(destLen); } - return expectedSizeOut; + if (expectedSizeOut != sizeOut) + { + throw("corrupted bzip2 buffer"); + } + + return sizeOut; +} +size_t CompressBZIP2::Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) +{ + size_t bufferInOffset = 1; // skip operator type + const uint8_t bufferVersion = + GetParameter(bufferIn, bufferInOffset); + bufferInOffset += 2; // skip two reserved bytes + + if (bufferVersion == 1) + { + // pass in the whole buffer as there is absolute positions saved in the + // buffer to determine the offsets and lengths for batches + return DecompressV1(bufferIn, sizeIn, dataOut); + } + else if (bufferVersion == 2) + { + // TODO: if a Version 2 bzip2 buffer is being implemented, put it here + // and keep the DecompressV1 routine for backward compatibility + } + else + { + throw("unknown bzip2 buffer version"); + } + + return 0; } bool CompressBZIP2::IsDataTypeValid(const DataType type) const { return true; } diff --git a/source/adios2/operator/compress/CompressBZIP2.h b/source/adios2/operator/compress/CompressBZIP2.h index 80a4acba9c..6e9388f914 100644 --- a/source/adios2/operator/compress/CompressBZIP2.h +++ b/source/adios2/operator/compress/CompressBZIP2.h @@ -31,33 +31,27 @@ class CompressBZIP2 : public Operator ~CompressBZIP2() = default; - size_t BufferMaxSize(const size_t sizeIn) const final; - /** - * Compression signature for legacy libraries that use void* * @param dataIn - * @param dimensions + * @param blockStart + * @param blockCount * @param type * @param bufferOut * @param parameters - * @return size of compressed buffer in bytes + * @return size of compressed buffer */ - size_t Compress(const void *dataIn, const Dims &dimensions, - const size_t elementSize, DataType type, void *bufferOut, - const Params ¶meters, Params &info) const final; + size_t Compress(const char *dataIn, const Dims &blockStart, + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) final; - using Operator::Decompress; /** - * Decompression signature for legacy libraries that use void* * @param bufferIn * @param sizeIn * @param dataOut - * @param dimensions - * @param type - * @return size of decompressed buffer in bytes + * @return size of decompressed buffer */ - size_t Decompress(const void *bufferIn, const size_t sizeIn, void *dataOut, - const size_t sizeOut, Params &info) const final; + size_t Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) final; bool IsDataTypeValid(const DataType type) const final; @@ -68,6 +62,18 @@ class CompressBZIP2 : public Operator * @param hint extra exception information */ void CheckStatus(const int status, const std::string hint) const; + + /** + * Decompress function for V1 buffer. Do NOT remove even if the buffer + * version is updated. Data might be still in lagacy formats. This function + * must be kept for backward compatibility + * @param bufferIn : compressed data buffer (V1 only) + * @param sizeIn : number of bytes in bufferIn + * @param dataOut : decompressed data buffer + * @return : number of bytes in dataOut + */ + size_t DecompressV1(const char *bufferIn, const size_t sizeIn, + char *dataOut); }; } // end namespace compress diff --git a/source/adios2/operator/compress/CompressBlosc.cpp b/source/adios2/operator/compress/CompressBlosc.cpp index 434e6ef0fe..eee5b9282e 100644 --- a/source/adios2/operator/compress/CompressBlosc.cpp +++ b/source/adios2/operator/compress/CompressBlosc.cpp @@ -42,35 +42,35 @@ CompressBlosc::CompressBlosc(const Params ¶meters) { } -size_t CompressBlosc::BufferMaxSize(const size_t sizeIn) const +size_t CompressBlosc::Compress(const char *dataIn, const Dims &blockStart, + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) { - const size_t maxInputPerChunk = BLOSC_MAX_BUFFERSIZE; - const size_t numFullChunks = sizeIn / maxInputPerChunk; - const size_t sizeLastChunk = sizeIn % maxInputPerChunk; + size_t bufferOutOffset = 0; + const uint8_t bufferVersion = 1; - const size_t maxOutputPerChunk = maxInputPerChunk + BLOSC_MAX_OVERHEAD; - const size_t maxOutputLastChunk = sizeLastChunk + BLOSC_MAX_OVERHEAD; + // Universal operator metadata + PutParameter(bufferOut, bufferOutOffset, OperatorType::BLOSC); + PutParameter(bufferOut, bufferOutOffset, bufferVersion); + PutParameter(bufferOut, bufferOutOffset, static_cast(0)); + // Universal operator metadata end - /* DataHeader is used to detect of old format which can only handle - * BLOSC_MAX_BUFFERSIZE (<2GiB) or the new adios2 chunked blosc format is - * used. - */ - const size_t maxRquiredDataMem = maxOutputPerChunk * numFullChunks + - maxOutputLastChunk + sizeof(DataHeader); - - return maxRquiredDataMem; -} - -size_t CompressBlosc::Compress(const void *dataIn, const Dims &dimensions, - const size_t elementSize, DataType type, - void *bufferOut, const Params ¶meters, - Params &info) const -{ const size_t sizeIn = - static_cast(helper::GetTotalSize(dimensions) * elementSize); + helper::GetTotalSize(blockCount, helper::GetDataTypeSize(type)); + + // blosc V1 metadata + PutParameter(bufferOut, bufferOutOffset, sizeIn); + PutParameter(bufferOut, bufferOutOffset, + static_cast(BLOSC_VERSION_MAJOR)); + PutParameter(bufferOut, bufferOutOffset, + static_cast(BLOSC_VERSION_MINOR)); + PutParameter(bufferOut, bufferOutOffset, + static_cast(BLOSC_VERSION_RELEASE)); + // blosc V1 metadata end bool useMemcpy = false; - /* input size under this bound would not compressed */ + + // input size under this bound will not compress size_t thresholdSize = 128; blosc_init(); @@ -150,21 +150,17 @@ size_t CompressBlosc::Compress(const void *dataIn, const Dims &dimensions, } // write header to detect new compression format (set first 8 byte to zero) - DataHeader *headerPtr = reinterpret_cast(bufferOut); + DataHeader *headerPtr = + reinterpret_cast(bufferOut + bufferOutOffset); // set default header *headerPtr = DataHeader{}; + bufferOutOffset += sizeof(DataHeader); - const uint8_t *inputDataBuff = reinterpret_cast(dataIn); - - int32_t typesize = elementSize; + int32_t typesize = helper::GetDataTypeSize(type); if (typesize > BLOSC_MAX_TYPESIZE) typesize = 1; - uint8_t *outputBuff = reinterpret_cast(bufferOut); - outputBuff += sizeof(DataHeader); - - size_t currentOutputSize = 0u; size_t inputOffset = 0u; if (sizeIn < thresholdSize) @@ -197,15 +193,13 @@ size_t CompressBlosc::Compress(const void *dataIn, const Dims &dimensions, bloscSize_t maxChunkSize = maxIntputSize + BLOSC_MAX_OVERHEAD; - const uint8_t *in_ptr = inputDataBuff + inputOffset; - uint8_t *out_ptr = outputBuff + currentOutputSize; - bloscSize_t compressedChunkSize = blosc_compress(compressionLevel, doShuffle, typesize, - maxIntputSize, in_ptr, out_ptr, maxChunkSize); + maxIntputSize, dataIn + inputOffset, + bufferOut + bufferOutOffset, maxChunkSize); if (compressedChunkSize > 0) - currentOutputSize += static_cast(compressedChunkSize); + bufferOutOffset += static_cast(compressedChunkSize); else { // something went wrong with the compression switch to memcopy @@ -226,41 +220,93 @@ size_t CompressBlosc::Compress(const void *dataIn, const Dims &dimensions, if (useMemcpy) { - std::memcpy(outputBuff, inputDataBuff, sizeIn); - currentOutputSize = sizeIn; + std::memcpy(bufferOut + bufferOutOffset, dataIn + inputOffset, sizeIn); + bufferOutOffset += sizeIn; headerPtr->SetNumChunks(0u); } blosc_destroy(); - return currentOutputSize + sizeof(DataHeader); + return bufferOutOffset; } -size_t CompressBlosc::Decompress(const void *bufferIn, const size_t sizeIn, - void *dataOut, const size_t sizeOut, - Params &info) const +size_t CompressBlosc::DecompressV1(const char *bufferIn, const size_t sizeIn, + char *dataOut) { - assert(sizeIn >= sizeof(DataHeader)); + // Do NOT remove even if the buffer version is updated. Data might be still + // in lagacy formats. This function must be kept for backward compatibility. + // If a newer buffer format is implemented, create another function, e.g. + // DecompressV2 and keep this function for decompressing lagacy data. + + size_t bufferInOffset = 0; + size_t sizeOut = GetParameter(bufferIn, bufferInOffset); + + m_VersionInfo = + " Data is compressed using BLOSC Version " + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + + ". Please make sure a compatible version is used for decompression."; + + if (sizeIn - bufferInOffset < sizeof(DataHeader)) + { + throw("corrupted blosc buffer header." + m_VersionInfo + "\n"); + } const bool isChunked = - reinterpret_cast(bufferIn)->IsChunked(); + reinterpret_cast(bufferIn + bufferInOffset) + ->IsChunked(); - size_t decompressedSize = 0u; + size_t decompressedSize = 0; if (isChunked) + { decompressedSize = - DecompressChunkedFormat(bufferIn, sizeIn, dataOut, sizeOut, info); + DecompressChunkedFormat(bufferIn + bufferInOffset, + sizeIn - bufferInOffset, dataOut, sizeOut); + } else + { decompressedSize = - DecompressOldFormat(bufferIn, sizeIn, dataOut, sizeOut, info); + DecompressOldFormat(bufferIn + bufferInOffset, + sizeIn - bufferInOffset, dataOut, sizeOut); + } + if (decompressedSize != sizeOut) + { + throw("corrupted blosc buffer." + m_VersionInfo + "\n"); + } + return sizeOut; +} + +size_t CompressBlosc::Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) +{ + size_t bufferInOffset = 1; // skip operator type + const uint8_t bufferVersion = + GetParameter(bufferIn, bufferInOffset); + bufferInOffset += 2; // skip two reserved bytes + + if (bufferVersion == 1) + { + return DecompressV1(bufferIn + bufferInOffset, sizeIn - bufferInOffset, + dataOut); + } + else if (bufferVersion == 2) + { + // TODO: if a Version 2 blosc buffer is being implemented, put it here + // and keep the DecompressV1 routine for backward compatibility + } + else + { + throw("unknown blosc buffer version"); + } - return decompressedSize; + return 0; } bool CompressBlosc::IsDataTypeValid(const DataType type) const { return true; } -size_t CompressBlosc::DecompressChunkedFormat(const void *bufferIn, +size_t CompressBlosc::DecompressChunkedFormat(const char *bufferIn, const size_t sizeIn, - void *dataOut, - const size_t sizeOut, - Params &info) const + char *dataOut, + const size_t sizeOut) const { const DataHeader *dataPtr = reinterpret_cast(bufferIn); uint32_t num_chunks = dataPtr->GetNumChunks(); @@ -273,20 +319,18 @@ size_t CompressBlosc::DecompressChunkedFormat(const void *bufferIn, size_t inputOffset = 0u; size_t currentOutputSize = 0u; - const uint8_t *inputDataBuff = - reinterpret_cast(bufferIn) + sizeof(DataHeader); + const char *inputDataBuff = bufferIn + sizeof(DataHeader); size_t uncompressedSize = sizeOut; if (isCompressed) { blosc_init(); - uint8_t *outputBuff = reinterpret_cast(dataOut); while (inputOffset < inputDataSize) { /* move over the size of the compressed data */ - const uint8_t *in_ptr = inputDataBuff + inputOffset; + const char *in_ptr = inputDataBuff + inputOffset; /** read the size of the compress block from the blosc meta data * @@ -301,10 +345,10 @@ size_t CompressBlosc::DecompressChunkedFormat(const void *bufferIn, * * we need only the compressed size ( source address + 12 byte) */ - bloscSize_t max_inputDataSize = - *reinterpret_cast(in_ptr + 12u); + bloscSize_t max_inputDataSize; + std::memcpy(&max_inputDataSize, in_ptr + 12, sizeof(bloscSize_t)); - uint8_t *out_ptr = outputBuff + currentOutputSize; + char *out_ptr = dataOut + currentOutputSize; size_t outputChunkSize = std::min(uncompressedSize - currentOutputSize, @@ -321,7 +365,8 @@ size_t CompressBlosc::DecompressChunkedFormat(const void *bufferIn, { throw std::runtime_error( "ERROR: ADIOS2 Blosc Decompress failed. Decompressed chunk " - "results in zero decompressed bytes.\n"); + "results in zero decompressed bytes." + + m_VersionInfo + "\n"); } inputOffset += static_cast(max_inputDataSize); } @@ -340,10 +385,9 @@ size_t CompressBlosc::DecompressChunkedFormat(const void *bufferIn, return currentOutputSize; } -size_t CompressBlosc::DecompressOldFormat(const void *bufferIn, - const size_t sizeIn, void *dataOut, - const size_t sizeOut, - Params &info) const +size_t CompressBlosc::DecompressOldFormat(const char *bufferIn, + const size_t sizeIn, char *dataOut, + const size_t sizeOut) const { blosc_init(); const int decompressedSize = blosc_decompress(bufferIn, dataOut, sizeOut); diff --git a/source/adios2/operator/compress/CompressBlosc.h b/source/adios2/operator/compress/CompressBlosc.h index 3e1f3b7705..8b05df38be 100644 --- a/source/adios2/operator/compress/CompressBlosc.h +++ b/source/adios2/operator/compress/CompressBlosc.h @@ -43,33 +43,28 @@ class CompressBlosc : public Operator ~CompressBlosc() = default; - size_t BufferMaxSize(const size_t sizeIn) const final; - /** - * Compression signature for legacy libraries that use void* * @param dataIn - * @param dimensions + * @param blockStart + * @param blockCount * @param type * @param bufferOut format will be: 'DataHeader ; (BloscCompressedChunk | * UncompressedData), [ BloscCompressedChunk, ...]' * @param parameters * @return size of compressed buffer in bytes */ - size_t Compress(const void *dataIn, const Dims &dimensions, - const size_t elementSize, DataType type, void *bufferOut, - const Params ¶meters, Params &info) const final; + size_t Compress(const char *dataIn, const Dims &blockStart, + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) final; /** - * Decompression signature for legacy libraries that use void* * @param bufferIn * @param sizeIn * @param dataOut - * @param dimensions - * @param type - * @return size of decompressed buffer in bytes + * @return size of decompressed buffer */ - size_t Decompress(const void *bufferIn, const size_t sizeIn, void *dataOut, - const size_t sizeOut, Params &info) const final; + size_t Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) final; bool IsDataTypeValid(const DataType type) const final; @@ -77,15 +72,25 @@ class CompressBlosc : public Operator using bloscSize_t = int32_t; /** Decompress chunked data */ - size_t DecompressChunkedFormat(const void *bufferIn, const size_t sizeIn, - void *dataOut, const size_t sizeOut, - Params &info) const; + size_t DecompressChunkedFormat(const char *bufferIn, const size_t sizeIn, + char *dataOut, const size_t sizeOut) const; /** Decompress data written before ADIOS2 supported large variables larger * 2GiB. */ - size_t DecompressOldFormat(const void *bufferIn, const size_t sizeIn, - void *dataOut, const size_t sizeOut, - Params &info) const; + size_t DecompressOldFormat(const char *bufferIn, const size_t sizeIn, + char *dataOut, const size_t sizeOut) const; + + /** + * Decompress function for V1 buffer. Do NOT remove even if the buffer + * version is updated. Data might be still in lagacy formats. This function + * must be kept for backward compatibility + * @param bufferIn : compressed data buffer (V1 only) + * @param sizeIn : number of bytes in bufferIn + * @param dataOut : decompressed data buffer + * @return : number of bytes in dataOut + */ + size_t DecompressV1(const char *bufferIn, const size_t sizeIn, + char *dataOut); ADIOS2_CLASS_PACKED(DataHeader) { @@ -121,6 +126,8 @@ class CompressBlosc : public Operator static const std::map m_Shuffles; static const std::set m_Compressors; + + std::string m_VersionInfo; }; } // end namespace compress diff --git a/source/adios2/operator/compress/CompressLibPressio.cpp b/source/adios2/operator/compress/CompressLibPressio.cpp index 868e56c938..7e696ad095 100644 --- a/source/adios2/operator/compress/CompressLibPressio.cpp +++ b/source/adios2/operator/compress/CompressLibPressio.cpp @@ -285,20 +285,40 @@ CompressLibPressio::CompressLibPressio(const Params ¶meters) { } -size_t CompressLibPressio::BufferMaxSize(const size_t sizeIn) const +size_t CompressLibPressio::Compress(const char *dataIn, const Dims &blockStart, + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) { - return static_cast(std::ceil(1.1 * sizeIn) + 600); -} + const uint8_t bufferVersion = 1; + size_t bufferOutOffset = 0; -size_t CompressLibPressio::Compress(const void *dataIn, const Dims &dimensions, - const size_t /*elementSize*/, - DataType varType, void *bufferOut, - const Params ¶meters, - Params &info) const -{ - auto inputs_dims = adios_to_libpressio_dims(dimensions); + // Universal operator metadata + PutParameter(bufferOut, bufferOutOffset, OperatorType::LIBPRESSIO); + PutParameter(bufferOut, bufferOutOffset, bufferVersion); + PutParameter(bufferOut, bufferOutOffset, static_cast(0)); + // Universal operator metadata end + + const size_t ndims = blockCount.size(); + + // zfp V1 metadata + PutParameter(bufferOut, bufferOutOffset, ndims); + for (const auto &d : blockCount) + { + PutParameter(bufferOut, bufferOutOffset, d); + } + PutParameter(bufferOut, bufferOutOffset, type); + PutParameter(bufferOut, bufferOutOffset, + static_cast(pressio_major_version())); + PutParameter(bufferOut, bufferOutOffset, + static_cast(pressio_minor_version())); + PutParameter(bufferOut, bufferOutOffset, + static_cast(pressio_patch_version())); + PutParameters(bufferOut, bufferOutOffset, parameters); + // zfp V1 metadata end + + auto inputs_dims = adios_to_libpressio_dims(blockCount); pressio_data *input_buf = pressio_data_new_nonowning( - adios_to_libpressio_dtype(varType), const_cast(dataIn), + adios_to_libpressio_dtype(type), const_cast(dataIn), inputs_dims.size(), inputs_dims.data()); pressio_data *output_buf = pressio_data_new_empty(pressio_byte_dtype, 0, nullptr); @@ -324,36 +344,59 @@ size_t CompressLibPressio::Compress(const void *dataIn, const Dims &dimensions, size_t size_in_bytes = 0; void *bytes = pressio_data_ptr(output_buf, &size_in_bytes); - memcpy(bufferOut, bytes, size_in_bytes); + memcpy(bufferOut + bufferOutOffset, bytes, size_in_bytes); + bufferOutOffset += size_in_bytes; pressio_data_free(input_buf); pressio_data_free(output_buf); - return static_cast(size_in_bytes); + return bufferOutOffset; } -size_t CompressLibPressio::Decompress(const void *bufferIn, const size_t sizeIn, - void *dataOut, const Dims &dimensions, - DataType varType, - const Params ¶ms) const +size_t CompressLibPressio::DecompressV1(const char *bufferIn, + const size_t sizeIn, char *dataOut) { - std::vector dims = adios_to_libpressio_dims(dimensions); + // Do NOT remove even if the buffer version is updated. Data might be still + // in lagacy formats. This function must be kept for backward compatibility. + // If a newer buffer format is implemented, create another function, e.g. + // DecompressV2 and keep this function for decompressing lagacy data. + + size_t bufferInOffset = 0; + + const size_t ndims = GetParameter(bufferIn, bufferInOffset); + Dims blockCount(ndims); + for (size_t i = 0; i < ndims; ++i) + { + blockCount[i] = GetParameter(bufferIn, bufferInOffset); + } + const DataType type = GetParameter(bufferIn, bufferInOffset); + m_VersionInfo = + " Data is compressed using LibPressio Version " + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + + ". Please make sure a compatible version is used for decompression."; + const Params parameters = GetParameters(bufferIn, bufferInOffset); + + std::vector dims = adios_to_libpressio_dims(blockCount); pressio_data *output_buf = pressio_data_new_owning( - adios_to_libpressio_dtype(varType), dims.size(), dims.data()); + adios_to_libpressio_dtype(type), dims.size(), dims.data()); + size_t newSizeIn = sizeIn - bufferInOffset; pressio_data *input_buf = pressio_data_new_nonowning( - pressio_byte_dtype, const_cast(bufferIn), 1, &sizeIn); + pressio_byte_dtype, const_cast(bufferIn + bufferInOffset), 1, + &newSizeIn); pressio_compressor *compressor = nullptr; try { - compressor = adios_to_libpressio_compressor(params); + compressor = adios_to_libpressio_compressor(parameters); } catch (std::exception &) { pressio_data_free(input_buf); pressio_data_free(output_buf); - throw; + throw std::runtime_error(m_VersionInfo + "\n"); } if (pressio_compressor_decompress(compressor, input_buf, output_buf) != 0) @@ -362,7 +405,7 @@ size_t CompressLibPressio::Decompress(const void *bufferIn, const size_t sizeIn, pressio_data_free(output_buf); throw std::runtime_error( std::string("pressio_compressor_decompress: ") + - pressio_compressor_error_msg(compressor)); + pressio_compressor_error_msg(compressor) + m_VersionInfo + "\n"); } size_t size_in_bytes = 0; @@ -374,6 +417,32 @@ size_t CompressLibPressio::Decompress(const void *bufferIn, const size_t sizeIn, return size_in_bytes; } +size_t CompressLibPressio::Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) +{ + size_t bufferInOffset = 1; // skip operator type + const uint8_t bufferVersion = + GetParameter(bufferIn, bufferInOffset); + bufferInOffset += 2; // skip two reserved bytes + + if (bufferVersion == 1) + { + return DecompressV1(bufferIn + bufferInOffset, sizeIn - bufferInOffset, + dataOut); + } + else if (bufferVersion == 2) + { + // TODO: if a Version 2 LibPressio buffer is being implemented, put it + // here and keep the DecompressV1 routine for backward compatibility + } + else + { + throw("unknown LibPressio buffer version"); + } + + return 0; +} + bool CompressLibPressio::IsDataTypeValid(const DataType type) const { #define declare_type(T) \ diff --git a/source/adios2/operator/compress/CompressLibPressio.h b/source/adios2/operator/compress/CompressLibPressio.h index 923400c055..925a17791a 100644 --- a/source/adios2/operator/compress/CompressLibPressio.h +++ b/source/adios2/operator/compress/CompressLibPressio.h @@ -31,37 +31,44 @@ class CompressLibPressio : public Operator ~CompressLibPressio() = default; - size_t BufferMaxSize(const size_t sizeIn) const final; - /** - * Compression signature for legacy libraries that use void* * @param dataIn - * @param dimensions + * @param blockStart + * @param blockCount * @param type * @param bufferOut * @param parameters - * @return size of compressed buffer in bytes + * @return size of compressed buffer */ - size_t Compress(const void *dataIn, const Dims &dimensions, - const size_t elementSize, DataType type, void *bufferOut, - const Params ¶meters, Params &info) const final; - - using Operator::Decompress; + size_t Compress(const char *dataIn, const Dims &blockStart, + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) final; /** - * Wrapper around zfp decompression * @param bufferIn * @param sizeIn * @param dataOut - * @param dimensions - * @param type - * @return size of decompressed data in dataOut + * @return size of decompressed buffer */ - size_t Decompress(const void *bufferIn, const size_t sizeIn, void *dataOut, - const Dims &dimensions, DataType type, - const Params ¶meters) const final; + size_t Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) final; bool IsDataTypeValid(const DataType type) const final; + +private: + /** + * Decompress function for V1 buffer. Do NOT remove even if the buffer + * version is updated. Data might be still in lagacy formats. This function + * must be kept for backward compatibility + * @param bufferIn : compressed data buffer (V1 only) + * @param sizeIn : number of bytes in bufferIn + * @param dataOut : decompressed data buffer + * @return : number of bytes in dataOut + */ + size_t DecompressV1(const char *bufferIn, const size_t sizeIn, + char *dataOut); + + std::string m_VersionInfo; }; } // end namespace compress diff --git a/source/adios2/operator/compress/CompressMGARD.cpp b/source/adios2/operator/compress/CompressMGARD.cpp index 070ae71717..885239ffb8 100644 --- a/source/adios2/operator/compress/CompressMGARD.cpp +++ b/source/adios2/operator/compress/CompressMGARD.cpp @@ -9,12 +9,10 @@ */ #include "CompressMGARD.h" - -#include //std::memcpy - -#include - #include "adios2/helper/adiosFunctions.h" +#include +#include +#include namespace adios2 { @@ -28,12 +26,35 @@ CompressMGARD::CompressMGARD(const Params ¶meters) { } -size_t CompressMGARD::Compress(const void *dataIn, const Dims &dimensions, - const size_t elementSize, DataType type, - void *bufferOut, const Params ¶meters, - Params &info) const +size_t CompressMGARD::Compress(const char *dataIn, const Dims &blockStart, + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) { - const size_t ndims = dimensions.size(); + const uint8_t bufferVersion = 1; + size_t bufferOutOffset = 0; + + // Universal operator metadata + PutParameter(bufferOut, bufferOutOffset, OperatorType::MGARD); + PutParameter(bufferOut, bufferOutOffset, bufferVersion); + PutParameter(bufferOut, bufferOutOffset, static_cast(0)); + // Universal operator metadata end + + const size_t ndims = blockCount.size(); + + // mgard V1 metadata + PutParameter(bufferOut, bufferOutOffset, ndims); + for (const auto &d : blockCount) + { + PutParameter(bufferOut, bufferOutOffset, d); + } + PutParameter(bufferOut, bufferOutOffset, type); + PutParameter(bufferOut, bufferOutOffset, + static_cast(MGARD_VERSION_MAJOR)); + PutParameter(bufferOut, bufferOutOffset, + static_cast(MGARD_VERSION_MINOR)); + PutParameter(bufferOut, bufferOutOffset, + static_cast(MGARD_VERSION_PATCH)); + // mgard V1 metadata end if (ndims > 3) { @@ -61,7 +82,7 @@ size_t CompressMGARD::Compress(const void *dataIn, const Dims &dimensions, for (size_t i = 0; i < ndims; i++) { - r[ndims - i - 1] = static_cast(dimensions[i]); + r[ndims - i - 1] = static_cast(blockCount[i]); } // Parameters @@ -92,32 +113,48 @@ size_t CompressMGARD::Compress(const void *dataIn, const Dims &dimensions, } int sizeOut = 0; - unsigned char *dataOutPtr = mgard_compress( - const_cast(static_cast(dataIn)), sizeOut, - r[0], r[1], r[2], tolerance, s); - - const size_t sizeOutT = static_cast(sizeOut); - std::memcpy(bufferOut, dataOutPtr, sizeOutT); + unsigned char *dataOutPtr = + mgard_compress(reinterpret_cast(const_cast(dataIn)), + sizeOut, r[0], r[1], r[2], tolerance, s); + std::memcpy(bufferOut + bufferOutOffset, dataOutPtr, sizeOut); free(dataOutPtr); dataOutPtr = nullptr; - return sizeOutT; + bufferOutOffset += sizeOut; + + return bufferOutOffset; } -size_t CompressMGARD::Decompress(const void *bufferIn, const size_t sizeIn, - void *dataOut, const Dims &dimensions, - DataType type, - const Params & /*parameters*/) const +size_t CompressMGARD::DecompressV1(const char *bufferIn, const size_t sizeIn, + char *dataOut) { + // Do NOT remove even if the buffer version is updated. Data might be still + // in lagacy formats. This function must be kept for backward compatibility. + // If a newer buffer format is implemented, create another function, e.g. + // DecompressV2 and keep this function for decompressing lagacy data. + + size_t bufferInOffset = 0; + + const size_t ndims = GetParameter(bufferIn, bufferInOffset); + Dims blockCount(ndims); + for (size_t i = 0; i < ndims; ++i) + { + blockCount[i] = GetParameter(bufferIn, bufferInOffset); + } + const DataType type = GetParameter(bufferIn, bufferInOffset); + m_VersionInfo = + " Data is compressed using MGARD Version " + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + + ". Please make sure a compatible version is used for decompression."; + int mgardType = -1; - size_t elementSize = 0; - double quantizer = 0.0; if (type == helper::GetDataType()) { mgardType = 1; - elementSize = 8; } else { @@ -126,7 +163,6 @@ size_t CompressMGARD::Decompress(const void *bufferIn, const size_t sizeIn, "MGARD only supports double precision, in call to Get\n"); } - const size_t ndims = dimensions.size(); int r[3]; r[0] = 1; r[1] = 1; @@ -134,20 +170,49 @@ size_t CompressMGARD::Decompress(const void *bufferIn, const size_t sizeIn, for (size_t i = 0; i < ndims; i++) { - r[ndims - i - 1] = static_cast(dimensions[i]); + r[ndims - i - 1] = static_cast(blockCount[i]); } void *dataPtr = mgard_decompress( - reinterpret_cast(const_cast(bufferIn)), - static_cast(sizeIn), r[0], r[1], r[2], 0.0); + reinterpret_cast( + const_cast(bufferIn + bufferInOffset)), + static_cast(sizeIn - bufferInOffset), r[0], r[1], r[2], 0.0); - const size_t dataSizeBytes = helper::GetTotalSize(dimensions) * elementSize; - std::memcpy(dataOut, dataPtr, dataSizeBytes); + const size_t sizeOut = + helper::GetTotalSize(blockCount, helper::GetDataTypeSize(type)); + + std::memcpy(dataOut, dataPtr, sizeOut); free(dataPtr); dataPtr = nullptr; - return static_cast(dataSizeBytes); + return sizeOut; +} + +size_t CompressMGARD::Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) +{ + size_t bufferInOffset = 1; // skip operator type + const uint8_t bufferVersion = + GetParameter(bufferIn, bufferInOffset); + bufferInOffset += 2; // skip two reserved bytes + + if (bufferVersion == 1) + { + return DecompressV1(bufferIn + bufferInOffset, sizeIn - bufferInOffset, + dataOut); + } + else if (bufferVersion == 2) + { + // TODO: if a Version 2 mgard buffer is being implemented, put it here + // and keep the DecompressV1 routine for backward compatibility + } + else + { + throw("unknown mgard buffer version"); + } + + return 0; } bool CompressMGARD::IsDataTypeValid(const DataType type) const diff --git a/source/adios2/operator/compress/CompressMGARD.h b/source/adios2/operator/compress/CompressMGARD.h index a9714f3b3f..a7147d68ca 100644 --- a/source/adios2/operator/compress/CompressMGARD.h +++ b/source/adios2/operator/compress/CompressMGARD.h @@ -32,33 +32,43 @@ class CompressMGARD : public Operator ~CompressMGARD() = default; /** - * Compression signature for legacy libraries that use void* * @param dataIn - * @param dimensions + * @param blockStart + * @param blockCount * @param type * @param bufferOut * @param parameters - * @return size of compressed buffer in bytes + * @return size of compressed buffer */ - size_t Compress(const void *dataIn, const Dims &dimensions, - const size_t elementSize, DataType type, void *bufferOut, - const Params ¶meters, Params &info) const final; + size_t Compress(const char *dataIn, const Dims &blockStart, + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) final; /** - * * @param bufferIn * @param sizeIn * @param dataOut - * @param dimensions - * @param varType - * @param - * @return + * @return size of decompressed buffer */ - size_t Decompress(const void *bufferIn, const size_t sizeIn, void *dataOut, - const Dims &dimensions, DataType varType, - const Params & /*parameters*/) const final; + size_t Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) final; bool IsDataTypeValid(const DataType type) const final; + +private: + /** + * Decompress function for V1 buffer. Do NOT remove even if the buffer + * version is updated. Data might be still in lagacy formats. This function + * must be kept for backward compatibility + * @param bufferIn : compressed data buffer (V1 only) + * @param sizeIn : number of bytes in bufferIn + * @param dataOut : decompressed data buffer + * @return : number of bytes in dataOut + */ + size_t DecompressV1(const char *bufferIn, const size_t sizeIn, + char *dataOut); + + std::string m_VersionInfo; }; } // end namespace compress diff --git a/source/adios2/operator/compress/CompressPNG.cpp b/source/adios2/operator/compress/CompressPNG.cpp index c5d2236cdf..565a9d69d4 100644 --- a/source/adios2/operator/compress/CompressPNG.cpp +++ b/source/adios2/operator/compress/CompressPNG.cpp @@ -47,11 +47,22 @@ CompressPNG::CompressPNG(const Params ¶meters) : Operator("png", parameters) { } -size_t CompressPNG::Compress(const void *dataIn, const Dims &dimensions, - const size_t elementSize, DataType /*type*/, - void *bufferOut, const Params ¶meters, - Params &info) const +size_t CompressPNG::Compress(const char *dataIn, const Dims &blockStart, + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) { + size_t bufferOutOffset = 0; + const uint8_t bufferVersion = 1; + + // Universal operator metadata + PutParameter(bufferOut, bufferOutOffset, OperatorType::PNG); + PutParameter(bufferOut, bufferOutOffset, bufferVersion); + PutParameter(bufferOut, bufferOutOffset, static_cast(0)); + // Universal operator metadata end + + size_t paramOffset = bufferOutOffset; + bufferOutOffset += sizeof(size_t) + 3; + auto lf_Write = [](png_structp png_ptr, png_bytep data, png_size_t length) { DestInfo *pDestInfo = reinterpret_cast(png_get_io_ptr(png_ptr)); @@ -59,7 +70,7 @@ size_t CompressPNG::Compress(const void *dataIn, const Dims &dimensions, pDestInfo->Offset += length; }; - const std::size_t ndims = dimensions.size(); + const std::size_t ndims = blockCount.size(); if (ndims != 3 && ndims != 2) { @@ -130,11 +141,12 @@ size_t CompressPNG::Compress(const void *dataIn, const Dims &dimensions, nullptr, nullptr, nullptr); png_infop pngInfo = png_create_info_struct(pngWrite); - const uint32_t bytesPerPixel = - ndims == 3 ? static_cast(dimensions[2]) : elementSize; + const uint32_t bytesPerPixel = ndims == 3 + ? static_cast(blockCount[2]) + : helper::GetDataTypeSize(type); - const uint32_t width = static_cast(dimensions[1]); - const uint32_t height = static_cast(dimensions[0]); + const uint32_t width = static_cast(blockCount[1]); + const uint32_t height = static_cast(blockCount[0]); png_set_IHDR(pngWrite, pngInfo, width, height, bitDepth, colorType, PNG_INTERLACE_NONE, PNG_COMPRESSION_TYPE_DEFAULT, @@ -152,14 +164,14 @@ size_t CompressPNG::Compress(const void *dataIn, const Dims &dimensions, std::vector rows(height); for (size_t r = 0; r < height; ++r) { - rows[r] = reinterpret_cast(const_cast(dataIn)) + + rows[r] = reinterpret_cast(const_cast(dataIn)) + r * width * bytesPerPixel; } png_set_rows(pngWrite, pngInfo, rows.data()); DestInfo destInfo; - destInfo.BufferOut = reinterpret_cast(bufferOut); - destInfo.Offset = 0; + destInfo.BufferOut = bufferOut; + destInfo.Offset = bufferOutOffset; png_set_write_fn(pngWrite, &destInfo, lf_Write, nullptr); png_write_png(pngWrite, pngInfo, PNG_TRANSFORM_IDENTITY, nullptr); @@ -167,24 +179,49 @@ size_t CompressPNG::Compress(const void *dataIn, const Dims &dimensions, // const size_t compressedSize = png_get_compression_buffer_size(pngWrite); png_destroy_write_struct(&pngWrite, &pngInfo); + + PutParameter(bufferOut, paramOffset, destInfo.Offset); + PutParameter(bufferOut, paramOffset, + static_cast(PNG_LIBPNG_VER_MAJOR)); + PutParameter(bufferOut, paramOffset, + static_cast(PNG_LIBPNG_VER_MINOR)); + PutParameter(bufferOut, paramOffset, + static_cast(PNG_LIBPNG_VER_RELEASE)); + return destInfo.Offset; } -size_t CompressPNG::Decompress(const void *bufferIn, const size_t sizeIn, - void *dataOut, const size_t sizeOut, - Params &info) const +size_t CompressPNG::DecompressV1(const char *bufferIn, const size_t sizeIn, + char *dataOut) { + // Do NOT remove even if the buffer version is updated. Data might be still + // in lagacy formats. This function must be kept for backward compatibility. + // If a newer buffer format is implemented, create another function, e.g. + // DecompressV2 and keep this function for decompressing lagacy data. + + size_t bufferInOffset = 0; + const size_t outSize = GetParameter(bufferIn, bufferInOffset); + + m_VersionInfo = + " Data is compressed using PNG Version " + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + + ". Please make sure a compatible version is used for decompression."; + png_image image; std::memset(&image, 0, sizeof(image)); image.version = PNG_IMAGE_VERSION; - int result = png_image_begin_read_from_memory(&image, bufferIn, sizeIn); + int result = png_image_begin_read_from_memory( + &image, bufferIn + bufferInOffset, sizeIn - bufferInOffset); if (result == 0) { throw std::runtime_error( "ERROR: png_image_begin_read_from_memory failed in call " - "to ADIOS2 PNG Decompress\n"); + "to ADIOS2 PNG Decompress." + + m_VersionInfo + "\n"); } // TODO might be needed from parameters? @@ -193,10 +230,38 @@ size_t CompressPNG::Decompress(const void *bufferIn, const size_t sizeIn, { throw std::runtime_error( "ERROR: png_image_finish_read_from_memory failed in call " - "to ADIOS2 PNG Decompress\n"); + "to ADIOS2 PNG Decompress." + + m_VersionInfo + "\n"); + } + return outSize; +} + +size_t CompressPNG::Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) +{ + size_t bufferInOffset = 1; // skip operator type + const uint8_t bufferVersion = + GetParameter(bufferIn, bufferInOffset); + bufferInOffset += 2; // skip two reserved bytes + + if (bufferVersion == 1) + { + // pass in the whole buffer as there is absolute positions saved in the + // buffer to determine the offsets and lengths for batches + return DecompressV1(bufferIn + bufferInOffset, sizeIn - bufferInOffset, + dataOut); + } + else if (bufferVersion == 2) + { + // TODO: if a Version 2 png buffer is being implemented, put it here + // and keep the DecompressV1 routine for backward compatibility + } + else + { + throw("unknown png buffer version"); } - return sizeOut; + return 0; } bool CompressPNG::IsDataTypeValid(const DataType type) const { return true; } diff --git a/source/adios2/operator/compress/CompressPNG.h b/source/adios2/operator/compress/CompressPNG.h index 903533b12c..003f8d9a79 100644 --- a/source/adios2/operator/compress/CompressPNG.h +++ b/source/adios2/operator/compress/CompressPNG.h @@ -34,33 +34,42 @@ class CompressPNG : public Operator ~CompressPNG() = default; /** - * Compression signature for legacy libraries that use void* * @param dataIn - * @param dimensions + * @param blockStart + * @param blockCount * @param type * @param bufferOut * @param parameters - * @return size of compressed buffer in bytes + * @return size of compressed buffer */ - size_t Compress(const void *dataIn, const Dims &dimensions, - const size_t elementSize, DataType type, void *bufferOut, - const Params ¶meters, Params &info) const final; + size_t Compress(const char *dataIn, const Dims &blockStart, + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) final; /** - * Decompression signature for legacy libraries that use void* * @param bufferIn * @param sizeIn * @param dataOut - * @param dimensions - * @param type - * @return size of decompressed buffer in bytes + * @return size of decompressed buffer */ - size_t Decompress(const void *bufferIn, const size_t sizeIn, void *dataOut, - const size_t sizeOut, Params &info) const final; + size_t Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) final; bool IsDataTypeValid(const DataType type) const final; private: + /** + * Decompress function for V1 buffer. Do NOT remove even if the buffer + * version is updated. Data might be still in lagacy formats. This function + * must be kept for backward compatibility + * @param bufferIn : compressed data buffer (V1 only) + * @param sizeIn : number of bytes in bufferIn + * @param dataOut : decompressed data buffer + * @return : number of bytes in dataOut + */ + size_t DecompressV1(const char *bufferIn, const size_t sizeIn, + char *dataOut); + /** * check status from PNG compression and decompression functions * @param status returned by PNG library @@ -77,6 +86,8 @@ class CompressPNG : public Operator char *BufferOut = nullptr; size_t Offset = 0; }; + + std::string m_VersionInfo; }; } // end namespace compress diff --git a/source/adios2/operator/compress/CompressSZ.cpp b/source/adios2/operator/compress/CompressSZ.cpp index 887f923e5a..82d8f51b80 100644 --- a/source/adios2/operator/compress/CompressSZ.cpp +++ b/source/adios2/operator/compress/CompressSZ.cpp @@ -29,19 +29,36 @@ namespace compress CompressSZ::CompressSZ(const Params ¶meters) : Operator("sz", parameters) {} -size_t CompressSZ::BufferMaxSize(const size_t sizeIn) const +size_t CompressSZ::Compress(const char *dataIn, const Dims &blockStart, + const Dims &blockCount, const DataType varType, + char *bufferOut, const Params ¶meters) { - return static_cast(std::ceil(1.1 * sizeIn) + 600); -} + const uint8_t bufferVersion = 1; + size_t bufferOutOffset = 0; -size_t CompressSZ::Compress(const void *dataIn, const Dims &dimensions, - const size_t elementSize, DataType varType, - void *bufferOut, const Params ¶meters, - Params &info) const -{ - Dims convertedDims = ConvertDims(dimensions, varType, 4); + // Universal operator metadata + PutParameter(bufferOut, bufferOutOffset, OperatorType::Sz); + PutParameter(bufferOut, bufferOutOffset, bufferVersion); + PutParameter(bufferOut, bufferOutOffset, static_cast(0)); + // Universal operator metadata end + + const size_t ndims = blockCount.size(); + + // sz V1 metadata + PutParameter(bufferOut, bufferOutOffset, ndims); + for (const auto &d : blockCount) + { + PutParameter(bufferOut, bufferOutOffset, d); + } + PutParameter(bufferOut, bufferOutOffset, varType); + for (uint8_t i = 0; i < 4; ++i) + { + PutParameter(bufferOut, bufferOutOffset, + static_cast(versionNumber[i])); + } + // sz V1 metadata end - const size_t ndims = convertedDims.size(); + Dims convertedDims = ConvertDims(blockCount, varType, 4); sz_params sz; memset(&sz, 0, sizeof(sz_params)); @@ -65,9 +82,7 @@ size_t CompressSZ::Compress(const void *dataIn, const Dims &dimensions, static_cast(std::pow(5., static_cast(ndims))); sz.pwr_type = SZ_PWR_MIN_TYPE; - convertedDims = ConvertDims(dimensions, varType, 4, true, 1); - - size_t outsize; + convertedDims = ConvertDims(blockCount, varType, 4, true, 1); /* SZ parameters */ int use_configfile = 0; @@ -233,7 +248,7 @@ size_t CompressSZ::Compress(const void *dataIn, const Dims &dimensions, if (use_configfile) { - SZ_Init((char *)sz_configfile.c_str()); + SZ_Init(sz_configfile.c_str()); } else { @@ -259,37 +274,85 @@ size_t CompressSZ::Compress(const void *dataIn, const Dims &dimensions, ToString(varType) + " is unsupported\n"); } - unsigned char *bytes = - SZ_compress(dtype, (void *)dataIn, &outsize, 0, convertedDims[0], - convertedDims[1], convertedDims[2], convertedDims[3]); - std::memcpy(bufferOut, bytes, outsize); - free(bytes); - bytes = nullptr; + size_t szBufferSize; + auto *szBuffer = SZ_compress( + dtype, const_cast(dataIn), &szBufferSize, 0, convertedDims[0], + convertedDims[1], convertedDims[2], convertedDims[3]); + std::memcpy(bufferOut + bufferOutOffset, szBuffer, szBufferSize); + bufferOutOffset += szBufferSize; + free(szBuffer); + szBuffer = nullptr; SZ_Finalize(); - return static_cast(outsize); + return bufferOutOffset; } -size_t CompressSZ::Decompress(const void *bufferIn, const size_t sizeIn, - void *dataOut, const Dims &dimensions, - DataType varType, - const Params & /*parameters*/) const +size_t CompressSZ::Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) { - Dims convertedDims = ConvertDims(dimensions, varType, 4, true, 1); + size_t bufferInOffset = 1; // skip operator type + const uint8_t bufferVersion = + GetParameter(bufferIn, bufferInOffset); + bufferInOffset += 2; // skip two reserved bytes + + if (bufferVersion == 1) + { + return DecompressV1(bufferIn + bufferInOffset, sizeIn - bufferInOffset, + dataOut); + } + else if (bufferVersion == 2) + { + // TODO: if a Version 2 sz buffer is being implemented, put it here + // and keep the DecompressV1 routine for backward compatibility + } + else + { + throw("unknown sz buffer version"); + } + + return 0; +} +size_t CompressSZ::DecompressV1(const char *bufferIn, const size_t sizeIn, + char *dataOut) +{ + // Do NOT remove even if the buffer version is updated. Data might be still + // in lagacy formats. This function must be kept for backward compatibility. + // If a newer buffer format is implemented, create another function, e.g. + // DecompressV2 and keep this function for decompressing lagacy data. + + size_t bufferInOffset = 0; + + const size_t ndims = GetParameter(bufferIn, bufferInOffset); + Dims blockCount(ndims); + for (size_t i = 0; i < ndims; ++i) + { + blockCount[i] = GetParameter(bufferIn, bufferInOffset); + } + const DataType type = GetParameter(bufferIn, bufferInOffset); + + m_VersionInfo = + " Data is compressed using SZ Version " + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + + ". Please make sure a compatible version is used for decompression."; + + Dims convertedDims = ConvertDims(blockCount, type, 4, true, 1); // Get type info int dtype = 0; - size_t typeSizeBytes = 0; - if (varType == helper::GetDataType() || - varType == helper::GetDataType>()) + size_t dataTypeSize; + if (type == helper::GetDataType() || + type == helper::GetDataType>()) { dtype = SZ_DOUBLE; - typeSizeBytes = 8; + dataTypeSize = 8; } - else if (varType == helper::GetDataType() || - varType == helper::GetDataType>()) + else if (type == helper::GetDataType() || + type == helper::GetDataType>()) { dtype = SZ_FLOAT; - typeSizeBytes = 4; + dataTypeSize = 4; } else { @@ -298,23 +361,25 @@ size_t CompressSZ::Decompress(const void *bufferIn, const size_t sizeIn, } const size_t dataSizeBytes = - helper::GetTotalSize(convertedDims) * typeSizeBytes; + helper::GetTotalSize(convertedDims, dataTypeSize); - void *result = SZ_decompress( - dtype, reinterpret_cast(const_cast(bufferIn)), - sizeIn, 0, convertedDims[0], convertedDims[1], convertedDims[2], - convertedDims[3]); + void *result = + SZ_decompress(dtype, + reinterpret_cast( + const_cast(bufferIn + bufferInOffset)), + sizeIn - bufferInOffset, 0, convertedDims[0], + convertedDims[1], convertedDims[2], convertedDims[3]); if (result == nullptr) { - throw std::runtime_error("ERROR: SZ_decompress failed\n"); + throw std::runtime_error("ERROR: SZ_decompress failed." + + m_VersionInfo + "\n"); } std::memcpy(dataOut, result, dataSizeBytes); free(result); result = nullptr; - return static_cast(dataSizeBytes); + return dataSizeBytes; } - bool CompressSZ::IsDataTypeValid(const DataType type) const { #define declare_type(T) \ diff --git a/source/adios2/operator/compress/CompressSZ.h b/source/adios2/operator/compress/CompressSZ.h index e415453303..e0e00e48af 100644 --- a/source/adios2/operator/compress/CompressSZ.h +++ b/source/adios2/operator/compress/CompressSZ.h @@ -31,37 +31,44 @@ class CompressSZ : public Operator ~CompressSZ() = default; - size_t BufferMaxSize(const size_t sizeIn) const final; - /** - * Compression signature for legacy libraries that use void* * @param dataIn - * @param dimensions + * @param blockStart + * @param blockCount * @param type * @param bufferOut * @param parameters - * @return size of compressed buffer in bytes + * @return size of compressed buffer */ - size_t Compress(const void *dataIn, const Dims &dimensions, - const size_t elementSize, DataType type, void *bufferOut, - const Params ¶meters, Params &info) const final; - - using Operator::Decompress; + size_t Compress(const char *dataIn, const Dims &blockStart, + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) final; /** - * Wrapper around zfp decompression * @param bufferIn * @param sizeIn * @param dataOut - * @param dimensions - * @param type - * @return size of decompressed data in dataOut + * @return size of decompressed buffer */ - size_t Decompress(const void *bufferIn, const size_t sizeIn, void *dataOut, - const Dims &dimensions, DataType type, - const Params ¶meters) const final; + size_t Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) final; bool IsDataTypeValid(const DataType type) const final; + +private: + /** + * Decompress function for V1 buffer. Do NOT remove even if the buffer + * version is updated. Data might be still in lagacy formats. This function + * must be kept for backward compatibility + * @param bufferIn : compressed data buffer (V1 only) + * @param sizeIn : number of bytes in bufferIn + * @param dataOut : decompressed data buffer + * @return : number of bytes in dataOut + */ + size_t DecompressV1(const char *bufferIn, const size_t sizeIn, + char *dataOut); + + std::string m_VersionInfo; }; } // end namespace compress diff --git a/source/adios2/operator/compress/CompressSirius.cpp b/source/adios2/operator/compress/CompressSirius.cpp new file mode 100644 index 0000000000..2047f005e8 --- /dev/null +++ b/source/adios2/operator/compress/CompressSirius.cpp @@ -0,0 +1,213 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * CompressSirius.cpp + * + * Created on: Jul 28, 2021 + * Author: Jason Wang jason.ruonan.wang@gmail.com + */ + +#include "CompressSirius.h" +#include "adios2/helper/adiosFunctions.h" + +namespace adios2 +{ +namespace core +{ +namespace compress +{ + +std::unordered_map CompressSirius::m_CurrentTierMap; +std::vector>> + CompressSirius::m_TierBuffersMap; +int CompressSirius::m_CurrentTier; +std::vector> CompressSirius::m_TierBuffers; +int CompressSirius::m_Tiers = 0; +bool CompressSirius::m_CurrentReadFinished = false; + +CompressSirius::CompressSirius(const Params ¶meters) +: Operator("sirius", parameters) +{ + helper::GetParameter(parameters, "Tiers", m_Tiers); + m_TierBuffersMap.resize(m_Tiers); + m_TierBuffers.resize(m_Tiers); +} + +size_t CompressSirius::Compress(const char *dataIn, const Dims &blockStart, + const Dims &blockCount, const DataType varType, + char *bufferOut, const Params ¶ms) +{ + const uint8_t bufferVersion = 1; + size_t bufferOutOffset = 0; + + // Universal operator metadata + PutParameter(bufferOut, bufferOutOffset, OperatorType::SIRIUS); + PutParameter(bufferOut, bufferOutOffset, bufferVersion); + PutParameter(bufferOut, bufferOutOffset, static_cast(0)); + // Universal operator metadata end + + const size_t ndims = blockCount.size(); + + // sirius V1 metadata + PutParameter(bufferOut, bufferOutOffset, ndims); + for (const auto &d : blockStart) + { + PutParameter(bufferOut, bufferOutOffset, d); + } + for (const auto &d : blockCount) + { + PutParameter(bufferOut, bufferOutOffset, d); + } + PutParameter(bufferOut, bufferOutOffset, varType); + // sirius V1 metadata end + + size_t totalInputBytes = + helper::GetTotalSize(blockCount, helper::GetDataTypeSize(varType)); + + // if called from Tier 0 sub-engine, then compute tier buffers and put into + // m_TierBuffers + size_t bytesPerTier = totalInputBytes / m_Tiers; + if (m_CurrentTier == 0) + { + for (size_t i = 0; i < m_TierBuffers.size(); i++) + { + m_TierBuffers[i].resize(bytesPerTier); + std::memcpy(m_TierBuffers[i].data(), dataIn + i * bytesPerTier, + bytesPerTier); + } + } + + // for all tiers' sub-engines, copy data from m_TierBuffers to output buffer + std::memcpy(bufferOut + bufferOutOffset, + m_TierBuffers[m_CurrentTier].data(), + m_TierBuffers[m_CurrentTier].size()); + + bufferOutOffset += bytesPerTier; + + m_CurrentTier++; + m_CurrentTier %= m_Tiers; + + return bufferOutOffset; +} + +size_t CompressSirius::DecompressV1(const char *bufferIn, const size_t sizeIn, + char *dataOut) +{ + // Do NOT remove even if the buffer version is updated. Data might be still + // in lagacy formats. This function must be kept for backward compatibility. + // If a newer buffer format is implemented, create another function, e.g. + // DecompressV2 and keep this function for decompressing lagacy data. + + size_t bufferInOffset = 0; + const size_t ndims = GetParameter(bufferIn, bufferInOffset); + Dims blockStart(ndims); + Dims blockCount(ndims); + for (size_t i = 0; i < ndims; ++i) + { + blockStart[i] = GetParameter(bufferIn, bufferInOffset); + } + for (size_t i = 0; i < ndims; ++i) + { + blockCount[i] = GetParameter(bufferIn, bufferInOffset); + } + const DataType type = GetParameter(bufferIn, bufferInOffset); + + const size_t outputBytes = + helper::GetTotalSize(blockCount, helper::GetDataTypeSize(type)); + + std::string blockId = + helper::DimsToString(blockStart) + helper::DimsToString(blockCount); + + // decompress data and copy back to m_TierBuffers + size_t bytesPerTier = outputBytes / m_Tiers; + auto ¤tBuffer = m_TierBuffersMap[m_CurrentTierMap[blockId]][blockId]; + auto ¤tTier = m_CurrentTierMap[blockId]; + currentBuffer.resize(bytesPerTier); + std::memcpy(currentBuffer.data(), bufferIn + bufferInOffset, bytesPerTier); + + // if called from the final tier, then merge all tier buffers and copy back + // to dataOut + size_t accumulatedBytes = 0; + + // TODO: it currently only copies output data back when the final tier is + // read. However, the real Sirius algorithm should instead decide when to + // copy back decompressed data based on required acuracy level. Once it's + // done, it should set m_CurrentReadFinished to true to inform the MHS + // engine that the current read is finished so that it won't read the next + // tier. + if (currentTier == m_Tiers - 1) + { + for (auto &bmap : m_TierBuffersMap) + { + auto &b = bmap[blockId]; + std::memcpy(dataOut + accumulatedBytes, b.data(), b.size()); + accumulatedBytes += b.size(); + } + // set m_CurrentReadFinished to true if after the current call, the + // required acuracy is already satisfied, so that the MHS engine knows + // it shouldn't continue reading the next tier. + m_CurrentReadFinished = true; + } + + // set m_CurrentReadFinished to false if the current tier does not satisfy + // the required acuracy, so the MHS engine will read the next tier. + m_CurrentReadFinished = false; + + currentTier++; + if (currentTier % m_Tiers == 0) + { + currentTier = 0; + } + + if (currentTier == 0) + { + return outputBytes; + } + else + { + return 0; + } +} + +size_t CompressSirius::Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) +{ + size_t bufferInOffset = 1; // skip operator type + const uint8_t bufferVersion = + GetParameter(bufferIn, bufferInOffset); + bufferInOffset += 2; // skip two reserved bytes + + if (bufferVersion == 1) + { + return DecompressV1(bufferIn + bufferInOffset, sizeIn - bufferInOffset, + dataOut); + } + else if (bufferVersion == 2) + { + // TODO: if a Version 2 sirius buffer is being implemented, put it here + // and keep the DecompressV1 routine for backward compatibility + } + else + { + throw("unknown sirius buffer version"); + } + + return 0; +} + +bool CompressSirius::IsDataTypeValid(const DataType type) const +{ +#define declare_type(T) \ + if (helper::GetDataType() == type) \ + { \ + return true; \ + } + ADIOS2_FOREACH_SIRIUS_TYPE_1ARG(declare_type) +#undef declare_type + return false; +} + +} // end namespace compress +} // end namespace core +} // end namespace adios2 diff --git a/source/adios2/operator/compress/CompressSirius.h b/source/adios2/operator/compress/CompressSirius.h new file mode 100644 index 0000000000..4c08aafedc --- /dev/null +++ b/source/adios2/operator/compress/CompressSirius.h @@ -0,0 +1,72 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * CompressSirius.h + * + * Created on: Jul 28, 2021 + * Author: Jason Wang jason.ruonan.wang@gmail.com + */ + +#ifndef ADIOS2_OPERATOR_COMPRESS_COMPRESSSIRIUS_H_ +#define ADIOS2_OPERATOR_COMPRESS_COMPRESSSIRIUS_H_ + +#include "adios2/core/Operator.h" +#include + +namespace adios2 +{ +namespace core +{ +namespace compress +{ + +class CompressSirius : public Operator +{ + +public: + CompressSirius(const Params ¶meters); + + ~CompressSirius() = default; + + size_t Compress(const char *dataIn, const Dims &blockStart, + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶ms) final; + + size_t Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) final; + + bool IsDataTypeValid(const DataType type) const final; + + static bool m_CurrentReadFinished; + +private: + static int m_Tiers; + + // for compress + static std::vector> m_TierBuffers; + static int m_CurrentTier; + + // for decompress + static std::vector>> + m_TierBuffersMap; + static std::unordered_map m_CurrentTierMap; + + /** + * Decompress function for V1 buffer. Do NOT remove even if the buffer + * version is updated. Data might be still in lagacy formats. This function + * must be kept for backward compatibility + * @param bufferIn : compressed data buffer (V1 only) + * @param sizeIn : number of bytes in bufferIn + * @param dataOut : decompressed data buffer + * @return : number of bytes in dataOut + */ + size_t DecompressV1(const char *bufferIn, const size_t sizeIn, + char *dataOut); +}; + +} // end namespace compress +} // end namespace core +} // end namespace adios2 + +#endif /* ADIOS2_TRANSFORM_COMPRESSION_COMPRESSSZ_H_ */ diff --git a/source/adios2/operator/compress/CompressZFP.cpp b/source/adios2/operator/compress/CompressZFP.cpp index 827a93871d..1ed8d5c312 100644 --- a/source/adios2/operator/compress/CompressZFP.cpp +++ b/source/adios2/operator/compress/CompressZFP.cpp @@ -22,32 +22,44 @@ CompressZFP::CompressZFP(const Params ¶meters) : Operator("zfp", parameters) { } -size_t CompressZFP::DoBufferMaxSize(const void *dataIn, const Dims &dimensions, - DataType type, - const Params ¶meters) const +size_t CompressZFP::Compress(const char *dataIn, const Dims &blockStart, + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) { - Dims convertedDims = ConvertDims(dimensions, type, 3); - zfp_field *field = GetZFPField(dataIn, convertedDims, type); - zfp_stream *stream = GetZFPStream(convertedDims, type, parameters); - const size_t maxSize = zfp_stream_maximum_size(stream, field); - zfp_field_free(field); - zfp_stream_close(stream); - return maxSize; -} -size_t CompressZFP::Compress(const void *dataIn, const Dims &dimensions, - const size_t elementSize, DataType type, - void *bufferOut, const Params ¶meters, - Params &info) const -{ + const uint8_t bufferVersion = 1; + size_t bufferOutOffset = 0; + + // Universal operator metadata + PutParameter(bufferOut, bufferOutOffset, OperatorType::ZFP); + PutParameter(bufferOut, bufferOutOffset, bufferVersion); + PutParameter(bufferOut, bufferOutOffset, static_cast(0)); + // Universal operator metadata end - Dims convertedDims = ConvertDims(dimensions, type, 3); + const size_t ndims = blockCount.size(); + // zfp V1 metadata + PutParameter(bufferOut, bufferOutOffset, ndims); + for (const auto &d : blockCount) + { + PutParameter(bufferOut, bufferOutOffset, d); + } + PutParameter(bufferOut, bufferOutOffset, type); + PutParameter(bufferOut, bufferOutOffset, + static_cast(ZFP_VERSION_MAJOR)); + PutParameter(bufferOut, bufferOutOffset, + static_cast(ZFP_VERSION_MINOR)); + PutParameter(bufferOut, bufferOutOffset, + static_cast(ZFP_VERSION_PATCH)); + PutParameters(bufferOut, bufferOutOffset, parameters); + // zfp V1 metadata end + + Dims convertedDims = ConvertDims(blockCount, type, 3); zfp_field *field = GetZFPField(dataIn, convertedDims, type); zfp_stream *stream = GetZFPStream(convertedDims, type, parameters); size_t maxSize = zfp_stream_maximum_size(stream, field); // associate bitstream - bitstream *bitstream = stream_open(bufferOut, maxSize); + bitstream *bitstream = stream_open(bufferOut + bufferOutOffset, maxSize); zfp_stream_set_bit_stream(stream, bitstream); zfp_stream_rewind(stream); @@ -59,36 +71,47 @@ size_t CompressZFP::Compress(const void *dataIn, const Dims &dimensions, "size is 0, in call to Compress"); } + bufferOutOffset += sizeOut; + zfp_field_free(field); zfp_stream_close(stream); stream_close(bitstream); - return sizeOut; + return bufferOutOffset; } -size_t CompressZFP::Decompress(const void *bufferIn, const size_t sizeIn, - void *dataOut, const Dims &dimensions, - DataType type, const Params ¶meters) const +size_t CompressZFP::DecompressV1(const char *bufferIn, const size_t sizeIn, + char *dataOut) { - auto lf_GetTypeSize = [](const zfp_type zfpType) -> size_t { - size_t size = 0; - if (zfpType == zfp_type_int32 || zfpType == zfp_type_float) - { - size = 4; - } - else if (zfpType == zfp_type_int64 || zfpType == zfp_type_double) - { - size = 8; - } - return size; - }; + // Do NOT remove even if the buffer version is updated. Data might be still + // in lagacy formats. This function must be kept for backward compatibility. + // If a newer buffer format is implemented, create another function, e.g. + // DecompressV2 and keep this function for decompressing lagacy data. + + size_t bufferInOffset = 0; - Dims convertedDims = ConvertDims(dimensions, type, 3); + const size_t ndims = GetParameter(bufferIn, bufferInOffset); + Dims blockCount(ndims); + for (size_t i = 0; i < ndims; ++i) + { + blockCount[i] = GetParameter(bufferIn, bufferInOffset); + } + const DataType type = GetParameter(bufferIn, bufferInOffset); + m_VersionInfo = + " Data is compressed using ZFP Version " + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + + ". Please make sure a compatible version is used for decompression."; + const Params parameters = GetParameters(bufferIn, bufferInOffset); + + Dims convertedDims = ConvertDims(blockCount, type, 3); zfp_field *field = GetZFPField(dataOut, convertedDims, type); zfp_stream *stream = GetZFPStream(convertedDims, type, parameters); // associate bitstream - bitstream *bitstream = stream_open(const_cast(bufferIn), sizeIn); + bitstream *bitstream = stream_open( + const_cast(bufferIn + bufferInOffset), sizeIn - bufferInOffset); zfp_stream_set_bit_stream(stream, bitstream); zfp_stream_rewind(stream); @@ -96,20 +119,42 @@ size_t CompressZFP::Decompress(const void *bufferIn, const size_t sizeIn, if (!status) { - throw std::invalid_argument("ERROR: zfp failed with status " + - std::to_string(status) + - ", in call to CompressZfp Decompress\n"); + throw std::runtime_error( + "ERROR: zfp failed with status " + std::to_string(status) + + ", in call to CompressZfp Decompress." + m_VersionInfo + "\n"); } zfp_field_free(field); zfp_stream_close(stream); stream_close(bitstream); - const size_t typeSizeBytes = lf_GetTypeSize(GetZfpType(type)); - const size_t dataSizeBytes = - helper::GetTotalSize(convertedDims) * typeSizeBytes; + return helper::GetTotalSize(convertedDims, helper::GetDataTypeSize(type)); +} - return dataSizeBytes; +size_t CompressZFP::Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) +{ + size_t bufferInOffset = 1; // skip operator type + const uint8_t bufferVersion = + GetParameter(bufferIn, bufferInOffset); + bufferInOffset += 2; // skip two reserved bytes + + if (bufferVersion == 1) + { + return DecompressV1(bufferIn + bufferInOffset, sizeIn - bufferInOffset, + dataOut); + } + else if (bufferVersion == 2) + { + // TODO: if a Version 2 zfp buffer is being implemented, put it here + // and keep the DecompressV1 routine for backward compatibility + } + else + { + throw std::runtime_error("unknown zfp buffer version"); + } + + return 0; } bool CompressZFP::IsDataTypeValid(const DataType type) const @@ -166,47 +211,42 @@ zfp_type CompressZFP::GetZfpType(DataType type) const return zfpType; } -zfp_field *CompressZFP::GetZFPField(const void *data, const Dims &dimensions, +zfp_field *CompressZFP::GetZFPField(const char *data, const Dims &dimensions, DataType type) const { - auto lf_CheckField = [](const zfp_field *field, - const std::string zfpFieldFunction, DataType type) { - if (field == nullptr || field == NULL) - { - throw std::invalid_argument( - "ERROR: " + zfpFieldFunction + " failed for data of type " + - ToString(type) + - ", data pointer might be corrupted, from " - "class CompressZfp Transform\n"); - } - }; - zfp_type zfpType = GetZfpType(type); zfp_field *field = nullptr; if (dimensions.size() == 1) { - field = zfp_field_1d(const_cast(data), zfpType, dimensions[0]); - lf_CheckField(field, "zfp_field_1d", type); + field = zfp_field_1d(const_cast(data), zfpType, dimensions[0]); } else if (dimensions.size() == 2) { - field = zfp_field_2d(const_cast(data), zfpType, dimensions[0], + field = zfp_field_2d(const_cast(data), zfpType, dimensions[0], dimensions[1]); - lf_CheckField(field, "zfp_field_2d", type); } else if (dimensions.size() == 3) { - field = zfp_field_3d(const_cast(data), zfpType, dimensions[0], + field = zfp_field_3d(const_cast(data), zfpType, dimensions[0], dimensions[1], dimensions[2]); - lf_CheckField(field, "zfp_field_3d", type); } else { throw std::invalid_argument( "ERROR: zfp_field* failed for data of type " + ToString(type) + ", only 1D, 2D and 3D dimensions are supported, from " - "class CompressZfp Transform\n"); + "class CompressZfp." + + m_VersionInfo + "\n"); + } + + if (field == nullptr) + { + throw std::invalid_argument( + "ERROR: zfp_field_" + std::to_string(dimensions.size()) + + "d failed for data of type " + ToString(type) + + ", data might be corrupted, from class CompressZfp." + + m_VersionInfo + "\n"); } return field; @@ -215,26 +255,16 @@ zfp_field *CompressZFP::GetZFPField(const void *data, const Dims &dimensions, zfp_stream *CompressZFP::GetZFPStream(const Dims &dimensions, DataType type, const Params ¶meters) const { - auto lf_HasKey = [](Params::const_iterator itKey, - const Params ¶meters) -> bool { - bool hasKey = false; - if (itKey != parameters.end()) - { - hasKey = true; - } - return hasKey; - }; - zfp_stream *stream = zfp_stream_open(NULL); auto itAccuracy = parameters.find("accuracy"); - const bool hasAccuracy = lf_HasKey(itAccuracy, parameters); + const bool hasAccuracy = itAccuracy != parameters.end(); auto itRate = parameters.find("rate"); - const bool hasRate = lf_HasKey(itRate, parameters); + const bool hasRate = itRate != parameters.end(); auto itPrecision = parameters.find("precision"); - const bool hasPrecision = lf_HasKey(itPrecision, parameters); + const bool hasPrecision = itPrecision != parameters.end(); if ((hasAccuracy && hasRate) || (hasAccuracy && hasPrecision) || (hasRate && hasPrecision) || !(hasAccuracy || hasRate || hasPrecision)) diff --git a/source/adios2/operator/compress/CompressZFP.h b/source/adios2/operator/compress/CompressZFP.h index 64c09c8ab7..ade19c7595 100644 --- a/source/adios2/operator/compress/CompressZFP.h +++ b/source/adios2/operator/compress/CompressZFP.h @@ -37,32 +37,26 @@ class CompressZFP : public Operator ~CompressZFP() = default; /** - * Wrapper around zfp compression * @param dataIn - * @param dimensions + * @param blockStart + * @param blockCount * @param type * @param bufferOut * @param parameters - * @return size of compressed buffer in bytes + * @return size of compressed buffer */ - size_t Compress(const void *dataIn, const Dims &dimensions, - const size_t elementSize, DataType type, void *bufferOut, - const Params ¶meters, Params &info) const final; - - using Operator::Decompress; + size_t Compress(const char *dataIn, const Dims &blockStart, + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) final; /** - * Wrapper around zfp decompression * @param bufferIn * @param sizeIn * @param dataOut - * @param dimensions - * @param type - * @return size of decompressed data in dataOut + * @return size of decompressed buffer */ - size_t Decompress(const void *bufferIn, const size_t sizeIn, void *dataOut, - const Dims &dimensions, DataType type, - const Params ¶meters) const final; + size_t Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) final; bool IsDataTypeValid(const DataType type) const final; @@ -83,21 +77,25 @@ class CompressZFP : public Operator * @param type * @return zfp_field* */ - zfp_field *GetZFPField(const void *data, const Dims &shape, + zfp_field *GetZFPField(const char *data, const Dims &shape, DataType type) const; zfp_stream *GetZFPStream(const Dims &dimensions, DataType type, const Params ¶meters) const; - size_t DoBufferMaxSize(const void *dataIn, const Dims &dimensions, - DataType type, const Params ¶meters) const final; - /** - * check status from BZip compression and decompression functions - * @param status returned by BZip2 library - * @param hint extra exception information + * Decompress function for V1 buffer. Do NOT remove even if the buffer + * version is updated. Data might be still in lagacy formats. This function + * must be kept for backward compatibility + * @param bufferIn : compressed data buffer (V1 only) + * @param sizeIn : number of bytes in bufferIn + * @param dataOut : decompressed data buffer + * @return : number of bytes in dataOut */ - void CheckStatus(const int status, const std::string hint) const; + size_t DecompressV1(const char *bufferIn, const size_t sizeIn, + char *dataOut); + + std::string m_VersionInfo; }; } // end namespace compress diff --git a/source/adios2/operator/compress/CompressorFactory.cpp b/source/adios2/operator/compress/CompressorFactory.cpp new file mode 100644 index 0000000000..d497e09bd6 --- /dev/null +++ b/source/adios2/operator/compress/CompressorFactory.cpp @@ -0,0 +1,210 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * CompressorFactory.cpp : + * + * Created on: Sep 29, 2021 + * Author: Jason Wang jason.ruonan.wang@gmail.com + */ + +#include "CompressorFactory.h" +#include "adios2/core/Operator.h" + +namespace adios2 +{ +namespace core +{ +namespace compress +{ +size_t CompressorFactory::Compress(const char *dataIn, const Dims &blockStart, + const Dims &blockCount, + const DataType dataType, char *bufferOut, + const Params ¶meters, + const std::string &compressorType) +{ + + if (compressorType == "blosc") + { +#ifdef ADIOS2_HAVE_BLOSC + CompressBlosc c({}); + return c.Compress(dataIn, blockStart, blockCount, dataType, bufferOut, + parameters); +#else + throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " + "with BLOSC, can't use compressor\n"); +#endif + } + else if (compressorType == "bzip2") + { +#ifdef ADIOS2_HAVE_BZIP2 + CompressBZIP2 c({}); + return c.Compress(dataIn, blockStart, blockCount, dataType, bufferOut, + parameters); +#else + throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " + "with BZIP2, can't use compressor\n"); +#endif + } + else if (compressorType == "libpressio") + { +#ifdef ADIOS2_HAVE_LIBPRESSIO + CompressLibPressio c({}); + return c.Compress(dataIn, blockStart, blockCount, dataType, bufferOut, + parameters); +#else + throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " + "with LibPressio, can't use compressor\n"); +#endif + } + else if (compressorType == "mgard") + { +#ifdef ADIOS2_HAVE_MGARD + CompressMGARD c({}); + return c.Compress(dataIn, blockStart, blockCount, dataType, bufferOut, + parameters); +#else + throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " + "with MGARD, can't use compressor\n"); +#endif + } + else if (compressorType == "png") + { +#ifdef ADIOS2_HAVE_PNG + CompressPNG c({}); + return c.Compress(dataIn, blockStart, blockCount, dataType, bufferOut, + parameters); +#else + throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " + "with PNG, can't use compressor\n"); +#endif + } + else if (compressorType == "sirius") + { +#ifdef ADIOS2_HAVE_MHS + CompressSirius c({}); + return c.Compress(dataIn, blockStart, blockCount, dataType, bufferOut, + parameters); +#else + throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " + "with MHS, can't use Sirius compressor\n"); +#endif + } + else if (compressorType == "sz") + { +#ifdef ADIOS2_HAVE_SZ + CompressSZ c({}); + return c.Compress(dataIn, blockStart, blockCount, dataType, bufferOut, + parameters); +#else + throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " + "with SZ, can't use compressor\n"); +#endif + } + else if (compressorType == "zfp") + { +#ifdef ADIOS2_HAVE_ZFP + CompressZFP c({}); + return c.Compress(dataIn, blockStart, blockCount, dataType, bufferOut, + parameters); +#else + throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " + "with ZFP, can't use compressor\n"); +#endif + } + return 0; +} + +size_t CompressorFactory::Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) +{ + Operator::OperatorType compressorType; + std::memcpy(&compressorType, bufferIn, 1); + + if (compressorType == Operator::OperatorType::BLOSC) + { +#ifdef ADIOS2_HAVE_BLOSC + compress::CompressBlosc op({}); + return op.Decompress(bufferIn, sizeIn, dataOut); +#else + throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " + "with BLOSC, can't use compressor\n"); +#endif + } + else if (compressorType == Operator::OperatorType::BZIP2) + { +#ifdef ADIOS2_HAVE_BZIP2 + compress::CompressBZIP2 op({}); + return op.Decompress(bufferIn, sizeIn, dataOut); +#else + throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " + "with BZIP2, can't use compressor\n"); +#endif + } + else if (compressorType == Operator::OperatorType::LIBPRESSIO) + { +#ifdef ADIOS2_HAVE_LIBPRESSIO + compress::CompressLibPressio op({}); + return op.Decompress(bufferIn, sizeIn, dataOut); +#else + throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " + "with LibPressio, can't use compressor\n"); +#endif + } + else if (compressorType == Operator::OperatorType::MGARD) + { +#ifdef ADIOS2_HAVE_MGARD + compress::CompressMGARD op({}); + return op.Decompress(bufferIn, sizeIn, dataOut); +#else + throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " + "with MGARD, can't use compressor\n"); +#endif + } + else if (compressorType == Operator::OperatorType::PNG) + { +#ifdef ADIOS2_HAVE_PNG + compress::CompressPNG op({}); + return op.Decompress(bufferIn, sizeIn, dataOut); +#else + throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " + "with PNG, can't use compressor\n"); +#endif + } + else if (compressorType == Operator::OperatorType::SIRIUS) + { +#ifdef ADIOS2_HAVE_MHS + compress::CompressSirius op({}); + return op.Decompress(bufferIn, sizeIn, dataOut); +#else + throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " + "with MHS, can't use Sirius compressor\n"); +#endif + } + else if (compressorType == Operator::OperatorType::Sz) + { +#ifdef ADIOS2_HAVE_SZ + compress::CompressSZ op({}); + return op.Decompress(bufferIn, sizeIn, dataOut); +#else + throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " + "with SZ, can't use compressor\n"); +#endif + } + else if (compressorType == Operator::OperatorType::ZFP) + { +#ifdef ADIOS2_HAVE_ZFP + compress::CompressZFP op({}); + return op.Decompress(bufferIn, sizeIn, dataOut); +#else + throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " + "with ZFP, can't use compressor\n"); +#endif + } + + return 0; +} + +} // end namespace compress +} // end namespace core +} // end namespace adios2 diff --git a/source/adios2/operator/compress/CompressorFactory.h b/source/adios2/operator/compress/CompressorFactory.h new file mode 100644 index 0000000000..e5b344cc91 --- /dev/null +++ b/source/adios2/operator/compress/CompressorFactory.h @@ -0,0 +1,65 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * CompressorFactory.h : + * + * Created on: Sep 29, 2021 + * Author: Jason Wang jason.ruonan.wang@gmail.com + */ + +#include "adios2/common/ADIOSTypes.h" + +#ifdef ADIOS2_HAVE_BLOSC +#include "adios2/operator/compress/CompressBlosc.h" +#endif + +#ifdef ADIOS2_HAVE_BZIP2 +#include "adios2/operator/compress/CompressBZIP2.h" +#endif + +#ifdef ADIOS2_HAVE_LIBPRESSIO +#include "adios2/operator/compress/CompressLibPressio.h" +#endif + +#ifdef ADIOS2_HAVE_MGARD +#include "adios2/operator/compress/CompressMGARD.h" +#endif + +#ifdef ADIOS2_HAVE_PNG +#include "adios2/operator/compress/CompressPNG.h" +#endif + +#ifdef ADIOS2_HAVE_MHS +#include "adios2/operator/compress/CompressSirius.h" +#endif + +#ifdef ADIOS2_HAVE_SZ +#include "adios2/operator/compress/CompressSZ.h" +#endif + +#ifdef ADIOS2_HAVE_ZFP +#include "adios2/operator/compress/CompressZFP.h" +#endif + +namespace adios2 +{ +namespace core +{ +namespace compress +{ + +class CompressorFactory +{ +public: + size_t Compress(const char *dataIn, const Dims &blockStart, + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters, + const std::string &compressorType); + + size_t Decompress(const char *bufferIn, const size_t sizeIn, char *dataOut); +}; + +} // end namespace compress +} // end namespace core +} // end namespace adios2 diff --git a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp index 7df033f849..7af472fc72 100644 --- a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp +++ b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp @@ -19,27 +19,14 @@ namespace aggregator MPIAggregator::MPIAggregator() {} -MPIAggregator::~MPIAggregator() -{ - if (m_IsActive) - { - m_Comm.Free("freeing aggregators comm in MPIAggregator " - "destructor, not recommended"); - } -} +MPIAggregator::~MPIAggregator() { Close(); } -void MPIAggregator::Init(const size_t subStreams, +void MPIAggregator::Init(const size_t numAggregators, const size_t subStreams, helper::Comm const &parentComm) -{ -} - -void MPIAggregator::SwapBuffers(const int step) noexcept {} -void MPIAggregator::ResetBuffers() noexcept {} - -format::Buffer &MPIAggregator::GetConsumerBuffer(format::Buffer &buffer) { - return buffer; + m_NumAggregators = numAggregators; + m_SubStreams = subStreams; } void MPIAggregator::Close() @@ -72,16 +59,16 @@ void MPIAggregator::InitComm(const size_t subStreams, if (process >= firstInSmallGroups) { m_SubStreamIndex = r + (process - firstInSmallGroups) / q; - m_ConsumerRank = + m_AggregatorRank = static_cast(firstInSmallGroups + (m_SubStreamIndex - r) * q); } else { m_SubStreamIndex = process / (q + 1); - m_ConsumerRank = static_cast(m_SubStreamIndex * (q + 1)); + m_AggregatorRank = static_cast(m_SubStreamIndex * (q + 1)); } - m_Comm = parentComm.Split(m_ConsumerRank, parentRank, + m_Comm = parentComm.Split(m_AggregatorRank, parentRank, "creating aggregators comm with split at Open"); m_Rank = m_Comm.Rank(); @@ -89,7 +76,7 @@ void MPIAggregator::InitComm(const size_t subStreams, if (m_Rank != 0) { - m_IsConsumer = false; + m_IsAggregator = false; } m_IsActive = true; @@ -104,7 +91,7 @@ void MPIAggregator::InitCommOnePerNode(helper::Comm const &parentComm) if (m_Rank != 0) { - m_IsConsumer = false; + m_IsAggregator = false; } m_IsActive = true; @@ -130,9 +117,9 @@ void MPIAggregator::InitCommOnePerNode(helper::Comm const &parentComm) /* Identify parent rank of aggregator process within each group */ if (!m_Rank) { - m_ConsumerRank = parentComm.Rank(); + m_AggregatorRank = parentComm.Rank(); } - m_ConsumerRank = m_Comm.BroadcastValue(m_ConsumerRank, 0); + m_AggregatorRank = m_Comm.BroadcastValue(m_AggregatorRank, 0); } void MPIAggregator::HandshakeRank(const int rank) diff --git a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h index 73c4bfb2f3..67f2b08770 100644 --- a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h +++ b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h @@ -31,6 +31,11 @@ class MPIAggregator /** current substream index from 0 to m_SubStreams-1 */ size_t m_SubStreamIndex = 0; + /** total number of aggregators + * (BP3/BP4 uses aggregators = substreams) + */ + size_t m_NumAggregators = 0; + /** split Communicator for a substream: producers and consumer (rank=0) */ helper::Comm m_Comm; @@ -44,54 +49,24 @@ class MPIAggregator * true: consumes data from itself or other processes and interacts with * transport managers */ - bool m_IsConsumer = true; + bool m_IsAggregator = true; /** true: doing aggregation, false: not doing aggregation */ bool m_IsActive = false; /** consumer original rank coming from the parent communicator in Init, * corresponds to m_Rank = 0 */ - int m_ConsumerRank = -1; + int m_AggregatorRank = -1; MPIAggregator(); virtual ~MPIAggregator(); - virtual void Init(const size_t subStreams, helper::Comm const &parentComm); - - struct ExchangeRequests - { - helper::Comm::Req m_SendSize; - helper::Comm::Req m_SendData; - helper::Comm::Req m_RecvData; - }; - - virtual ExchangeRequests IExchange(format::Buffer &buffer, - const int step) = 0; - - struct ExchangeAbsolutePositionRequests - { - helper::Comm::Req m_Send; - helper::Comm::Req m_Recv; - }; - - virtual ExchangeAbsolutePositionRequests - IExchangeAbsolutePosition(format::Buffer &buffer, const int step) = 0; - - virtual void - WaitAbsolutePosition(ExchangeAbsolutePositionRequests &requests, - const int step) = 0; - - virtual void Wait(ExchangeRequests &requests, const int step) = 0; - - virtual void SwapBuffers(const int step) noexcept; - - virtual void ResetBuffers() noexcept; - - virtual format::Buffer &GetConsumerBuffer(format::Buffer &buffer); + virtual void Init(const size_t numAggregators, const size_t subStreams, + helper::Comm const &parentComm); /** closes current aggregator, frees m_Comm */ - void Close(); + virtual void Close(); protected: /** Init m_Comm splitting assigning ranks to subStreams (balanced except for diff --git a/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp b/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp index eea9c50f15..b81be2ba7b 100644 --- a/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp +++ b/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp @@ -18,8 +18,11 @@ namespace aggregator MPIChain::MPIChain() : MPIAggregator() {} -void MPIChain::Init(const size_t subStreams, helper::Comm const &parentComm) +void MPIChain::Init(const size_t numAggregators, const size_t subStreams, + helper::Comm const &parentComm) { + /* numAggregators ignored here as BP3/BP4 uses substreams = aggregators */ + m_NumAggregators = subStreams; if (subStreams > 0) { InitComm(subStreams, parentComm); @@ -39,6 +42,8 @@ void MPIChain::Init(const size_t subStreams, helper::Comm const &parentComm) } } +void MPIChain::Close() { MPIAggregator::Close(); } + MPIChain::ExchangeRequests MPIChain::IExchange(format::Buffer &buffer, const int step) { diff --git a/source/adios2/toolkit/aggregator/mpi/MPIChain.h b/source/adios2/toolkit/aggregator/mpi/MPIChain.h index a26895d8d2..9a8570c243 100644 --- a/source/adios2/toolkit/aggregator/mpi/MPIChain.h +++ b/source/adios2/toolkit/aggregator/mpi/MPIChain.h @@ -22,27 +22,44 @@ class MPIChain : public MPIAggregator { public: + /* Chain aggregator used by BP3/BP4 */ MPIChain(); ~MPIChain() = default; - void Init(const size_t subStreams, helper::Comm const &parentComm) final; + void Init(const size_t numAggregators, const size_t subStreams, + helper::Comm const &parentComm) final; - ExchangeRequests IExchange(format::Buffer &buffer, const int step) final; + void Close() final; - ExchangeAbsolutePositionRequests - IExchangeAbsolutePosition(format::Buffer &buffer, const int step) final; + struct ExchangeRequests + { + helper::Comm::Req m_SendSize; + helper::Comm::Req m_SendData; + helper::Comm::Req m_RecvData; + }; + + ExchangeRequests IExchange(format::Buffer &buffer, const int step); - void Wait(ExchangeRequests &requests, const int step) final; + struct ExchangeAbsolutePositionRequests + { + helper::Comm::Req m_Send; + helper::Comm::Req m_Recv; + }; + + ExchangeAbsolutePositionRequests + IExchangeAbsolutePosition(format::Buffer &buffer, const int step); void WaitAbsolutePosition(ExchangeAbsolutePositionRequests &requests, - const int step) final; + const int step); + + void Wait(ExchangeRequests &requests, const int step); - void SwapBuffers(const int step) noexcept final; + void SwapBuffers(const int step) noexcept; - void ResetBuffers() noexcept final; + void ResetBuffers() noexcept; - format::Buffer &GetConsumerBuffer(format::Buffer &buffer) final; + format::Buffer &GetConsumerBuffer(format::Buffer &buffer); private: bool m_IsInExchangeAbsolutePosition = false; diff --git a/source/adios2/toolkit/aggregator/mpi/MPIShmChain.cpp b/source/adios2/toolkit/aggregator/mpi/MPIShmChain.cpp new file mode 100644 index 0000000000..a05ed71a83 --- /dev/null +++ b/source/adios2/toolkit/aggregator/mpi/MPIShmChain.cpp @@ -0,0 +1,426 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * MPIShmChain.h + * + * Created on: July 5, 2021 + * Author: Norbert Podhorszki pnorbert@ornl.gov + */ +#include "MPIShmChain.h" + +#include "adios2/helper/adiosMemory.h" // PaddingToAlignOffset + +#include + +namespace adios2 +{ +namespace aggregator +{ + +MPIShmChain::MPIShmChain() : MPIAggregator() {} + +MPIShmChain::~MPIShmChain() { Close(); } + +void MPIShmChain::Close() +{ + if (m_IsActive) + { + m_NodeComm.Free("free per-node comm in ~MPIShmChain()"); + m_OnePerNodeComm.Free("free chain of nodes in ~MPIShmChain()"); + m_AllAggregatorsComm.Free( + "free comm of all aggregators in ~MPIShmChain()"); + m_AggregatorChainComm.Free( + "free chains of aggregators in ~MPIShmChain()"); + } + MPIAggregator::Close(); +} + +size_t MPIShmChain::PreInit(helper::Comm const &parentComm) +{ + /* Communicator connecting ranks on each Compute Node */ + m_NodeComm = parentComm.GroupByShm("creating per-node comm at Open"); + int NodeRank = m_NodeComm.Rank(); + + /* + * Communicators connecting rank N of each node + * We are only interested in the chain of rank 0s + */ + int color = (NodeRank ? 1 : 0); + m_OnePerNodeComm = + parentComm.Split(color, 0, "creating chain of nodes at Open"); + + /* Number of nodes */ + if (!NodeRank) + { + m_NumNodes = static_cast(m_OnePerNodeComm.Size()); + } + m_NumNodes = m_NodeComm.BroadcastValue(m_NumNodes, 0); + PreInitCalled = true; + return m_NumNodes; +} + +void MPIShmChain::Init(const size_t numAggregators, const size_t subStreams, + helper::Comm const &parentComm) +{ + if (!PreInitCalled) + { + PreInit(parentComm); + } + + // int AllRank = parentComm.Rank(); + // int AllSize = parentComm.Size(); + int NodeRank = m_NodeComm.Rank(); + size_t NodeSize = static_cast(m_NodeComm.Size()); + + /* Number of aggregators per node */ + size_t aggregatorPerNode = numAggregators / m_NumNodes; + if (aggregatorPerNode == 0) + { + aggregatorPerNode = 1; /* default */ + } + if (aggregatorPerNode > NodeSize) + { + aggregatorPerNode = NodeSize; + } + + /* Create main communicator that splits the node comm into one or more + * aggregator chains */ + float k = + static_cast(NodeSize) / static_cast(aggregatorPerNode); + float c = static_cast(NodeRank) / k; + int color = static_cast(c); + m_Comm = m_NodeComm.Split(color, 0, "creating aggregator groups at Open"); + m_Rank = m_Comm.Rank(); + m_Size = m_Comm.Size(); + if (m_Rank != 0) + { + m_IsAggregator = false; + m_IsMasterAggregator = false; + } + + /* Identify parent rank of aggregator process within each chain */ + if (!m_Rank) + { + m_AggregatorRank = parentComm.Rank(); + } + m_AggregatorRank = m_Comm.BroadcastValue(m_AggregatorRank, 0); + + /* Communicator for all Aggregators */ + color = (m_Rank ? 1 : 0); + m_AllAggregatorsComm = + parentComm.Split(color, 0, "creating comm of all aggregators at Open"); + + /* Total number of aggregators */ + if (!NodeRank) + { + m_NumAggregators = static_cast(m_AllAggregatorsComm.Size()); + } + m_NumAggregators = m_NodeComm.BroadcastValue(m_NumAggregators); + + /* Number of substreams */ + m_SubStreams = subStreams; + if (m_SubStreams == 0) + { + m_SubStreams = m_NumAggregators; /* default */ + } + if (m_SubStreams > m_NumAggregators) + { + m_SubStreams = m_NumAggregators; + } + + if (!m_Rank) + { + k = static_cast(m_NumAggregators) / + static_cast(m_SubStreams); + /* 1.0 <= k <= m_NumAggregators */ + c = static_cast(m_AllAggregatorsComm.Rank()) / k; + m_SubStreamIndex = static_cast(c); + } + m_SubStreamIndex = m_Comm.BroadcastValue(m_SubStreamIndex); + + /* Create the communicator to connect aggregators writing to the same + * substream */ + color = static_cast(m_SubStreamIndex); + m_AggregatorChainComm = m_AllAggregatorsComm.Split( + color, 0, "creating chains of aggregators at Open"); + + if (m_AggregatorChainComm.Rank() != 0) + { + m_IsMasterAggregator = false; + } + HandshakeStruct hsAC; // need persistence until complete + if (m_AggregatorChainComm.Size() > 1) + { + HandshakeLinks_Start(m_AggregatorChainComm, hsAC); + } + + m_IsActive = true; + + if (m_Comm.Size() > 1) + { + HandshakeStruct hs; // need persistence until complete + HandshakeLinks_Start(m_Comm, hs); + + /* Create the shared memory segment */ + // CreateShm(); + + HandshakeLinks_Complete(hs); + } + + if (m_AggregatorChainComm.Size() > 1) + { + HandshakeLinks_Complete(hsAC); + } +} + +// PRIVATE +void MPIShmChain::HandshakeLinks_Start(helper::Comm &comm, HandshakeStruct &hs) +{ + int rank = comm.Rank(); + hs.sendToken = rank; + if (rank < comm.Size() - 1) // send to next + { + hs.sendRequest = comm.Isend( + &hs.sendToken, 1, rank + 1, 0, + "Isend handshake with neighbor, MPIChain aggregator, at Open"); + } + else // send to 0 to close the loop + { + hs.sendRequest = comm.Isend( + &hs.sendToken, 1, 0, 0, + "Isend handshake with rank 0, MPIChain aggregator, at Open"); + } + + if (comm.Rank() > 0) // receive from previous + { + hs.recvRequest = comm.Irecv( + &hs.recvToken, 1, rank - 1, 0, + "Irecv handshake with neighbor, MPIChain aggregator, at Open"); + } + else // rank 0 receives from last + { + hs.recvRequest = comm.Irecv( + &hs.recvToken, 1, rank - 1, 0, + "Irecv handshake with neighbor, MPIChain aggregator, at Open"); + } +} + +void MPIShmChain::HandshakeLinks_Complete(HandshakeStruct &hs) +{ + hs.recvRequest.Wait("Wait handshake with neighbor (recv), MPIChain " + "aggregator, at Open"); + hs.sendRequest.Wait("Wait handshake with neighbor (send), MPIChain " + "aggregator, at Open"); +} + +void MPIShmChain::CreateShm(size_t blocksize, const size_t maxsegmentsize) +{ + if (!m_Comm.IsMPI()) + { + throw std::runtime_error("Coding Error: MPIShmChain::CreateShm was " + "called with a non-MPI communicator"); + } + char *ptr; + size_t structsize = sizeof(ShmSegment); + structsize += helper::PaddingToAlignOffset(structsize, sizeof(max_align_t)); + if (!m_Rank) + { + blocksize += + helper::PaddingToAlignOffset(blocksize, sizeof(max_align_t)); + size_t totalsize = structsize + 2 * blocksize; + if (totalsize > maxsegmentsize) + { + // roll back and calculate sizes from maxsegmentsize + totalsize = maxsegmentsize - sizeof(max_align_t) + 1; + totalsize += + helper::PaddingToAlignOffset(totalsize, sizeof(max_align_t)); + blocksize = (totalsize - structsize) / 2 - sizeof(max_align_t) + 1; + blocksize += + helper::PaddingToAlignOffset(blocksize, sizeof(max_align_t)); + totalsize = structsize + 2 * blocksize; + } + m_Win = m_Comm.Win_allocate_shared(totalsize, 1, &ptr); + } + else + { + m_Win = m_Comm.Win_allocate_shared(0, 1, &ptr); + size_t shmsize; + int disp_unit; + m_Comm.Win_shared_query(m_Win, 0, &shmsize, &disp_unit, &ptr); + blocksize = (shmsize - structsize) / 2; + } + m_Shm = reinterpret_cast(ptr); + m_ShmBufA = ptr + structsize; + m_ShmBufB = m_ShmBufA + blocksize; + + if (!m_Rank) + { + m_Shm->producerBuffer = LastBufferUsed::None; + m_Shm->consumerBuffer = LastBufferUsed::None; + m_Shm->NumBuffersFull = 0; + m_Shm->sdbA.buf = nullptr; + m_Shm->sdbA.max_size = blocksize; + m_Shm->sdbB.buf = nullptr; + m_Shm->sdbB.max_size = blocksize; + } + /*std::cout << "Rank " << m_Rank << " shm = " << ptr + << " bufA = " << static_cast(m_Shm->bufA) + << " bufB = " << static_cast(m_Shm->bufB) << std::endl;*/ +} + +void MPIShmChain::DestroyShm() { m_Comm.Win_free(m_Win); } + +/* + The buffering strategy is the following. + Assumptions: 1. Only one Producer (and one Consumer) is active at a time. + + The first Producer fills buffer A first then B and then is always + alternating, blocking when Consumer is behind (NumBuffersFull == 2). + The next Producer will continue with the alternating pattern where the + previous Producer has finished. + + The Consumer is blocked until there is at least one buffer available. + It takes buffer A at the first call, then it alternates between the two + buffers. + + C++ atomic locks in Shm for IPC locking: + - lockSegment is used to modify the m_Shm variables exclusively (short time), + - lockA and lockB are used to give long term exclusive access to one buffer + to be filled or consumed. + + The sleeping phases, to wait on the other party to catch up, are outside of + the locking code areas. + + Note: the m_Shm->sdbX.buf pointers must be set on the local process every + time, even tough it is stored on the shared memory segment, because the + address of the segment is different on every process. Failing to set on the + local process causes this pointer pointing to an invalid address (set on + another process). + + Note: the sdbA and sdbB structs are stored on the shared memory segment + because they contain 'actual_size' which is set on the Producer and used by + the Consumer. + +*/ + +MPIShmChain::ShmDataBuffer *MPIShmChain::LockProducerBuffer() +{ + MPIShmChain::ShmDataBuffer *sdb = nullptr; + + // Sleep until there is a buffer available at all + while (m_Shm->NumBuffersFull == 2) + { + std::this_thread::sleep_for(std::chrono::duration(0.00001)); + } + + m_Shm->lockSegment.lock(); + if (m_Shm->producerBuffer == LastBufferUsed::A) + + { + m_Shm->producerBuffer = LastBufferUsed::B; + sdb = &m_Shm->sdbB; + // point to shm data buffer (in local process memory) + sdb->buf = m_ShmBufB; + } + else // None or B + { + m_Shm->producerBuffer = LastBufferUsed::A; + sdb = &m_Shm->sdbA; + // point to shm data buffer (in local process memory) + sdb->buf = m_ShmBufA; + } + m_Shm->lockSegment.unlock(); + + // We determined we want a specific buffer + // Now we need to get a lock on it in case consumer is using it + if (m_Shm->producerBuffer == LastBufferUsed::A) + { + m_Shm->lockA.lock(); + } + else + { + m_Shm->lockB.lock(); + } + + return sdb; +} + +void MPIShmChain::UnlockProducerBuffer() +{ + m_Shm->lockSegment.lock(); + ++m_Shm->NumBuffersFull; + m_Shm->lockSegment.unlock(); + + if (m_Shm->producerBuffer == LastBufferUsed::A) + { + m_Shm->lockA.unlock(); + } + else + { + m_Shm->lockB.unlock(); + } +} + +MPIShmChain::ShmDataBuffer *MPIShmChain::LockConsumerBuffer() +{ + MPIShmChain::ShmDataBuffer *sdb = nullptr; + + // Sleep until there is at least one buffer filled + while (m_Shm->NumBuffersFull < 1) + { + std::this_thread::sleep_for(std::chrono::duration(0.00001)); + } + // At this point we know buffer A has content or going to have content + // when we successfully lock it + + m_Shm->lockSegment.lock(); + if (m_Shm->consumerBuffer == LastBufferUsed::A) + + { + m_Shm->consumerBuffer = LastBufferUsed::B; + sdb = &m_Shm->sdbB; + // point to shm data buffer (in local process memory) + sdb->buf = m_ShmBufB; + } + else // None or B + { + m_Shm->consumerBuffer = LastBufferUsed::A; + sdb = &m_Shm->sdbA; + // point to shm data buffer (in local process memory) + sdb->buf = m_ShmBufA; + } + m_Shm->lockSegment.unlock(); + + // We determined we want a specific buffer + // Now we need to get a lock on it in case producer is using it + if (m_Shm->consumerBuffer == LastBufferUsed::A) + { + m_Shm->lockA.lock(); + } + else + { + m_Shm->lockB.lock(); + } + + return sdb; +} + +void MPIShmChain::UnlockConsumerBuffer() +{ + m_Shm->lockSegment.lock(); + --m_Shm->NumBuffersFull; + m_Shm->lockSegment.unlock(); + + if (m_Shm->consumerBuffer == LastBufferUsed::A) + { + m_Shm->lockA.unlock(); + } + else + { + m_Shm->lockB.unlock(); + } +} + +} // end namespace aggregator +} // end namespace adios2 diff --git a/source/adios2/toolkit/aggregator/mpi/MPIShmChain.h b/source/adios2/toolkit/aggregator/mpi/MPIShmChain.h new file mode 100644 index 0000000000..50ed905c31 --- /dev/null +++ b/source/adios2/toolkit/aggregator/mpi/MPIShmChain.h @@ -0,0 +1,181 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * MPIShmChain.h + * + * Created on: July 5, 2021 + * Author: Norbert Podhorszki pnorbert@ornl.gov + * + */ + +#ifndef ADIOS2_TOOLKIT_AGGREGATOR_MPI_MPISHMCHAIN_H_ +#define ADIOS2_TOOLKIT_AGGREGATOR_MPI_MPISHMCHAIN_H_ + +#include "adios2/common/ADIOSConfig.h" +#include "adios2/toolkit/aggregator/mpi/MPIAggregator.h" + +#include +#include +#include + +namespace adios2 +{ +namespace aggregator +{ + +class Spinlock +{ + /* from + * https://wang-yimu.com/a-tutorial-on-shared-memory-inter-process-communication + */ +public: + Spinlock() { flag_.clear(); } + void lock() + { + while (!try_lock()) + { + std::this_thread::sleep_for(std::chrono::duration(0.00001)); + } + } + void unlock() { flag_.clear(); } + +private: + inline bool try_lock() { return !flag_.test_and_set(); } + std::atomic_flag flag_; //{ATOMIC_FLAG_INIT}; +}; + +// constexpr size_t SHM_BUF_SIZE = 4194304; // 4MB +// we allocate 2x this size + a bit for shared memory segment + +/** A one- or two-layer aggregator chain for using Shared memory within a + * compute node. + * Use MPI split type to group processes within a node into one chain. + * Depending on the number of aggregators, multiple nodes may be merged into + * a two-layer chain, (multiple Aggregators and one Master). + * Otherwise its a simple chain with one Aggregator=Master. + * + * m_Comm is the communicator that split for each node + * + */ + +class MPIShmChain : public MPIAggregator +{ + +public: + MPIShmChain(); + + ~MPIShmChain(); + + /* Create a per-node communicator and return number of nodes */ + size_t PreInit(helper::Comm const &parentComm); + + void Init(const size_t numAggregators, const size_t subStreams, + helper::Comm const &parentComm) final; + + void Close() final; + + /** + * true: the Master (aggregator) process in the chain + * always m_Rank == m_Comm.Rank() == 0 for a master aggregator + * same as (m_AggregatorChainComm.Rank() == 0) + */ + bool m_IsMasterAggregator = true; + + /* + Variables set in PreInit + */ + + bool PreInitCalled = false; + /* Communicator per compute node */ + helper::Comm m_NodeComm; + /* Communicator connecting rank 0 on each node + Useful only on rank 0s of m_NodeComm */ + helper::Comm m_OnePerNodeComm; + /* Number of Compute Nodes + * (size of m_OnePerNodeComm created from rank 0s of m_NodeComm) + */ + size_t m_NumNodes; + + /* + Variables set in Init + */ + + /* Communicator connecting all aggregators + (rank 0 of each aggregator group) + Useful only on aggregators themselves + */ + helper::Comm m_AllAggregatorsComm; + + /* Communicator connecting the aggregators + that write to the same substream. + rank 0 becomes a MasterAggregator + Useful only on aggregators themselves + */ + helper::Comm m_AggregatorChainComm; + + struct ShmDataBuffer + { + size_t max_size; // max size for buf + size_t actual_size; // size of actual content + // points to data buffer in shared memory + // Warning: This is a different address on every process + char *buf; + }; + + ShmDataBuffer *LockProducerBuffer(); + void UnlockProducerBuffer(); + ShmDataBuffer *LockConsumerBuffer(); + void UnlockConsumerBuffer(); + void ResetBuffers() noexcept; + + // 2*blocksize+some is allocated but only up to maxsegmentsize + void CreateShm(size_t blocksize, const size_t maxsegmentsize); + void DestroyShm(); + +private: + struct HandshakeStruct + { + int sendToken; + int recvToken; + helper::Comm::Req sendRequest; + helper::Comm::Req recvRequest; + }; + + void HandshakeLinks_Start(helper::Comm &comm, HandshakeStruct &hs); + void HandshakeLinks_Complete(HandshakeStruct &hs); + + helper::Comm::Win m_Win; + + enum class LastBufferUsed + { + None, + A, + B + }; + + struct ShmSegment + { + LastBufferUsed producerBuffer; + LastBufferUsed consumerBuffer; + unsigned int NumBuffersFull; + // user facing structs + ShmDataBuffer sdbA; + ShmDataBuffer sdbB; + aggregator::Spinlock lockSegment; + // locks for individual buffers (sdb and buf) + aggregator::Spinlock lockA; + aggregator::Spinlock lockB; + // the actual data buffers + // char bufA[SHM_BUF_SIZE]; + // char bufB[SHM_BUF_SIZE]; + }; + ShmSegment *m_Shm; + char *m_ShmBufA; + char *m_ShmBufB; +}; + +} // end namespace aggregator +} // end namespace adios2 + +#endif /* ADIOS2_TOOLKIT_AGGREGATOR_MPI_MPISHMCHAIN_H_ */ diff --git a/source/adios2/toolkit/burstbuffer/FileDrainerSingleThread.cpp b/source/adios2/toolkit/burstbuffer/FileDrainerSingleThread.cpp index b2a37e6ca4..91c9603eee 100644 --- a/source/adios2/toolkit/burstbuffer/FileDrainerSingleThread.cpp +++ b/source/adios2/toolkit/burstbuffer/FileDrainerSingleThread.cpp @@ -20,6 +20,7 @@ #include #include "../../common/ADIOSTypes.h" +#include "../../core/CoreTypes.h" /// \endcond #if defined(__has_feature) @@ -54,23 +55,17 @@ void FileDrainerSingleThread::Finish() finishMutex.unlock(); } -typedef std::chrono::duration Seconds; -typedef std::chrono::time_point< - std::chrono::steady_clock, - std::chrono::duration> - TimePoint; - void FileDrainerSingleThread::Join() { if (th.joinable()) { - const auto tTotalStart = std::chrono::steady_clock::now(); - Seconds timeTotal = Seconds(0.0); + const auto tTotalStart = core::Now(); + core::Seconds timeTotal(0.0); Finish(); th.join(); - const auto tTotalEnd = std::chrono::steady_clock::now(); + const auto tTotalEnd = core::Now(); timeTotal = tTotalEnd - tTotalStart; if (m_Verbose) { @@ -89,13 +84,13 @@ void FileDrainerSingleThread::Join() */ void FileDrainerSingleThread::DrainThread() { - const auto tTotalStart = std::chrono::steady_clock::now(); - Seconds timeTotal = Seconds(0.0); - Seconds timeSleep = Seconds(0.0); - Seconds timeRead = Seconds(0.0); - Seconds timeWrite = Seconds(0.0); - Seconds timeClose = Seconds(0.0); - TimePoint ts, te; + const auto tTotalStart = core::Now(); + core::Seconds timeTotal(0.0); + core::Seconds timeSleep(0.0); + core::Seconds timeRead(0.0); + core::Seconds timeWrite(0.0); + core::Seconds timeClose(0.0); + core::TimePoint ts, te; size_t maxQueueSize = 0; std::vector buffer; // fixed, preallocated buffer to read/write data buffer.resize(bufferSize); @@ -110,18 +105,18 @@ void FileDrainerSingleThread::DrainThread() auto lf_Copy = [&](FileDrainOperation &fdo, InputFile fdr, OutputFile fdw, size_t count) { nReadBytesTasked += count; - ts = std::chrono::steady_clock::now(); + ts = core::Now(); std::pair ret = Read(fdr, count, buffer.data(), fdo.fromFileName); - te = std::chrono::steady_clock::now(); + te = core::Now(); timeRead += te - ts; nReadBytesSucc += ret.first; sleptForWaitingOnRead += ret.second; nWriteBytesTasked += count; - ts = std::chrono::steady_clock::now(); + ts = core::Now(); size_t n = Write(fdw, count, buffer.data(), fdo.toFileName); - te = std::chrono::steady_clock::now(); + te = core::Now(); timeWrite += te - ts; nWriteBytesSucc += n; }; @@ -141,9 +136,9 @@ void FileDrainerSingleThread::DrainThread() { break; } - ts = std::chrono::steady_clock::now(); + ts = core::Now(); std::this_thread::sleep_for(d); - te = std::chrono::steady_clock::now(); + te = core::Now(); timeSleep += te - ts; continue; } @@ -162,15 +157,15 @@ void FileDrainerSingleThread::DrainThread() case DrainOperation::CopyAt: case DrainOperation::Copy: { - ts = std::chrono::steady_clock::now(); + ts = core::Now(); auto fdr = GetFileForRead(fdo.fromFileName); - te = std::chrono::steady_clock::now(); + te = core::Now(); timeRead += te - ts; - ts = std::chrono::steady_clock::now(); + ts = core::Now(); bool append = (fdo.op == DrainOperation::Copy); auto fdw = GetFileForWrite(fdo.toFileName, append); - te = std::chrono::steady_clock::now(); + te = core::Now(); timeWrite += te - ts; if (m_Verbose >= 2) @@ -198,14 +193,14 @@ void FileDrainerSingleThread::DrainThread() { if (fdo.op == DrainOperation::CopyAt) { - ts = std::chrono::steady_clock::now(); + ts = core::Now(); Seek(fdr, fdo.fromOffset, fdo.fromFileName); - te = std::chrono::steady_clock::now(); + te = core::Now(); timeRead += te - ts; - ts = std::chrono::steady_clock::now(); + ts = core::Now(); Seek(fdw, fdo.toOffset, fdo.toFileName); - te = std::chrono::steady_clock::now(); + te = core::Now(); timeWrite += te - ts; } const size_t batches = fdo.countBytes / bufferSize; @@ -236,10 +231,10 @@ void FileDrainerSingleThread::DrainThread() << fdo.toFileName << std::endl; #endif } - ts = std::chrono::steady_clock::now(); + ts = core::Now(); auto fdw = GetFileForWrite(fdo.toFileName); SeekEnd(fdw); - te = std::chrono::steady_clock::now(); + te = core::Now(); timeWrite += te - ts; break; } @@ -255,12 +250,12 @@ void FileDrainerSingleThread::DrainThread() #endif } nWriteBytesTasked += fdo.countBytes; - ts = std::chrono::steady_clock::now(); + ts = core::Now(); auto fdw = GetFileForWrite(fdo.toFileName); Seek(fdw, fdo.toOffset, fdo.toFileName); size_t n = Write(fdw, fdo.countBytes, fdo.dataToWrite.data(), fdo.toFileName); - te = std::chrono::steady_clock::now(); + te = core::Now(); timeWrite += te - ts; nWriteBytesSucc += n; break; @@ -277,11 +272,11 @@ void FileDrainerSingleThread::DrainThread() #endif } nWriteBytesTasked += fdo.countBytes; - ts = std::chrono::steady_clock::now(); + ts = core::Now(); auto fdw = GetFileForWrite(fdo.toFileName); size_t n = Write(fdw, fdo.countBytes, fdo.dataToWrite.data(), fdo.toFileName); - te = std::chrono::steady_clock::now(); + te = core::Now(); timeWrite += te - ts; nWriteBytesSucc += n; break; @@ -295,9 +290,9 @@ void FileDrainerSingleThread::DrainThread() << fdo.toFileName << std::endl; #endif } - ts = std::chrono::steady_clock::now(); + ts = core::Now(); GetFileForWrite(fdo.toFileName, false); - te = std::chrono::steady_clock::now(); + te = core::Now(); timeWrite += te - ts; break; } @@ -310,9 +305,9 @@ void FileDrainerSingleThread::DrainThread() << fdo.toFileName << " for append " << std::endl; #endif } - ts = std::chrono::steady_clock::now(); + ts = core::Now(); GetFileForWrite(fdo.toFileName, true); - te = std::chrono::steady_clock::now(); + te = core::Now(); timeWrite += te - ts; break; } @@ -325,10 +320,10 @@ void FileDrainerSingleThread::DrainThread() << fdo.toFileName << std::endl; #endif } - ts = std::chrono::steady_clock::now(); + ts = core::Now(); auto fdw = GetFileForWrite(fdo.toFileName, true); Delete(fdw, fdo.toFileName); - te = std::chrono::steady_clock::now(); + te = core::Now(); timeWrite += te - ts; break; } @@ -349,12 +344,12 @@ void FileDrainerSingleThread::DrainThread() #endif } - ts = std::chrono::steady_clock::now(); + ts = core::Now(); CloseAll(); - te = std::chrono::steady_clock::now(); + te = core::Now(); timeClose += te - ts; - const auto tTotalEnd = std::chrono::steady_clock::now(); + const auto tTotalEnd = core::Now(); timeTotal = tTotalEnd - tTotalStart; const bool shouldReport = (m_Verbose || (nReadBytesTasked != nReadBytesSucc) || diff --git a/source/adios2/toolkit/format/bp/BPBase.cpp b/source/adios2/toolkit/format/bp/BPBase.cpp index 0ec23a9a02..6b669b5b75 100644 --- a/source/adios2/toolkit/format/bp/BPBase.cpp +++ b/source/adios2/toolkit/format/bp/BPBase.cpp @@ -13,14 +13,6 @@ #include "adios2/helper/adiosFunctions.h" -#include "adios2/toolkit/format/bp/bpOperation/compress/BPBZIP2.h" -#include "adios2/toolkit/format/bp/bpOperation/compress/BPBlosc.h" -#include "adios2/toolkit/format/bp/bpOperation/compress/BPLIBPRESSIO.h" -#include "adios2/toolkit/format/bp/bpOperation/compress/BPMGARD.h" -#include "adios2/toolkit/format/bp/bpOperation/compress/BPPNG.h" -#include "adios2/toolkit/format/bp/bpOperation/compress/BPSZ.h" -#include "adios2/toolkit/format/bp/bpOperation/compress/BPZFP.h" - namespace adios2 { namespace format @@ -281,11 +273,6 @@ void BPBase::Init(const Params ¶meters, const std::string hint, profiling::Timer("mkdir", timeUnit)); m_Profiler.m_Bytes.emplace("buffering", 0); } - - // set initial buffer size - m_Profiler.Start("buffering"); - m_Data.Resize(m_Parameters.InitialBufferSize, hint); - m_Profiler.Stop("buffering"); } BPBase::ResizeResult BPBase::ResizeBuffer(const size_t dataIn, @@ -393,7 +380,7 @@ BPBase::GetTransportIDs(const std::vector &transportsTypes) const std::vector transportsIDs; transportsIDs.reserve(transportsTypes.size()); - for (const std::string transportType : transportsTypes) + for (const std::string &transportType : transportsTypes) { transportsIDs.push_back(lf_GetTransportID(transportType)); } @@ -452,14 +439,14 @@ std::string BPBase::ReadBPString(const std::vector &buffer, // static members const std::set BPBase::m_TransformTypes = { {"unknown", "none", "identity", "bzip2", "sz", "zfp", "mgard", "png", - "blosc"}}; + "blosc", "sirius"}}; const std::map BPBase::m_TransformTypesToNames = { {transform_unknown, "unknown"}, {transform_none, "none"}, {transform_identity, "identity"}, {transform_sz, "sz"}, {transform_zfp, "zfp"}, {transform_mgard, "mgard"}, {transform_png, "png"}, {transform_bzip2, "bzip2"}, - {transform_blosc, "blosc"}}; + {transform_blosc, "blosc"}, {transform_sirius, "sirius"}}; BPBase::TransformTypes BPBase::TransformTypeEnum(const std::string transformType) const noexcept @@ -477,60 +464,6 @@ BPBase::TransformTypeEnum(const std::string transformType) const noexcept return transformEnum; } -std::shared_ptr -BPBase::SetBPOperation(const std::string type) const noexcept -{ - std::shared_ptr bpOp; - if (type == "sz") - { - bpOp = std::make_shared(); - } - else if (type == "zfp") - { - bpOp = std::make_shared(); - } - else if (type == "mgard") - { - bpOp = std::make_shared(); - } - else if (type == "bzip2") - { - bpOp = std::make_shared(); - } - else if (type == "png") - { - bpOp = std::make_shared(); - } - else if (type == "blosc") - { - bpOp = std::make_shared(); - } - else if (type == "libpressio") - { - bpOp = std::make_shared(); - } - - return bpOp; -} - -std::map> BPBase::SetBPOperations( - const std::vector &operations) const -{ - std::map> bpOperations; - - for (size_t i = 0; i < operations.size(); ++i) - { - const std::string type = operations[i].Op->m_Type; - std::shared_ptr bpOperation = SetBPOperation(type); - - if (bpOperation) // if the result is a supported type - { - bpOperations.emplace(i, bpOperation); - } - } - return bpOperations; -} - #define declare_template_instantiation(T) \ template BPBase::Characteristics \ BPBase::ReadElementIndexCharacteristics(const std::vector &, \ diff --git a/source/adios2/toolkit/format/bp/BPBase.h b/source/adios2/toolkit/format/bp/BPBase.h index 6a8cb16393..52a8bb26bf 100644 --- a/source/adios2/toolkit/format/bp/BPBase.h +++ b/source/adios2/toolkit/format/bp/BPBase.h @@ -451,7 +451,8 @@ class BPBase transform_lz4 = 10, transform_blosc = 11, transform_mgard = 12, - transform_png = 13 + transform_png = 13, + transform_sirius = 14 }; /** Supported transform types */ @@ -469,22 +470,6 @@ class BPBase TransformTypes TransformTypeEnum(const std::string transformType) const noexcept; - /** - * Returns the proper derived class for BPOperation based on type - * @param type input, must be a supported type under BPOperation - * @return derived class if supported, false pointer if type not supported - */ - std::shared_ptr SetBPOperation(const std::string type) const - noexcept; - - /** - * Holds requested BPOperations - * @param operations input operations form user - * @return map holding only valid operations - */ - std::map> SetBPOperations( - const std::vector &operations) const; - struct ProcessGroupIndex { uint64_t Offset; diff --git a/source/adios2/toolkit/format/bp/BPSerializer.cpp b/source/adios2/toolkit/format/bp/BPSerializer.cpp index d933ac3cb1..69c89b85a8 100644 --- a/source/adios2/toolkit/format/bp/BPSerializer.cpp +++ b/source/adios2/toolkit/format/bp/BPSerializer.cpp @@ -350,7 +350,7 @@ void BPSerializer::UpdateOffsetsInMetadata() }; // BODY OF FUNCTION STARTS HERE - if (m_Aggregator.m_IsConsumer) + if (m_Aggregator.m_IsAggregator) { return; } diff --git a/source/adios2/toolkit/format/bp/BPSerializer.tcc b/source/adios2/toolkit/format/bp/BPSerializer.tcc index 64968b7440..0a7b151548 100644 --- a/source/adios2/toolkit/format/bp/BPSerializer.tcc +++ b/source/adios2/toolkit/format/bp/BPSerializer.tcc @@ -106,8 +106,8 @@ void BPSerializer::UpdateIndexOffsetsCharacteristics(size_t ¤tPosition, std::vector &buffer) { const bool isLittleEndian = helper::IsLittleEndian(); - const uint8_t characteristicsCount = - helper::ReadValue(buffer, currentPosition, isLittleEndian); + + helper::ReadValue(buffer, currentPosition, isLittleEndian); const uint32_t characteristicsLength = helper::ReadValue(buffer, currentPosition, isLittleEndian); @@ -354,14 +354,7 @@ void BPSerializer::PutCharacteristicOperation( const typename core::Variable::BPInfo &blockInfo, std::vector &buffer) noexcept { - // TODO: we only take the first operation for now - const std::map> bpOperations = - SetBPOperations(blockInfo.Operations); - - const size_t operationIndex = bpOperations.begin()->first; - std::shared_ptr bpOperation = bpOperations.begin()->second; - - auto &operation = blockInfo.Operations[operationIndex]; + auto &operation = blockInfo.Operations[0]; const std::string type = operation.Op->m_Type; const uint8_t typeLength = static_cast(type.size()); @@ -379,7 +372,8 @@ void BPSerializer::PutCharacteristicOperation( PutDimensionsRecord(blockInfo.Count, blockInfo.Shape, blockInfo.Start, buffer); // here put the metadata info depending on operation - bpOperation->SetMetadata(variable, blockInfo, operation, buffer); + BPOperation bpOperation; + bpOperation.SetMetadata(variable, blockInfo, operation, buffer); } template @@ -387,24 +381,16 @@ void BPSerializer::PutOperationPayloadInBuffer( const core::Variable &variable, const typename core::Variable::BPInfo &blockInfo) { - // TODO: we only take the first operation for now - const std::map> bpOperations = - SetBPOperations(blockInfo.Operations); - - const size_t operationIndex = bpOperations.begin()->first; - const std::shared_ptr bpOperation = - bpOperations.begin()->second; - bpOperation->SetData(variable, blockInfo, - blockInfo.Operations[operationIndex], m_Data); + BPOperation bpOperation; + bpOperation.SetData(variable, blockInfo, blockInfo.Operations[0], m_Data); // update metadata bool isFound = false; SerialElementIndex &variableIndex = GetSerialElementIndex( variable.m_Name, m_MetadataSet.VarsIndices, isFound); - bpOperation->UpdateMetadata(variable, blockInfo, - blockInfo.Operations[operationIndex], - variableIndex.Buffer); + bpOperation.UpdateMetadata(variable, blockInfo, blockInfo.Operations[0], + variableIndex.Buffer); } } // end namespace format diff --git a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.cpp b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.cpp index af9cec95fe..34d9d07189 100644 --- a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.cpp +++ b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.cpp @@ -29,7 +29,7 @@ namespace format std::mutex BP3Deserializer::m_Mutex; BP3Deserializer::BP3Deserializer(helper::Comm const &comm) -: BP3Base(comm), BPBase(comm), m_Minifooter(3) +: BPBase(comm), BP3Base(comm), m_Minifooter(3) { } @@ -38,7 +38,10 @@ void BP3Deserializer::ParseMetadata(const BufferSTL &bufferSTL, { ParseMinifooter(bufferSTL); - ParsePGIndex(bufferSTL, engine.m_IO.m_HostLanguage); + ParsePGIndex(bufferSTL, + (engine.m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) + ? "C++" + : "Fortran"); ParseVariablesIndex(bufferSTL, engine); ParseAttributesIndex(bufferSTL, engine); } @@ -139,8 +142,7 @@ void BP3Deserializer::ParsePGIndex(const BufferSTL &bufferSTL, m_MetadataSet.DataPGCount = helper::ReadValue( buffer, position, m_Minifooter.IsLittleEndian); - const size_t length = helper::ReadValue( - buffer, position, m_Minifooter.IsLittleEndian); + helper::ReadValue(buffer, position, m_Minifooter.IsLittleEndian); size_t localPosition = 0; @@ -210,10 +212,8 @@ void BP3Deserializer::ParseVariablesIndex(const BufferSTL &bufferSTL, m_Minifooter.VarsIndexStart, m_Minifooter.PGIndexStart, " BP3 variable index start < pg index start, in call to Open"); - const uint32_t count = helper::ReadValue( - buffer, position, m_Minifooter.IsLittleEndian); - const uint64_t length = helper::ReadValue( - buffer, position, m_Minifooter.IsLittleEndian); + helper::ReadValue(buffer, position, m_Minifooter.IsLittleEndian); + helper::ReadValue(buffer, position, m_Minifooter.IsLittleEndian); const size_t startPosition = position; size_t localPosition = 0; @@ -319,10 +319,8 @@ void BP3Deserializer::ParseAttributesIndex(const BufferSTL &bufferSTL, m_Minifooter.AttributesIndexStart, m_Minifooter.PGIndexStart, " BP3 attributes index start < pg index start, in call to Open"); - const uint32_t count = helper::ReadValue( - buffer, position, m_Minifooter.IsLittleEndian); - const uint64_t length = helper::ReadValue( - buffer, position, m_Minifooter.IsLittleEndian); + helper::ReadValue(buffer, position, m_Minifooter.IsLittleEndian); + helper::ReadValue(buffer, position, m_Minifooter.IsLittleEndian); const size_t startPosition = position; size_t localPosition = 0; diff --git a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc index fb561cdff4..bdd525e696 100644 --- a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc +++ b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc @@ -17,6 +17,7 @@ #include #include "adios2/helper/adiosFunctions.h" +#include "adios2/operator/compress/CompressorFactory.h" namespace adios2 { @@ -148,8 +149,8 @@ void BP3Deserializer::SetVariableBlockInfo( blockOperation.PreSizeOf = sizeof(T); // read metadata from supported type and populate Info - std::shared_ptr bpOp = SetBPOperation(bpOpInfo.Type); - bpOp->GetMetadata(bpOpInfo.Metadata, blockOperation.Info); + BPOperation bpOp; + bpOp.GetMetadata(bpOpInfo.Metadata, blockOperation.Info); blockOperation.PayloadSize = static_cast( std::stoull(blockOperation.Info.at("OutputSize"))); @@ -517,14 +518,12 @@ void BP3Deserializer::PostDataRead( blockOperationInfo.PreSizeOf; m_ThreadBuffers[threadID][0].resize(preOpPayloadSize); - // get the right bp3Op - std::shared_ptr bp3Op = - SetBPOperation(blockOperationInfo.Info.at("Type")); - // get original block back char *preOpData = m_ThreadBuffers[threadID][0].data(); const char *postOpData = m_ThreadBuffers[threadID][1].data(); - bp3Op->GetData(postOpData, blockOperationInfo, preOpData); + + core::compress::CompressorFactory of; + of.Decompress(postOpData, blockOperationInfo.PayloadSize, preOpData); // clip block to match selection helper::ClipVector(m_ThreadBuffers[threadID][0], diff --git a/source/adios2/toolkit/format/bp/bp3/BP3Serializer.tcc b/source/adios2/toolkit/format/bp/bp3/BP3Serializer.tcc index c8e7a86360..9a91b6e960 100644 --- a/source/adios2/toolkit/format/bp/bp3/BP3Serializer.tcc +++ b/source/adios2/toolkit/format/bp/bp3/BP3Serializer.tcc @@ -30,7 +30,7 @@ void BP3Serializer::PutVariableMetadata( const bool sourceRowMajor, typename core::Variable::Span *span) noexcept { auto lf_SetOffset = [&](uint64_t &offset) { - if (m_Aggregator.m_IsActive && !m_Aggregator.m_IsConsumer) + if (m_Aggregator.m_IsActive && !m_Aggregator.m_IsAggregator) { offset = static_cast(m_Data.m_Position); } diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Base.cpp b/source/adios2/toolkit/format/bp/bp4/BP4Base.cpp index f58440ba9c..2a35e807ea 100644 --- a/source/adios2/toolkit/format/bp/bp4/BP4Base.cpp +++ b/source/adios2/toolkit/format/bp/bp4/BP4Base.cpp @@ -83,6 +83,28 @@ std::string BP4Base::GetBPMetadataIndexFileName(const std::string &name) const return bpMetaDataIndexRankName; } +std::vector +BP4Base::GetBPVersionFileNames(const std::vector &names) const + noexcept +{ + std::vector versionFileNames; + versionFileNames.reserve(names.size()); + for (const auto &name : names) + { + versionFileNames.push_back(GetBPVersionFileName(name)); + } + return versionFileNames; +} + +std::string BP4Base::GetBPVersionFileName(const std::string &name) const + noexcept +{ + const std::string bpName = helper::RemoveTrailingSlash(name); + /* the name of the version file is ".bpversion" */ + const std::string bpVersionFileName(bpName + PathSeparator + ".bpversion"); + return bpVersionFileName; +} + std::vector BP4Base::GetBPActiveFlagFileNames(const std::vector &names) const noexcept diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Base.h b/source/adios2/toolkit/format/bp/bp4/BP4Base.h index c5fb29bb4d..afe4704ec5 100644 --- a/source/adios2/toolkit/format/bp/bp4/BP4Base.h +++ b/source/adios2/toolkit/format/bp/bp4/BP4Base.h @@ -104,6 +104,11 @@ class BP4Base : virtual public BPBase std::string GetBPMetadataIndexFileName(const std::string &name) const noexcept; + std::vector + GetBPVersionFileNames(const std::vector &names) const noexcept; + + std::string GetBPVersionFileName(const std::string &name) const noexcept; + std::vector GetBPActiveFlagFileNames(const std::vector &names) const noexcept; diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.cpp b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.cpp index 05728c9a41..a09a9013f1 100644 --- a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.cpp +++ b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.cpp @@ -33,7 +33,7 @@ namespace format std::mutex BP4Deserializer::m_Mutex; BP4Deserializer::BP4Deserializer(helper::Comm const &comm) -: BP4Base(comm), BPBase(comm), m_Minifooter(4) +: BPBase(comm), BP4Base(comm), m_Minifooter(4) { } @@ -67,8 +67,12 @@ size_t BP4Deserializer::ParseMetadata(const BufferSTL &bufferSTL, std::find(selectedSteps.begin(), selectedSteps.end(), i) != selectedSteps.end()) { - ParsePGIndexPerStep(bufferSTL, engine.m_IO.m_HostLanguage, 0, - i + 1); + ParsePGIndexPerStep( + bufferSTL, + (engine.m_IO.m_ArrayOrder == ArrayOrdering::RowMajor) + ? "C++" + : "Fortran", + 0, i + 1); ParseVariablesIndexPerStep(bufferSTL, engine, 0, i + 1); ParseAttributesIndexPerStep(bufferSTL, engine, 0, i + 1); lastposition = m_MetadataIndexTable[0][i + 1][3]; @@ -335,8 +339,7 @@ void BP4Deserializer::ParseVariablesIndexPerStep(const BufferSTL &bufferSTL, const auto &buffer = bufferSTL.m_Buffer; size_t position = m_MetadataIndexTable[submetadatafileId][step][1]; - const uint32_t count = helper::ReadValue( - buffer, position, m_Minifooter.IsLittleEndian); + helper::ReadValue(buffer, position, m_Minifooter.IsLittleEndian); const uint64_t length = helper::ReadValue( buffer, position, m_Minifooter.IsLittleEndian); @@ -528,8 +531,7 @@ void BP4Deserializer::ParseAttributesIndexPerStep(const BufferSTL &bufferSTL, const auto &buffer = bufferSTL.m_Buffer; size_t position = m_MetadataIndexTable[submetadatafileId][step][2]; - const uint32_t count = helper::ReadValue( - buffer, position, m_Minifooter.IsLittleEndian); + helper::ReadValue(buffer, position, m_Minifooter.IsLittleEndian); const uint64_t length = helper::ReadValue( buffer, position, m_Minifooter.IsLittleEndian); diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc index 7ffa95d3d4..22c67a449e 100644 --- a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc +++ b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc @@ -20,6 +20,7 @@ #include #include "adios2/helper/adiosFunctions.h" +#include "adios2/operator/compress/CompressorFactory.h" namespace adios2 { @@ -151,8 +152,8 @@ void BP4Deserializer::SetVariableBlockInfo( blockOperation.PreSizeOf = sizeof(T); // read metadata from supported type and populate Info - std::shared_ptr bpOp = SetBPOperation(bpOpInfo.Type); - bpOp->GetMetadata(bpOpInfo.Metadata, blockOperation.Info); + BPOperation bpOp; + bpOp.GetMetadata(bpOpInfo.Metadata, blockOperation.Info); blockOperation.PayloadSize = static_cast( std::stoull(blockOperation.Info.at("OutputSize"))); @@ -521,14 +522,12 @@ void BP4Deserializer::PostDataRead( blockOperationInfo.PreSizeOf; m_ThreadBuffers[threadID][0].resize(preOpPayloadSize); - // get the right bp4Op - std::shared_ptr bp4Op = - SetBPOperation(blockOperationInfo.Info.at("Type")); - // get original block back char *preOpData = m_ThreadBuffers[threadID][0].data(); const char *postOpData = m_ThreadBuffers[threadID][1].data(); - bp4Op->GetData(postOpData, blockOperationInfo, preOpData); + + core::compress::CompressorFactory of; + of.Decompress(postOpData, blockOperationInfo.PayloadSize, preOpData); // clip block to match selection helper::ClipVector(m_ThreadBuffers[threadID][0], @@ -1039,14 +1038,14 @@ void BP4Deserializer::DefineAttributeInEngineIO( if (characteristics.Statistics.IsValue) { - engine.m_IO.DefineAttribute(attributeName, - characteristics.Statistics.Value); + engine.m_IO.DefineAttribute( + attributeName, characteristics.Statistics.Value, "", "", true); } else { engine.m_IO.DefineAttribute( attributeName, characteristics.Statistics.Values.data(), - characteristics.Statistics.Values.size()); + characteristics.Statistics.Values.size(), "", "", true); } } diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Serializer.tcc b/source/adios2/toolkit/format/bp/bp4/BP4Serializer.tcc index e3e86089f2..9045dd500a 100644 --- a/source/adios2/toolkit/format/bp/bp4/BP4Serializer.tcc +++ b/source/adios2/toolkit/format/bp/bp4/BP4Serializer.tcc @@ -37,7 +37,7 @@ inline void BP4Serializer::PutVariableMetadata( const bool sourceRowMajor, typename core::Variable::Span *span) noexcept { auto lf_SetOffset = [&](uint64_t &offset) { - if (m_Aggregator.m_IsActive && !m_Aggregator.m_IsConsumer) + if (m_Aggregator.m_IsActive && !m_Aggregator.m_IsAggregator) { offset = static_cast(m_Data.m_Position); } @@ -633,14 +633,12 @@ void BP4Serializer::PutBoundsRecord(const bool singleValue, static_cast(stats.SubBlockInfo.SubBlockSize); helper::InsertToBuffer(buffer, &subBlockSize); - const uint16_t N = - static_cast(stats.SubBlockInfo.Div.size()); for (auto const d : stats.SubBlockInfo.Div) { helper::InsertToBuffer(buffer, &d); } // insert min+max (alternating) elements (2*M values) - for (auto const m : stats.MinMaxs) + for (auto const &m : stats.MinMaxs) { helper::InsertToBuffer(buffer, &m); } @@ -687,14 +685,12 @@ void BP4Serializer::PutBoundsRecord(const bool singleValue, static_cast(stats.SubBlockInfo.SubBlockSize); helper::CopyToBuffer(buffer, position, &subBlockSize); - const uint16_t N = - static_cast(stats.SubBlockInfo.Div.size()); for (auto const d : stats.SubBlockInfo.Div) { helper::CopyToBuffer(buffer, position, &d); } // insert min+max (alternating) elements (2*M values) - for (auto const m : stats.MinMaxs) + for (auto const &m : stats.MinMaxs) { helper::CopyToBuffer(buffer, position, &m); } diff --git a/source/adios2/toolkit/format/bp/bpOperation/BPOperation.cpp b/source/adios2/toolkit/format/bp/bpOperation/BPOperation.cpp index 0f7b497063..894d8021d1 100644 --- a/source/adios2/toolkit/format/bp/bpOperation/BPOperation.cpp +++ b/source/adios2/toolkit/format/bp/bpOperation/BPOperation.cpp @@ -16,47 +16,29 @@ namespace adios2 namespace format { -#define declare_type(T) \ - void BPOperation::SetData( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - BufferSTL &bufferSTL) const noexcept \ - { \ - } \ - \ - void BPOperation::SetMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept \ - { \ - } \ - \ - void BPOperation::UpdateMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept \ - { \ - } - -ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) -#undef declare_type +void BPOperation::GetMetadata(const std::vector &buffer, + Params &info) const noexcept +{ + size_t position = 0; + info["InputSize"] = + std::to_string(helper::ReadValue(buffer, position)); + info["OutputSize"] = + std::to_string(helper::ReadValue(buffer, position)); +} #define declare_type(T) \ \ - template void BPOperation::SetDataDefault( \ + template void BPOperation::SetData( \ const core::Variable &, const typename core::Variable::BPInfo &, \ const typename core::Variable::Operation &, BufferSTL &bufferSTL) \ const noexcept; \ \ - template void BPOperation::SetMetadataDefault( \ + template void BPOperation::SetMetadata( \ const core::Variable &, const typename core::Variable::BPInfo &, \ const typename core::Variable::Operation &, std::vector &) \ const noexcept; \ \ - template void BPOperation::UpdateMetadataDefault( \ + template void BPOperation::UpdateMetadata( \ const core::Variable &, const typename core::Variable::BPInfo &, \ const typename core::Variable::Operation &, std::vector &) \ const noexcept; diff --git a/source/adios2/toolkit/format/bp/bpOperation/BPOperation.h b/source/adios2/toolkit/format/bp/bpOperation/BPOperation.h index d022231171..5882aa6125 100644 --- a/source/adios2/toolkit/format/bp/bpOperation/BPOperation.h +++ b/source/adios2/toolkit/format/bp/bpOperation/BPOperation.h @@ -30,75 +30,46 @@ class BPOperation BPOperation() = default; virtual ~BPOperation() = default; -#define declare_type(T) \ - virtual void SetData( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - BufferSTL &bufferSTL) const noexcept; \ - \ - virtual void SetMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept; \ - \ - virtual void UpdateMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept; - - ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) -#undef declare_type - /** * Deserializes metadata in the form of parameters * @param buffer contains serialized metadata buffer * @param info parameters info from metadata buffer */ - virtual void GetMetadata(const std::vector &buffer, - Params &info) const noexcept = 0; + void GetMetadata(const std::vector &buffer, Params &info) const + noexcept; - virtual void GetData(const char *input, - const helper::BlockOperationInfo &blockOperationInfo, - char *dataOutput) const = 0; - -protected: template - void SetDataDefault(const core::Variable &variable, - const typename core::Variable::BPInfo &blockInfo, - const typename core::Variable::Operation &operation, - BufferSTL &bufferSTL) const noexcept; + void SetData(const core::Variable &variable, + const typename core::Variable::BPInfo &blockInfo, + const typename core::Variable::Operation &operation, + BufferSTL &bufferSTL) const noexcept; template - void - SetMetadataDefault(const core::Variable &variable, - const typename core::Variable::BPInfo &blockInfo, - const typename core::Variable::Operation &operation, - std::vector &buffer) const noexcept; + void SetMetadata(const core::Variable &variable, + const typename core::Variable::BPInfo &blockInfo, + const typename core::Variable::Operation &operation, + std::vector &buffer) const noexcept; template - void UpdateMetadataDefault( - const core::Variable &variable, - const typename core::Variable::BPInfo &blockInfo, - const typename core::Variable::Operation &operation, - std::vector &buffer) const noexcept; + void UpdateMetadata(const core::Variable &variable, + const typename core::Variable::BPInfo &blockInfo, + const typename core::Variable::Operation &operation, + std::vector &buffer) const noexcept; }; #define declare_type(T) \ \ - extern template void BPOperation::SetDataDefault( \ + extern template void BPOperation::SetData( \ const core::Variable &, const typename core::Variable::BPInfo &, \ const typename core::Variable::Operation &, BufferSTL &bufferSTL) \ const noexcept; \ \ - extern template void BPOperation::SetMetadataDefault( \ + extern template void BPOperation::SetMetadata( \ const core::Variable &, const typename core::Variable::BPInfo &, \ const typename core::Variable::Operation &, std::vector &) \ const noexcept; \ \ - extern template void BPOperation::UpdateMetadataDefault( \ + extern template void BPOperation::UpdateMetadata( \ const core::Variable &, const typename core::Variable::BPInfo &, \ const typename core::Variable::Operation &, std::vector &) \ const noexcept; diff --git a/source/adios2/toolkit/format/bp/bpOperation/BPOperation.tcc b/source/adios2/toolkit/format/bp/bpOperation/BPOperation.tcc index ffc13c2692..f83653761a 100644 --- a/source/adios2/toolkit/format/bp/bpOperation/BPOperation.tcc +++ b/source/adios2/toolkit/format/bp/bpOperation/BPOperation.tcc @@ -17,24 +17,23 @@ namespace adios2 { namespace format { -// DEFAULTS only saves input and output payload sizes in metadata -// PROTECTED + template -void BPOperation::SetDataDefault( +void BPOperation::SetData( const core::Variable &variable, const typename core::Variable::BPInfo &blockInfo, const typename core::Variable::Operation &operation, BufferSTL &bufferSTL) const noexcept { - const core::Operator &op = *operation.Op; + core::Operator &op = *operation.Op; const Params ¶meters = operation.Parameters; // being naughty here Params &info = const_cast(operation.Info); const size_t outputSize = op.Compress( - blockInfo.Data, blockInfo.Count, variable.m_ElementSize, - variable.m_Type, bufferSTL.m_Buffer.data() + bufferSTL.m_Position, - parameters, info); + reinterpret_cast(blockInfo.Data), blockInfo.Start, + blockInfo.Count, variable.m_Type, + bufferSTL.m_Buffer.data() + bufferSTL.m_Position, parameters); info["OutputSize"] = std::to_string(outputSize); @@ -43,7 +42,7 @@ void BPOperation::SetDataDefault( } template -void BPOperation::SetMetadataDefault( +void BPOperation::SetMetadata( const core::Variable &variable, const typename core::Variable::BPInfo &blockInfo, const typename core::Variable::Operation &operation, @@ -65,7 +64,7 @@ void BPOperation::SetMetadataDefault( } template -void BPOperation::UpdateMetadataDefault( +void BPOperation::UpdateMetadata( const core::Variable &variable, const typename core::Variable::BPInfo &blockInfo, const typename core::Variable::Operation &operation, diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPBZIP2.cpp b/source/adios2/toolkit/format/bp/bpOperation/compress/BPBZIP2.cpp deleted file mode 100644 index f1f08b4a32..0000000000 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPBZIP2.cpp +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * BPBZIP2.cpp - * - * Created on: Jun 10, 2019 - * Author: William F Godoy godoywf@ornl.gov - */ - -#include "BPBZIP2.h" -#include "BPBZIP2.tcc" - -#include "adios2/helper/adiosFunctions.h" - -#ifdef ADIOS2_HAVE_BZIP2 -#include "adios2/operator/compress/CompressBZIP2.h" -#endif - -namespace adios2 -{ -namespace format -{ - -#define declare_type(T) \ - void BPBZIP2::SetData( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - BufferSTL &bufferSTL) const noexcept \ - { \ - SetDataDefault(variable, blockInfo, operation, bufferSTL); \ - } \ - \ - void BPBZIP2::SetMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept \ - { \ - SetMetadataCommon(variable, blockInfo, operation, buffer); \ - } \ - \ - void BPBZIP2::UpdateMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept \ - { \ - UpdateMetadataCommon(variable, blockInfo, operation, buffer); \ - } - -ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type) -#undef declare_type - -void BPBZIP2::GetMetadata(const std::vector &buffer, Params &info) const - noexcept -{ - size_t position = 0; - info["InputSize"] = - std::to_string(helper::ReadValue(buffer, position)); - info["OutputSize"] = - std::to_string(helper::ReadValue(buffer, position)); - - const uint16_t batches = helper::ReadValue(buffer, position); - info["batches"] = std::to_string(batches); - - for (auto b = 0; b < batches; ++b) - { - const std::string bStr = std::to_string(b); - - info["OriginalOffset_" + bStr] = - std::to_string(helper::ReadValue(buffer, position)); - info["OriginalSize_" + bStr] = - std::to_string(helper::ReadValue(buffer, position)); - info["CompressedOffset_" + bStr] = - std::to_string(helper::ReadValue(buffer, position)); - info["CompressedSize_" + bStr] = - std::to_string(helper::ReadValue(buffer, position)); - } -} - -void BPBZIP2::GetData(const char *input, - const helper::BlockOperationInfo &blockOperationInfo, - char *dataOutput) const -{ -#ifdef ADIOS2_HAVE_BZIP2 - core::compress::CompressBZIP2 op((Params())); - const size_t sizeOut = (sizeof(size_t) == 8) - ? static_cast(helper::StringTo( - blockOperationInfo.Info.at("InputSize"), - "when reading BZIP2 input size")) - : static_cast(helper::StringTo( - blockOperationInfo.Info.at("InputSize"), - "when reading BZIP2 input size")); - - Params &info = const_cast(blockOperationInfo.Info); - op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput, sizeOut, - info); - -#else - throw std::runtime_error( - "ERROR: current ADIOS2 library didn't compile " - "with BZIP2, can't read BZIP2 compressed data, in call " - "to Get\n"); -#endif -} - -} // end namespace format -} // end namespace adios2 diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPBZIP2.h b/source/adios2/toolkit/format/bp/bpOperation/compress/BPBZIP2.h deleted file mode 100644 index b5c5a433d5..0000000000 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPBZIP2.h +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * BPBZIP2.h - * - * Created on: Jun 10, 2019 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPBZIP2_H_ -#define ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPBZIP2_H_ - -#include "adios2/toolkit/format/bp/bpOperation/BPOperation.h" - -namespace adios2 -{ -namespace format -{ - -class BPBZIP2 : public BPOperation -{ -public: - BPBZIP2() = default; - - ~BPBZIP2() = default; - - using BPOperation::SetData; - using BPOperation::SetMetadata; - using BPOperation::UpdateMetadata; - // using override due to PGI compiler warnings -#define declare_type(T) \ - void SetData(const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - BufferSTL &bufferSTL) const noexcept override; \ - \ - void SetMetadata(const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept override; \ - \ - void UpdateMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept override; - - ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type) -#undef declare_type - - void GetMetadata(const std::vector &buffer, Params &info) const - noexcept final; - - void GetData(const char *input, - const helper::BlockOperationInfo &blockOperationInfo, - char *dataOutput) const final; - -private: - template - void - SetMetadataCommon(const core::Variable &variable, - const typename core::Variable::BPInfo &blockInfo, - const typename core::Variable::Operation &operation, - std::vector &buffer) const noexcept; - - template - void - UpdateMetadataCommon(const core::Variable &variable, - const typename core::Variable::BPInfo &blockInfo, - const typename core::Variable::Operation &operation, - std::vector &buffer) const noexcept; -}; - -} // end namespace format -} // end namespace adios2 - -#endif /* ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPBZIP2_H_ */ diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPBZIP2.tcc b/source/adios2/toolkit/format/bp/bpOperation/compress/BPBZIP2.tcc deleted file mode 100644 index e361ca7003..0000000000 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPBZIP2.tcc +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * BPBZIP2.tcc - * - * Created on: Jun 10, 2019 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPBZIP2_TCC_ -#define ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPBZIP2_TCC_ - -#include "BPBZIP2.h" - -#include "adios2/helper/adiosFunctions.h" - -namespace adios2 -{ -namespace format -{ - -template -void BPBZIP2::SetMetadataCommon( - const core::Variable &variable, - const typename core::Variable::BPInfo &blockInfo, - const typename core::Variable::Operation &operation, - std::vector &buffer) const noexcept -{ - const uint64_t inputSize = static_cast( - helper::GetTotalSize(blockInfo.Count) * sizeof(T)); - // being naughty here - Params &info = const_cast(operation.Info); - info["InputSize"] = std::to_string(inputSize); - - // fixed size - const uint16_t batches = - static_cast(inputSize / DefaultMaxFileBatchSize + 1); - - const uint16_t metadataSize = 8 + 8 + 2 + batches * (4 * 8); - helper::InsertToBuffer(buffer, &metadataSize); - helper::InsertToBuffer(buffer, &inputSize); - info["OutputSizeMetadataPosition"] = std::to_string(buffer.size()); - - constexpr uint64_t outputSize = 0; - // dummy - helper::InsertToBuffer(buffer, &outputSize); - // additional batch info - helper::InsertToBuffer(buffer, &batches); - info["BatchesMetadataPosition"] = std::to_string(buffer.size()); - - // inserting dummies to preallocate, updated in UpdateMetadataCommon - buffer.resize(buffer.size() + batches * 4 * 8); -} - -template -void BPBZIP2::UpdateMetadataCommon( - const core::Variable &variable, - const typename core::Variable::BPInfo &blockInfo, - const typename core::Variable::Operation &operation, - std::vector &buffer) const noexcept -{ - const uint64_t inputSize = static_cast( - helper::GetTotalSize(blockInfo.Count) * sizeof(T)); - - const uint64_t outputSize = - static_cast(std::stoll(operation.Info.at("OutputSize"))); - - size_t backPosition = static_cast( - std::stoull(operation.Info.at("OutputSizeMetadataPosition"))); - - helper::CopyToBuffer(buffer, backPosition, &outputSize); - // being naughty here - Params &info = const_cast(operation.Info); - - // additional metadata for supporting 64-bit count - const uint16_t batches = - static_cast(inputSize / DefaultMaxFileBatchSize + 1); - - backPosition = static_cast( - std::stoull(operation.Info.at("BatchesMetadataPosition"))); - - for (auto b = 0; b < batches; ++b) - { - const std::string bStr = std::to_string(b); - - const uint64_t originalOffset = - std::stoull(info["OriginalOffset_" + bStr]); - const uint64_t originalSize = std::stoull(info["OriginalSize_" + bStr]); - const uint64_t compressedOffset = - std::stoull(info["CompressedOffset_" + bStr]); - const uint64_t compressedSize = - std::stoull(info["CompressedSize_" + bStr]); - - helper::CopyToBuffer(buffer, backPosition, &originalOffset); - helper::CopyToBuffer(buffer, backPosition, &originalSize); - helper::CopyToBuffer(buffer, backPosition, &compressedOffset); - helper::CopyToBuffer(buffer, backPosition, &compressedSize); - } - - info.erase("OutputSizeMetadataPosition"); - info.erase("BatchesMetadataPosition"); -} - -} // end namespace format -} // end namespace adios2 - -#endif /** ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPBZIP2_TCC_ */ diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPBlosc.cpp b/source/adios2/toolkit/format/bp/bpOperation/compress/BPBlosc.cpp deleted file mode 100644 index 48ee23d789..0000000000 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPBlosc.cpp +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * BPBlosc.cpp - * - * Created on: Jun 21, 2019 - * Author: William F Godoy godoywf@ornl.gov - */ - -#include "BPBlosc.h" - -#include "adios2/helper/adiosFunctions.h" - -#ifdef ADIOS2_HAVE_BLOSC -#include "adios2/operator/compress/CompressBlosc.h" -#endif - -namespace adios2 -{ -namespace format -{ - -#define declare_type(T) \ - void BPBlosc::SetData( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - BufferSTL &bufferSTL) const noexcept \ - { \ - SetDataDefault(variable, blockInfo, operation, bufferSTL); \ - } \ - \ - void BPBlosc::SetMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept \ - { \ - SetMetadataDefault(variable, blockInfo, operation, buffer); \ - } \ - \ - void BPBlosc::UpdateMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept \ - { \ - UpdateMetadataDefault(variable, blockInfo, operation, buffer); \ - } - -ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type) -#undef declare_type - -void BPBlosc::GetMetadata(const std::vector &buffer, Params &info) const - noexcept -{ - size_t position = 0; - info["InputSize"] = - std::to_string(helper::ReadValue(buffer, position)); - info["OutputSize"] = - std::to_string(helper::ReadValue(buffer, position)); -} - -void BPBlosc::GetData(const char *input, - const helper::BlockOperationInfo &blockOperationInfo, - char *dataOutput) const -{ -#ifdef ADIOS2_HAVE_BLOSC - core::compress::CompressBlosc op((Params())); - const size_t sizeOut = (sizeof(size_t) == 8) - ? static_cast(helper::StringTo( - blockOperationInfo.Info.at("InputSize"), - "when reading Blosc input size")) - : static_cast(helper::StringTo( - blockOperationInfo.Info.at("InputSize"), - "when reading Blosc input size")); - - Params &info = const_cast(blockOperationInfo.Info); - op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput, sizeOut, - info); - -#else - throw std::runtime_error( - "ERROR: current ADIOS2 library didn't compile " - "with Blosc, can't read Blosc compressed data, in call " - "to Get\n"); -#endif -} - -} // end namespace format -} // end namespace adios2 diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPBlosc.h b/source/adios2/toolkit/format/bp/bpOperation/compress/BPBlosc.h deleted file mode 100644 index a1d619347f..0000000000 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPBlosc.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * BPBlosc.h - * - * Created on: Jun 21, 2019 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPBLOSC_H_ -#define ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPBLOSC_H_ - -#include "adios2/toolkit/format/bp/bpOperation/BPOperation.h" - -namespace adios2 -{ -namespace format -{ - -class BPBlosc : public BPOperation -{ -public: - BPBlosc() = default; - - ~BPBlosc() = default; - - using BPOperation::SetData; - using BPOperation::SetMetadata; - using BPOperation::UpdateMetadata; -#define declare_type(T) \ - void SetData(const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - BufferSTL &bufferSTL) const noexcept override; \ - \ - void SetMetadata(const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept override; \ - \ - void UpdateMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept override; - - ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type) -#undef declare_type - - void GetMetadata(const std::vector &buffer, Params &info) const - noexcept final; - - void GetData(const char *input, - const helper::BlockOperationInfo &blockOperationInfo, - char *dataOutput) const final; -}; - -} // end namespace format -} // end namespace adios2 - -#endif /* ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPBLOSC_H_ */ diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPLIBPRESSIO.cpp b/source/adios2/toolkit/format/bp/bpOperation/compress/BPLIBPRESSIO.cpp deleted file mode 100644 index bbf848cec3..0000000000 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPLIBPRESSIO.cpp +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * BPLIBPRESSIO.cpp : - * - * Created on: Apr 13, 2021 - * Author: Robert Underwood - */ - -#include "BPLIBPRESSIO.h" - -#include "adios2/helper/adiosFunctions.h" -#include "adios2/helper/adiosType.h" - -#ifdef ADIOS2_HAVE_LIBPRESSIO -#include "adios2/operator/compress/CompressLibPressio.h" -#endif - -namespace adios2 -{ -namespace format -{ - -#define declare_type(T) \ - void BPLIBPRESSIO::SetData( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - BufferSTL &bufferSTL) const noexcept \ - { \ - SetDataDefault(variable, blockInfo, operation, bufferSTL); \ - } \ - \ - void BPLIBPRESSIO::SetMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept \ - { \ - SetMetadataDefault(variable, blockInfo, operation, buffer); \ - } \ - \ - void BPLIBPRESSIO::UpdateMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept \ - { \ - UpdateMetadataDefault(variable, blockInfo, operation, buffer); \ - } - -ADIOS2_FOREACH_LIBPRESSIO_TYPE_1ARG(declare_type) -#undef declare_type - -void BPLIBPRESSIO::GetMetadata(const std::vector &buffer, - Params &info) const noexcept -{ - size_t position = 0; - info["InputSize"] = - std::to_string(helper::ReadValue(buffer, position)); - info["OutputSize"] = - std::to_string(helper::ReadValue(buffer, position)); -} - -void BPLIBPRESSIO::GetData(const char *input, - const helper::BlockOperationInfo &blockOperationInfo, - char *dataOutput) const -{ -#ifdef ADIOS2_HAVE_LIBPRESSIO - core::compress::CompressLibPressio op((Params())); - op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput, - blockOperationInfo.PreCount, - helper::GetDataTypeFromString( - blockOperationInfo.Info.at("PreDataType")), - blockOperationInfo.Info); - -#else - throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " - "with SZ, can't read SZ compressed data, in call " - "to Get\n"); -#endif -} - -} // end namespace format -} // end namespace adios2 diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPLIBPRESSIO.h b/source/adios2/toolkit/format/bp/bpOperation/compress/BPLIBPRESSIO.h deleted file mode 100644 index 9059a73e45..0000000000 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPLIBPRESSIO.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * BPLIBPRESSIO.h - * - * Created on: Apr 13, 2021 - * Author: Robert Underwood - */ - -#ifndef ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPLIBPRESSIO_H_ -#define ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPLIBPRESSIO_H_ - -#include "adios2/toolkit/format/bp/bpOperation/BPOperation.h" - -namespace adios2 -{ -namespace format -{ - -class BPLIBPRESSIO : public BPOperation -{ -public: - BPLIBPRESSIO() = default; - - ~BPLIBPRESSIO() = default; - - using BPOperation::SetData; - using BPOperation::SetMetadata; - using BPOperation::UpdateMetadata; -#define declare_type(T) \ - void SetData(const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - BufferSTL &bufferSTL) const noexcept override; \ - \ - void SetMetadata(const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept override; \ - \ - void UpdateMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept override; - - ADIOS2_FOREACH_LIBPRESSIO_TYPE_1ARG(declare_type) -#undef declare_type - - void GetMetadata(const std::vector &buffer, Params &info) const - noexcept final; - - void GetData(const char *input, - const helper::BlockOperationInfo &blockOperationInfo, - char *dataOutput) const final; -}; - -} // end namespace format -} // end namespace adios2 - -#endif /* ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPLIBPRESSIO_H_ */ diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPMGARD.cpp b/source/adios2/toolkit/format/bp/bpOperation/compress/BPMGARD.cpp deleted file mode 100644 index 744a013155..0000000000 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPMGARD.cpp +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * BPMGARD.cpp - * - * Created on: Nov 16, 2018 - * Author: William F Godoy godoywf@ornl.gov - */ - -#include "BPMGARD.h" - -#include "adios2/helper/adiosFunctions.h" -#include "adios2/helper/adiosType.h" - -#ifdef ADIOS2_HAVE_MGARD -#include "adios2/operator/compress/CompressMGARD.h" -#endif - -namespace adios2 -{ -namespace format -{ - -#define declare_type(T) \ - void BPMGARD::SetData( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - BufferSTL &bufferSTL) const noexcept \ - { \ - SetDataDefault(variable, blockInfo, operation, bufferSTL); \ - } \ - \ - void BPMGARD::SetMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept \ - { \ - SetMetadataDefault(variable, blockInfo, operation, buffer); \ - } \ - \ - void BPMGARD::UpdateMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept \ - { \ - UpdateMetadataDefault(variable, blockInfo, operation, buffer); \ - } - -ADIOS2_FOREACH_MGARD_TYPE_1ARG(declare_type) -#undef declare_type - -void BPMGARD::GetMetadata(const std::vector &buffer, Params &info) const - noexcept -{ - size_t position = 0; - info["InputSize"] = - std::to_string(helper::ReadValue(buffer, position)); - info["OutputSize"] = - std::to_string(helper::ReadValue(buffer, position)); -} - -void BPMGARD::GetData(const char *input, - const helper::BlockOperationInfo &blockOperationInfo, - char *dataOutput) const -{ -#ifdef ADIOS2_HAVE_MGARD - core::compress::CompressMGARD op((Params())); - op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput, - blockOperationInfo.PreCount, - helper::GetDataTypeFromString( - blockOperationInfo.Info.at("PreDataType")), - blockOperationInfo.Info); - -#else - throw std::runtime_error( - "ERROR: current ADIOS2 library didn't compile " - "with MGARD, can't read MGARD compressed data, in call " - "to Get\n"); -#endif -} - -} // end namespace format -} // end namespace adios2 diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPMGARD.h b/source/adios2/toolkit/format/bp/bpOperation/compress/BPMGARD.h deleted file mode 100644 index bbc397c147..0000000000 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPMGARD.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * BPMGARD.h - * - * Created on: Nov 16, 2018 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPMGARD_H_ -#define ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPMGARD_H_ - -#include "adios2/toolkit/format/bp/bpOperation/BPOperation.h" - -namespace adios2 -{ -namespace format -{ - -class BPMGARD : public BPOperation -{ -public: - BPMGARD() = default; - - ~BPMGARD() = default; - - using BPOperation::SetData; - using BPOperation::SetMetadata; - using BPOperation::UpdateMetadata; -#define declare_type(T) \ - void SetData(const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - BufferSTL &bufferSTL) const noexcept override; \ - \ - void SetMetadata(const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept override; \ - \ - void UpdateMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept override; - - ADIOS2_FOREACH_MGARD_TYPE_1ARG(declare_type) -#undef declare_type - - void GetMetadata(const std::vector &buffer, Params &info) const - noexcept final; - - void GetData(const char *input, - const helper::BlockOperationInfo &blockOperationInfo, - char *dataOutput) const final; -}; - -} // end namespace format -} // end namespace adios2 - -#endif /* ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPMGARD_H_ */ diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPPNG.cpp b/source/adios2/toolkit/format/bp/bpOperation/compress/BPPNG.cpp deleted file mode 100644 index 366b044a48..0000000000 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPPNG.cpp +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * BPPNG.cpp - * - * Created on: Jun 10, 2019 - * Author: William F Godoy godoywf@ornl.gov - */ - -#include "BPPNG.h" - -#include "adios2/helper/adiosFunctions.h" - -#ifdef ADIOS2_HAVE_PNG -#include "adios2/operator/compress/CompressPNG.h" -#endif - -namespace adios2 -{ -namespace format -{ - -#define declare_type(T) \ - void BPPNG::SetData( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - BufferSTL &bufferSTL) const noexcept \ - { \ - SetDataDefault(variable, blockInfo, operation, bufferSTL); \ - } \ - \ - void BPPNG::SetMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept \ - { \ - SetMetadataDefault(variable, blockInfo, operation, buffer); \ - } \ - \ - void BPPNG::UpdateMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept \ - { \ - UpdateMetadataDefault(variable, blockInfo, operation, buffer); \ - } - -ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type) -#undef declare_type - -void BPPNG::GetMetadata(const std::vector &buffer, Params &info) const - noexcept -{ - size_t position = 0; - info["InputSize"] = - std::to_string(helper::ReadValue(buffer, position)); - info["OutputSize"] = - std::to_string(helper::ReadValue(buffer, position)); -} - -void BPPNG::GetData(const char *input, - const helper::BlockOperationInfo &blockOperationInfo, - char *dataOutput) const -{ -#ifdef ADIOS2_HAVE_PNG - core::compress::CompressPNG op((Params())); - const size_t sizeOut = (sizeof(size_t) == 8) - ? static_cast(helper::StringTo( - blockOperationInfo.Info.at("InputSize"), - "when reading PNG input size")) - : static_cast(helper::StringTo( - blockOperationInfo.Info.at("InputSize"), - "when reading PNG input size")); - - Params &info = const_cast(blockOperationInfo.Info); - op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput, sizeOut, - info); - -#else - throw std::runtime_error( - "ERROR: current ADIOS2 library didn't compile " - "with PNG, can't read PNG compressed data, in call " - "to Get\n"); -#endif -} - -} // end namespace format -} // end namespace adios2 diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPPNG.h b/source/adios2/toolkit/format/bp/bpOperation/compress/BPPNG.h deleted file mode 100644 index d81ea94c01..0000000000 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPPNG.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * BPPNG.h - * - * Created on: Jun 10, 2019 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPPNG_H_ -#define ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPPNG_H_ - -#include "adios2/toolkit/format/bp/bpOperation/BPOperation.h" - -namespace adios2 -{ -namespace format -{ - -class BPPNG : public BPOperation -{ -public: - BPPNG() = default; - - ~BPPNG() = default; - - using BPOperation::SetData; - using BPOperation::SetMetadata; - using BPOperation::UpdateMetadata; -#define declare_type(T) \ - void SetData(const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - BufferSTL &bufferSTL) const noexcept override; \ - \ - void SetMetadata(const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept override; \ - \ - void UpdateMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept override; - - ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type) -#undef declare_type - - void GetMetadata(const std::vector &buffer, Params &info) const - noexcept final; - - void GetData(const char *input, - const helper::BlockOperationInfo &blockOperationInfo, - char *dataOutput) const final; -}; - -} // end namespace format -} // end namespace adios2 - -#endif /* ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPPNG_H_ */ diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPSZ.cpp b/source/adios2/toolkit/format/bp/bpOperation/compress/BPSZ.cpp deleted file mode 100644 index 4083c70d86..0000000000 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPSZ.cpp +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * BPSZ.cpp : - * - * Created on: Jul 17, 2018 - * Author: William F Godoy godoywf@ornl.gov - */ - -#include "BPSZ.h" - -#include "adios2/helper/adiosFunctions.h" -#include "adios2/helper/adiosType.h" - -#ifdef ADIOS2_HAVE_SZ -#include "adios2/operator/compress/CompressSZ.h" -#endif - -namespace adios2 -{ -namespace format -{ - -#define declare_type(T) \ - void BPSZ::SetData(const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - BufferSTL &bufferSTL) const noexcept \ - { \ - SetDataDefault(variable, blockInfo, operation, bufferSTL); \ - } \ - \ - void BPSZ::SetMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept \ - { \ - SetMetadataDefault(variable, blockInfo, operation, buffer); \ - } \ - \ - void BPSZ::UpdateMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept \ - { \ - UpdateMetadataDefault(variable, blockInfo, operation, buffer); \ - } - -ADIOS2_FOREACH_SZ_TYPE_1ARG(declare_type) -#undef declare_type - -void BPSZ::GetMetadata(const std::vector &buffer, Params &info) const - noexcept -{ - size_t position = 0; - info["InputSize"] = - std::to_string(helper::ReadValue(buffer, position)); - info["OutputSize"] = - std::to_string(helper::ReadValue(buffer, position)); -} - -void BPSZ::GetData(const char *input, - const helper::BlockOperationInfo &blockOperationInfo, - char *dataOutput) const -{ -#ifdef ADIOS2_HAVE_SZ - core::compress::CompressSZ op((Params())); - op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput, - blockOperationInfo.PreCount, - helper::GetDataTypeFromString( - blockOperationInfo.Info.at("PreDataType")), - blockOperationInfo.Info); - -#else - throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " - "with SZ, can't read SZ compressed data, in call " - "to Get\n"); -#endif -} - -} // end namespace format -} // end namespace adios2 diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPSZ.h b/source/adios2/toolkit/format/bp/bpOperation/compress/BPSZ.h deleted file mode 100644 index deef59bbdf..0000000000 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPSZ.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * BPSZ.h - * - * Created on: Jul 20, 2018 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPSZ_H_ -#define ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPSZ_H_ - -#include "adios2/toolkit/format/bp/bpOperation/BPOperation.h" - -namespace adios2 -{ -namespace format -{ - -class BPSZ : public BPOperation -{ -public: - BPSZ() = default; - - ~BPSZ() = default; - - using BPOperation::SetData; - using BPOperation::SetMetadata; - using BPOperation::UpdateMetadata; -#define declare_type(T) \ - void SetData(const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - BufferSTL &bufferSTL) const noexcept override; \ - \ - void SetMetadata(const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept override; \ - \ - void UpdateMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept override; - - ADIOS2_FOREACH_SZ_TYPE_1ARG(declare_type) -#undef declare_type - - void GetMetadata(const std::vector &buffer, Params &info) const - noexcept final; - - void GetData(const char *input, - const helper::BlockOperationInfo &blockOperationInfo, - char *dataOutput) const final; -}; - -} // end namespace format -} // end namespace adios2 - -#endif /* ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPSZ_H_ */ diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPZFP.cpp b/source/adios2/toolkit/format/bp/bpOperation/compress/BPZFP.cpp deleted file mode 100644 index b5f1b020f9..0000000000 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPZFP.cpp +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * BPZFP.cpp : - * - * Created on: Jul 17, 2018 - * Author: William F Godoy godoywf@ornl.gov - */ - -#include "BPZFP.h" -#include "BPZFP.tcc" - -#include "adios2/helper/adiosFunctions.h" -#include "adios2/helper/adiosType.h" - -#ifdef ADIOS2_HAVE_ZFP -#include "adios2/operator/compress/CompressZFP.h" -#endif - -namespace adios2 -{ -namespace format -{ - -#define declare_type(T) \ - void BPZFP::SetData( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - BufferSTL &bufferSTL) const noexcept \ - { \ - SetDataDefault(variable, blockInfo, operation, bufferSTL); \ - } \ - \ - void BPZFP::SetMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept \ - { \ - SetMetadataCommon(variable, blockInfo, operation, buffer); \ - } \ - \ - void BPZFP::UpdateMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept \ - { \ - UpdateMetadataDefault(variable, blockInfo, operation, buffer); \ - } - -ADIOS2_FOREACH_ZFP_TYPE_1ARG(declare_type) -#undef declare_type - -void BPZFP::GetMetadata(const std::vector &buffer, Params &info) const - noexcept -{ - size_t position = 0; - info["InputSize"] = - std::to_string(helper::ReadValue(buffer, position)); - info["OutputSize"] = - std::to_string(helper::ReadValue(buffer, position)); - const int mode = - static_cast(helper::ReadValue(buffer, position)); - - const std::string modeStr(buffer.data() + position); - - switch (mode) - { - case zfp_mode_precision: - info["precision"] = modeStr; - break; - - case zfp_mode_accuracy: - info["accuracy"] = modeStr; - break; - - case zfp_mode_rate: - info["rate"] = modeStr; - break; - } -} - -void BPZFP::GetData(const char *input, - const helper::BlockOperationInfo &blockOperationInfo, - char *dataOutput) const -{ -#ifdef ADIOS2_HAVE_ZFP - core::compress::CompressZFP op((Params())); - op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput, - blockOperationInfo.PreCount, - helper::GetDataTypeFromString( - blockOperationInfo.Info.at("PreDataType")), - blockOperationInfo.Info); -#else - throw std::runtime_error( - "ERROR: current ADIOS2 library didn't compile " - "with ZFP, can't read ZFP compressed data, in call " - "to Get\n"); -#endif -} - -} // end namespace format -} // end namespace adios2 diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPZFP.h b/source/adios2/toolkit/format/bp/bpOperation/compress/BPZFP.h deleted file mode 100644 index 90fee2201b..0000000000 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPZFP.h +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * BPZFP.h : - * - * Created on: Jul 17, 2018 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPZFP_H_ -#define ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPZFP_H_ - -#include "adios2/toolkit/format/bp/bpOperation/BPOperation.h" - -namespace adios2 -{ -namespace format -{ - -class BPZFP : public BPOperation -{ -public: - BPZFP() = default; - - ~BPZFP() = default; - - using BPOperation::SetData; - using BPOperation::SetMetadata; - using BPOperation::UpdateMetadata; -#define declare_type(T) \ - void SetData(const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - BufferSTL &bufferSTL) const noexcept override; \ - \ - void SetMetadata(const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept override; \ - \ - void UpdateMetadata( \ - const core::Variable &variable, \ - const typename core::Variable::BPInfo &blockInfo, \ - const typename core::Variable::Operation &operation, \ - std::vector &buffer) const noexcept override; - - ADIOS2_FOREACH_ZFP_TYPE_1ARG(declare_type) -#undef declare_type - - void GetMetadata(const std::vector &buffer, Params &info) const - noexcept final; - - void GetData(const char *input, - const helper::BlockOperationInfo &blockOperationInfo, - char *dataOutput) const final; - -private: - enum Mode - { - zfp_mode_accuracy = 0, - zfp_mode_precision = 1, - zfp_mode_rate = 2 - }; - - template - void - SetMetadataCommon(const core::Variable &variable, - const typename core::Variable::BPInfo &blockInfo, - const typename core::Variable::Operation &operation, - std::vector &buffer) const noexcept; -}; - -} // end namespace format -} // end namespace adios2 - -#endif /* ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPZFP_H_ */ diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPZFP.tcc b/source/adios2/toolkit/format/bp/bpOperation/compress/BPZFP.tcc deleted file mode 100644 index 8b03a70900..0000000000 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPZFP.tcc +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * BPZFP.tcc : - * - * Created on: Jul 18, 2018 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPZFP_TCC_ -#define ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPZFP_TCC_ - -#include "BPZFP.h" - -#include "adios2/helper/adiosFunctions.h" - -namespace adios2 -{ -namespace format -{ - -template -void BPZFP::SetMetadataCommon( - const core::Variable &variable, - const typename core::Variable::BPInfo &blockInfo, - const typename core::Variable::Operation &operation, - std::vector &buffer) const noexcept -{ - const uint64_t inputSize = - helper::GetTotalSize(blockInfo.Count) * sizeof(T); - // being naughty here - Params &info = const_cast(operation.Info); - info["InputSize"] = std::to_string(inputSize); - - const uint64_t outputSize = 0; // not known yet - - auto itMode = operation.Parameters.find("accuracy"); - int32_t mode = -1; - - if (itMode != operation.Parameters.end()) - { - mode = static_cast(zfp_mode_accuracy); - } - else - { - itMode = operation.Parameters.find("precision"); - if (itMode != operation.Parameters.end()) - { - mode = static_cast(zfp_mode_precision); - } - else - { - itMode = operation.Parameters.find("rate"); - if (itMode != operation.Parameters.end()) - { - mode = static_cast(zfp_mode_rate); - } - } - } - const std::string modeStr = itMode->second; - - // fixed size - constexpr uint16_t metadataSize = 532; - helper::InsertToBuffer(buffer, &metadataSize); - helper::InsertToBuffer(buffer, &inputSize); - // to be filled out after operation is applied on data - info["OutputSizeMetadataPosition"] = std::to_string(buffer.size()); - helper::InsertToBuffer(buffer, &outputSize); - helper::InsertToBuffer(buffer, &mode); - - const size_t fixedRecordsPosition = buffer.size(); - buffer.resize(fixedRecordsPosition + 512, '\0'); - size_t backPosition = fixedRecordsPosition; - helper::CopyToBuffer(buffer, backPosition, modeStr.data(), modeStr.size()); - backPosition = fixedRecordsPosition + 256; - helper::CopyToBuffer(buffer, backPosition, variable.m_Name.data(), - variable.m_Name.size()); -} - -} // end namespace format -} // end namespace adios2 - -#endif /* ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPZFP_TCC_ */ diff --git a/source/adios2/toolkit/format/bp5/BP5Base.cpp b/source/adios2/toolkit/format/bp5/BP5Base.cpp index 8a7b974e9a..211a93ecfe 100644 --- a/source/adios2/toolkit/format/bp5/BP5Base.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Base.cpp @@ -25,9 +25,9 @@ namespace format void BP5Base::FFSBitfieldSet(struct FFSMetadataInfoStruct *MBase, int Bit) { - int Element = Bit / (sizeof(size_t) * 8); + size_t Element = Bit / (sizeof(size_t) * 8); int ElementBit = Bit % (sizeof(size_t) * 8); - if (Element >= MBase->BitFieldCount) + if (static_cast(Element) >= MBase->BitFieldCount) { MBase->BitField = (size_t *)realloc(MBase->BitField, sizeof(size_t) * (Element + 1)); @@ -40,9 +40,9 @@ void BP5Base::FFSBitfieldSet(struct FFSMetadataInfoStruct *MBase, int Bit) int BP5Base::FFSBitfieldTest(struct FFSMetadataInfoStruct *MBase, int Bit) { - int Element = Bit / (sizeof(size_t) * 8); + size_t Element = Bit / (sizeof(size_t) * 8); int ElementBit = Bit % (sizeof(size_t) * 8); - if (Element >= MBase->BitFieldCount) + if (static_cast(Element) >= MBase->BitFieldCount) { MBase->BitField = (size_t *)realloc(MBase->BitField, sizeof(size_t) * (Element + 1)); @@ -53,6 +53,5 @@ int BP5Base::FFSBitfieldTest(struct FFSMetadataInfoStruct *MBase, int Bit) return ((MBase->BitField[Element] & ((size_t)1 << ElementBit)) == ((size_t)1 << ElementBit)); } - } } diff --git a/source/adios2/toolkit/format/bp5/BP5Base.h b/source/adios2/toolkit/format/bp5/BP5Base.h index 3767d705c0..c4a5e1e469 100644 --- a/source/adios2/toolkit/format/bp5/BP5Base.h +++ b/source/adios2/toolkit/format/bp5/BP5Base.h @@ -37,13 +37,13 @@ class BP5Base typedef struct _MetaArrayRec { - size_t Dims; // How many dimensions does this array have - size_t BlockCount; // How many blocks are written - size_t DBCount; // Dimens * BlockCount - size_t *Shape; // Global dimensionality [Dims] NULL for local - size_t *Count; // Per-block Counts [DBCount] - size_t *Offsets; // Per-block Offsets [DBCount] NULL for local - size_t *DataLocation; + size_t Dims; // How many dimensions does this array have + size_t BlockCount; // How many blocks are written + size_t DBCount; // Dimens * BlockCount + size_t *Shape; // Global dimensionality [Dims] NULL for local + size_t *Count; // Per-block Counts [DBCount] + size_t *Offsets; // Per-block Offsets [DBCount] NULL for local + size_t *DataLocation; // Per-block Offsets [BlockCount] } MetaArrayRec; struct FFSMetadataInfoStruct diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp index b66b18089e..027acc8008 100644 --- a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp @@ -9,10 +9,13 @@ #include "adios2/core/Attribute.h" #include "adios2/core/Engine.h" #include "adios2/core/IO.h" +#include "adios2/core/VariableBase.h" #include "BP5Deserializer.h" #include "BP5Deserializer.tcc" +#include +#include #include #ifdef _WIN32 @@ -54,6 +57,12 @@ bool BP5Deserializer::NameIndicatesArray(const char *Name) return (strcmp("Dims", Name + Len - 4) == 0); } +bool BP5Deserializer::NameIndicatesAttrArray(const char *Name) +{ + int Len = strlen(Name); + return (strcmp("ElemCount", Name + Len - 9) == 0); +} + DataType BP5Deserializer::TranslateFFSType2ADIOS(const char *Type, int size) { if (strcmp(Type, "integer") == 0) @@ -103,7 +112,7 @@ DataType BP5Deserializer::TranslateFFSType2ADIOS(const char *Type, int size) else if ((sizeof(long double) != sizeof(double)) && (size == sizeof(long double))) { - return DataType::Double; + return DataType::LongDouble; } else { @@ -118,6 +127,11 @@ DataType BP5Deserializer::TranslateFFSType2ADIOS(const char *Type, int size) { return DataType::DoubleComplex; } + else if (strcmp(Type, "string") == 0) + { + return DataType::String; + } + return DataType::None; } @@ -160,10 +174,16 @@ BP5Deserializer::BP5VarRec *BP5Deserializer::LookupVarByName(const char *Name) BP5Deserializer::BP5VarRec *BP5Deserializer::CreateVarRec(const char *ArrayName) { - BP5VarRec *Ret = new BP5VarRec(m_WriterCohortSize); + BP5VarRec *Ret = new BP5VarRec(); Ret->VarName = strdup(ArrayName); Ret->Variable = nullptr; + Ret->VarNum = m_VarCount++; VarByName[Ret->VarName] = Ret; + if (!m_RandomAccessMode) + { + Ret->PerWriterMetaFieldOffset.resize(m_WriterCohortSize); + Ret->PerWriterBlockStart.resize(m_WriterCohortSize); + } return Ret; } @@ -180,6 +200,7 @@ BP5Deserializer::ControlInfo *BP5Deserializer::BuildControl(FMFormat Format) int ControlCount = 0; ControlInfo *ret = (BP5Deserializer::ControlInfo *)malloc(sizeof(*ret)); ret->Format = Format; + ret->MetaFieldOffset = new std::vector(); while (FieldList[i].field_name) { ret = (ControlInfo *)realloc( @@ -187,22 +208,18 @@ BP5Deserializer::ControlInfo *BP5Deserializer::BuildControl(FMFormat Format) struct ControlStruct *C = &(ret->Controls[ControlCount]); ControlCount++; - C->FieldIndex = i; C->FieldOffset = FieldList[i].field_offset; + BP5VarRec *VarRec = nullptr; if (NameIndicatesArray(FieldList[i].field_name)) { char *ArrayName; DataType Type; - BP5VarRec *VarRec = nullptr; int ElementSize; C->IsArray = 1; BreakdownArrayName(FieldList[i].field_name, &ArrayName, &Type, &ElementSize); - // if (WriterRank != 0) - // { VarRec = LookupVarByName(ArrayName); - // } if (!VarRec) { VarRec = CreateVarRec(ArrayName); @@ -218,7 +235,6 @@ BP5Deserializer::ControlInfo *BP5Deserializer::BuildControl(FMFormat Format) { /* simple field */ char *FieldName = strdup(FieldList[i].field_name + 4); // skip SST_ - BP5VarRec *VarRec = NULL; C->IsArray = 0; VarRec = LookupVarByName(FieldName); if (!VarRec) @@ -236,6 +252,9 @@ BP5Deserializer::ControlInfo *BP5Deserializer::BuildControl(FMFormat Format) free(FieldName); i++; } + if (ret->MetaFieldOffset->size() <= VarRec->VarNum) + ret->MetaFieldOffset->resize(VarRec->VarNum + 1); + (*ret->MetaFieldOffset)[VarRec->VarNum] = C->FieldOffset; } ret->ControlCount = ControlCount; ret->Next = ControlBlocks; @@ -330,26 +349,34 @@ void BP5Deserializer::SetupForTimestep(size_t Timestep) { CurTimestep = Timestep; PendingRequests.clear(); + for (auto RecPair : VarByKey) { + m_Engine->m_IO.RemoveVariable(RecPair.second->VarName); RecPair.second->Variable = NULL; } } + void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen, - size_t WriterRank) + size_t WriterRank, size_t Step) { FFSTypeHandle FFSformat; void *BaseData; static int DumpMetadata = -1; FFSformat = FFSTypeHandle_from_encode(ReaderFFSContext, (char *)MetadataBlock); + if (!FFSformat) + { + throw std::logic_error("Internal error or file corruption, no know " + "format for Metadata Block"); + } if (!FFShas_conversion(FFSformat)) { FMContext FMC = FMContext_from_FFS(ReaderFFSContext); FMFormat Format = FMformat_from_ID(FMC, (char *)MetadataBlock); FMStructDescList List = FMcopy_struct_list(format_list_of_FMFormat(Format)); - FMlocalize_structs(List); + // GSE - restrict to homogenous FTM FMlocalize_structs(List); establish_conversion(ReaderFFSContext, FFSformat, List); FMfree_struct_list(List); } @@ -376,26 +403,58 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen, printf("\n\n"); } struct ControlInfo *Control; - struct ControlStruct *ControlArray; + struct ControlStruct *ControlFields; Control = GetPriorControl(FMFormat_of_original(FFSformat)); if (!Control) { Control = BuildControl(FMFormat_of_original(FFSformat)); } - ControlArray = &Control->Controls[0]; - ActiveControl[WriterRank] = Control; + ControlFields = &Control->Controls[0]; + + if (m_RandomAccessMode) + { + if (m_ControlArray.size() < Step + 1) + { + m_ControlArray.resize(Step + 1); + } + if (m_ControlArray[Step].size() == 0) + { + m_ControlArray[Step].resize(m_WriterCohortSize); + } + m_ControlArray[Step][WriterRank] = Control; + + MetadataBaseArray.resize(Step + 1); + if (MetadataBaseArray[Step] == nullptr) + { + m_MetadataBaseAddrs = new std::vector(); + m_MetadataBaseAddrs->resize(m_WriterCohortSize); + MetadataBaseArray[Step] = m_MetadataBaseAddrs; + m_FreeableMBA = nullptr; + } + } + else + { + if (!m_MetadataBaseAddrs) + { + m_MetadataBaseAddrs = new std::vector(); + m_FreeableMBA = m_MetadataBaseAddrs; + m_MetadataBaseAddrs->resize(m_WriterCohortSize); + } + } + (*m_MetadataBaseAddrs)[WriterRank] = BaseData; - MetadataBaseAddrs[WriterRank] = BaseData; for (int i = 0; i < Control->ControlCount; i++) { - int FieldOffset = ControlArray[i].FieldOffset; - BP5VarRec *VarRec = ControlArray[i].VarRec; + size_t FieldOffset = ControlFields[i].FieldOffset; + BP5VarRec *VarRec = ControlFields[i].VarRec; void *field_data = (char *)BaseData + FieldOffset; if (!FFSBitfieldTest((FFSMetadataInfoStruct *)BaseData, i)) { continue; } - if (ControlArray[i].IsArray) + if (!m_RandomAccessMode) + VarRec->PerWriterMetaFieldOffset[WriterRank] = FieldOffset; + if (ControlFields[i].IsArray) { MetaArrayRec *meta_base = (MetaArrayRec *)field_data; if ((meta_base->Dims > 1) && @@ -407,8 +466,9 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen, ReverseDimensions(meta_base->Count, meta_base->Dims); ReverseDimensions(meta_base->Offsets, meta_base->Dims); } - if (WriterRank == 0) + if ((WriterRank == 0) || (VarRec->GlobalDims == NULL)) { + // use the shape from rank 0 (or first non-NULL) VarRec->GlobalDims = meta_base->Shape; } if (!VarRec->Variable) @@ -416,37 +476,45 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen, VarRec->Variable = ArrayVarSetup( m_Engine, VarRec->VarName, VarRec->Type, meta_base->Dims, meta_base->Shape, meta_base->Offsets, meta_base->Count); + static_cast(VarRec->Variable)->m_Engine = + m_Engine; VarByKey[VarRec->Variable] = VarRec; + VarRec->LastTSAdded = Step; // starts at 1 } + VarRec->DimCount = meta_base->Dims; - VarRec->PerWriterBlockCount[WriterRank] = + size_t BlockCount = meta_base->Dims ? meta_base->DBCount / meta_base->Dims : 1; - VarRec->PerWriterStart[WriterRank] = meta_base->Offsets; - VarRec->PerWriterCounts[WriterRank] = meta_base->Count; - VarRec->PerWriterDataLocation[WriterRank] = meta_base->DataLocation; - if (WriterRank == 0) - { - VarRec->PerWriterBlockStart[WriterRank] = 0; - } - if (WriterRank < m_WriterCohortSize - 1) + if (!m_RandomAccessMode) { - VarRec->PerWriterBlockStart[WriterRank + 1] = - VarRec->PerWriterBlockStart[WriterRank] + - VarRec->PerWriterBlockCount[WriterRank]; + if (WriterRank == 0) + { + VarRec->PerWriterBlockStart[WriterRank] = 0; + if (m_WriterCohortSize > 1) + VarRec->PerWriterBlockStart[WriterRank + 1] = + BlockCount; + } + if (WriterRank < static_cast(m_WriterCohortSize - 1)) + { + VarRec->PerWriterBlockStart[WriterRank + 1] = + VarRec->PerWriterBlockStart[WriterRank] + BlockCount; + } } -#ifdef NOTDEF - // needs to be replaced with Simple Blocks Info - for (int i = 0; i < VarRec->PerWriterBlockCount[WriterRank]; i++) + else { - size_t *Offsets = NULL; - if (meta_base->Offsets) - Offsets = meta_base->Offsets + (i * meta_base->Dims); - ArrayBlocksInfoUpcall(m_Engine, VarRec->Variable, VarRec->Type, - WriterRank, meta_base->Dims, - meta_base->Shape, Offsets, - meta_base->Count); + // Random access, add to m_AvailableShapes + if ((VarRec->LastShapeAdded != Step) && meta_base->Shape) + { + std::vector shape; + for (size_t i = 0; i < meta_base->Dims; i++) + { + shape.push_back(meta_base->Shape[i]); + } + static_cast(VarRec->Variable) + ->m_AvailableShapes[Step + 1] = shape; + VarRec->LastShapeAdded = Step; + } } -#endif } else { @@ -455,14 +523,20 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen, VarRec->Variable = VarSetup(m_Engine, VarRec->VarName, VarRec->Type, field_data); VarByKey[VarRec->Variable] = VarRec; + VarRec->LastTSAdded = Step; // starts at 1 } - VarRec->PerWriterMetaFieldOffset[WriterRank] = FieldOffset; + } + if (m_RandomAccessMode && (VarRec->LastTSAdded != Step)) + { + static_cast(VarRec->Variable) + ->m_AvailableStepsCount++; + VarRec->LastTSAdded = Step; } } } void BP5Deserializer::InstallAttributeData(void *AttributeBlock, - size_t BlockLen) + size_t BlockLen, size_t Step) { static int DumpMetadata = -1; FMFieldList FieldList; @@ -476,13 +550,18 @@ void BP5Deserializer::InstallAttributeData(void *AttributeBlock, m_Engine->m_IO.RemoveAllAttributes(); FFSformat = FFSTypeHandle_from_encode(ReaderFFSContext, (char *)AttributeBlock); + if (!FFSformat) + { + throw std::logic_error("Internal error or file corruption, no know " + "format for Attribute Block"); + } if (!FFShas_conversion(FFSformat)) { FMContext FMC = FMContext_from_FFS(ReaderFFSContext); FMFormat Format = FMformat_from_ID(FMC, (char *)AttributeBlock); FMStructDescList List = FMcopy_struct_list(format_list_of_FMFormat(Format)); - FMlocalize_structs(List); + // GSE - restrict to homogenous FTM FMlocalize_structs(List); establish_conversion(ReaderFFSContext, FFSformat, List); FMfree_struct_list(List); } @@ -519,48 +598,153 @@ void BP5Deserializer::InstallAttributeData(void *AttributeBlock, char *FieldName; void *field_data = (char *)BaseData + FieldList[i].field_offset; - DataType Type; - int ElemSize; - BreakdownVarName(FieldList[i].field_name, &FieldName, &Type, &ElemSize); - if (Type == adios2::DataType::Compound) + if (!NameIndicatesAttrArray(FieldList[i].field_name)) { - return; - } - else if (Type == helper::GetDataType()) - { - m_Engine->m_IO.DefineAttribute(FieldName, - *(char **)field_data); - } + DataType Type; + int ElemSize; + BreakdownVarName(FieldList[i].field_name, &FieldName, &Type, + &ElemSize); + if (Type == adios2::DataType::Compound) + { + return; + } + else if (Type == helper::GetDataType()) + { + m_Engine->m_IO.DefineAttribute( + FieldName, *(char **)field_data); + } #define declare_type(T) \ else if (Type == helper::GetDataType()) \ { \ m_Engine->m_IO.DefineAttribute(FieldName, *(T *)field_data); \ } - ADIOS2_FOREACH_ATTRIBUTE_PRIMITIVE_STDTYPE_1ARG(declare_type) + ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type) #undef declare_type + else + { + std::cout << "Loading attribute matched no type " + << ToString(Type) << std::endl; + } + free(FieldName); + i++; + } else { - std::cout << "Loading attribute matched no type " << ToString(Type) - << std::endl; + DataType Type; + size_t ElemCount = *(size_t *)field_data; + field_data = (void *)((char *)field_data + sizeof(size_t)); + i++; + char *FieldName = strdup(FieldList[i].field_name + 4); // skip SST_ + char *FieldType = strdup(FieldList[i].field_type); + *index(FieldType, '[') = 0; + Type = (DataType)TranslateFFSType2ADIOS(FieldType, + FieldList[i].field_size); + if (Type == adios2::DataType::Compound) + { + return; + } + else if (Type == helper::GetDataType()) + { + std::vector array; + array.resize(ElemCount); + char **str_array = *(char ***)field_data; + for (size_t i = 0; i < ElemCount; i++) + { + array[i].assign(str_array[i]); + } + m_Engine->m_IO.DefineAttribute( + FieldName, array.data(), array.size()); + } +#define declare_type(T) \ + else if (Type == helper::GetDataType()) \ + { \ + T **array = *(T ***)field_data; \ + m_Engine->m_IO.DefineAttribute(FieldName, (T *)array, ElemCount); \ + } + + ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type) +#undef declare_type + else + { + std::cout << "Loading attribute matched no type " + << ToString(Type) << std::endl; + } + free(FieldName); + i++; } - free(FieldName); - i++; } } bool BP5Deserializer::QueueGet(core::VariableBase &variable, void *DestData) +{ + if (!m_RandomAccessMode) + { + return QueueGetSingle(variable, DestData, CurTimestep); + } + else + { + bool ret = false; + if (variable.m_StepsStart + variable.m_StepsCount > + variable.m_AvailableStepsCount) + { + throw std::invalid_argument( + "ERROR: offset " + std::to_string(variable.m_StepsCount) + + " from steps start " + std::to_string(variable.m_StepsStart) + + " in variable " + variable.m_Name + + " is beyond the largest available step = " + + std::to_string(variable.m_AvailableStepsCount + + variable.m_AvailableStepsStart) + + ", check Variable SetStepSelection argument stepsCount " + "(random access), or " + "number of BeginStep calls (streaming), in call to Get"); + } + for (size_t i = 0; i < variable.m_StepsCount; i++) + { + ret = QueueGetSingle(variable, DestData, variable.m_StepsStart + i); + size_t increment = variable.TotalSize() * variable.m_ElementSize; + DestData = (void *)((char *)DestData + increment); + } + return ret; + } +} + +bool BP5Deserializer::QueueGetSingle(core::VariableBase &variable, + void *DestData, size_t Step) { if (variable.m_SingleValue) { + char *src; + BP5VarRec *VarRec = VarByKey[&variable]; int WriterRank = 0; + if (m_RandomAccessMode) + { + ControlInfo *CI = + m_ControlArray[Step][WriterRank]; // writer 0 control array + size_t MetadataFieldOffset = (*CI->MetaFieldOffset)[VarRec->VarNum]; + char *MetadataBase = + (char *)((*MetadataBaseArray[Step])[WriterRank]); + src = MetadataBase + MetadataFieldOffset; + } + else + { + src = ((char *)(*m_MetadataBaseAddrs)[WriterRank]) + + VarRec->PerWriterMetaFieldOffset[WriterRank]; + } if (variable.m_SelectionType == adios2::SelectionType::WriteBlock) WriterRank = variable.m_BlockID; - BP5VarRec *VarRec = VarByKey[&variable]; - char *src = ((char *)MetadataBaseAddrs[WriterRank]) + - VarRec->PerWriterMetaFieldOffset[WriterRank]; - memcpy(DestData, src, variable.m_ElementSize); + if (variable.m_Type != DataType::String) + { + std::cout << "Performing get for var " << variable.m_Name << " TS " + << Step << " Mdatabase " << (void *)src << std::endl; + memcpy(DestData, src, variable.m_ElementSize); + } + else + { + std::string *TmpStr = static_cast(DestData); + TmpStr->assign(*(const char **)src); + } return false; } if (variable.m_SelectionType == adios2::SelectionType::BoundingBox) @@ -571,6 +755,7 @@ bool BP5Deserializer::QueueGet(core::VariableBase &variable, void *DestData) Req.BlockID = variable.m_BlockID; Req.Count = variable.m_Count; Req.Start = variable.m_Start; + Req.Step = Step; Req.Data = DestData; PendingRequests.push_back(Req); } @@ -582,44 +767,69 @@ bool BP5Deserializer::QueueGet(core::VariableBase &variable, void *DestData) Req.BlockID = variable.m_BlockID; Req.Count = variable.m_Count; Req.Data = DestData; + Req.Step = Step; PendingRequests.push_back(Req); } else { } - return false; + return true; } -bool BP5Deserializer::NeedWriter(BP5ArrayRequest Req, int i) +bool BP5Deserializer::NeedWriter(BP5ArrayRequest Req, size_t WriterRank) { + MetaArrayRec *writer_meta_base; + if (m_RandomAccessMode) + { + ControlInfo *CI = + m_ControlArray[Req.Step][WriterRank]; // writer 0 control array + size_t MetadataFieldOffset = (*CI->MetaFieldOffset)[Req.VarRec->VarNum]; + writer_meta_base = + (MetaArrayRec + *)(((char *)(*MetadataBaseArray[Req.Step])[WriterRank]) + + MetadataFieldOffset); + } + else + { + writer_meta_base = + (MetaArrayRec *)(((char *)(*m_MetadataBaseAddrs)[WriterRank]) + + Req.VarRec->PerWriterMetaFieldOffset[WriterRank]); + } + if (Req.RequestType == Local) { - size_t NodeFirst = Req.VarRec->PerWriterBlockStart[i]; - size_t NodeLast = Req.VarRec->PerWriterBlockCount[i] + NodeFirst - 1; - return (NodeFirst <= Req.BlockID) && (NodeLast >= Req.BlockID); + size_t WriterBlockCount = + writer_meta_base->Dims + ? writer_meta_base->DBCount / writer_meta_base->Dims + : 1; + size_t NodeFirst = Req.VarRec->PerWriterBlockStart[WriterRank]; + size_t NodeLast = WriterBlockCount + NodeFirst - 1; + bool res = (NodeFirst <= Req.BlockID) && (NodeLast >= Req.BlockID); + return res; } // else Global case - for (int j = 0; j < Req.VarRec->DimCount; j++) + for (size_t i = 0; i < writer_meta_base->BlockCount; i++) { - size_t SelOffset = Req.Start[j]; - size_t SelSize = Req.Count[j]; - size_t RankOffset; - size_t RankSize; - if (Req.VarRec->PerWriterStart[i] == NULL) - /* this writer didn't write */ + for (size_t j = 0; j < writer_meta_base->Dims; j++) { - return false; - } - RankOffset = Req.VarRec->PerWriterStart[i][j]; - RankSize = Req.VarRec->PerWriterCounts[i][j]; - if ((SelSize == 0) || (RankSize == 0)) - { - return false; - } - if ((RankOffset < SelOffset && (RankOffset + RankSize) <= SelOffset) || - (RankOffset >= SelOffset + SelSize)) - { - return false; + size_t SelOffset = Req.Start[j]; + size_t SelSize = Req.Count[j]; + size_t RankOffset; + size_t RankSize; + + RankOffset = + writer_meta_base->Offsets[i * writer_meta_base->Dims + j]; + RankSize = writer_meta_base->Count[i * writer_meta_base->Dims + j]; + if ((SelSize == 0) || (RankSize == 0)) + { + return false; + } + if ((RankOffset < SelOffset && + (RankOffset + RankSize) <= SelOffset) || + (RankOffset >= SelOffset + SelSize)) + { + return false; + } } } return true; @@ -629,38 +839,43 @@ std::vector BP5Deserializer::GenerateReadRequests() { std::vector Ret; - for (auto &W : WriterInfo) - { - W.Status = Empty; - W.RawBuffer = NULL; - } + std::vector WriterInfo(m_WriterCohortSize); + typedef std::pair pair; + std::map WriterTSNeeded; for (const auto &Req : PendingRequests) { - for (int i = 0; i < m_WriterCohortSize; i++) + for (size_t i = 0; i < m_WriterCohortSize; i++) { - if ((WriterInfo[i].Status != Needed) && (NeedWriter(Req, i))) + if (WriterTSNeeded.count(std::make_pair(Req.Step, i)) == 0) { - WriterInfo[i].Status = Needed; + WriterTSNeeded[std::make_pair(Req.Step, i)] = true; } } } - for (int i = 0; i < m_WriterCohortSize; i++) + for (std::pair element : WriterTSNeeded) { - if (WriterInfo[i].Status == Needed) + ReadRequest RR; + RR.Timestep = element.first.first; + RR.WriterRank = element.first.second; + RR.StartOffset = 0; + if (m_RandomAccessMode) { - ReadRequest RR; - RR.Timestep = CurTimestep; - RR.WriterRank = i; - RR.StartOffset = 0; RR.ReadLength = - ((struct FFSMetadataInfoStruct *)MetadataBaseAddrs[i]) + ((struct FFSMetadataInfoStruct *)(( + *MetadataBaseArray[RR.Timestep])[RR.WriterRank])) ->DataBlockSize; - RR.DestinationAddr = (char *)malloc(RR.ReadLength); - RR.Internal = NULL; - Ret.push_back(RR); } + else + { + RR.ReadLength = ((struct FFSMetadataInfoStruct + *)(*m_MetadataBaseAddrs)[RR.WriterRank]) + ->DataBlockSize; + } + RR.DestinationAddr = (char *)malloc(RR.ReadLength); + RR.Internal = NULL; + Ret.push_back(RR); } return Ret; } @@ -670,72 +885,98 @@ void BP5Deserializer::FinalizeGets(std::vector Requests) for (const auto &Req : PendingRequests) { // ImplementGapWarning(Reqs); - for (int i = 0; i < m_WriterCohortSize; i++) + for (size_t WriterRank = 0; WriterRank < m_WriterCohortSize; + WriterRank++) { - if (NeedWriter(Req, i)) + if (NeedWriter(Req, WriterRank)) { /* if needed this writer fill destination with acquired data */ int ElementSize = Req.VarRec->ElementSize; - int DimCount = Req.VarRec->DimCount; size_t *GlobalDimensions = Req.VarRec->GlobalDims; - size_t *RankOffset = Req.VarRec->PerWriterStart[i]; - const size_t *RankSize = Req.VarRec->PerWriterCounts[i]; - std::vector ZeroSel(DimCount); - std::vector ZeroRankOffset(DimCount); - std::vector ZeroGlobalDimensions(DimCount); - const size_t *SelOffset = NULL; - const size_t *SelSize = Req.Count.data(); - int ReqIndex = 0; - while (Requests[ReqIndex].WriterRank != i) - ReqIndex++; - char *IncomingData = - (char *)Requests[ReqIndex].DestinationAddr + - Req.VarRec->PerWriterDataLocation[i][0]; - - if (Req.Start.size()) + MetaArrayRec *writer_meta_base; + if (m_RandomAccessMode) + { + ControlInfo *CI = + m_ControlArray[Req.Step] + [WriterRank]; // writer 0 control array + size_t MetadataFieldOffset = + (*CI->MetaFieldOffset)[Req.VarRec->VarNum]; + writer_meta_base = + (MetaArrayRec *)(((char *)(*MetadataBaseArray[Req.Step]) + [WriterRank]) + + MetadataFieldOffset); + } + else { - SelOffset = Req.Start.data(); + writer_meta_base = + (MetaArrayRec + *)(((char *)(*m_MetadataBaseAddrs)[WriterRank]) + + Req.VarRec + ->PerWriterMetaFieldOffset[WriterRank]); } - if (Req.RequestType == Local) + int DimCount = writer_meta_base->Dims; + for (size_t i = 0; i < writer_meta_base->BlockCount; i++) { - int LocalBlockID = - Req.BlockID - Req.VarRec->PerWriterBlockStart[i]; - size_t DataOffset = 0; - for (int i = 0; i < LocalBlockID; i++) + size_t *RankOffset = + &writer_meta_base->Offsets[i * writer_meta_base->Dims]; + const size_t *RankSize = + &writer_meta_base->Count[i * writer_meta_base->Dims]; + std::vector ZeroSel(DimCount); + std::vector ZeroRankOffset(DimCount); + std::vector ZeroGlobalDimensions(DimCount); + const size_t *SelOffset = NULL; + const size_t *SelSize = Req.Count.data(); + int ReqIndex = 0; + while (Requests[ReqIndex].WriterRank != + static_cast(WriterRank) || + (Requests[ReqIndex].Timestep != Req.Step)) + ReqIndex++; + if (writer_meta_base->DataLocation == NULL) + { + // No Data from this writer + continue; + } + char *IncomingData = + (char *)Requests[ReqIndex].DestinationAddr + + writer_meta_base->DataLocation[i]; + if (Req.Start.size()) + { + SelOffset = Req.Start.data(); + } + if (Req.RequestType == Local) { - int BlockElemCount = 1; - for (int j = 0; j < DimCount; j++) + int LocalBlockID = + Req.BlockID - + Req.VarRec->PerWriterBlockStart[WriterRank]; + IncomingData = + (char *)Requests[ReqIndex].DestinationAddr + + writer_meta_base->DataLocation[LocalBlockID]; + + RankOffset = ZeroRankOffset.data(); + GlobalDimensions = ZeroGlobalDimensions.data(); + if (SelOffset == NULL) { - BlockElemCount *= RankSize[j]; + SelOffset = ZeroSel.data(); + } + for (int i = 0; i < DimCount; i++) + { + GlobalDimensions[i] = RankSize[i]; } - DataOffset += BlockElemCount * ElementSize; - RankSize += DimCount; } - RankOffset = ZeroRankOffset.data(); - GlobalDimensions = ZeroGlobalDimensions.data(); - if (SelOffset == NULL) + if (m_ReaderIsRowMajor) { - SelOffset = ZeroSel.data(); + ExtractSelectionFromPartialRM( + ElementSize, DimCount, GlobalDimensions, RankOffset, + RankSize, SelOffset, SelSize, IncomingData, + (char *)Req.Data); } - for (int i = 0; i < DimCount; i++) + else { - GlobalDimensions[i] = RankSize[i]; + ExtractSelectionFromPartialCM( + ElementSize, DimCount, GlobalDimensions, RankOffset, + RankSize, SelOffset, SelSize, IncomingData, + (char *)Req.Data); } - IncomingData = IncomingData + DataOffset; - } - if (m_ReaderIsRowMajor) - { - ExtractSelectionFromPartialRM( - ElementSize, DimCount, GlobalDimensions, RankOffset, - RankSize, SelOffset, SelSize, IncomingData, - (char *)Req.Data); - } - else - { - ExtractSelectionFromPartialCM( - ElementSize, DimCount, GlobalDimensions, RankOffset, - RankSize, SelOffset, SelSize, IncomingData, - (char *)Req.Data); } } } @@ -752,7 +993,7 @@ void BP5Deserializer::MapGlobalToLocalIndex(size_t Dims, const size_t *LocalOffsets, size_t *LocalIndex) { - for (int i = 0; i < Dims; i++) + for (size_t i = 0; i < Dims; i++) { LocalIndex[i] = GlobalIndex[i] - LocalOffsets[i]; } @@ -762,7 +1003,7 @@ int BP5Deserializer::FindOffset(size_t Dims, const size_t *Size, const size_t *Index) { int Offset = 0; - for (int i = 0; i < Dims; i++) + for (size_t i = 0; i < Dims; i++) { Offset = Index[i] + (Size[i] * Offset); } @@ -771,12 +1012,13 @@ int BP5Deserializer::FindOffset(size_t Dims, const size_t *Size, static int FindOffsetCM(size_t Dims, const size_t *Size, const size_t *Index) { - int Offset = 0; - for (int i = Dims - 1; i >= 0; i--) + size_t Offset = 0; + for (int i = static_cast(Dims - 1); i >= 0; i--) { Offset = Index[i] + (Size[i] * Offset); } - return Offset; + + return std::min(static_cast(INT_MAX), Offset); } #define MAX(x, y) (((x) > (y)) ? (x) : (y)) @@ -830,8 +1072,8 @@ void BP5Deserializer::ExtractSelectionFromPartialRM( BlockSize = 1; OperantDims = Dims; - OperantElementSize = ElementSize; - for (int Dim = Dims - 1; Dim >= 0; Dim--) + OperantElementSize = static_cast(ElementSize); + for (int Dim = static_cast(Dims - 1); Dim >= 0; Dim--) { if ((GlobalDims[Dim] == PartialCounts[Dim]) && (SelectionCounts[Dim] == PartialCounts[Dim])) @@ -859,7 +1101,7 @@ void BP5Deserializer::ExtractSelectionFromPartialRM( /* calculate first selected element and count */ BlockCount = 1; size_t *FirstIndex = (size_t *)malloc(Dims * sizeof(FirstIndex[0])); - for (int Dim = 0; Dim < Dims; Dim++) + for (size_t Dim = 0; Dim < Dims; Dim++) { size_t Left = MAX(PartialOffsets[Dim], SelectionOffsets[Dim]); size_t Right = MIN(PartialOffsets[Dim] + PartialCounts[Dim], @@ -911,7 +1153,7 @@ void BP5Deserializer::ExtractSelectionFromPartialCM( BlockSize = 1; OperantElementSize = ElementSize; - for (int Dim = 0; Dim < Dims; Dim++) + for (size_t Dim = 0; Dim < Dims; Dim++) { if ((GlobalDims[Dim] == PartialCounts[Dim]) && (SelectionCounts[Dim] == PartialCounts[Dim])) @@ -947,7 +1189,7 @@ void BP5Deserializer::ExtractSelectionFromPartialCM( /* calculate first selected element and count */ BlockCount = 1; size_t *FirstIndex = (size_t *)malloc(Dims * sizeof(FirstIndex[0])); - for (int Dim = 0; Dim < Dims; Dim++) + for (size_t Dim = 0; Dim < Dims; Dim++) { int Left = MAX(PartialOffsets[Dim], SelectionOffsets[Dim]); int Right = MIN(PartialOffsets[Dim] + PartialCounts[Dim], @@ -983,45 +1225,25 @@ void BP5Deserializer::ExtractSelectionFromPartialCM( } BP5Deserializer::BP5Deserializer(int WriterCount, bool WriterIsRowMajor, - bool ReaderIsRowMajor) -: m_WriterCohortSize{WriterCount}, m_WriterIsRowMajor{WriterIsRowMajor}, - m_ReaderIsRowMajor{ReaderIsRowMajor} + bool ReaderIsRowMajor, bool RandomAccessMode) +: m_WriterIsRowMajor{WriterIsRowMajor}, m_ReaderIsRowMajor{ReaderIsRowMajor}, + m_WriterCohortSize{static_cast(WriterCount)}, m_RandomAccessMode{ + RandomAccessMode} { FMContext Tmp = create_local_FMcontext(); ReaderFFSContext = create_FFSContext_FM(Tmp); free_FMcontext(Tmp); - WriterInfo.resize(m_WriterCohortSize); - MetadataBaseAddrs.resize(m_WriterCohortSize); - MetadataFieldLists.resize(m_WriterCohortSize); - DataBaseAddrs.resize(m_WriterCohortSize); - ActiveControl.resize(m_WriterCohortSize); } BP5Deserializer::~BP5Deserializer() { - free_FFSContext(ReaderFFSContext); - for (int i = 0; i < m_WriterCohortSize; i++) - { - if (WriterInfo[i].RawBuffer) - free(WriterInfo[i].RawBuffer); - } - // for (int i = 0; i < Info.VarCount; i++) - // { - // free(Info.VarList[i]->VarName); - // free(Info.VarList[i]->PerWriterMetaFieldOffset); - // free(Info.VarList[i]->PerWriterBlockCount); - // free(Info.VarList[i]->PerWriterBlockStart); - // free(Info.VarList[i]->PerWriterStart); - // free(Info.VarList[i]->PerWriterCounts); - // free(Info.VarList[i]->PerWriterIncomingData); - // free(Info.VarList[i]->PerWriterIncomingSize); - // free(Info.VarList[i]); - // } struct ControlInfo *tmp = ControlBlocks; + free_FFSContext(ReaderFFSContext); ControlBlocks = NULL; while (tmp) { struct ControlInfo *next = tmp->Next; + delete tmp->MetaFieldOffset; free(tmp); tmp = next; } @@ -1030,16 +1252,73 @@ BP5Deserializer::~BP5Deserializer() free(VarRec.second->VarName); delete VarRec.second; } + if (m_FreeableMBA) + delete m_FreeableMBA; + for (auto &step : MetadataBaseArray) + { + delete step; + } } -#define declare_template_instantiation(T) \ - \ - template std::vector::BPInfo> \ - BP5Deserializer::BlocksInfo(const core::Variable &, const size_t) \ - const; -ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation) -#undef declare_template_instantiation +Engine::MinVarInfo *BP5Deserializer::MinBlocksInfo(const VariableBase &Var, + const size_t Step) +{ + BP5VarRec *VarRec = LookupVarByKey((void *)&Var); + MetaArrayRec *meta_base = + (MetaArrayRec *)(((char *)(*m_MetadataBaseAddrs)[0]) + + VarRec->PerWriterMetaFieldOffset[0]); + Engine::MinVarInfo *MV = + new Engine::MinVarInfo(meta_base->Dims, meta_base->Shape); + + MV->Dims = meta_base->Dims; + MV->Shape = meta_base->Shape; + MV->IsReverseDims = + ((meta_base->Dims > 1) && (m_WriterIsRowMajor != m_ReaderIsRowMajor)); + int Id = 0; + for (size_t WriterRank = 0; WriterRank < m_WriterCohortSize; WriterRank++) + { + MetaArrayRec *writer_meta_base = + (MetaArrayRec *)(((char *)(*m_MetadataBaseAddrs)[WriterRank]) + + VarRec->PerWriterMetaFieldOffset[WriterRank]); + size_t WriterBlockCount = + writer_meta_base->Dims + ? writer_meta_base->DBCount / writer_meta_base->Dims + : 1; + Id += WriterBlockCount; + } + MV->BlocksInfo.reserve(Id); + + Id = 0; + for (size_t WriterRank = 0; WriterRank < m_WriterCohortSize; WriterRank++) + { + MetaArrayRec *writer_meta_base = + (MetaArrayRec *)(((char *)(*m_MetadataBaseAddrs)[WriterRank]) + + VarRec->PerWriterMetaFieldOffset[WriterRank]); + + size_t WriterBlockCount = + meta_base->Dims ? meta_base->DBCount / meta_base->Dims : 1; + for (size_t i = 0; i < WriterBlockCount; i++) + { + size_t *Offsets = NULL; + size_t *Count = NULL; + if (writer_meta_base->Offsets) + Offsets = writer_meta_base->Offsets + (i * meta_base->Dims); + if (writer_meta_base->Count) + Count = writer_meta_base->Count + (i * meta_base->Dims); + Engine::MinBlockInfo Blk; + Blk.WriterID = WriterRank; + Blk.BlockID = Id++; + Blk.Start = Offsets; + Blk.Count = Count; + // Blk.MinUnion + // Blk.MaxUnion + // Blk.BufferP + MV->BlocksInfo.push_back(Blk); + } + } + return MV; } } +} diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.h b/source/adios2/toolkit/format/bp5/BP5Deserializer.h index 0503bea936..495827620b 100644 --- a/source/adios2/toolkit/format/bp5/BP5Deserializer.h +++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.h @@ -27,12 +27,14 @@ namespace adios2 namespace format { +using namespace core; + class BP5Deserializer : virtual public BP5Base { public: BP5Deserializer(int WriterCount, bool WriterIsRowMajor, - bool ReaderIsRowMajor); + bool ReaderIsRowMajor, bool RandomAccessMode = false); ~BP5Deserializer(); @@ -47,56 +49,46 @@ class BP5Deserializer : virtual public BP5Base }; void InstallMetaMetaData(MetaMetaInfoBlock &MMList); void InstallMetaData(void *MetadataBlock, size_t BlockLen, - size_t WriterRank); - void InstallAttributeData(void *AttributeBlock, size_t BlockLen); + size_t WriterRank, size_t Step = SIZE_MAX); + void InstallAttributeData(void *AttributeBlock, size_t BlockLen, + size_t Step = SIZE_MAX); void SetupForTimestep(size_t t); // return from QueueGet is true if a sync is needed to fill the data bool QueueGet(core::VariableBase &variable, void *DestData); + bool QueueGetSingle(core::VariableBase &variable, void *DestData, + size_t Step); std::vector GenerateReadRequests(); void FinalizeGets(std::vector); + Engine::MinVarInfo *AllRelativeStepsMinBlocksInfo(const VariableBase &var); + Engine::MinVarInfo *AllStepsMinBlocksInfo(const VariableBase &var); + Engine::MinVarInfo *MinBlocksInfo(const VariableBase &Var, + const size_t Step); + bool m_WriterIsRowMajor = 1; bool m_ReaderIsRowMajor = 1; core::Engine *m_Engine = NULL; - template - std::vector::BPInfo> - BlocksInfo(const core::Variable &variable, const size_t step) const; - private: + size_t m_VarCount = 0; struct BP5VarRec { + size_t VarNum; void *Variable = NULL; char *VarName = NULL; size_t DimCount = 0; DataType Type; int ElementSize = 0; size_t *GlobalDims = NULL; + size_t LastTSAdded = SIZE_MAX; + size_t LastShapeAdded = SIZE_MAX; std::vector PerWriterMetaFieldOffset; std::vector PerWriterBlockStart; - std::vector PerWriterBlockCount; - std::vector PerWriterStart; - std::vector PerWriterCounts; - std::vector PerWriterIncomingData; - std::vector PerWriterIncomingSize; // important for compression - std::vector PerWriterDataLocation; - BP5VarRec(int WriterSize) - { - PerWriterMetaFieldOffset.resize(WriterSize); - PerWriterBlockStart.resize(WriterSize); - PerWriterBlockCount.resize(WriterSize); - PerWriterStart.resize(WriterSize); - PerWriterCounts.resize(WriterSize); - PerWriterIncomingData.resize(WriterSize); - PerWriterIncomingSize.resize(WriterSize); - PerWriterDataLocation.resize(WriterSize); - } }; struct ControlStruct { - int FieldIndex; int FieldOffset; BP5VarRec *VarRec; int IsArray; @@ -109,6 +101,7 @@ class BP5Deserializer : virtual public BP5Base FMFormat Format; int ControlCount; struct ControlInfo *Next; + std::vector *MetaFieldOffset; struct ControlStruct Controls[1]; }; @@ -123,26 +116,31 @@ class BP5Deserializer : virtual public BP5Base struct FFSReaderPerWriterRec { enum WriterDataStatusEnum Status = Empty; - char *RawBuffer = NULL; }; FFSContext ReaderFFSContext; - int m_WriterCohortSize; + size_t m_WriterCohortSize; + bool m_RandomAccessMode; + std::unordered_map VarByName; std::unordered_map VarByKey; - FMContext LocalFMContext; - // Ffsarrayrequest PendingVarRequests; - std::vector MetadataBaseAddrs; - std::vector MetadataFieldLists; - std::vector DataBaseAddrs; - std::vector WriterInfo; - // struct ControlInfo *ControlBlocks; + std::vector *m_MetadataBaseAddrs = + nullptr; // may be a pointer into MetadataBaseArray or m_FreeableMBA + std::vector *m_FreeableMBA = nullptr; + + // for random access mode, for each timestep, for each writerrank, what + // metameta info applies to the metadata + std::vector> m_ControlArray; + // for random access mode, for each timestep, for each writerrank, base + // address of the metadata + std::vector *> MetadataBaseArray; ControlInfo *ControlBlocks = nullptr; ControlInfo *GetPriorControl(FMFormat Format); ControlInfo *BuildControl(FMFormat Format); bool NameIndicatesArray(const char *Name); + bool NameIndicatesAttrArray(const char *Name); DataType TranslateFFSType2ADIOS(const char *Type, int size); BP5VarRec *LookupVarByKey(void *Key); BP5VarRec *LookupVarByName(const char *Name); @@ -185,25 +183,17 @@ class BP5Deserializer : virtual public BP5Base { BP5VarRec *VarRec = NULL; enum RequestTypeEnum RequestType; + size_t Step; size_t BlockID; Dims Start; Dims Count; void *Data; }; std::vector PendingRequests; - bool NeedWriter(BP5ArrayRequest Req, int i); + bool NeedWriter(BP5ArrayRequest Req, size_t i); size_t CurTimestep = 0; - std::vector ActiveControl; }; -#define declare_template_instantiation(T) \ - extern template std::vector::BPInfo> \ - BP5Deserializer::BlocksInfo(const core::Variable &, const size_t) \ - const; - -ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation) -#undef declare_template_instantiation - } // end namespace format } // end namespace adios2 diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.tcc b/source/adios2/toolkit/format/bp5/BP5Deserializer.tcc index d52e6f0d9f..772786d371 100644 --- a/source/adios2/toolkit/format/bp5/BP5Deserializer.tcc +++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.tcc @@ -19,71 +19,7 @@ namespace adios2 { namespace format -{ - -template -std::vector::BPInfo> -BP5Deserializer::BlocksInfo(const core::Variable &variable, - const size_t step) const -{ - auto VarRec = VarByKey.find((void *)&variable)->second; - if (!VarRec) - { - return std::vector::BPInfo>(); - } - std::vector::BPInfo> Ret; - for (int WriterRank = 0; WriterRank < m_WriterCohortSize; WriterRank++) - { - const void *BaseData = MetadataBaseAddrs[WriterRank]; - struct ControlStruct *ControlArray = - &ActiveControl[WriterRank]->Controls[0]; - int i = 0; - while (ControlArray[i].VarRec != VarRec) - i++; - int FieldOffset = ControlArray[i].FieldOffset; - void *field_data = (char *)BaseData + FieldOffset; - MetaArrayRec *meta_base = (MetaArrayRec *)field_data; - for (int i = 0; i < VarRec->PerWriterBlockCount[WriterRank]; i++) - { - size_t *Offsets = NULL; - if (meta_base->Offsets) - Offsets = meta_base->Offsets + (i * meta_base->Dims); - typename core::Variable::BPInfo Tmp; - std::vector VecShape; - std::vector VecStart; - std::vector VecCount; - size_t DimCount = meta_base->Dims; - size_t *Start = Offsets; - size_t *Shape = meta_base->Shape; - size_t *Count = meta_base->Count; - if (Shape) - { - for (int i = 0; i < DimCount; i++) - { - VecShape.push_back(Shape[i]); - VecStart.push_back(Start[i]); - VecCount.push_back(Count[i]); - } - } - else - { - VecShape = {}; - VecStart = {}; - for (int i = 0; i < DimCount; i++) - { - VecCount.push_back(Count[i]); - } - } - Tmp.Shape = VecShape; - Tmp.Start = VecStart; - Tmp.Count = VecCount; - Ret.push_back(Tmp); - } - } - return Ret; -} - -} // end namespace format +{} // end namespace format } // end namespace adios2 #endif /* ADIOS2_TOOLKIT_FORMAT_BP5_BP5DESERIALIZER_TCC_ */ diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp index cd8794db44..9659d07194 100644 --- a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp @@ -8,8 +8,10 @@ #include "adios2/core/Attribute.h" #include "adios2/core/IO.h" +#include "adios2/core/VariableBase.h" #include "adios2/helper/adiosMemory.h" #include "adios2/toolkit/format/buffer/ffs/BufferFFS.h" +#include // max_align_t #include @@ -49,7 +51,8 @@ BP5Serializer::~BP5Serializer() void BP5Serializer::Init() { - memset(&Info, 0, sizeof(Info)); + // Re-init Info to zero + Info = FFSWriterMarshalBase(); Info.RecCount = 0; Info.RecList = (BP5Serializer::BP5WriterRec)malloc(sizeof(Info.RecList[0])); Info.MetaFieldCount = 0; @@ -375,14 +378,6 @@ BP5Serializer::CreateWriterRec(void *Variable, const char *Name, DataType Type, free(LocationsName); RecalcMarshalStorageSize(); -#ifdef NDEF - if ((ConfigParams->CompressionMethod == SstCompressZFP) && - ZFPcompressionPossible(Type, DimCount)) - { - Type = Int8; - ElemSize = 1; - } -#endif // To Data, add FMFields for ElemCount and Array matching _ArrayRec char *ElemCountName = ConcatName(Name, "ElemCount"); AddField(&Info.DataFields, &Info.DataFieldCount, ElemCountName, @@ -429,34 +424,79 @@ size_t BP5Serializer::CalcSize(const size_t Count, const size_t *Vals) return Elems; } +void BP5Serializer::PerformPuts() +{ + // Dump data for externs into iovec + DumpDeferredBlocks(); + + CurDataBuffer->CopyExternalToInternal(); +} + +void BP5Serializer::DumpDeferredBlocks() +{ + for (auto &Def : DeferredExterns) + { + MetaArrayRec *MetaEntry = + (MetaArrayRec *)((char *)(MetadataBuf) + Def.MetaOffset); + size_t DataOffset = m_PriorDataBufferSizeTotal + + CurDataBuffer->AddToVec(Def.DataSize, Def.Data, + Def.AlignReq, false); + MetaEntry->DataLocation[Def.BlockID] = DataOffset; + } + DeferredExterns.clear(); +} + void BP5Serializer::Marshal(void *Variable, const char *Name, const DataType Type, size_t ElemSize, size_t DimCount, const size_t *Shape, const size_t *Count, const size_t *Offsets, - const void *Data, bool Sync) + const void *Data, bool Sync, + BufferV::BufferPos *Span) { + core::VariableBase *VB = static_cast(Variable); + FFSMetadataInfoStruct *MBase; BP5WriterRec Rec = LookupWriterRec(Variable); + bool DeferAddToVec; + + if (VB->m_SingleValue) + { + DimCount = 0; + } if (!Rec) { Rec = CreateWriterRec(Variable, Name, Type, ElemSize, DimCount); } + if (!Sync && (Rec->DimCount != 0) && !Span) + { + /* + * If this is a big external block, we'll do everything except add it to + * the BufferV now, saving enough information to add it and patch back + * the DataLocation in the metadata in DumpDeferredBlocks() + */ + DeferAddToVec = true; + } + else + { + DeferAddToVec = false; + } + MBase = (struct FFSMetadataInfoStruct *)MetadataBuf; int AlreadyWritten = FFSBitfieldTest(MBase, Rec->FieldID); FFSBitfieldSet(MBase, Rec->FieldID); - if (Rec->DimCount == 0) + if (VB->m_SingleValue) { if (Type != DataType::String) memcpy((char *)(MetadataBuf) + Rec->MetaOffset, Data, ElemSize); else { char **StrPtr = (char **)((char *)(MetadataBuf) + Rec->MetaOffset); - if (AlreadyWritten) + if (AlreadyWritten && (*StrPtr != NULL)) free(*StrPtr); *StrPtr = strdup(*(char **)Data); } @@ -466,16 +506,30 @@ void BP5Serializer::Marshal(void *Variable, const char *Name, MetaArrayRec *MetaEntry = (MetaArrayRec *)((char *)(MetadataBuf) + Rec->MetaOffset); size_t ElemCount = CalcSize(DimCount, Count); - size_t DataOffset; + size_t DataOffset = 0; /* handle metadata */ MetaEntry->Dims = DimCount; if (CurDataBuffer == NULL) { - CurDataBuffer = new BufferV("data buffer"); + throw std::logic_error( + "BP5Serializer:: Marshall without Prior Init"); + } + + if (Span == nullptr) + { + if (!DeferAddToVec) + { + DataOffset = m_PriorDataBufferSizeTotal + + CurDataBuffer->AddToVec(ElemCount * ElemSize, Data, + ElemSize, Sync); + } + } + else + { + *Span = CurDataBuffer->Allocate(ElemCount * ElemSize, ElemSize); + DataOffset = m_PriorDataBufferSizeTotal + Span->globalPos; } - DataOffset = - CurDataBuffer->AddToVec(ElemCount * ElemSize, Data, ElemSize, Sync); if (!AlreadyWritten) { @@ -492,6 +546,12 @@ void BP5Serializer::Marshal(void *Variable, const char *Name, MetaEntry->Offsets = CopyDims(DimCount, Offsets); else MetaEntry->Offsets = NULL; + if (DeferAddToVec) + { + DeferredExtern rec = {Rec->MetaOffset, 0, Data, + ElemCount * ElemSize, ElemSize}; + DeferredExterns.push_back(rec); + } } else { @@ -501,6 +561,11 @@ void BP5Serializer::Marshal(void *Variable, const char *Name, // dimensions can change ) // Also assume Dims is always right and consistent, otherwise, // bad things + if (Shape && MetaEntry->Shape) + { + // Shape can change with later writes, so must overwrite + memcpy(MetaEntry->Shape, Shape, DimCount * sizeof(Shape[0])); + } MetaEntry->DBCount += DimCount; MetaEntry->BlockCount++; MetaEntry->Count = @@ -509,26 +574,16 @@ void BP5Serializer::Marshal(void *Variable, const char *Name, (size_t *)realloc(MetaEntry->DataLocation, MetaEntry->BlockCount * sizeof(size_t)); MetaEntry->DataLocation[MetaEntry->BlockCount - 1] = DataOffset; + if (DeferAddToVec) + { + DeferredExterns.push_back({Rec->MetaOffset, + MetaEntry->BlockCount - 1, Data, + ElemCount * ElemSize, ElemSize}); + } if (Offsets) MetaEntry->Offsets = AppendDims( MetaEntry->Offsets, PreviousDBCount, DimCount, Offsets); } - - // if ((Stream->ConfigParams->CompressionMethod == - // SstCompressZFP) && - // ZFPcompressionPossible(Type, DimCount)) - // { -#ifdef ADIOS2_HAVE_ZFP - // /* this should never be true if ZFP is not available - // */ size_t ByteCount; char *Output = - // FFS_ZFPCompress(Stream, Rec->DimCount, Rec->Type, - // (void *)Data, Count, - // &ByteCount); - // DataEntry->ElemCount = ByteCount; - // DataEntry->Array = Output; -#endif - // } - // else } } @@ -562,57 +617,55 @@ void BP5Serializer::MarshalAttribute(const char *Name, const DataType Type, } else { - /* // Array field. To Metadata, add FMFields for DimCount, Shape, Count - */ - /* // and Offsets matching _MetaArrayRec */ - /* char *ArrayName = BuildStaticArrayName(Name, Type, ElemCount); */ - /* AddField(&Info->AttributeFields, &Info->AttributeFieldCount, - * ArrayName, Type, */ - /* sizeof(size_t)); */ - /* free(ArrayName); */ - /* Rec->MetaOffset = */ - /* Info->MetaFields[Info->MetaFieldCount - 1].field_offset; */ - /* char *ShapeName = ConcatName(Name, "Shape"); */ - /* char *CountName = ConcatName(Name, "Count"); */ - /* char *OffsetsName = ConcatName(Name, "Offsets"); */ - /* AddFixedArrayField(&Info->MetaFields, &Info->MetaFieldCount, - * ShapeName, */ - /* "integer", sizeof(size_t), DimCount); */ - /* AddFixedArrayField(&Info->MetaFields, &Info->MetaFieldCount, - * CountName, */ - /* "integer", sizeof(size_t), DimCount); */ - /* AddFixedArrayField(&Info->MetaFields, &Info->MetaFieldCount, */ - /* OffsetsName, "integer", sizeof(size_t), DimCount); - */ - /* free(ShapeName); */ - /* free(CountName); */ - /* free(OffsetsName); */ - /* RecalcMarshalStorageSize(Stream); */ - - /* if ((Stream->ConfigParams->CompressionMethod == SstCompressZFP) && */ - /* ZFPcompressionPossible(Type, DimCount)) */ - /* { */ - /* Type = "char"; */ - /* ElemSize = 1; */ - /* } */ - /* // To Data, add FMFields for ElemCount and Array matching _ArrayRec - */ - /* char *ElemCountName = ConcatName(Name, "ElemCount"); */ - /* AddField(&Info->DataFields, &Info->DataFieldCount, ElemCountName, */ - /* "integer", sizeof(size_t)); */ - /* Rec->DataOffset = */ - /* Info->DataFields[Info->DataFieldCount - 1].field_offset; */ - /* char *SstName = ConcatName(Name, ""); */ - /* AddVarArrayField(&Info->DataFields, &Info->DataFieldCount, SstName, - */ - /* Type, ElemSize, ElemCountName); */ - /* free(SstName); */ - /* free(ElemCountName); */ - /* RecalcMarshalStorageSize(Stream); */ - /* // Changing the formats renders these invalid */ - /* Info->MetaFormat = NULL; */ - /* Info->DataFormat = NULL; */ + // Array field. To attribute data add dimension field and dynamic array + // field + char *ElemCountName = ConcatName(Name, "ElemCount"); + char *ArrayName = ConcatName(Name, ""); + AddField(&Info.AttributeFields, &Info.AttributeFieldCount, + ElemCountName, DataType::Int64, sizeof(int64_t)); + int CountOffset = + Info.AttributeFields[Info.AttributeFieldCount - 1].field_offset; + AddVarArrayField(&Info.AttributeFields, &Info.AttributeFieldCount, + ArrayName, Type, ElemSize, ElemCountName); + int DataOffset = + Info.AttributeFields[Info.AttributeFieldCount - 1].field_offset; + free(ElemCountName); + free(ArrayName); + + RecalcAttributeStorageSize(); + + memcpy((char *)(Info.AttributeData) + CountOffset, &ElemCount, + sizeof(size_t)); + memcpy((char *)(Info.AttributeData) + DataOffset, &Data, + sizeof(void *)); + } +} + +void BP5Serializer::InitStep(BufferV *DataBuffer) +{ + if (CurDataBuffer != NULL) + { + throw std::logic_error("BP5Serializer:: InitStep without prior close"); + } + CurDataBuffer = DataBuffer; + m_PriorDataBufferSizeTotal = 0; +} + +BufferV *BP5Serializer::ReinitStepData(BufferV *DataBuffer) +{ + if (CurDataBuffer == NULL) + { + throw std::logic_error("BP5Serializer:: ReinitStep without prior Init"); } + // Dump data for externs into iovec + DumpDeferredBlocks(); + + m_PriorDataBufferSizeTotal += CurDataBuffer->AddToVec( + 0, NULL, sizeof(max_align_t), true); // output block size aligned + + BufferV *tmp = CurDataBuffer; + CurDataBuffer = DataBuffer; + return tmp; } BP5Serializer::TimestepInfo BP5Serializer::CloseTimestep(int timestep) @@ -644,11 +697,18 @@ BP5Serializer::TimestepInfo BP5Serializer::CloseTimestep(int timestep) if (NewAttribute && Info.AttributeFields) { MetaMetaInfoBlock Block; - char *tmpName = strdup("Attributes"); - FMFormat Format = FMregister_simple_format( - Info.LocalFMContext, tmpName, Info.AttributeFields, - FMstruct_size_field_list(Info.AttributeFields, sizeof(char *))); - free(tmpName); + FMStructDescRec struct_list[4] = { + {NULL, NULL, 0, NULL}, + {"complex4", fcomplex_field_list, sizeof(fcomplex_struct), NULL}, + {"complex8", dcomplex_field_list, sizeof(dcomplex_struct), NULL}, + {NULL, NULL, 0, NULL}}; + struct_list[0].format_name = "Attributes"; + struct_list[0].field_list = Info.AttributeFields; + struct_list[0].struct_size = + FMstruct_size_field_list(Info.AttributeFields, sizeof(char *)); + + FMFormat Format = + register_data_format(Info.LocalFMContext, &struct_list[0]); Info.AttributeFormat = Format; int size; Block.MetaMetaInfo = get_server_rep_FMformat(Format, &size); @@ -667,10 +727,17 @@ BP5Serializer::TimestepInfo BP5Serializer::CloseTimestep(int timestep) if (CurDataBuffer == NULL) { - CurDataBuffer = new BufferV("data buffer"); + throw std::logic_error( + "BP5Serializer:: CloseTimestep without Prior Init"); } + + // Dump data for externs into iovec + DumpDeferredBlocks(); + MBase->DataBlockSize = CurDataBuffer->AddToVec( - 0, NULL, 8, true); // output block size multiple of 8, offset is size + 0, NULL, sizeof(max_align_t), true); // output block size aligned + + MBase->DataBlockSize += m_PriorDataBufferSizeTotal; void *MetaDataBlock = FFSencode(MetaEncodeBuffer, Info.MetaFormat, MetadataBuf, &MetaDataSize); @@ -686,8 +753,7 @@ BP5Serializer::TimestepInfo BP5Serializer::CloseTimestep(int timestep) Info.AttributeData, &AttributeSize); AttrData = new BufferFFS(AttributeEncodeBuffer, AttributeBlock, AttributeSize); - // FMdump_encoded_data(Info.AttributeFormat, AttributeBlock, - // 1024000); + FMdump_encoded_data(Info.AttributeFormat, AttributeBlock, 1024000); } // FMdump_encoded_data(Info.MetaFormat, MetaDataBlock, 1024000); @@ -725,7 +791,8 @@ BP5Serializer::TimestepInfo BP5Serializer::CloseTimestep(int timestep) std::vector BP5Serializer::CopyMetadataToContiguous( const std::vector NewMetaMetaBlocks, const format::Buffer *MetaEncodeBuffer, - const format::Buffer *AttributeEncodeBuffer, uint64_t DataSize) const + const format::Buffer *AttributeEncodeBuffer, uint64_t DataSize, + uint64_t WriterDataPos) const { std::vector Ret; uint64_t RetSize = 0; @@ -751,6 +818,7 @@ std::vector BP5Serializer::CopyMetadataToContiguous( RetSize += AttributeEncodeBufferAlignedSize; } RetSize += sizeof(DataSize); + RetSize += sizeof(WriterDataPos); Ret.resize(RetSize); helper::CopyToBuffer(Ret, Position, &NMMBCount); @@ -794,20 +862,21 @@ std::vector BP5Serializer::CopyMetadataToContiguous( } } helper::CopyToBuffer(Ret, Position, &DataSize); + helper::CopyToBuffer(Ret, Position, &WriterDataPos); return Ret; } -std::vector BP5Serializer::BreakoutContiguousMetadata( +std::vector BP5Serializer::BreakoutContiguousMetadata( std::vector *Aggregate, const std::vector Counts, std::vector &UniqueMetaMetaBlocks, - std::vector &AttributeBlocks, - std::vector &DataSizes) const + std::vector &AttributeBlocks, std::vector &DataSizes, + std::vector &WriterDataPositions) const { size_t Position = 0; - std::vector MetadataBlocks; + std::vector MetadataBlocks; MetadataBlocks.reserve(Counts.size()); DataSizes.resize(Counts.size()); - for (int Rank = 0; Rank < Counts.size(); Rank++) + for (size_t Rank = 0; Rank < Counts.size(); Rank++) { int32_t NMMBCount; helper::CopyFromBuffer(*Aggregate, Position, &NMMBCount); @@ -846,9 +915,16 @@ std::vector BP5Serializer::BreakoutContiguousMetadata( AttributeBlocks.push_back({Aggregate->data() + Position, AEBSize}); Position += AEBSize; helper::CopyFromBuffer(*Aggregate, Position, &DataSizes[Rank]); + helper::CopyFromBuffer(*Aggregate, Position, + &WriterDataPositions[Rank]); } return MetadataBlocks; } +void *BP5Serializer::GetPtr(int bufferIdx, size_t posInBuffer) +{ + return CurDataBuffer->GetPtr(bufferIdx, posInBuffer); } -} + +} // end namespace format +} // end namespace adios2 diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.h b/source/adios2/toolkit/format/bp5/BP5Serializer.h index e341ae1cba..f8335a53fa 100644 --- a/source/adios2/toolkit/format/bp5/BP5Serializer.h +++ b/source/adios2/toolkit/format/bp5/BP5Serializer.h @@ -10,6 +10,7 @@ #include "BP5Base.h" #include "adios2/core/Attribute.h" +#include "adios2/core/CoreTypes.h" #include "adios2/core/IO.h" #include "adios2/toolkit/format/buffer/BufferV.h" #include "adios2/toolkit/format/buffer/heap/BufferSTL.h" @@ -62,23 +63,46 @@ class BP5Serializer : virtual public BP5Base void Marshal(void *Variable, const char *Name, const DataType Type, size_t ElemSize, size_t DimCount, const size_t *Shape, const size_t *Count, const size_t *Offsets, const void *Data, - bool Sync); + bool Sync, BufferV::BufferPos *span); void MarshalAttribute(const char *Name, const DataType Type, size_t ElemSize, size_t ElemCount, const void *Data); + + /* + * InitStep must be called with an appropriate BufferV subtype before a + * step can begin + */ + void InitStep(BufferV *DataBuffer); + + /* + * ReinitStepData can be called to "flush" out already written + * data it returns a BufferV representing already-written data and + * provides the serializer with a new, empty BufferV This call + * does *not* reset the data offsets generated with Marshal, so + * those offsets are relative to the entire sequence of data + * produced by a writer rank. + */ + BufferV *ReinitStepData(BufferV *DataBuffer); + TimestepInfo CloseTimestep(int timestep); + void PerformPuts(); core::Engine *m_Engine = NULL; std::vector CopyMetadataToContiguous( const std::vector NewmetaMetaBlocks, const format::Buffer *MetaEncodeBuffer, - const format::Buffer *AttributeEncodeBuffer, uint64_t DataSize) const; + const format::Buffer *AttributeEncodeBuffer, uint64_t DataSize, + uint64_t WriterDataPos) const; - std::vector BreakoutContiguousMetadata( + std::vector BreakoutContiguousMetadata( std::vector *Aggregate, const std::vector Counts, std::vector &UniqueMetaMetaBlocks, - std::vector &AttributeBlocks, - std::vector &DataSizes) const; + std::vector &AttributeBlocks, + std::vector &DataSizes, + std::vector &WriterDataPositions) const; + + void *GetPtr(int bufferIdx, size_t posInBuffer); + size_t CalcSize(const size_t Count, const size_t *Vals); private: void Init(); @@ -96,7 +120,7 @@ class BP5Serializer : virtual public BP5Base { int RecCount = 0; BP5WriterRec RecList = NULL; - FMContext LocalFMContext; + FMContext LocalFMContext = {0}; int MetaFieldCount = 0; FMFieldList MetaFields = NULL; FMFormat MetaFormat; @@ -108,10 +132,18 @@ class BP5Serializer : virtual public BP5Base FMFormat AttributeFormat = NULL; void *AttributeData = NULL; int AttributeSize = 0; - int CompressZFP = 0; - attr_list ZFPParams = NULL; }; + struct DeferredExtern + { + size_t MetaOffset; + size_t BlockID; + const void *Data; + size_t DataSize; + size_t AlignReq; + }; + std::vector DeferredExterns; + FFSWriterMarshalBase Info; void *MetadataBuf = NULL; bool NewAttribute = false; @@ -120,6 +152,8 @@ class BP5Serializer : virtual public BP5Base BufferV *CurDataBuffer = NULL; std::vector PreviousMetaMetaInfoBlocks; + size_t m_PriorDataBufferSizeTotal = 0; + BP5WriterRec LookupWriterRec(void *Key); BP5WriterRec CreateWriterRec(void *Variable, const char *Name, DataType Type, size_t ElemSize, @@ -150,7 +184,8 @@ class BP5Serializer : virtual public BP5Base size_t *CopyDims(const size_t Count, const size_t *Vals); size_t *AppendDims(size_t *OldDims, const size_t OldCount, const size_t Count, const size_t *Vals); - size_t CalcSize(const size_t Count, const size_t *Vals); + + void DumpDeferredBlocks(); typedef struct _ArrayRec { diff --git a/source/adios2/toolkit/format/buffer/BufferV.cpp b/source/adios2/toolkit/format/buffer/BufferV.cpp index 494f30c29e..7e488acec7 100644 --- a/source/adios2/toolkit/format/buffer/BufferV.cpp +++ b/source/adios2/toolkit/format/buffer/BufferV.cpp @@ -7,6 +7,8 @@ */ #include "BufferV.h" +#include +#include // max_align_t #include namespace adios2 @@ -14,68 +16,37 @@ namespace adios2 namespace format { -BufferV::BufferV(const std::string type) : m_Type(type) {} +char BufferV::zero[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; -size_t BufferV::AddToVec(const size_t size, const void *buf, int align, - bool CopyReqd) +BufferV::BufferV(const std::string type, const bool AlwaysCopy) +: m_Type(type), m_AlwaysCopy(AlwaysCopy) { - int badAlign = CurOffset % align; - if (badAlign) - { - int addAlign = align - badAlign; - char zero[16] = {0}; - AddToVec(addAlign, zero, 1, true); - } - size_t retOffset = CurOffset; +} - if (size == 0) - return CurOffset; +BufferV::~BufferV() {} - if (!CopyReqd) - { - // just add buf to internal version of output vector - VecEntry entry = {true, buf, 0, size}; - DataV.push_back(entry); - } - else - { - InternalBlock.Resize(m_internalPos + size, ""); - memcpy(InternalBlock.Data() + m_internalPos, buf, size); - if (DataV.size() && !DataV.back().External && - (m_internalPos == (DataV.back().Offset + DataV.back().Size))) - { - // just add to the size of the existing tail entry - DataV.back().Size += size; - } - else - { - DataV.push_back({false, NULL, m_internalPos, size}); - } - m_internalPos += size; - } - CurOffset = retOffset + size; - return retOffset; +void BufferV::Reset() +{ + CurOffset = 0; + m_internalPos = 0; + DataV.clear(); } uint64_t BufferV::Size() noexcept { return CurOffset; } -BufferV::BufferV_iovec BufferV::DataVec() noexcept +void BufferV::AlignBuffer(const size_t align) { - BufferV_iovec ret = new iovec[DataV.size() + 1]; - for (std::size_t i = 0; i < DataV.size(); ++i) + size_t badAlign = CurOffset % align; + if (badAlign) { - if (DataV[i].External) - { - ret[i].iov_base = DataV[i].Base; - } - else - { - ret[i].iov_base = InternalBlock.Data() + DataV[i].Offset; - } - ret[i].iov_len = DataV[i].Size; + size_t addAlign = align - badAlign; + assert(addAlign < sizeof(max_align_t)); + AddToVec(addAlign, zero, 1, true); } - ret[DataV.size()] = {NULL, 0}; - return ret; } + } // end namespace format } // end namespace adios2 diff --git a/source/adios2/toolkit/format/buffer/BufferV.h b/source/adios2/toolkit/format/buffer/BufferV.h index f64f342402..d857044957 100644 --- a/source/adios2/toolkit/format/buffer/BufferV.h +++ b/source/adios2/toolkit/format/buffer/BufferV.h @@ -9,7 +9,8 @@ #include "adios2/common/ADIOSConfig.h" #include "adios2/common/ADIOSTypes.h" -#include "heap/BufferSTL.h" +#include "adios2/core/CoreTypes.h" +#include namespace adios2 { @@ -21,25 +22,52 @@ class BufferV public: const std::string m_Type; - typedef struct iovec + uint64_t Size() noexcept; + + BufferV(const std::string type, const bool AlwaysCopy = false); + virtual ~BufferV(); + + virtual std::vector DataVec() noexcept = 0; + + /* + * This is used in PerformPuts() to copy externally referenced data so that + * it can be modified by the application + */ + virtual void CopyExternalToInternal() = 0; + + /** + * Reset the buffer to initial state (without freeing internal buffers) + */ + virtual void Reset(); + + virtual size_t AddToVec(const size_t size, const void *buf, size_t align, + bool CopyReqd) = 0; + + struct BufferPos { - const void - *iov_base; // Base address of a memory region for input or output. - size_t iov_len; // The size of the memory pointed to by iov_base. - } * BufferV_iovec; + int bufferIdx = -1; // buffer index + size_t posInBuffer = 0; // position in buffer[idx] + size_t globalPos = 0; // global position in virtual buffer + BufferPos(int idx, size_t pos, size_t globalPos) + : bufferIdx(idx), posInBuffer(pos), globalPos(globalPos){}; + }; - uint64_t Size() noexcept; + /** Allocate size bytes and return BufferPos position. + * Used by Span functions to allocate memory on behalf of the user + * Return both the position in the virtual memory buffer as well + * as all info needed to retrieve a valid pointer any time + * during execution (even after reallocs) + */ + virtual BufferPos Allocate(const size_t size, size_t align) = 0; - BufferV(const std::string type); - virtual ~BufferV() = default; + void AlignBuffer(const size_t align); - virtual BufferV_iovec DataVec() noexcept; - // virtual const BufferV_iovec DataVec() const noexcept; + virtual void *GetPtr(int bufferIdx, size_t posInBuffer) = 0; - virtual size_t AddToVec(const size_t size, const void *buf, int align, - bool CopyReqd); +protected: + static char zero[64]; + const bool m_AlwaysCopy = false; -private: struct VecEntry { bool External; @@ -50,7 +78,6 @@ class BufferV std::vector DataV; size_t CurOffset = 0; size_t m_internalPos = 0; - BufferSTL InternalBlock; }; } // end namespace format diff --git a/source/adios2/toolkit/format/buffer/chunk/ChunkV.cpp b/source/adios2/toolkit/format/buffer/chunk/ChunkV.cpp new file mode 100644 index 0000000000..8814a70473 --- /dev/null +++ b/source/adios2/toolkit/format/buffer/chunk/ChunkV.cpp @@ -0,0 +1,234 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * BufferV.cpp + * + */ + +#include "ChunkV.h" +#include "adios2/toolkit/format/buffer/BufferV.h" + +#include +#include +#include +#include // max_align_t +#include + +namespace adios2 +{ +namespace format +{ + +ChunkV::ChunkV(const std::string type, const bool AlwaysCopy, + const size_t ChunkSize) +: BufferV(type, AlwaysCopy), m_ChunkSize(ChunkSize) +{ +} + +ChunkV::~ChunkV() +{ + for (const auto &Chunk : m_Chunks) + { + free((void *)Chunk); + } +} + +void ChunkV::CopyExternalToInternal() +{ + for (std::size_t i = 0; i < DataV.size(); ++i) + { + if (DataV[i].External) + { + size_t size = DataV[i].Size; + // we can possibly append this entry to the tail if the tail entry + // is internal + bool AppendPossible = DataV.size() && !DataV.back().External; + + if (AppendPossible && (m_TailChunkPos + size > m_ChunkSize)) + { + // No room in current chunk, close it out + // realloc down to used size (helpful?) and set size in array + m_Chunks.back() = + (char *)realloc(m_Chunks.back(), m_TailChunkPos); + + m_TailChunkPos = 0; + m_TailChunk = NULL; + AppendPossible = false; + } + if (AppendPossible) + { + // We can use current chunk, just append the data and modify the + // DataV entry + memcpy(m_TailChunk + m_TailChunkPos, DataV[i].Base, size); + DataV[i].External = false; + DataV[i].Base = m_TailChunk + m_TailChunkPos; + m_TailChunkPos += size; + } + else + { + // We need a new chunk, get the larger of size or m_ChunkSize + size_t NewSize = m_ChunkSize; + if (size > m_ChunkSize) + NewSize = size; + m_TailChunk = (char *)malloc(NewSize); + m_Chunks.push_back(m_TailChunk); + memcpy(m_TailChunk, DataV[i].Base, size); + m_TailChunkPos = size; + DataV[i] = {false, m_TailChunk, 0, size}; + } + } + } +} + +size_t ChunkV::AddToVec(const size_t size, const void *buf, size_t align, + bool CopyReqd) +{ + if (size == 0) + { + return CurOffset; + } + + AlignBuffer(align); + size_t retOffset = CurOffset; + + if (!CopyReqd && !m_AlwaysCopy) + { + // just add buf to internal version of output vector + VecEntry entry = {true, buf, 0, size}; + DataV.push_back(entry); + } + else + { + // we can possibly append this entry to the last if the last was + // internal + bool AppendPossible = + DataV.size() && !DataV.back().External && + (m_TailChunk + m_TailChunkPos - DataV.back().Size == + DataV.back().Base); + + if (AppendPossible && (m_TailChunkPos + size > m_ChunkSize)) + { + // No room in current chunk, close it out + // realloc down to used size (helpful?) and set size in array + m_Chunks.back() = (char *)realloc(m_Chunks.back(), m_TailChunkPos); + + m_TailChunkPos = 0; + m_TailChunk = NULL; + AppendPossible = false; + } + if (AppendPossible) + { + // We can use current chunk, just append the data; + memcpy(m_TailChunk + m_TailChunkPos, buf, size); + DataV.back().Size += size; + m_TailChunkPos += size; + } + else + { + // We need a new chunk, get the larger of size or m_ChunkSize + size_t NewSize = m_ChunkSize; + if (size > m_ChunkSize) + NewSize = size; + m_TailChunk = (char *)malloc(NewSize); + m_Chunks.push_back(m_TailChunk); + memcpy(m_TailChunk, buf, size); + m_TailChunkPos = size; + VecEntry entry = {false, m_TailChunk, 0, size}; + DataV.push_back(entry); + } + } + CurOffset = retOffset + size; + return retOffset; +} + +BufferV::BufferPos ChunkV::Allocate(const size_t size, size_t align) +{ + if (size == 0) + { + return BufferPos(-1, 0, CurOffset); + } + + AlignBuffer(align); + + // we can possibly append this entry to the last if the last was + // internal + bool AppendPossible = + DataV.size() && !DataV.back().External && + (m_TailChunk + m_TailChunkPos - DataV.back().Size == DataV.back().Base); + + if (AppendPossible && (m_TailChunkPos + size > m_ChunkSize)) + { + // No room in current chunk, close it out + // realloc down to used size (helpful?) and set size in array + m_Chunks.back() = (char *)realloc(m_Chunks.back(), m_TailChunkPos); + + m_TailChunkPos = 0; + m_TailChunk = NULL; + AppendPossible = false; + } + + size_t bufferPos = 0; + if (AppendPossible) + { + // We can use current chunk, just append the data; + bufferPos = m_TailChunkPos; + DataV.back().Size += size; + m_TailChunkPos += size; + } + else + { + // We need a new chunk, get the larger of size or m_ChunkSize + size_t NewSize = m_ChunkSize; + if (size > m_ChunkSize) + NewSize = size; + m_TailChunk = (char *)malloc(NewSize); + m_Chunks.push_back(m_TailChunk); + bufferPos = 0; + m_TailChunkPos = size; + VecEntry entry = {false, m_TailChunk, 0, size}; + DataV.push_back(entry); + } + + BufferPos bp(static_cast(DataV.size() - 1), bufferPos, CurOffset); + // valid ptr anytime <-- DataV[idx] + bufferPos; + + CurOffset += size; + + return bp; +} + +void *ChunkV::GetPtr(int bufferIdx, size_t posInBuffer) +{ + if (bufferIdx == -1) + { + return nullptr; + } + else if (static_cast(bufferIdx) > DataV.size() || + DataV[bufferIdx].External) + { + throw std::invalid_argument( + "ChunkV::GetPtr(" + std::to_string(bufferIdx) + ", " + + std::to_string(posInBuffer) + + ") refers to a non-existing or deferred memory chunk."); + } + else + { + return (void *)((char *)DataV[bufferIdx].Base + posInBuffer); + } +} + +std::vector ChunkV::DataVec() noexcept +{ + std::vector iov(DataV.size()); + for (std::size_t i = 0; i < DataV.size(); ++i) + { + // For ChunkV, all entries in DataV are actual iov entries. + iov[i].iov_base = DataV[i].Base; + iov[i].iov_len = DataV[i].Size; + } + return iov; +} + +} // end namespace format +} // end namespace adios2 diff --git a/source/adios2/toolkit/format/buffer/chunk/ChunkV.h b/source/adios2/toolkit/format/buffer/chunk/ChunkV.h new file mode 100644 index 0000000000..5fe0b1c900 --- /dev/null +++ b/source/adios2/toolkit/format/buffer/chunk/ChunkV.h @@ -0,0 +1,52 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + */ + +#ifndef ADIOS2_TOOLKIT_FORMAT_BUFFER_MALLOC_CHUNKV_H_ +#define ADIOS2_TOOLKIT_FORMAT_BUFFER_MALLOC_CHUNKV_H_ + +#include "adios2/common/ADIOSConfig.h" +#include "adios2/common/ADIOSTypes.h" +#include "adios2/core/CoreTypes.h" + +#include "adios2/toolkit/format/buffer/BufferV.h" + +namespace adios2 +{ +namespace format +{ + +class ChunkV : public BufferV +{ +public: + uint64_t Size() noexcept; + + const size_t m_ChunkSize; + + ChunkV(const std::string type, const bool AlwaysCopy = false, + const size_t ChunkSize = DefaultBufferChunkSize); + virtual ~ChunkV(); + + virtual std::vector DataVec() noexcept; + + virtual size_t AddToVec(const size_t size, const void *buf, size_t align, + bool CopyReqd); + + virtual BufferPos Allocate(const size_t size, size_t align); + + virtual void *GetPtr(int bufferIdx, size_t posInBuffer); + + void CopyExternalToInternal(); + +private: + std::vector m_Chunks; + size_t m_TailChunkPos = 0; + char *m_TailChunk = NULL; +}; + +} // end namespace format +} // end namespace adios2 + +#endif /* ADIOS2_TOOLKIT_FORMAT_BUFFER_MALLOC_MALLOCV_H_ */ diff --git a/source/adios2/toolkit/format/buffer/malloc/MallocV.cpp b/source/adios2/toolkit/format/buffer/malloc/MallocV.cpp new file mode 100644 index 0000000000..a871d2b45a --- /dev/null +++ b/source/adios2/toolkit/format/buffer/malloc/MallocV.cpp @@ -0,0 +1,217 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * BufferV.cpp + * + */ + +#include "MallocV.h" +#include "adios2/toolkit/format/buffer/BufferV.h" + +#include +#include +#include // max_align_t +#include + +namespace adios2 +{ +namespace format +{ + +MallocV::MallocV(const std::string type, const bool AlwaysCopy, + size_t InitialBufferSize, double GrowthFactor) +: BufferV(type, AlwaysCopy), m_InitialBufferSize(InitialBufferSize), + m_GrowthFactor(GrowthFactor) +{ +} + +MallocV::~MallocV() +{ + if (m_InternalBlock) + free(m_InternalBlock); +} + +void MallocV::Reset() +{ + CurOffset = 0; + m_internalPos = 0; + DataV.clear(); +} + +/* + * This is used in PerformPuts() to copy externally referenced data + * so that it can be modified by the application. It does *not* + * change the metadata offset that was originally returned by + * AddToVec. That is, it relocates the data from application memory + * into the internal buffer, but it does not change the position of + * that data in the write order, which may result in non-contiguous + * writes from the internal buffer. + */ +void MallocV::CopyExternalToInternal() +{ + for (std::size_t i = 0; i < DataV.size(); ++i) + { + if (DataV[i].External) + { + size_t size = DataV[i].Size; + + /* force internal buffer alignment */ + (void)AddToVec(0, NULL, sizeof(max_align_t), true); + + if (m_internalPos + size > m_AllocatedSize) + { + // need to resize + size_t NewSize; + if (m_internalPos + size > m_AllocatedSize * m_GrowthFactor) + { + // just grow as needed (more than GrowthFactor) + NewSize = m_internalPos + size; + } + else + { + NewSize = (size_t)(m_AllocatedSize * m_GrowthFactor); + } + m_InternalBlock = (char *)realloc(m_InternalBlock, NewSize); + m_AllocatedSize = NewSize; + } + memcpy(m_InternalBlock + m_internalPos, DataV[i].Base, size); + DataV[i].External = false; + DataV[i].Base = NULL; + DataV[i].Offset = m_internalPos; + m_internalPos += size; + } + } +} + +size_t MallocV::AddToVec(const size_t size, const void *buf, size_t align, + bool CopyReqd) +{ + if (size == 0) + { + return CurOffset; + } + + AlignBuffer(align); + size_t retOffset = CurOffset; + + if (!CopyReqd && !m_AlwaysCopy) + { + // just add buf to internal version of output vector + VecEntry entry = {true, buf, 0, size}; + DataV.push_back(entry); + } + else + { + if (m_internalPos + size > m_AllocatedSize) + { + // need to resize + size_t NewSize; + if (m_internalPos + size > m_AllocatedSize * m_GrowthFactor) + { + // just grow as needed (more than GrowthFactor) + NewSize = m_internalPos + size; + } + else + { + NewSize = (size_t)(m_AllocatedSize * m_GrowthFactor); + } + m_InternalBlock = (char *)realloc(m_InternalBlock, NewSize); + m_AllocatedSize = NewSize; + } + memcpy(m_InternalBlock + m_internalPos, buf, size); + + if (DataV.size() && !DataV.back().External && + (m_internalPos == (DataV.back().Offset + DataV.back().Size))) + { + // just add to the size of the existing tail entry + DataV.back().Size += size; + } + else + { + DataV.push_back({false, NULL, m_internalPos, size}); + } + m_internalPos += size; + } + CurOffset = retOffset + size; + return retOffset; +} + +BufferV::BufferPos MallocV::Allocate(const size_t size, size_t align) +{ + if (size == 0) + { + return BufferPos(-1, 0, CurOffset); + } + + AlignBuffer(align); + + if (m_internalPos + size > m_AllocatedSize) + { + // need to resize + size_t NewSize; + if (m_internalPos + size > m_AllocatedSize * m_GrowthFactor) + { + // just grow as needed (more than GrowthFactor) + NewSize = m_internalPos + size; + } + else + { + NewSize = (size_t)(m_AllocatedSize * m_GrowthFactor); + } + m_InternalBlock = (char *)realloc(m_InternalBlock, NewSize); + m_AllocatedSize = NewSize; + } + + if (DataV.size() && !DataV.back().External && + (m_internalPos == (DataV.back().Offset + DataV.back().Size))) + { + // just add to the size of the existing tail entry + DataV.back().Size += size; + } + else + { + DataV.push_back({false, NULL, m_internalPos, size}); + } + + BufferPos bp(0, m_internalPos, CurOffset); + // valid ptr anytime <-- m_InternalBlock + m_internalPos; + + CurOffset += size; + m_internalPos += size; + + return bp; +} + +void *MallocV::GetPtr(int bufferIdx, size_t posInBuffer) +{ + if (bufferIdx == -1) + { + return nullptr; + } + else + { + return m_InternalBlock + posInBuffer; + } +} + +std::vector MallocV::DataVec() noexcept +{ + std::vector iov(DataV.size()); + for (std::size_t i = 0; i < DataV.size(); ++i) + { + if (DataV[i].External) + { + iov[i].iov_base = DataV[i].Base; + } + else + { + iov[i].iov_base = m_InternalBlock + DataV[i].Offset; + } + iov[i].iov_len = DataV[i].Size; + } + return iov; +} + +} // end namespace format +} // end namespace adios2 diff --git a/source/adios2/toolkit/format/buffer/malloc/MallocV.h b/source/adios2/toolkit/format/buffer/malloc/MallocV.h new file mode 100644 index 0000000000..d121adedd2 --- /dev/null +++ b/source/adios2/toolkit/format/buffer/malloc/MallocV.h @@ -0,0 +1,57 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + */ + +#ifndef ADIOS2_TOOLKIT_FORMAT_BUFFER_MALLOC_MALLOCV_H_ +#define ADIOS2_TOOLKIT_FORMAT_BUFFER_MALLOC_MALLOCV_H_ + +#include "adios2/common/ADIOSConfig.h" +#include "adios2/common/ADIOSTypes.h" +#include "adios2/core/CoreTypes.h" + +#include "adios2/toolkit/format/buffer/BufferV.h" + +namespace adios2 +{ +namespace format +{ + +class MallocV : public BufferV +{ +public: + uint64_t Size() noexcept; + + MallocV(const std::string type, const bool AlwaysCopy = false, + size_t InitialBufferSize = DefaultInitialBufferSize, + double GrowthFactor = DefaultBufferGrowthFactor); + virtual ~MallocV(); + + virtual std::vector DataVec() noexcept; + + /** + * Reset the buffer to initial state (without freeing internal buffers) + */ + virtual void Reset(); + + virtual size_t AddToVec(const size_t size, const void *buf, size_t align, + bool CopyReqd); + + virtual BufferPos Allocate(const size_t size, size_t align); + + virtual void *GetPtr(int bufferIdx, size_t posInBuffer); + + void CopyExternalToInternal(); + +private: + char *m_InternalBlock = NULL; + size_t m_AllocatedSize = 0; + const size_t m_InitialBufferSize = 16 * 1024; + const double m_GrowthFactor = 1.05; +}; + +} // end namespace format +} // end namespace adios2 + +#endif /* ADIOS2_TOOLKIT_FORMAT_BUFFER_MALLOC_MALLOCV_H_ */ diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp index f6f66729f0..904a0118b9 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp @@ -670,5 +670,127 @@ void DataManSerializer::Log(const int level, const std::string &message, } } +template <> +void DataManSerializer::PutData( + const std::string *inputData, const std::string &varName, + const Dims &varShape, const Dims &varStart, const Dims &varCount, + const Dims &varMemStart, const Dims &varMemCount, const std::string &doid, + const size_t step, const int rank, const std::string &address, + const std::vector &ops, VecPtr localBuffer, + JsonPtr metadataJson) +{ + PERFSTUBS_SCOPED_TIMER_FUNC(); + Log(1, + "DataManSerializer::PutData begin with Step " + std::to_string(step) + + " Var " + varName, + true, true); + + if (localBuffer == nullptr) + { + localBuffer = m_LocalBuffer; + } + + nlohmann::json metaj; + + metaj["N"] = varName; + metaj["O"] = varStart; + metaj["C"] = varCount; + metaj["S"] = varShape; + metaj["Y"] = "string"; + metaj["P"] = localBuffer->size(); + + if (not address.empty()) + { + metaj["A"] = address; + } + + if (not m_IsRowMajor) + { + metaj["M"] = m_IsRowMajor; + } + if (not m_IsLittleEndian) + { + metaj["E"] = m_IsLittleEndian; + } + + metaj["I"] = inputData->size(); + + if (localBuffer->capacity() < localBuffer->size() + inputData->size()) + { + localBuffer->reserve((localBuffer->size() + inputData->size()) * 2); + } + + localBuffer->resize(localBuffer->size() + inputData->size()); + + std::memcpy(localBuffer->data() + localBuffer->size() - inputData->size(), + inputData->data(), inputData->size()); + + if (metadataJson == nullptr) + { + m_MetadataJson[std::to_string(step)][std::to_string(rank)].emplace_back( + std::move(metaj)); + } + else + { + (*metadataJson)[std::to_string(step)][std::to_string(rank)] + .emplace_back(std::move(metaj)); + } + + Log(1, + "DataManSerializer::PutData end with Step " + std::to_string(step) + + " Var " + varName, + true, true); +} + +template <> +int DataManSerializer::GetData(std::string *outputData, + const std::string &varName, const Dims &varStart, + const Dims &varCount, const size_t step, + const Dims &varMemStart, const Dims &varMemCount) +{ + PERFSTUBS_SCOPED_TIMER_FUNC(); + + DmvVecPtr vec = nullptr; + + { + std::lock_guard l(m_DataManVarMapMutex); + const auto &i = m_DataManVarMap.find(step); + if (i == m_DataManVarMap.end()) + { + return -1; // step not found + } + else + { + vec = i->second; + } + } + + if (vec == nullptr) + { + return -2; // step found but variable not found + } + + char *input_data = nullptr; + + for (const auto &j : *vec) + { + if (j.name == varName) + { + if (j.buffer == nullptr) + { + continue; + } + else + { + input_data = reinterpret_cast(j.buffer->data()); + } + + input_data += j.position; + + *outputData = std::string(input_data, j.size); + } + } + return 0; +} } // namespace format } // namespace adios2 diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.h b/source/adios2/toolkit/format/dataman/DataManSerializer.h index 64179b43c4..5699604b14 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.h +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.h @@ -19,7 +19,7 @@ #include #include -#include +#include // A - Address // C - Count diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.tcc b/source/adios2/toolkit/format/dataman/DataManSerializer.tcc index da2db22e13..ae485a02ae 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.tcc +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.tcc @@ -235,6 +235,7 @@ void DataManSerializer::PutData( datasize = std::accumulate(varCount.begin(), varCount.end(), sizeof(T), std::multiplies()); } + metaj["I"] = datasize; if (localBuffer->capacity() < localBuffer->size() + datasize) @@ -335,8 +336,7 @@ int DataManSerializer::GetData(T *outputData, const std::string &varName, try { decompressor.Decompress(j.buffer->data() + j.position, - j.size, decompressBuffer.data(), - j.count, j.type, j.params); + j.size, decompressBuffer.data()); decompressed = true; } catch (std::exception &e) @@ -362,8 +362,7 @@ int DataManSerializer::GetData(T *outputData, const std::string &varName, try { decompressor.Decompress(j.buffer->data() + j.position, - j.size, decompressBuffer.data(), - j.count, j.type, j.params); + j.size, decompressBuffer.data()); decompressed = true; } catch (std::exception &e) @@ -387,10 +386,8 @@ int DataManSerializer::GetData(T *outputData, const std::string &varName, decompressBuffer.reserve(datasize); try { - Params info; decompressor.Decompress(j.buffer->data() + j.position, - j.size, decompressBuffer.data(), - datasize, info); + j.size, decompressBuffer.data()); decompressed = true; } catch (std::exception &e) @@ -415,8 +412,7 @@ int DataManSerializer::GetData(T *outputData, const std::string &varName, try { decompressor.Decompress(j.buffer->data() + j.position, - j.size, decompressBuffer.data(), - j.count, j.type, j.params); + j.size, decompressBuffer.data()); decompressed = true; } catch (std::exception &e) @@ -460,7 +456,8 @@ int DataManSerializer::GetData(T *outputData, const std::string &varName, } if (j.shape.empty() or (j.shape.size() == 1 and j.shape[0] == 1)) { - std::memcpy(outputData, input_data, sizeof(T)); + std::memcpy(reinterpret_cast(outputData), input_data, + sizeof(T)); } } } @@ -480,10 +477,9 @@ void DataManSerializer::PutZfp(nlohmann::json &metaj, size_t &datasize, std::multiplies())); try { - Params info; - datasize = compressor.Compress(inputData, varCount, sizeof(T), - helper::GetDataType(), - m_CompressBuffer.data(), params, info); + datasize = compressor.Compress( + reinterpret_cast(inputData), {}, varCount, + helper::GetDataType(), m_CompressBuffer.data(), params); } catch (std::exception &e) { @@ -508,10 +504,9 @@ void DataManSerializer::PutSz(nlohmann::json &metaj, size_t &datasize, core::compress::CompressSZ compressor(params); try { - Params info; - datasize = compressor.Compress(inputData, varCount, sizeof(T), - helper::GetDataType(), - m_CompressBuffer.data(), params, info); + datasize = compressor.Compress( + reinterpret_cast(inputData), {}, varCount, + helper::GetDataType(), m_CompressBuffer.data(), params); } catch (std::exception &e) { @@ -536,10 +531,9 @@ void DataManSerializer::PutBZip2(nlohmann::json &metaj, size_t &datasize, core::compress::CompressBZIP2 compressor(params); try { - Params info; - datasize = compressor.Compress(inputData, varCount, sizeof(T), - helper::GetDataType(), - m_CompressBuffer.data(), params, info); + datasize = compressor.Compress( + reinterpret_cast(inputData), {}, varCount, + helper::GetDataType(), m_CompressBuffer.data(), params); } catch (std::exception &e) { @@ -564,10 +558,9 @@ void DataManSerializer::PutMgard(nlohmann::json &metaj, size_t &datasize, std::multiplies())); try { - Params info; - datasize = compressor.Compress(inputData, varCount, sizeof(T), - helper::GetDataType(), - m_CompressBuffer.data(), params, info); + datasize = compressor.Compress( + reinterpret_cast(inputData), {}, varCount, + helper::GetDataType(), m_CompressBuffer.data(), params); } catch (std::exception &e) { diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp index 450d8c94b8..5bea4f8618 100644 --- a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp +++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp @@ -149,7 +149,7 @@ void HDF5Common::ParseParameters(core::IO &io) } } - m_OrderByC = helper::IsRowMajor(io.m_HostLanguage); + m_OrderByC = (io.m_ArrayOrder == ArrayOrdering::RowMajor); } void HDF5Common::Append(const std::string &name, helper::Comm const &comm) @@ -619,7 +619,7 @@ void HDF5Common::AddVar(core::IO &io, std::string const &name, hid_t datasetId, shape.resize(ndims); if (ndims > 0) { - bool isOrderC = helper::IsRowMajor(io.m_HostLanguage); + bool isOrderC = (io.m_ArrayOrder == ArrayOrdering::RowMajor); for (int i = 0; i < ndims; i++) { if (isOrderC) diff --git a/source/adios2/toolkit/profiling/iochrono/IOChrono.cpp b/source/adios2/toolkit/profiling/iochrono/IOChrono.cpp index d1d9264ba5..6d8aaabac1 100644 --- a/source/adios2/toolkit/profiling/iochrono/IOChrono.cpp +++ b/source/adios2/toolkit/profiling/iochrono/IOChrono.cpp @@ -9,6 +9,7 @@ */ #include "IOChrono.h" +#include "adios2/helper/adiosMemory.h" namespace adios2 { @@ -31,5 +32,136 @@ void IOChrono::Stop(const std::string process) } } +// +// class JSON Profiler +// +JSONProfiler::JSONProfiler(helper::Comm const &comm) : m_Comm(comm) +{ + m_Profiler.m_IsActive = true; // default is true + + AddTimerWatch("buffering"); + // xAddTimerWatch("memcpy"); + AddTimerWatch("endstep"); + AddTimerWatch("PP"); + // AddTimerWatch("meta_merge"); + AddTimerWatch("meta_gather"); + // AddTimerWatch("meta_ds"); + // AddTimerWatch("meta_s"); + // AddTimerWatch("meta_sort_merge"); + + AddTimerWatch("AWD"); + + m_Profiler.m_Bytes.emplace("buffering", 0); + + m_RankMPI = m_Comm.Rank(); +} + +void JSONProfiler::AddTimerWatch(const std::string &name) +{ + const TimeUnit timerUnit = DefaultTimeUnitEnum; + m_Profiler.m_Timers.emplace(name, profiling::Timer(name, timerUnit)); +} + +std::string JSONProfiler::GetRankProfilingJSON( + const std::vector &transportsTypes, + const std::vector &transportsProfilers) noexcept +{ + auto lf_WriterTimer = [](std::string &rankLog, + const profiling::Timer &timer) { + // rankLog += "\"" + timer.m_Process + "_" + timer.GetShortUnits() + + // "\": " + std::to_string(timer.m_ProcessTime) + ", "; + timer.AddToJsonStr(rankLog); + }; + + // prepare string dictionary per rank + std::string rankLog("{ \"rank\": " + std::to_string(m_RankMPI) + ", "); + + auto &profiler = m_Profiler; + + std::string timeDate(profiler.m_Timers.at("buffering").m_LocalTimeDate); + timeDate.pop_back(); + // avoid whitespace + std::replace(timeDate.begin(), timeDate.end(), ' ', '_'); + + rankLog += "\"start\": \"" + timeDate + "\", "; + // rankLog += "\"threads\": " + std::to_string(m_Parameters.Threads) + ", "; + rankLog += + "\"bytes\": " + std::to_string(profiler.m_Bytes.at("buffering")) + ", "; + + for (const auto &timerPair : profiler.m_Timers) + { + const profiling::Timer &timer = timerPair.second; + // rankLog += "\"" + timer.m_Process + "_" + timer.GetShortUnits() + + // "\": " + std::to_string(timer.m_ProcessTime) + ", "; + timer.AddToJsonStr(rankLog); + } + + const size_t transportsSize = transportsTypes.size(); + + for (unsigned int t = 0; t < transportsSize; ++t) + { + rankLog += "\"transport_" + std::to_string(t) + "\": { "; + rankLog += "\"type\": \"" + transportsTypes[t] + "\", "; + + for (const auto &transportTimerPair : transportsProfilers[t]->m_Timers) + { + lf_WriterTimer(rankLog, transportTimerPair.second); + } + // replace last comma with space + rankLog.pop_back(); + rankLog.pop_back(); + rankLog += " "; + + if (t == transportsSize - 1) // last element + { + rankLog += "}"; + } + else + { + rankLog += "},"; + } + } + rankLog += " }"; // end rank entry + + return rankLog; +} + +std::vector +JSONProfiler::AggregateProfilingJSON(const std::string &rankLog) const +{ + // Gather sizes + const size_t rankLogSize = rankLog.size(); + std::vector rankLogsSizes = m_Comm.GatherValues(rankLogSize); + + // Gatherv JSON per rank + std::vector profilingJSON(3); + const std::string header("[\n"); + const std::string footer("\n]\n"); + size_t gatheredSize = 0; + size_t position = 0; + + if (m_RankMPI == 0) // pre-allocate in destination + { + gatheredSize = std::accumulate(rankLogsSizes.begin(), + rankLogsSizes.end(), size_t(0)); + + profilingJSON.resize(gatheredSize + header.size() + footer.size() - 2); + adios2::helper::CopyToBuffer(profilingJSON, position, header.c_str(), + header.size()); + } + + m_Comm.GathervArrays(rankLog.c_str(), rankLog.size(), rankLogsSizes.data(), + rankLogsSizes.size(), &profilingJSON[position]); + + if (m_RankMPI == 0) // add footer to close JSON + { + position += gatheredSize - 2; + helper::CopyToBuffer(profilingJSON, position, footer.c_str(), + footer.size()); + } + + return profilingJSON; +} + } // end namespace profiling } // end namespace adios2 diff --git a/source/adios2/toolkit/profiling/iochrono/IOChrono.h b/source/adios2/toolkit/profiling/iochrono/IOChrono.h index 2e08ab157c..638b903721 100644 --- a/source/adios2/toolkit/profiling/iochrono/IOChrono.h +++ b/source/adios2/toolkit/profiling/iochrono/IOChrono.h @@ -17,6 +17,7 @@ /// \endcond #include "adios2/common/ADIOSConfig.h" +#include "adios2/helper/adiosComm.h" #include "adios2/toolkit/profiling/iochrono/Timer.h" namespace adios2 @@ -58,6 +59,29 @@ class IOChrono void Stop(const std::string process); }; +class JSONProfiler +{ +public: + JSONProfiler(helper::Comm const &comm); + void Gather(); + void AddTimerWatch(const std::string &); + + void Start(const std::string process) { m_Profiler.Start(process); }; + void Stop(const std::string process) { m_Profiler.Stop(process); }; + + std::string + GetRankProfilingJSON(const std::vector &transportsTypes, + const std::vector + &transportsProfilers) noexcept; + + std::vector AggregateProfilingJSON(const std::string &rankLog) const; + +private: + IOChrono m_Profiler; + int m_RankMPI = 0; + helper::Comm const &m_Comm; +}; + } // end namespace profiling } // end namespace adios diff --git a/source/adios2/toolkit/profiling/iochrono/Timer.cpp b/source/adios2/toolkit/profiling/iochrono/Timer.cpp index 0f6122c3e0..c8f01db724 100644 --- a/source/adios2/toolkit/profiling/iochrono/Timer.cpp +++ b/source/adios2/toolkit/profiling/iochrono/Timer.cpp @@ -21,6 +21,9 @@ Timer::Timer(const std::string process, const TimeUnit timeUnit) : m_Process(process), m_TimeUnit(timeUnit), m_LocalTimeDate(helper::LocalTimeDate()) { + std::size_t found = m_Process.find("gather"); + if (found != std::string::npos) + m_Always = true; } void Timer::Resume() noexcept @@ -33,6 +36,8 @@ void Timer::Pause() { m_ElapsedTime = std::chrono::high_resolution_clock::now(); m_ProcessTime += GetElapsedTime(); + + AddDetail(); } std::string Timer::GetShortUnits() const noexcept diff --git a/source/adios2/toolkit/profiling/iochrono/Timer.h b/source/adios2/toolkit/profiling/iochrono/Timer.h index 115289a2ed..122386235d 100644 --- a/source/adios2/toolkit/profiling/iochrono/Timer.h +++ b/source/adios2/toolkit/profiling/iochrono/Timer.h @@ -19,11 +19,16 @@ #include "adios2/common/ADIOSConfig.h" #include "adios2/common/ADIOSTypes.h" +#include // myTimer + namespace adios2 { namespace profiling { +static std::chrono::time_point + m_ADIOS2ProgStart = std::chrono::high_resolution_clock::now(); + class Timer { @@ -31,6 +36,7 @@ class Timer /** process name */ const std::string m_Process; + bool m_Always = false; /** process elapsed time */ int64_t m_ProcessTime = 0; @@ -61,6 +67,51 @@ class Timer /** Returns TimeUnit as a short std::string */ std::string GetShortUnits() const noexcept; + void AddDetail() + { + m_nCalls++; + auto relative = std::chrono::duration_cast( + m_InitialTime - m_ADIOS2ProgStart) + .count(); + auto micros = std::chrono::duration_cast( + m_ElapsedTime - m_InitialTime) + .count(); + + if ((micros > 10000) || m_Always) + { + if (m_Details.size() > 0) + m_Details += ","; + + std::ostringstream ss; + + ss << "\"" << relative / 1000.0 << "+" << micros / 1000.0 << "\""; + + m_Details += ss.str(); + } + } + + void AddToJsonStr(std::string &rankLog) const + { + if (0 == m_nCalls) + return; + + rankLog += + "\"" + m_Process + "\":{ \"mus\":" + std::to_string(m_ProcessTime); + rankLog += ", \"nCalls\":" + std::to_string(m_nCalls); + + if (500 > m_nCalls) + { + if (m_Details.size() > 2) + { + rankLog += ", \"trace\":[" + m_Details + "]"; + } + } + rankLog += "}, "; + } + + std::string m_Details; + uint64_t m_nCalls = 0; + private: /** Set at Resume */ std::chrono::time_point m_InitialTime; diff --git a/source/adios2/toolkit/query/JsonWorker.cpp b/source/adios2/toolkit/query/JsonWorker.cpp index 10c4d0caec..c538d69fb7 100644 --- a/source/adios2/toolkit/query/JsonWorker.cpp +++ b/source/adios2/toolkit/query/JsonWorker.cpp @@ -1,5 +1,6 @@ #include "Worker.h" -#include + +#include namespace adios2 { diff --git a/source/adios2/toolkit/query/Query.cpp b/source/adios2/toolkit/query/Query.cpp index 21844cb7a0..f7f856163a 100644 --- a/source/adios2/toolkit/query/Query.cpp +++ b/source/adios2/toolkit/query/Query.cpp @@ -247,12 +247,14 @@ bool QueryVar::IsSelectionValid(adios2::Dims &shape) const return false; // different dimension } + /* for (size_t i = 0; i < shape.size(); i++) { if ((m_Selection.first[i] > shape[i]) || (m_Selection.second[i] > shape[i])) return false; } + */ return true; } diff --git a/source/adios2/toolkit/query/XmlWorker.cpp b/source/adios2/toolkit/query/XmlWorker.cpp index dbecccabcf..fcf0656b54 100644 --- a/source/adios2/toolkit/query/XmlWorker.cpp +++ b/source/adios2/toolkit/query/XmlWorker.cpp @@ -169,7 +169,7 @@ void XmlWorker::ConstructTree(RangeTree &host, const pugi::xml_node &node) std::string relationStr = adios2::helper::XMLAttribute("value", node, "in query")->value(); host.SetRelation(adios2::query::strToRelation(relationStr)); - for (const pugi::xml_node rangeNode : node.children("range")) + for (const pugi::xml_node &rangeNode : node.children("range")) { std::string valStr = adios2::helper::XMLAttribute("value", rangeNode, "in query") @@ -181,7 +181,7 @@ void XmlWorker::ConstructTree(RangeTree &host, const pugi::xml_node &node) host.AddLeaf(adios2::query::strToQueryOp(opStr), valStr); } - for (const pugi::xml_node opNode : node.children("op")) + for (const pugi::xml_node &opNode : node.children("op")) { adios2::query::RangeTree subNode; ConstructTree(subNode, opNode); @@ -196,8 +196,6 @@ void XmlWorker::ConstructQuery(QueryVar &simpleQ, const pugi::xml_node &node) if (bbNode) { - adios2::Box box = - adios2::Box({100, 100}, {200, 200}); std::string startStr = adios2::helper::XMLAttribute("start", bbNode, "in query")->value(); std::string countStr = diff --git a/source/adios2/toolkit/sst/cp/cp_internal.h b/source/adios2/toolkit/sst/cp/cp_internal.h index 2af2246344..d3addea00d 100644 --- a/source/adios2/toolkit/sst/cp/cp_internal.h +++ b/source/adios2/toolkit/sst/cp/cp_internal.h @@ -226,6 +226,7 @@ struct _SstStream FFSContext ReaderFFSContext; VarSetupUpcallFunc VarSetupUpcall; ArraySetupUpcallFunc ArraySetupUpcall; + MinArraySetupUpcallFunc MinArraySetupUpcall; AttrSetupUpcallFunc AttrSetupUpcall; ArrayBlocksInfoUpcallFunc ArrayBlocksInfoUpcall; void *SetupUpcallReader; diff --git a/source/adios2/toolkit/sst/cp/cp_reader.c b/source/adios2/toolkit/sst/cp/cp_reader.c index 38c56e9c77..e6de65e6d7 100644 --- a/source/adios2/toolkit/sst/cp/cp_reader.c +++ b/source/adios2/toolkit/sst/cp/cp_reader.c @@ -1448,18 +1448,24 @@ extern SstMetaMetaList SstGetNewMetaMetaData(SstStream Stream, long Timestep) { int RetCount = 0; STREAM_MUTEX_LOCK(Stream); + int64_t LastRetTimestep = -1; for (int i = 0; i < Stream->InternalMetaMetaCount; i++) { - if (Stream->InternalMetaMetaInfo[i].TimestepAdded >= Timestep) + if ((LastRetTimestep == -1) || + (Stream->InternalMetaMetaInfo[i].TimestepAdded >= LastRetTimestep)) RetCount++; } if (RetCount == 0) + { + STREAM_MUTEX_UNLOCK(Stream); return NULL; + } SstMetaMetaList ret = malloc(sizeof(ret[0]) * (RetCount + 1)); int j = 0; for (int i = 0; i < Stream->InternalMetaMetaCount; i++) { - if (Stream->InternalMetaMetaInfo[i].TimestepAdded >= Timestep) + if ((LastRetTimestep == -1) || + (Stream->InternalMetaMetaInfo[i].TimestepAdded >= LastRetTimestep)) { // no copies, keep memory ownership in SST ret[j].BlockData = Stream->InternalMetaMetaInfo[i].BlockData; @@ -1470,6 +1476,7 @@ extern SstMetaMetaList SstGetNewMetaMetaData(SstStream Stream, long Timestep) } } memset(&ret[j], 0, sizeof(ret[j])); + LastRetTimestep = Timestep; STREAM_MUTEX_UNLOCK(Stream); return ret; } diff --git a/source/adios2/toolkit/sst/cp/ffs_marshal.c b/source/adios2/toolkit/sst/cp/ffs_marshal.c index 950590b08a..678350ca73 100644 --- a/source/adios2/toolkit/sst/cp/ffs_marshal.c +++ b/source/adios2/toolkit/sst/cp/ffs_marshal.c @@ -695,11 +695,13 @@ extern int SstFFSWriterBeginStep(SstStream Stream, int mode, */ void NO_SANITIZE_THREAD SstReaderInitFFSCallback( SstStream Stream, void *Reader, VarSetupUpcallFunc VarCallback, - ArraySetupUpcallFunc ArrayCallback, AttrSetupUpcallFunc AttrCallback, + ArraySetupUpcallFunc ArrayCallback, + MinArraySetupUpcallFunc MinArrayCallback, AttrSetupUpcallFunc AttrCallback, ArrayBlocksInfoUpcallFunc BlocksInfoCallback) { Stream->VarSetupUpcall = VarCallback; Stream->ArraySetupUpcall = ArrayCallback; + Stream->MinArraySetupUpcall = MinArrayCallback; Stream->AttrSetupUpcall = AttrCallback; Stream->ArrayBlocksInfoUpcall = BlocksInfoCallback; Stream->SetupUpcallReader = Reader; @@ -744,6 +746,38 @@ extern int SstFFSGetDeferred(SstStream Stream, void *Variable, const char *Name, } } +extern void *SstFFSGetBlocksInfo(SstStream Stream, void *Variable) +{ + struct FFSReaderMarshalBase *Info = Stream->ReaderMarshalData; + FFSVarRec VarRec = LookupVarByKey(Stream, Variable); + MetaArrayRec *meta_base = + (MetaArrayRec *)(((char *)Info->MetadataBaseAddrs[0]) + + VarRec->PerWriterMetaFieldOffset[0]); + if (!Stream->MinArraySetupUpcall) + return NULL; + + void *Ret = Stream->MinArraySetupUpcall(Stream->SetupUpcallReader, + meta_base->Dims, meta_base->Shape); + for (int WriterRank = 0; WriterRank < Stream->WriterCohortSize; + WriterRank++) + { + meta_base = + (MetaArrayRec *)(((char *)Info->MetadataBaseAddrs[WriterRank]) + + VarRec->PerWriterMetaFieldOffset[WriterRank]); + + for (int i = 0; i < VarRec->PerWriterBlockCount[WriterRank]; i++) + { + size_t *Offsets = NULL; + if (meta_base->Offsets) + Offsets = meta_base->Offsets + (i * meta_base->Dims); + Stream->ArrayBlocksInfoUpcall( + Stream->SetupUpcallReader, Ret, VarRec->Type, WriterRank, + meta_base->Dims, meta_base->Shape, Offsets, meta_base->Count); + } + } + return Ret; +} + extern int SstFFSGetLocalDeferred(SstStream Stream, void *Variable, const char *Name, size_t DimCount, const int BlockID, const size_t *Count, @@ -2008,15 +2042,32 @@ static void BuildVarList(SstStream Stream, TSMetadataMsg MetaData, VarRec->PerWriterBlockStart[WriterRank] + VarRec->PerWriterBlockCount[WriterRank]; } - for (int i = 0; i < VarRec->PerWriterBlockCount[WriterRank]; i++) + static int UseMin = 1; + if (UseMin == -1) { - size_t *Offsets = NULL; - if (meta_base->Offsets) - Offsets = meta_base->Offsets + (i * meta_base->Dims); - Stream->ArrayBlocksInfoUpcall( - Stream->SetupUpcallReader, VarRec->Variable, VarRec->Type, - WriterRank, meta_base->Dims, meta_base->Shape, Offsets, - meta_base->Count); + if (getenv("OldBlocksInfo") == NULL) + { + UseMin = 0; + } + else + { + UseMin = 1; + } + } + if (!UseMin) + { + for (int i = 0; i < VarRec->PerWriterBlockCount[WriterRank]; + i++) + { + size_t *Offsets = NULL; + if (meta_base->Offsets) + Offsets = meta_base->Offsets + (i * meta_base->Dims); + void *Variable = VarRec->Variable; + Stream->ArrayBlocksInfoUpcall( + Stream->SetupUpcallReader, Variable, VarRec->Type, + WriterRank, meta_base->Dims, meta_base->Shape, Offsets, + meta_base->Count); + } } } else @@ -2027,8 +2078,8 @@ static void BuildVarList(SstStream Stream, TSMetadataMsg MetaData, Stream->SetupUpcallReader, VarRec->VarName, VarRec->Type, field_data); } - VarRec->PerWriterMetaFieldOffset[WriterRank] = FieldOffset; } + VarRec->PerWriterMetaFieldOffset[WriterRank] = FieldOffset; } } @@ -2100,6 +2151,7 @@ extern void SstFFSSetZFPParams(SstStream Stream, attr_list Attrs) } } +/* GetDeferred calls return true if need later sync */ extern void SstFFSMarshal(SstStream Stream, void *Variable, const char *Name, const int Type, size_t ElemSize, size_t DimCount, const size_t *Shape, const size_t *Count, diff --git a/source/adios2/toolkit/sst/dp/rdma_dp.c b/source/adios2/toolkit/sst/dp/rdma_dp.c index a6060b38bf..d54480b7cd 100644 --- a/source/adios2/toolkit/sst/dp/rdma_dp.c +++ b/source/adios2/toolkit/sst/dp/rdma_dp.c @@ -1139,7 +1139,7 @@ static void *RdmaReadRemoteMemory(CP_Services Svcs, DP_RS_Stream Stream_v, size_t Length, void *Buffer, void *DP_TimestepInfo) { - RdmaCompletionHandle ret; + RdmaCompletionHandle ret = {0}; Rdma_RS_Stream RS_Stream = (Rdma_RS_Stream)Stream_v; RdmaBufferHandle Info = (RdmaBufferHandle)DP_TimestepInfo; RdmaStepLogEntry StepLog; @@ -1539,9 +1539,10 @@ static void RdmaDestroyReader(CP_Services Svcs, DP_RS_Stream RS_Stream_v) static void RdmaDestroyWriterPerReader(CP_Services Svcs, DP_WSR_Stream WSR_Stream_v) { - Rdma_WSR_Stream WSR_Stream = (Rdma_WSR_Stream)WSR_Stream_v; + Rdma_WSR_Stream WSR_Stream = {0}; + memcpy(&WSR_Stream, &WSR_Stream_v, sizeof(Rdma_WSR_Stream)); Rdma_WS_Stream WS_Stream = WSR_Stream->WS_Stream; - RdmaWriterContactInfo WriterContactInfo; + RdmaWriterContactInfo WriterContactInfo = {0}; pthread_mutex_lock(&wsr_mutex); for (int i = 0; i < WS_Stream->ReaderCount; i++) @@ -2067,7 +2068,7 @@ static void PullSelection(CP_Services Svcs, Rdma_WSR_Stream Stream) Rdma_WS_Stream WS_Stream = Stream->WS_Stream; FabricState Fabric = WS_Stream->Fabric; RdmaBuffer ReaderRoll = (RdmaBuffer)Stream->ReaderRoll->Handle.Block; - struct _RdmaBuffer ReqBuffer = {0}; + struct _RdmaBuffer ReqBuffer = {{0}}; struct fi_cq_data_entry CQEntry = {0}; struct fid_mr *rrmr = NULL; void *rrdesc = NULL; diff --git a/source/adios2/toolkit/sst/sst.h b/source/adios2/toolkit/sst/sst.h index 49dea22ecc..2a428e491b 100644 --- a/source/adios2/toolkit/sst/sst.h +++ b/source/adios2/toolkit/sst/sst.h @@ -135,14 +135,19 @@ typedef void *(*ArraySetupUpcallFunc)(void *Reader, const char *Name, const int Type, int DimsCount, size_t *Shape, size_t *Start, size_t *Count); +typedef void *(*MinArraySetupUpcallFunc)(void *Reader, int DimsCount, + size_t *Shape); typedef void (*ArrayBlocksInfoUpcallFunc)(void *Reader, void *Variable, const int Type, int WriterRank, int DimsCount, size_t *Shape, size_t *Start, size_t *Count); -extern void SstReaderInitFFSCallback( - SstStream stream, void *Reader, VarSetupUpcallFunc VarCallback, - ArraySetupUpcallFunc ArrayCallback, AttrSetupUpcallFunc AttrCallback, - ArrayBlocksInfoUpcallFunc BlocksInfoCallback); +extern void +SstReaderInitFFSCallback(SstStream stream, void *Reader, + VarSetupUpcallFunc VarCallback, + ArraySetupUpcallFunc ArrayCallback, + MinArraySetupUpcallFunc MinArraySetupUpcall, + AttrSetupUpcallFunc AttrCallback, + ArrayBlocksInfoUpcallFunc BlocksInfoCallback); /* * Calls that support SST-external writer-side aggregation of metadata @@ -174,6 +179,8 @@ extern int SstFFSGetLocalDeferred(SstStream Stream, void *Variable, const char *Name, size_t DimCount, const int BlockID, const size_t *Count, void *Data); +/* GetDeferred calls return true if need later sync */ +extern void *SstFFSGetBlocksInfo(SstStream Stream, void *Variable); extern SstStatusValue SstFFSPerformGets(SstStream Stream); diff --git a/source/adios2/toolkit/transport/Transport.cpp b/source/adios2/toolkit/transport/Transport.cpp index 73277246c5..7df1afe647 100644 --- a/source/adios2/toolkit/transport/Transport.cpp +++ b/source/adios2/toolkit/transport/Transport.cpp @@ -27,6 +27,23 @@ void Transport::IWrite(const char *buffer, size_t size, Status &status, throw std::invalid_argument("ERROR: this class doesn't implement IWrite\n"); } +void Transport::WriteV(const core::iovec *iov, const int iovcnt, size_t start) +{ + if (iovcnt > 0) + { + Write(static_cast(iov[0].iov_base), iov[0].iov_len, + start); + for (int c = 1; c < iovcnt; ++c) + { + Write(static_cast(iov[c].iov_base), iov[c].iov_len); + } + } + else if (start != MaxSizeT) + { + Seek(start); + } +} + void Transport::IRead(char *buffer, size_t size, Status &status, size_t start) { throw std::invalid_argument("ERROR: this class doesn't implement IRead\n"); @@ -72,6 +89,14 @@ void Transport::InitProfiler(const Mode openMode, const TimeUnit timeUnit) "close", profiling::Timer("close", TimeUnit::Microseconds)); } +void Transport::OpenChain(const std::string &name, const Mode openMode, + const helper::Comm &chainComm, const bool async) +{ + std::invalid_argument("ERROR: " + m_Name + " transport type " + m_Type + + " using library " + m_Library + + " doesn't implement the OpenChain function\n"); +} + void Transport::SetParameters(const Params ¶meters) {} void Transport::SetBuffer(char * /*buffer*/, size_t /*size*/) diff --git a/source/adios2/toolkit/transport/Transport.h b/source/adios2/toolkit/transport/Transport.h index cd7663da73..939299ef21 100644 --- a/source/adios2/toolkit/transport/Transport.h +++ b/source/adios2/toolkit/transport/Transport.h @@ -18,6 +18,7 @@ #include "adios2/common/ADIOSConfig.h" #include "adios2/common/ADIOSTypes.h" +#include "adios2/core/CoreTypes.h" #include "adios2/helper/adiosComm.h" #include "adios2/toolkit/profiling/iochrono/IOChrono.h" @@ -66,6 +67,19 @@ class Transport virtual void Open(const std::string &name, const Mode openMode, const bool async = false) = 0; + /** + * Opens transport, possibly asynchronously, in a chain to avoid + * DOS attack to the file system. + * Required before SetBuffer, Write, Read, Flush, Close + * @param name + * @param openMode + * @param chainComm + * @param async + */ + virtual void OpenChain(const std::string &name, Mode openMode, + const helper::Comm &chainComm, + const bool async = false); + /** * If OS buffered (FILE* or fstream), sets the buffer size * @param buffer address for OS buffering @@ -93,6 +107,17 @@ class Transport virtual void IWrite(const char *buffer, size_t size, Status &status, size_t start = MaxSizeT); + /** + * Writes to transport, writev version. Note that size is non-const due to + * the nature of underlying transport libraries + * @param iovec array pointer + * @param iovcnt number of entries + * @param start starting position for writing (to allow rewind), if not + * passed then start at current stream position + */ + virtual void WriteV(const core::iovec *iov, const int iovcnt, + size_t start = MaxSizeT); + /** * Reads from transport "size" bytes from a certain position. Note that size * and position and non-const due to the nature of underlying transport @@ -127,6 +152,8 @@ class Transport virtual void SeekToBegin() = 0; + virtual void Seek(const size_t start = MaxSizeT) = 0; + protected: virtual void MkDir(const std::string &fileName); diff --git a/source/adios2/toolkit/transport/file/FileDaos.cpp b/source/adios2/toolkit/transport/file/FileDaos.cpp index 6bf2a09f62..5415723f4c 100644 --- a/source/adios2/toolkit/transport/file/FileDaos.cpp +++ b/source/adios2/toolkit/transport/file/FileDaos.cpp @@ -338,5 +338,27 @@ void FileDaos::SeekToBegin() } } +void FileDaos::Seek(const size_t start) +{ + if (start != MaxSizeT) + { + WaitForOpen(); + errno = 0; + const int status = lseek(m_FileDescriptor, start, SEEK_SET); + m_Errno = errno; + if (status == -1) + { + throw std::ios_base::failure("ERROR: couldn't seek to offset " + + std::to_string(start) + " of file " + + m_Name + ", in call to DAOS IO lseek" + + SysErrMsg()); + } + } + else + { + SeekToEnd(); + } +} + } // end namespace transport } // end namespace adios2 diff --git a/source/adios2/toolkit/transport/file/FileDaos.h b/source/adios2/toolkit/transport/file/FileDaos.h index 0d30c11322..e9123e2c7a 100644 --- a/source/adios2/toolkit/transport/file/FileDaos.h +++ b/source/adios2/toolkit/transport/file/FileDaos.h @@ -52,6 +52,8 @@ class FileDaos : public Transport void SeekToBegin() final; + void Seek(const size_t start = MaxSizeT) final; + private: /** DAOS file handle returned by Open */ int m_FileDescriptor = -1; diff --git a/source/adios2/toolkit/transport/file/FileFStream.cpp b/source/adios2/toolkit/transport/file/FileFStream.cpp index 50e7169567..76ab5979e1 100644 --- a/source/adios2/toolkit/transport/file/FileFStream.cpp +++ b/source/adios2/toolkit/transport/file/FileFStream.cpp @@ -101,6 +101,88 @@ void FileFStream::Open(const std::string &name, const Mode openMode, } } +void FileFStream::OpenChain(const std::string &name, Mode openMode, + const helper::Comm &chainComm, const bool async) +{ + auto lf_AsyncOpenWrite = [&](const std::string &name) -> void { + ProfilerStart("open"); + m_FileStream.open(name, std::fstream::out | std::fstream::binary | + std::fstream::trunc); + ProfilerStop("open"); + }; + + int token = 1; + m_Name = name; + CheckName(); + + if (chainComm.Rank() > 0) + { + chainComm.Recv(&token, 1, chainComm.Rank() - 1, 0, + "Chain token in FileFStream::OpenChain"); + } + + m_OpenMode = openMode; + switch (m_OpenMode) + { + case (Mode::Write): + if (async && chainComm.Size() == 1) + { + m_IsOpening = true; + m_OpenFuture = + std::async(std::launch::async, lf_AsyncOpenWrite, name); + } + else + { + ProfilerStart("open"); + if (chainComm.Rank() == 0) + { + m_FileStream.open(name, std::fstream::out | + std::fstream::binary | + std::fstream::trunc); + } + else + { + m_FileStream.open(name, + std::fstream::out | std::fstream::binary); + } + ProfilerStop("open"); + } + break; + + case (Mode::Append): + ProfilerStart("open"); + m_FileStream.open(name, std::fstream::in | std::fstream::out | + std::fstream::binary); + m_FileStream.seekp(0, std::ios_base::end); + ProfilerStop("open"); + break; + + case (Mode::Read): + ProfilerStart("open"); + m_FileStream.open(name, std::fstream::in | std::fstream::binary); + ProfilerStop("open"); + break; + + default: + CheckFile("unknown open mode for file " + m_Name + + ", in call to stream open"); + } + + if (!m_IsOpening) + { + CheckFile( + "couldn't open file " + m_Name + + ", check permissions or path existence, in call to fstream open"); + m_IsOpen = true; + } + + if (chainComm.Rank() < chainComm.Size() - 1) + { + chainComm.Isend(&token, 1, chainComm.Rank() + 1, 0, + "Sending Chain token in FileFStream::OpenChain"); + } +} + void FileFStream::SetBuffer(char *buffer, size_t size) { if (!buffer && size != 0) @@ -257,5 +339,20 @@ void FileFStream::SeekToBegin() ", in call to fstream seekp"); } +void FileFStream::Seek(const size_t start) +{ + if (start != MaxSizeT) + { + WaitForOpen(); + m_FileStream.seekp(start, std::ios_base::beg); + CheckFile("couldn't move to offset " + std::to_string(start) + + " of file " + m_Name + ", in call to fstream seekp"); + } + else + { + SeekToEnd(); + } +} + } // end namespace transport } // end namespace adios2 diff --git a/source/adios2/toolkit/transport/file/FileFStream.h b/source/adios2/toolkit/transport/file/FileFStream.h index 1c44e9bd9b..140fe4c45f 100644 --- a/source/adios2/toolkit/transport/file/FileFStream.h +++ b/source/adios2/toolkit/transport/file/FileFStream.h @@ -35,6 +35,10 @@ class FileFStream : public Transport void Open(const std::string &name, const Mode openMode, const bool async = false) final; + void OpenChain(const std::string &name, Mode openMode, + const helper::Comm &chainComm, + const bool async = false) final; + void SetBuffer(char *buffer, size_t size) final; void Write(const char *buffer, size_t size, size_t start = MaxSizeT) final; @@ -53,6 +57,8 @@ class FileFStream : public Transport void SeekToBegin() final; + void Seek(const size_t start = MaxSizeT) final; + private: /** file stream using fstream library */ std::fstream m_FileStream; diff --git a/source/adios2/toolkit/transport/file/FileIME.cpp b/source/adios2/toolkit/transport/file/FileIME.cpp index 9aec7eff62..6fd03aecbf 100644 --- a/source/adios2/toolkit/transport/file/FileIME.cpp +++ b/source/adios2/toolkit/transport/file/FileIME.cpp @@ -336,5 +336,24 @@ void FileIME::SeekToBegin() } } +void FileIME::Seek(const size_t start) +{ + if (start != MaxSizeT) + { + const int status = + ime_client_native2_lseek(m_FileDescriptor, start, SEEK_SET); + if (status == -1) + { + throw std::ios_base::failure( + "ERROR: couldn't seek to offset " + std::to_string(start) + + " of file " + m_Name + ", in call to IME IO lseek\n"); + } + } + else + { + SeekToEnd(); + } +} + } // end namespace transport } // end namespace adios2 diff --git a/source/adios2/toolkit/transport/file/FileIME.h b/source/adios2/toolkit/transport/file/FileIME.h index 211ddca240..80d2aeed96 100644 --- a/source/adios2/toolkit/transport/file/FileIME.h +++ b/source/adios2/toolkit/transport/file/FileIME.h @@ -57,6 +57,8 @@ class FileIME : public Transport void SeekToBegin() final; + void Seek(const size_t start = MaxSizeT) final; + private: /** POSIX file handle returned by Open */ int m_FileDescriptor = -1; diff --git a/source/adios2/toolkit/transport/file/FilePOSIX.cpp b/source/adios2/toolkit/transport/file/FilePOSIX.cpp index 24540d2ff8..b4b9e828c9 100644 --- a/source/adios2/toolkit/transport/file/FilePOSIX.cpp +++ b/source/adios2/toolkit/transport/file/FilePOSIX.cpp @@ -16,8 +16,11 @@ #include // write output #include // open, fstat #include // open +#include // writev #include // write, close +#include + /// \cond EXCLUDE_FROM_DOXYGEN #include //std::ios_base::failure /// \endcond @@ -120,6 +123,102 @@ void FilePOSIX::Open(const std::string &name, const Mode openMode, } } +void FilePOSIX::OpenChain(const std::string &name, Mode openMode, + const helper::Comm &chainComm, const bool async) +{ + auto lf_AsyncOpenWrite = [&](const std::string &name) -> int { + ProfilerStart("open"); + errno = 0; + int FD = open(m_Name.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666); + m_Errno = errno; + ProfilerStop("open"); + return FD; + }; + + int token = 1; + m_Name = name; + CheckName(); + + if (chainComm.Rank() > 0) + { + chainComm.Recv(&token, 1, chainComm.Rank() - 1, 0, + "Chain token in FilePOSIX::OpenChain"); + } + + m_OpenMode = openMode; + switch (m_OpenMode) + { + + case (Mode::Write): + if (async && chainComm.Size() == 1) + { + // only when process is a single writer, can create the file + // asynchronously, otherwise other processes are waiting on it + m_IsOpening = true; + m_OpenFuture = + std::async(std::launch::async, lf_AsyncOpenWrite, name); + } + else + { + ProfilerStart("open"); + errno = 0; + if (chainComm.Rank() == 0) + { + m_FileDescriptor = + open(m_Name.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666); + } + else + { + m_FileDescriptor = open(m_Name.c_str(), O_WRONLY, 0666); + lseek(m_FileDescriptor, 0, SEEK_SET); + } + m_Errno = errno; + ProfilerStop("open"); + } + break; + + case (Mode::Append): + ProfilerStart("open"); + errno = 0; + if (chainComm.Rank() == 0) + { + m_FileDescriptor = open(m_Name.c_str(), O_RDWR | O_CREAT, 0666); + } + else + { + m_FileDescriptor = open(m_Name.c_str(), O_RDWR); + } + lseek(m_FileDescriptor, 0, SEEK_END); + m_Errno = errno; + ProfilerStop("open"); + break; + + case (Mode::Read): + ProfilerStart("open"); + errno = 0; + m_FileDescriptor = open(m_Name.c_str(), O_RDONLY); + m_Errno = errno; + ProfilerStop("open"); + break; + + default: + CheckFile("unknown open mode for file " + m_Name + + ", in call to POSIX open"); + } + + if (!m_IsOpening) + { + CheckFile("couldn't open file " + m_Name + ", in call to POSIX open"); + m_IsOpen = true; + } + + if (chainComm.Rank() < chainComm.Size() - 1) + { + chainComm.Isend(&token, 1, chainComm.Rank() + 1, 0, + "Sending Chain token in FilePOSIX::OpenChain"); + } +} + void FilePOSIX::Write(const char *buffer, size_t size, size_t start) { auto lf_Write = [&](const char *buffer, size_t size) { @@ -163,6 +262,11 @@ void FilePOSIX::Write(const char *buffer, size_t size, size_t start) ", in call to POSIX lseek" + SysErrMsg()); } } + else + { + const auto pos = lseek(m_FileDescriptor, 0, SEEK_CUR); + start = static_cast(pos); + } if (size > DefaultMaxFileBatchSize) { @@ -183,6 +287,100 @@ void FilePOSIX::Write(const char *buffer, size_t size, size_t start) } } +#ifdef REALLY_WANT_WRITEV +void FilePOSIX::WriteV(const core::iovec *iov, const int iovcnt, size_t start) +{ + auto lf_Write = [&](const core::iovec *iov, const int iovcnt) { + ProfilerStart("write"); + errno = 0; + size_t nBytesExpected = 0; + for (int i = 0; i < iovcnt; ++i) + { + nBytesExpected += iov[i].iov_len; + } + const iovec *v = reinterpret_cast(iov); + const auto ret = writev(m_FileDescriptor, v, iovcnt); + m_Errno = errno; + ProfilerStop("write"); + + size_t written; + if (ret == -1) + { + if (errno != EINTR) + { + throw std::ios_base::failure( + "ERROR: couldn't write to file " + m_Name + + ", in call to POSIX Write(iovec)" + SysErrMsg()); + } + written = 0; + } + else + { + written = static_cast(ret); + } + + if (written < nBytesExpected) + { + /* Fall back to write calls with individual buffers */ + // find where the writing has ended + int c = 0; + size_t n = 0; + size_t pos = 0; + while (n < written) + { + if (n + iov[c].iov_len <= written) + { + n += iov[c].iov_len; + ++c; + } + else + { + pos = written - n; + n = written; + } + } + + // write the rest one by one + Write(static_cast(iov[c].iov_base) + pos, + iov[c].iov_len - pos); + for (; c < iovcnt; ++c) + { + Write(static_cast(iov[c].iov_base), + iov[c].iov_len); + } + } + }; + + WaitForOpen(); + if (start != MaxSizeT) + { + errno = 0; + const auto newPosition = lseek(m_FileDescriptor, start, SEEK_SET); + m_Errno = errno; + + if (static_cast(newPosition) != start) + { + throw std::ios_base::failure( + "ERROR: couldn't move to start position " + + std::to_string(start) + " in file " + m_Name + + ", in call to POSIX lseek" + SysErrMsg()); + } + } + + int cntTotal = 0; + while (cntTotal < iovcnt) + { + int cnt = iovcnt - cntTotal; + if (cnt > 8) + { + cnt = 8; + } + lf_Write(iov + cntTotal, cnt); + cntTotal += cnt; + } +} +#endif + void FilePOSIX::Read(char *buffer, size_t size, size_t start) { auto lf_Read = [&](char *buffer, size_t size) { @@ -335,5 +533,27 @@ void FilePOSIX::SeekToBegin() } } +void FilePOSIX::Seek(const size_t start) +{ + if (start != MaxSizeT) + { + WaitForOpen(); + errno = 0; + const int status = lseek(m_FileDescriptor, start, SEEK_SET); + m_Errno = errno; + if (status == -1) + { + throw std::ios_base::failure( + "ERROR: couldn't seek to offset " + std::to_string(start) + + " of file " + m_Name + ", in call to POSIX IO lseek" + + SysErrMsg()); + } + } + else + { + SeekToEnd(); + } +} + } // end namespace transport } // end namespace adios2 diff --git a/source/adios2/toolkit/transport/file/FilePOSIX.h b/source/adios2/toolkit/transport/file/FilePOSIX.h index 4834d02b8d..c6d96ca83a 100644 --- a/source/adios2/toolkit/transport/file/FilePOSIX.h +++ b/source/adios2/toolkit/transport/file/FilePOSIX.h @@ -37,8 +37,18 @@ class FilePOSIX : public Transport void Open(const std::string &name, const Mode openMode, const bool async = false) final; + void OpenChain(const std::string &name, Mode openMode, + const helper::Comm &chainComm, + const bool async = false) final; + void Write(const char *buffer, size_t size, size_t start = MaxSizeT) final; +#ifdef REALLY_WANT_WRITEV + /* Actual writev() function, inactive for now */ + void WriteV(const core::iovec *iov, const int iovcnt, + size_t start = MaxSizeT) final; +#endif + void Read(char *buffer, size_t size, size_t start = MaxSizeT) final; size_t GetSize() final; @@ -54,6 +64,8 @@ class FilePOSIX : public Transport void SeekToBegin() final; + void Seek(const size_t start = MaxSizeT) final; + private: /** POSIX file handle returned by Open */ int m_FileDescriptor = -1; diff --git a/source/adios2/toolkit/transport/file/FileStdio.cpp b/source/adios2/toolkit/transport/file/FileStdio.cpp index 1437837205..af6d26609e 100644 --- a/source/adios2/toolkit/transport/file/FileStdio.cpp +++ b/source/adios2/toolkit/transport/file/FileStdio.cpp @@ -353,5 +353,24 @@ void FileStdio::SeekToBegin() } } +void FileStdio::Seek(const size_t start) +{ + if (start != MaxSizeT) + { + WaitForOpen(); + const auto status = std::fseek(m_File, 0, SEEK_SET); + if (status == -1) + { + throw std::ios_base::failure("ERROR: couldn't seek to offset " + + std::to_string(start) + " of file " + + m_Name + ", in call to stdio fseek\n"); + } + } + else + { + SeekToEnd(); + } +} + } // end namespace transport } // end namespace adios2 diff --git a/source/adios2/toolkit/transport/file/FileStdio.h b/source/adios2/toolkit/transport/file/FileStdio.h index ddce5ef100..39eb83b9ab 100644 --- a/source/adios2/toolkit/transport/file/FileStdio.h +++ b/source/adios2/toolkit/transport/file/FileStdio.h @@ -55,6 +55,8 @@ class FileStdio : public Transport void SeekToBegin() final; + void Seek(const size_t start) final; + private: /** C File pointer */ FILE *m_File = nullptr; diff --git a/source/adios2/toolkit/transport/null/NullTransport.cpp b/source/adios2/toolkit/transport/null/NullTransport.cpp index 5ec8878490..41825fd3df 100644 --- a/source/adios2/toolkit/transport/null/NullTransport.cpp +++ b/source/adios2/toolkit/transport/null/NullTransport.cpp @@ -131,6 +131,16 @@ void NullTransport::SeekToBegin() Impl->CurPos = 0; } +void NullTransport::Seek(const size_t start) +{ + if (!Impl->IsOpen) + { + throw std::runtime_error( + "ERROR: NullTransport::SeekToEnd: The transport is not open."); + } + Impl->CurPos = start; +} + void NullTransport::MkDir(const std::string &fileName) { return; } void NullTransport::CheckName() const { return; } diff --git a/source/adios2/toolkit/transport/null/NullTransport.h b/source/adios2/toolkit/transport/null/NullTransport.h index 1dbb1d2fa4..f8b95c78a6 100644 --- a/source/adios2/toolkit/transport/null/NullTransport.h +++ b/source/adios2/toolkit/transport/null/NullTransport.h @@ -56,6 +56,8 @@ class NullTransport : public Transport void SeekToBegin() override; + void Seek(const size_t start = MaxSizeT) override; + protected: struct NullTransportImpl; std::unique_ptr Impl; diff --git a/source/adios2/toolkit/transport/shm/ShmSystemV.cpp b/source/adios2/toolkit/transport/shm/ShmSystemV.cpp index d3040886f9..0fddde0a62 100644 --- a/source/adios2/toolkit/transport/shm/ShmSystemV.cpp +++ b/source/adios2/toolkit/transport/shm/ShmSystemV.cpp @@ -145,6 +145,11 @@ void ShmSystemV::SeekToBegin() // empty function. seek operation is meaningless for shared memory } +void ShmSystemV::Seek(const size_t start) +{ + // empty function. seek operation is meaningless for shared memory +} + // PRIVATE void ShmSystemV::CheckShmID(const std::string hint) const { diff --git a/source/adios2/toolkit/transport/shm/ShmSystemV.h b/source/adios2/toolkit/transport/shm/ShmSystemV.h index 479fe9f87f..ff58f0df9e 100644 --- a/source/adios2/toolkit/transport/shm/ShmSystemV.h +++ b/source/adios2/toolkit/transport/shm/ShmSystemV.h @@ -51,6 +51,8 @@ class ShmSystemV : public Transport void SeekToBegin() final; + void Seek(const size_t start = MaxSizeT) final; + private: /** 1st argument of ftok to create shared memory segment key, from Open */ std::string m_PathName; diff --git a/source/adios2/toolkit/transportman/TransportMan.cpp b/source/adios2/toolkit/transportman/TransportMan.cpp index d2b1a4ec32..c23b4c0749 100644 --- a/source/adios2/toolkit/transportman/TransportMan.cpp +++ b/source/adios2/toolkit/transportman/TransportMan.cpp @@ -93,8 +93,27 @@ void TransportMan::OpenFiles(const std::vector &fileNames, if (type == "File" || type == "file") { - std::shared_ptr file = - OpenFileTransport(fileNames[i], openMode, parameters, profile); + std::shared_ptr file = OpenFileTransport( + fileNames[i], openMode, parameters, profile, false, m_Comm); + m_Transports.insert({i, file}); + } + } +} + +void TransportMan::OpenFiles(const std::vector &fileNames, + const Mode openMode, + const std::vector ¶metersVector, + const bool profile, const helper::Comm &chainComm) +{ + for (size_t i = 0; i < fileNames.size(); ++i) + { + const Params ¶meters = parametersVector[i]; + const std::string type = parameters.at("transport"); + + if (type == "File" || type == "file") + { + std::shared_ptr file = OpenFileTransport( + fileNames[i], openMode, parameters, profile, true, chainComm); m_Transports.insert({i, file}); } } @@ -105,7 +124,7 @@ void TransportMan::OpenFileID(const std::string &name, const size_t id, const bool profile) { std::shared_ptr file = - OpenFileTransport(name, mode, parameters, profile); + OpenFileTransport(name, mode, parameters, profile, false, m_Comm); m_Transports.insert({id, file}); } @@ -224,6 +243,53 @@ void TransportMan::WriteFileAt(const char *buffer, const size_t size, } } +void TransportMan::WriteFiles(const core::iovec *iov, const size_t iovcnt, + const int transportIndex) +{ + if (transportIndex == -1) + { + for (auto &transportPair : m_Transports) + { + auto &transport = transportPair.second; + if (transport->m_Type == "File") + { + // make this truly asynch? + transport->WriteV(iov, static_cast(iovcnt)); + } + } + } + else + { + auto itTransport = m_Transports.find(transportIndex); + CheckFile(itTransport, ", in call to WriteFiles with index " + + std::to_string(transportIndex)); + itTransport->second->WriteV(iov, static_cast(iovcnt)); + } +} + +void TransportMan::WriteFileAt(const core::iovec *iov, const size_t iovcnt, + const size_t start, const int transportIndex) +{ + if (transportIndex == -1) + { + for (auto &transportPair : m_Transports) + { + auto &transport = transportPair.second; + if (transport->m_Type == "File") + { + transport->WriteV(iov, static_cast(iovcnt), start); + } + } + } + else + { + auto itTransport = m_Transports.find(transportIndex); + CheckFile(itTransport, ", in call to WriteFileAt with index " + + std::to_string(transportIndex)); + itTransport->second->WriteV(iov, static_cast(iovcnt), start); + } +} + void TransportMan::SeekToFileEnd(const int transportIndex) { if (transportIndex == -1) @@ -376,8 +442,8 @@ bool TransportMan::FileExists(const std::string &name, const Params ¶meters, bool exists = false; try { - std::shared_ptr file = - OpenFileTransport(name, Mode::Read, parameters, profile); + std::shared_ptr file = OpenFileTransport( + name, Mode::Read, parameters, profile, false, m_Comm); exists = true; file->Close(); } @@ -388,10 +454,9 @@ bool TransportMan::FileExists(const std::string &name, const Params ¶meters, } // PRIVATE -std::shared_ptr -TransportMan::OpenFileTransport(const std::string &fileName, - const Mode openMode, const Params ¶meters, - const bool profile) +std::shared_ptr TransportMan::OpenFileTransport( + const std::string &fileName, const Mode openMode, const Params ¶meters, + const bool profile, const bool useComm, const helper::Comm &chainComm) { auto lf_GetBuffered = [&](const std::string bufferedDefault) -> bool { bool bufferedValue; @@ -515,7 +580,15 @@ TransportMan::OpenFileTransport(const std::string &fileName, transport->SetParameters(parameters); // open - transport->Open(fileName, openMode, lf_GetAsync("false", parameters)); + if (useComm) + { + transport->OpenChain(fileName, openMode, chainComm, + lf_GetAsync("true", parameters)); + } + else + { + transport->Open(fileName, openMode, lf_GetAsync("false", parameters)); + } return transport; } diff --git a/source/adios2/toolkit/transportman/TransportMan.h b/source/adios2/toolkit/transportman/TransportMan.h index 35544cf70c..29e8903678 100644 --- a/source/adios2/toolkit/transportman/TransportMan.h +++ b/source/adios2/toolkit/transportman/TransportMan.h @@ -17,6 +17,7 @@ #include #include +#include "adios2/core/CoreTypes.h" #include "adios2/toolkit/transport/Transport.h" namespace adios2 @@ -71,6 +72,20 @@ class TransportMan const std::vector ¶metersVector, const bool profile); + /** + * OpenFiles passed from fileNames, in a chain to avoid DOS attacking of the + * file system + * @param fileNames + * @param openMode + * @param parametersVector from IO + * @param profile + * @oaram chainComm + */ + void OpenFiles(const std::vector &fileNames, + const Mode openMode, + const std::vector ¶metersVector, + const bool profile, const helper::Comm &chainComm); + /** * Used for sub-files defined by index * @param name @@ -123,10 +138,30 @@ class TransportMan * @param transportIndex * @param buffer * @param size + * @param start offset in file */ void WriteFileAt(const char *buffer, const size_t size, const size_t start, const int transportIndex = -1); + /** + * Write to file transports, writev version + * @param transportIndex + * @param iovec array pointer + * @param iovcnt number of entries + */ + void WriteFiles(const core::iovec *iov, const size_t iovcnt, + const int transportIndex = -1); + + /** + * Write data to a specific location in files, writev version + * @param transportIndex + * @param iovec array pointer + * @param iovcnt number of entries + * @param start offset in file + */ + void WriteFileAt(const core::iovec *iov, const size_t iovcnt, + const size_t start, const int transportIndex = -1); + size_t GetFileSize(const size_t transportIndex = 0) const; /** @@ -178,10 +213,10 @@ class TransportMan protected: helper::Comm const &m_Comm; - std::shared_ptr OpenFileTransport(const std::string &fileName, - const Mode openMode, - const Params ¶meters, - const bool profile); + std::shared_ptr + OpenFileTransport(const std::string &fileName, const Mode openMode, + const Params ¶meters, const bool profile, + const bool useComm, const helper::Comm &chainComm); void CheckFile( std::unordered_map>::const_iterator diff --git a/source/adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.h b/source/adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.h index 6afa8d266f..56cae2b2aa 100644 --- a/source/adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.h +++ b/source/adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.h @@ -45,7 +45,6 @@ class ZmqReqRep void SendReply(const void *reply, const size_t size); private: - int m_Verbosity = 0; int m_Timeout; std::vector m_ReceiverBuffer; diff --git a/source/h5vol/H5VolError.h b/source/h5vol/H5VolError.h index 1437bac82a..824a45f5ad 100644 --- a/source/h5vol/H5VolError.h +++ b/source/h5vol/H5VolError.h @@ -11,7 +11,12 @@ void *safe_malloc(size_t n, unsigned long line); #define SAFE_MALLOC(n) safe_malloc(n, __LINE__) void safe_free(void *p); -#define SAFE_FREE(ptr) safe_free(ptr) + +#ifdef __cplusplus +#define SAFE_FREE(ptr) safe_free(reinterpret_cast(ptr)) +#else +#define SAFE_FREE(ptr) safe_free((void *)(ptr)) +#endif void *safe_ralloc(void *ptr, size_t newsize, unsigned long line); #define SAFE_REALLOC(ptr, newsize) safe_ralloc(ptr, newsize, __LINE__) diff --git a/source/utils/CMakeLists.txt b/source/utils/CMakeLists.txt index b8acbf7f59..26fa3f3b7b 100644 --- a/source/utils/CMakeLists.txt +++ b/source/utils/CMakeLists.txt @@ -71,6 +71,7 @@ endif() if(Python_Interpreter_FOUND) add_subdirectory(bp4dbg) + add_subdirectory(bp5dbg) endif() install(PROGRAMS adios_deactivate_bp.sh diff --git a/source/utils/adios_iotest/adiosStream.cpp b/source/utils/adios_iotest/adiosStream.cpp index feac5beee8..eac0905365 100644 --- a/source/utils/adios_iotest/adiosStream.cpp +++ b/source/utils/adios_iotest/adiosStream.cpp @@ -60,17 +60,20 @@ void adiosStream::defineADIOSArray(const std::shared_ptr ov) { adios2::Variable v = io.DefineVariable( ov->name, ov->shape, ov->start, ov->count, true); + (void)v; // v = io->InquireVariable(ov->name); } else if (ov->type == "float") { adios2::Variable v = io.DefineVariable( ov->name, ov->shape, ov->start, ov->count, true); + (void)v; } else if (ov->type == "int") { adios2::Variable v = io.DefineVariable( ov->name, ov->shape, ov->start, ov->count, true); + (void)v; } } diff --git a/source/utils/bp5dbg/CMakeLists.txt b/source/utils/bp5dbg/CMakeLists.txt new file mode 100644 index 0000000000..7cd62825dd --- /dev/null +++ b/source/utils/bp5dbg/CMakeLists.txt @@ -0,0 +1,13 @@ +install(PROGRAMS bp5dbg.py + RENAME bp5dbg + DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT adios2_scripts-runtime +) +install( + FILES + adios2/bp5dbg/__init__.py + adios2/bp5dbg/data.py + adios2/bp5dbg/utils.py + adios2/bp5dbg/metadata.py + adios2/bp5dbg/idxtable.py + DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2/bp5dbg COMPONENT adios2_scripts-runtime +) diff --git a/source/utils/bp5dbg/adios2/bp5dbg/__init__.py b/source/utils/bp5dbg/adios2/bp5dbg/__init__.py new file mode 100644 index 0000000000..eb13414f08 --- /dev/null +++ b/source/utils/bp5dbg/adios2/bp5dbg/__init__.py @@ -0,0 +1,3 @@ +from .data import * +from .idxtable import * +from .metadata import * diff --git a/source/utils/bp5dbg/adios2/bp5dbg/data.py b/source/utils/bp5dbg/adios2/bp5dbg/data.py new file mode 100644 index 0000000000..6b77fbd865 --- /dev/null +++ b/source/utils/bp5dbg/adios2/bp5dbg/data.py @@ -0,0 +1,635 @@ +import numpy as np +from os import fstat +from .utils import * + + +def ReadEncodedString(f, ID, limit, lensize=2): + if lensize == 2: + # 2 bytes length + string without \0 + namelen = np.fromfile(f, dtype=np.uint16, count=1)[0] + elif lensize == 4: + # 2 bytes length + string without \0 + namelen = np.fromfile(f, dtype=np.uint32, count=1)[0] + else: + print("CODING ERROR: bp5dbp_data.ReadEncodedString: " + "lensize must be 2 or 4") + return False, "" + if namelen > limit: + print("ERROR: " + ID + " string length ({0}) is longer than the " + "limit to stay inside the block ({1})".format( + namelen, limit)) + return False, "" + name = f.read(namelen).decode('ascii') + return True, name + + +def ReadEncodedStringArray(f, ID, limit, nStrings): + s = [] + for i in range(nStrings): + # 2 bytes length + string + # !!! String here INCLUDES Terminating \0 !!! + namelen = np.fromfile(f, dtype=np.uint32, count=1)[0] + if namelen > limit - 4: + print("ERROR: " + ID + " string length ({0}) is longer than the " + "limit to stay inside the block ({1})".format( + namelen, limit - 4)) + return False, s + name = f.read(namelen).decode('ascii') + limit = limit - namelen - 4 + s.append(name[0:-1]) # omit the terminating \0 + return True, s + + +def readDataToNumpyArray(f, typeName, nElements): + if typeName == 'byte': + return np.fromfile(f, dtype=np.int8, count=nElements) + elif typeName == 'char': + return np.fromfile(f, dtype=np.uint8, count=nElements) + elif typeName == 'short': + return np.fromfile(f, dtype=np.int16, count=nElements) + elif typeName == 'integer': + return np.fromfile(f, dtype=np.int32, count=nElements) + elif typeName == 'long': + return np.fromfile(f, dtype=np.int64, count=nElements) + + elif typeName == 'unsigned_byte': + return np.fromfile(f, dtype=np.uint8, count=nElements) + elif typeName == 'unsigned_short': + return np.fromfile(f, dtype=np.uint16, count=nElements) + elif typeName == 'unsigned_integer': + return np.fromfile(f, dtype=np.uint32, count=nElements) + elif typeName == 'unsigned_long': + return np.fromfile(f, dtype=np.uint64, count=nElements) + + elif typeName == 'real': + return np.fromfile(f, dtype=np.float32, count=nElements) + elif typeName == 'double': + return np.fromfile(f, dtype=np.float64, count=nElements) + elif typeName == 'long_double': + return np.fromfile(f, dtype=np.float128, count=nElements) + + elif typeName == 'complex': + return np.fromfile(f, dtype=np.complex64, count=nElements) + elif typeName == 'double_complex': + return np.fromfile(f, dtype=np.complex128, count=nElements) + + else: + return np.zeros(1, dtype=np.uint32) + + +def ReadCharacteristicsFromData(f, limit, typeID, ndim): + cStartPosition = f.tell() + dataTypeName = GetTypeName(typeID) + # 1 byte NCharacteristics + nCharacteristics = np.fromfile(f, dtype=np.uint8, count=1)[0] + print(" # of Characteristics : {0}".format(nCharacteristics)) + # 4 bytes length + charLen = np.fromfile(f, dtype=np.uint32, count=1)[0] + print(" Characteristics Length : {0}".format(charLen)) + + for i in range(nCharacteristics): + print(" Characteristics[{0}]".format(i)) + # 1 byte TYPE + cID = np.fromfile(f, dtype=np.uint8, count=1)[0] + cName = GetCharacteristicName(cID) + print(" Type : {0} ({1}) ".format(cName, cID)) + if cName == 'value' or cName == 'min' or cName == 'max': + if dataTypeName == 'string': + namelimit = limit - (f.tell() - cStartPosition) + status, s = ReadEncodedString(f, "String Value", namelimit) + if not status: + return False + print(" Value : '" + s + "'") + else: + data = readDataToNumpyArray(f, dataTypeName, 1) + print(" Value : {0}".format(data[0])) + elif cName == 'offset' or cName == 'payload_offset': + data = readDataToNumpyArray(f, 'unsigned_long', 1) + print(" Value : {0}".format(data[0])) + elif cName == 'time_index' or cName == 'file_index': + data = readDataToNumpyArray(f, 'unsigned_integer', 1) + print(" Value : {0}".format(data[0])) + elif cName == 'minmax': + nBlocks = np.fromfile(f, + dtype=np.uint16, count=1)[0] + print(" nBlocks : {0}".format(nBlocks)) + bminmax = readDataToNumpyArray(f, dataTypeName, 2) + print(" Min/max : {0} / {1}".format( + bminmax[0], bminmax[1])) + if nBlocks > 1: + method = np.fromfile(f, dtype=np.uint8, + count=1)[0] + print(" Division method: {0}".format(method)) + blockSize = np.fromfile(f, dtype=np.uint64, + count=1)[0] + print(" Block size : {0}".format(blockSize)) + div = np.fromfile(f, dtype=np.uint16, + count=ndim) + print(" Division vector: (", end="") + for d in range(ndim): + print("{0}".format(div[d]), end="") + if d < ndim - 1: + print(", ", end="") + else: + print(")") + minmax = readDataToNumpyArray( + f, dataTypeName, 2 * nBlocks) + for i in range(nBlocks): + print(" Min/max : {0} / {1}".format( + minmax[2 * i], minmax[2 * i + 1])) + else: + print(" ERROR: could not understand this " + "characteristics type '{0}' id {1}".format(cName, cID)) + return True + +# Read String Variable data + + +def ReadStringVarData(f, expectedSize, + varsStartPosition): + # 2 bytes String Length + len = np.fromfile(f, dtype=np.uint16, count=1)[0] + if len != expectedSize - 2: + print("ERROR: Variable data block size does not equal the size " + "calculated from var block length") + print("Expected size = {0} calculated size " + "from encoded length info {1}". + format(expectedSize, len + 2)) + return False + + str = f.read(len).decode('ascii') + print(" Variable Data : '" + str + "'") + return True + +# Read Variable data + + +def ReadVarData(f, nElements, typeID, ldims, varLen, + varsStartPosition, varsTotalLength): + if typeID == 9: # string type + return ReadStringVarData(f, varLen, varsStartPosition) + typeSize = GetTypeSize(typeID) + if typeSize == 0: + print("ERROR: Cannot process variable data block with " + "unknown type size") + return False + + currentPosition = f.tell() + print(" Payload offset : {0}".format(currentPosition)) + + if currentPosition + varLen > varsStartPosition + varsTotalLength: + print("ERROR: Variable data block of size would reach beyond all " + "variable blocks") + print("VarsStartPosition = {0} varsTotalLength = {1}".format( + varsStartPosition, varsTotalLength)) + print("current Position = {0} var block length = {1}".format( + currentPosition, varLen)) + return False + + nBytes = int(varLen.item()) + + if nElements == 1: + # single value. read and print + value = readDataToNumpyArray(f, GetTypeName(typeID), + nElements) + print(" Payload (value) : {0} ({1} bytes)".format( + value[0], nBytes)) + else: + # seek instead of reading for now + # f.read(nBytes) + f.seek(nBytes, 1) + # data = readDataToNumpyArray(f, GetTypeName(typeID), + # nElements) + print(" Payload (array) : {0} bytes".format(nBytes)) + + return True + +# Read a variable's metadata + + +def ReadVMD(f, varidx, varsStartPosition, varsTotalLength): + startPosition = f.tell() + print(" Var {0:5d}".format(varidx)) + print(" Starting offset : {0}".format(startPosition)) + # 4 bytes TAG + tag = f.read(4) + if tag != b"[VMD": + print(" Tag: " + str(tag)) + print("ERROR: VAR group does not start with [VMD") + return False + print(" Tag : " + tag.decode('ascii')) + + # 8 bytes VMD Length + vmdlen = np.fromfile(f, dtype=np.uint64, count=1)[0] + print(" Var block size : {0} bytes (+4 for Tag)".format(vmdlen)) + expectedVarBlockLength = vmdlen + 4 # [VMD is not included in vmdlen + + if startPosition + expectedVarBlockLength > \ + varsStartPosition + varsTotalLength: + print("ERROR: There is not enough bytes inside this PG to read " + "this Var block") + print("VarsStartPosition = {0} varsTotalLength = {1}".format( + varsStartPosition, varsTotalLength)) + print("current var's start position = {0} var block length = {1}". + format(startPosition, expectedVarBlockLength)) + return False + + # 4 bytes VAR MEMBER ID + memberID = np.fromfile(f, dtype=np.uint32, count=1)[0] + print(" Member ID : {0}".format(memberID)) + + # VAR NAME, 2 bytes length + string without \0 + sizeLimit = expectedVarBlockLength - (f.tell() - startPosition) + status, varname = ReadEncodedString(f, "Var Name", sizeLimit) + if not status: + return False + print(" Var Name : " + varname) + + # VAR PATH, 2 bytes length + string without \0 + # sizeLimit = expectedVarBlockLength - (f.tell() - startPosition) + # status, varpath = ReadEncodedString(f, "Var Path", sizeLimit) + # if not status: + # return False + # print(" Var Path : " + varpath) + + # 1 byte ORDER (K, C, F) + order = f.read(1) + if (order != b'K' and order != b'C' and order != b'F' and order != b'\x00'): + print( + "ERROR: Next byte for Order must be 'K', 'C', or 'F' " + "but it isn't = {0}".format(order)) + return False + if (order == b'\x00'): + order = b'0' + print(" Order : " + order.decode('ascii')) + + # 1 byte UNUSED + unused = f.read(1) + print(" Unused byte : {0}".format(ord(unused))) + + # 1 byte TYPE + typeID = np.fromfile(f, dtype=np.uint8, count=1)[0] + print(" Type : {0} ({1}) ".format( + GetTypeName(typeID), typeID)) + + # ISDIMENSIONS 1 byte, 'y' or 'n' + isDimensionVar = f.read(1) + if (isDimensionVar != b'y' and isDimensionVar != b'n'): + print( + "ERROR: Next byte for isDimensionVar must be 'y' or 'n' " + "but it isn't = {0}".format(isDimensionVar)) + return False + print(" isDimensionVar : " + isDimensionVar.decode('ascii')) + + # 1 byte NDIMENSIONS + ndims = np.fromfile(f, dtype=np.uint8, count=1)[0] + print(" # of Dimensions : {0}".format( + ndims)) + + # DIMLENGTH + dimsLen = np.fromfile(f, dtype=np.uint16, count=1)[0] + print(" Dims Length : {0}".format( + dimsLen)) + + nElements = np.uint64(1) + ldims = np.zeros(ndims, dtype=np.uint64) + isLocalValueArray = False + for i in range(ndims): + print(" Dim[{0}]".format(i)) + # Read Local Dimensions (1 byte flag + 8 byte value) + # Is Dimension a variable ID 1 byte, 'y' or 'n' or '\0' + isDimensionVarID = f.read(1) + if isDimensionVarID != b'y' and isDimensionVarID != b'n' and \ + isDimensionVarID != b'\0': + print( + "ERROR: Next byte for isDimensionVarID must be 'y' or 'n' " + "but it isn't = {0}".format(isDimensionVarID)) + return False + if isDimensionVarID == b'\0': + isDimensionVarID = b'n' + ldims[i] = np.fromfile(f, dtype=np.uint64, count=1)[0] + print(" local dim : {0}".format(ldims[i])) + nElements = nElements * ldims[i] + # Read Global Dimensions (1 byte flag + 8 byte value) + # Is Dimension a variable ID 1 byte, 'y' or 'n' or '\0' + isDimensionVarID = f.read(1) + if isDimensionVarID != b'y' and isDimensionVarID != b'n' \ + and isDimensionVarID != b'\0': + print( + "ERROR: Next byte for isDimensionVarID must be 'y' or 'n' " + "but it isn't = {0}".format(isDimensionVarID)) + return False + if isDimensionVarID == b'\0': + isDimensionVarID = b'n' + gdim = np.fromfile(f, dtype=np.uint64, count=1)[0] + if i == 0 and ldims[i] == 0 and gdim == LocalValueDim: + print(" global dim : LocalValueDim ({0})".format(gdim)) + isLocalValueArray = True + else: + print(" global dim : {0}".format(gdim)) + + # Read Offset Dimensions (1 byte flag + 8 byte value) + # Is Dimension a variable ID 1 byte, 'y' or 'n' or '\0' + isDimensionVarID = f.read(1) + if isDimensionVarID != b'y' and isDimensionVarID != b'n' and \ + isDimensionVarID != b'\0': + print( + "ERROR: Next byte for isDimensionVarID must be 'y' or 'n' " + "but it isn't = {0}".format(isDimensionVarID)) + return False + if isDimensionVarID == b'\0': + isDimensionVarID = b'n' + offset = np.fromfile(f, dtype=np.uint64, count=1)[0] + print(" offset dim : {0}".format(offset)) + + sizeLimit = expectedVarBlockLength - (f.tell() - startPosition) + status = ReadCharacteristicsFromData(f, sizeLimit, typeID, ndims) + if not status: + return False + + # Padded end TAG + # 1 byte length of tag + endTagLen = np.fromfile(f, dtype=np.uint8, count=1)[0] + tag = f.read(endTagLen) + if not tag.endswith(b"VMD]"): + print(" Tag: " + str(tag)) + print("ERROR: VAR group metadata does not end with VMD]") + return False + print(" Tag (pad {0:2d}) : {1}".format( + endTagLen - 4, tag.decode('ascii'))) + + # special case: LocalValueDim: local values turned into 1D global array + # but it seems there is no data block at all for these variables + if isLocalValueArray: + ldims[0] = 1 + nElements = np.uint64(1) + else: + expectedVarDataSize = expectedVarBlockLength - \ + (f.tell() - startPosition) + status = ReadVarData(f, nElements, typeID, ldims, expectedVarDataSize, + varsStartPosition, varsTotalLength) + if not status: + return False + + return True + +# Read an attribute's metadata and value + + +def ReadAMD(f, attridx, attrsStartPosition, attrsTotalLength): + startPosition = f.tell() + print(" attr {0:5d}".format(attridx)) + print(" Starting offset : {0}".format(startPosition)) + # 4 bytes TAG + tag = f.read(4) + if tag != b"[AMD": + print(" Tag: " + str(tag)) + print("ERROR: ATTR group does not start with [AMD") + return False + print(" Tag : " + tag.decode('ascii')) + + # 8 bytes AMD Length + amdlen = np.fromfile(f, dtype=np.uint32, count=1)[0] + print(" Attr block size : {0} bytes (+4 for Tag)".format(amdlen)) + expectedAttrBlockLength = amdlen + 4 # [AMD is not included in amdlen + if startPosition + expectedAttrBlockLength > \ + attrsStartPosition + attrsTotalLength: + print("ERROR: There is not enough bytes inside this PG " + "to read this Attr block") + print("AttrsStartPosition = {0} attrsTotalLength = {1}".format( + attrsStartPosition, attrsTotalLength)) + print("current attr's start position = {0} " + "attr block length = {1}".format( + startPosition, expectedAttrBlockLength)) + return False + + # 4 bytes ATTR MEMBER ID + memberID = np.fromfile(f, dtype=np.uint32, count=1)[0] + print(" Member ID : {0}".format(memberID)) + + # ATTR NAME, 2 bytes length + string without \0 + sizeLimit = expectedAttrBlockLength - (f.tell() - startPosition) + status, attrname = ReadEncodedString(f, "Attr Name", sizeLimit) + if not status: + return False + print(" Attr Name : " + attrname) + + # ATTR PATH, 2 bytes length + string without \0 + sizeLimit = expectedAttrBlockLength - (f.tell() - startPosition) + status, attrpath = ReadEncodedString(f, "Attr Path", sizeLimit) + if not status: + return False + print(" Attr Path : " + attrpath) + + # isAttrAVar 1 byte, 'y' or 'n' + isAttrAVar = f.read(1) + if isAttrAVar != b'y' and isAttrAVar != b'n': + print( + "ERROR: Next byte for isAttrAVar must be 'y' or 'n' " + "but it isn't = {0}".format(isAttrAVar)) + return False + print(" Refers to Var? : " + isAttrAVar.decode('ascii')) + + # 1 byte TYPE + typeID = np.fromfile(f, dtype=np.uint8, count=1)[0] + typeName = GetTypeName(typeID) + print(" Type : {0} ({1}) ".format(typeName, typeID)) + + # Read Attribute data + if typeName == 'string': + sizeLimit = expectedAttrBlockLength - (f.tell() - startPosition) + status, s = ReadEncodedString( + f, "Attribute String Value", sizeLimit, 4) + if not status: + return False + print(" Value : '" + s + "'") + + elif typeName == 'string_array': + nElems = np.fromfile(f, dtype=np.uint32, count=1)[0] + sizeLimit = expectedAttrBlockLength - (f.tell() - startPosition) + status, strList = ReadEncodedStringArray( + f, "Attribute String Array", sizeLimit, nElems) + if not status: + return False + print(" Value : [", end="") + for j in range(len(strList)): + print("'" + strList[j] + "'", end="") + if j < len(strList) - 1: + print(", ", end="") + print("]") + else: + nBytes = np.fromfile(f, dtype=np.uint32, count=1)[0] + typeSize = GetTypeSize(typeID) + nElems = int(nBytes / typeSize) + data = readDataToNumpyArray(f, typeName, nElems) + print(" Value : [", end="") + for j in range(nElems): + print("{0}".format(data[j]), end="") + if j < nElems - 1: + print(", ", end="") + print("]") + + # End TAG AMD] + tag = f.read(4) + if tag != b"AMD]": + print(" Tag: " + str(tag)) + print("ERROR: PG group metadata does not end with AMD]") + return False + print(" Tag : {0}".format(tag.decode('ascii'))) + + return True + +# Read one PG process group (variables and attributes from one process in +# one step) + + +def ReadPG(f, fileSize, pgidx): + pgStartPosition = f.tell() + if pgidx > 0: + print("========================================================") + print("Process Group {0}: ".format(pgidx)) + print(" Starting offset : {0}".format(pgStartPosition)) + tag = f.read(4) + if tag != b"[PGI": + print(" Tag: " + str(tag)) + print("ERROR: PG group does not start with [PGI") + return False + + print(" Tag : " + tag.decode('ascii')) + + # 8 bytes PG Length + pglen = np.fromfile(f, dtype=np.uint64, count=1)[0] + print(" PG length : {0} bytes (+4 for Tag)".format(pglen)) + # pglen does not include the opening tag 4 bytes: + expectedPGLength = pglen + 4 + if pgStartPosition + expectedPGLength > fileSize: + print("ERROR: There is not enough bytes in file to read this PG") + return False + + # ColumnMajor (host language Fortran) 1 byte, 'y' or 'n' + isColumnMajor = f.read(1) + if isColumnMajor != b'y' and isColumnMajor != b'n': + print( + "ERROR: Next byte for isColumnMajor must be 'y' or 'n' " + "but it isn't = {0}".format(isColumnMajor)) + return False + print(" isColumnMajor : " + isColumnMajor.decode('ascii')) + + # PG Name, 2 bytes length + string without \0 + sizeLimit = expectedPGLength - (f.tell() - pgStartPosition) + status, pgname = ReadEncodedString(f, "PG Name", sizeLimit) + if not status: + return False + print(" PG Name : " + pgname) + + # 4 bytes unused (for Coordination variable) + tag = f.read(4) + print(" Unused 4 bytes : " + str(tag)) + + # Timestep name + sizeLimit = expectedPGLength - (f.tell() - pgStartPosition) + status, tsname = ReadEncodedString(f, "Timestep Name", sizeLimit) + if not status: + return False + print(" Step Name : " + tsname) + + # STEP 4 bytes + step = np.fromfile(f, dtype=np.uint32, count=1)[0] + print(" Step Value : {0}".format(step)) + + # Method Count 1 byte1 + nMethods = np.fromfile(f, dtype=np.uint8, count=1)[0] + print(" Methods count : {0}".format(nMethods)) + + # Method Length 2 byte1 + lenMethods = np.fromfile(f, dtype=np.uint16, count=1)[0] + print(" Methods length : {0}".format(lenMethods)) + + print(" Methods info") + for i in range(nMethods): + # Method ID + methodID = np.fromfile(f, dtype=np.uint8, count=1)[0] + print(" Method ID : {0}".format(methodID)) + sizeLimit = expectedPGLength - (f.tell() - pgStartPosition) + status, methodParams = ReadEncodedString( + f, "Method Parameters", sizeLimit) + if not status: + return False + print(' M. params : "' + methodParams + '"') + + # VARIABLES + + # VARS COUNT 4 bytes + nVars = np.fromfile(f, dtype=np.uint32, count=1)[0] + print(" # of Variables : {0}".format(nVars)) + + # VARS SIZE 8 bytes + varlen = np.fromfile(f, dtype=np.uint64, count=1)[0] + print(" Vars length : {0} bytes".format(varlen)) + sizeLimit = expectedPGLength - (f.tell() - pgStartPosition) + expectedVarsLength = varlen # need to read this more + if expectedVarsLength > sizeLimit: + print("ERROR: There is not enough bytes in PG to read the variables") + return False + + varsStartPosition = f.tell() + for i in range(nVars): + # VMD block + status = ReadVMD(f, i, varsStartPosition, expectedVarsLength) + if not status: + return False + + # ATTRIBUTES + + # ATTRS COUNT 4 bytes + nAttrs = np.fromfile(f, dtype=np.uint32, count=1)[0] + print(" # of Attributes : {0}".format(nAttrs)) + + attrsStartPosition = f.tell() + # ATTS SIZE 8 bytes + # attlen includes the 8 bytes of itself, so remember position before this + attlen = np.fromfile(f, dtype=np.uint64, count=1)[0] + print(" Attrs length : {0} bytes".format(attlen)) + sizeLimit = expectedPGLength - (attrsStartPosition - pgStartPosition) - 4 + expectedAttrsLength = attlen # need to read this more before reaching PGI] + + if expectedAttrsLength > sizeLimit: + print("ERROR: There is not enough bytes in PG to read the attributes") + return False + + attrsStartPosition = f.tell() + for i in range(nAttrs): + # AMD block + status = ReadAMD(f, i, attrsStartPosition, expectedAttrsLength) + if not status: + return False + + # End TAG PGI] + tag = f.read(4) + if tag != b"PGI]": + print(" Tag: " + str(tag)) + print("ERROR: PG group metadata does not end with PGI]") + return False + print(" Tag : {0}".format(tag.decode('ascii'))) + + return True + + +def DumpData(fileName): + print("========================================================") + print(" Data File: " + fileName) + print("========================================================") + with open(fileName, "rb") as f: + fileSize = fstat(f.fileno()).st_size + status = ReadHeader(f, fileSize, "Data") + if not status: + return status + pgidx = 0 + while (f.tell() < fileSize - 12 and status): + status = ReadPG(f, fileSize, pgidx) + pgidx = pgidx + 1 + return status + + +if __name__ == "__main__": + print("ERROR: Utility main program is bp5dbg.py") diff --git a/source/utils/bp5dbg/adios2/bp5dbg/idxtable.py b/source/utils/bp5dbg/adios2/bp5dbg/idxtable.py new file mode 100644 index 0000000000..0d68354bf7 --- /dev/null +++ b/source/utils/bp5dbg/adios2/bp5dbg/idxtable.py @@ -0,0 +1,87 @@ +import numpy as np +from os import fstat +from .utils import * + +WriterCount = -1 + +def ReadWriterArray(f, fileSize, WriterCount): + + print("Writer count is " + str(WriterCount)) + array = f.read(WriterCount * 8) + print("=====================") + print("| Rank | Subfile |") + print("=====================") + for r in range(0, WriterCount): + pos = r * 8 + data = np.frombuffer(array, dtype=np.uint64, count=1, offset=pos) + rank = str(r).rjust(7) + sub = str(data[0]).rjust(9) + print("|" + rank + " | FlushCount = " + sub + " |") + print("=====================") + return True + +def ReadIndex(f, fileSize, WriterCount): + nBytes = fileSize - f.tell() + if nBytes <= 0: + return True + table = f.read(nBytes) + pos = 0 + step = 0 + while pos < nBytes: + print("-----------------------------------------------" + + "---------------------------------------------------") + data = np.frombuffer(table, dtype=np.uint64, count=3, + offset=pos) + stepstr = str(step).ljust(6) + mdatapos = str(data[0]).ljust(10) + mdatasize = str(data[1]).ljust(10) + flushcount = str(data[2]).ljust(3) + FlushCount = data[2] + print("| Step = " + stepstr + "| MetadataPos = " + mdatapos + + " | MetadataSize = " + mdatasize + " |" + flushcount + "|") + + pos = pos + 3 * 8 + for Writer in range(0, WriterCount): + start = " Writer " + str(Writer) + " data " + thiswriter = np.frombuffer(table, dtype=np.uint64, + count=int(FlushCount * 2 + 1), + offset=pos) + + for i in range(0, FlushCount): # for flushes + start += ("loc:" + str(thiswriter[int(i * 2)]) + " siz:" + + str(thiswriter[i * 2 + 1]) + "; ") + start += "loc:" + str(thiswriter[int(FlushCount * 2)]) + pos = int(pos + (FlushCount * 2 + 1) * 8) + print(start) + print("---------------------------------------------------" + + "-----------------------------------------------") + step = step + 1 + + if fileSize - f.tell() > 1: + print("ERROR: There are {0} bytes at the end of file" + " that cannot be interpreted".format(fileSize - f.tell() - 1)) + return False + + return True + + +def DumpIndexTable(fileName): + print("========================================================") + print(" Index Table File: " + fileName) + print("========================================================") + status = False + with open(fileName, "rb") as f: + fileSize = fstat(f.fileno()).st_size + status = ReadHeader(f, fileSize, "Index Table") + if isinstance(status, list): + WriterCount = status[1] + status = status[0] + if status: + status = ReadWriterArray(f, fileSize, WriterCount) + if status: + status = ReadIndex(f, fileSize, WriterCount) + return status + + +if __name__ == "__main__": + print("ERROR: Utility main program is bp5dbg.py") diff --git a/source/utils/bp5dbg/adios2/bp5dbg/metadata.py b/source/utils/bp5dbg/adios2/bp5dbg/metadata.py new file mode 100644 index 0000000000..e460e41bd0 --- /dev/null +++ b/source/utils/bp5dbg/adios2/bp5dbg/metadata.py @@ -0,0 +1,572 @@ +import numpy as np +from os import fstat +from .utils import * + + +def ReadEncodedStringFromBuffer(buf, pos, ID, limit, lenbytes=2): + # 'lenbytes' bytes length + string without \0 + if lenbytes == 1: + dt = np.dtype(np.uint8) + else: + dt = np.dtype(np.uint16) + namelen = np.frombuffer(buf, dtype=dt, count=1, offset=pos)[0] + pos = pos + lenbytes + if namelen > limit - lenbytes: + print("ERROR: " + ID + " string length ({0}) is longer than the " + "limit to stay inside the block ({1})".format( + namelen, limit - lenbytes)) + return False, "", namelen, pos + name = buf[pos:pos + namelen].decode('ascii') + pos = pos + namelen + return True, name, namelen, pos + + +def ReadEncodedStringArrayFromBuffer(buf, pos, ID, limit, nStrings): + s = [] + for i in range(nStrings): + # 2 bytes length + string without \0 + namelen = np.frombuffer(buf, dtype=np.uint16, count=1, offset=pos)[0] + pos = pos + 2 + if namelen > limit - 2: + print("ERROR: " + ID + " string length ({0}) is longer than the " + "limit to stay inside the block ({1})".format( + namelen, limit - 2)) + return False, s, pos + name = buf[pos:pos + namelen].decode('ascii') + pos = pos + namelen + limit = limit - namelen - 2 + s.append(name) + return True, s, pos + + +def ReadDimensionCharacteristics(buf, pos): + ndim = np.frombuffer(buf, dtype=np.uint8, count=1, offset=pos)[0] + pos = pos + 1 + lgo = np.zeros(ndim, dtype=np.uint64) + dimLen = np.frombuffer(buf, dtype=np.uint16, count=1, offset=pos)[0] + pos = pos + 2 + if dimLen != 24 * ndim: + print("ERROR: Encoded dimension length expected size = {0} bytes, " + "but found {1} bytes".format(24 * ndim, dimLen)) + return False, pos, ndim, lgo + + lgo = np.frombuffer(buf, dtype=np.uint64, count=3 * ndim, offset=pos) + pos = pos + 24 * ndim + return True, pos, ndim, lgo + + +def bDataToNumpyArray(cData, typeName, nElements, startPos=0): + if typeName == 'byte': + return np.frombuffer(cData, dtype=np.int8, count=nElements, + offset=startPos) + elif typeName == 'char': + return np.frombuffer(cData, dtype=np.uint8, count=nElements, + offset=startPos) + elif typeName == 'short': + return np.frombuffer(cData, dtype=np.int16, count=nElements, + offset=startPos) + elif typeName == 'integer': + return np.frombuffer(cData, dtype=np.int32, count=nElements, + offset=startPos) + elif typeName == 'long': + return np.frombuffer(cData, dtype=np.int64, count=nElements, + offset=startPos) + + elif typeName == 'unsigned_byte': + return np.frombuffer(cData, dtype=np.uint8, count=nElements, + offset=startPos) + elif typeName == 'unsigned_short': + return np.frombuffer(cData, dtype=np.uint16, count=nElements, + offset=startPos) + elif typeName == 'unsigned_integer': + return np.frombuffer(cData, dtype=np.uint32, count=nElements, + offset=startPos) + elif typeName == 'unsigned_long': + return np.frombuffer(cData, dtype=np.uint64, count=nElements, + offset=startPos) + + elif typeName == 'real': + return np.frombuffer(cData, dtype=np.float32, count=nElements, + offset=startPos) + elif typeName == 'double': + return np.frombuffer(cData, dtype=np.float64, count=nElements, + offset=startPos) + elif typeName == 'long_double': + return np.frombuffer(cData, dtype=np.float128, count=nElements, + offset=startPos) + + elif typeName == 'complex': + return np.frombuffer(cData, dtype=np.complex64, count=nElements, + offset=startPos) + elif typeName == 'double_complex': + return np.frombuffer(cData, dtype=np.complex128, count=nElements, + offset=startPos) + + else: + return np.zeros(1, dtype=np.uint32) + + +def ReadCharacteristicsFromMetaData(buf, idx, pos, limit, typeID, + fileOffset, isVarCharacteristics): + cStartPosition = pos + dataTypeName = GetTypeName(typeID) + print(" Block {0}: ".format(idx)) + print(" Starting offset : {0}".format(fileOffset)) + # 1 byte NCharacteristics + nChars = np.frombuffer(buf, dtype=np.uint8, count=1, offset=pos)[0] + pos = pos + 1 + print(" # of Characteristics : {0}".format(nChars)) + # 4 bytes length + charLen = np.frombuffer(buf, dtype=np.uint8, count=32, offset=pos)[0] + pos = pos + 4 + print(" Characteristics Length : {0}".format(charLen)) + + # For attributes, we need to remember the dimensions and size + # when reading the value + ndim = 0 + nElems = 1 + + for i in range(nChars): + print(" Characteristics[{0}]".format(i)) + # 1 byte TYPE + cID = np.frombuffer(buf, dtype=np.uint8, count=1, offset=pos)[0] + pos = pos + 1 + cName = GetCharacteristicName(cID) + print(" Type : {0} ({1}) ".format( + cName, cID)) + cLen = GetCharacteristicDataLength(cID, typeID) + + if cName == 'dimensions': + status, pos, ndim, lgo = ReadDimensionCharacteristics(buf, pos) + if not status: + return status, pos + print(" # of Dims : {0}".format(ndim)) + if ndim > 0: + print(" Dims (lgo) : (", end="") + for d in range(ndim): + p = 3 * d + nElems = int(nElems * lgo[p]) # need for value later + print("{0}:{1}:{2}".format(lgo[p], lgo[p + 1], lgo[p + 2]), + end="") + if d < ndim - 1: + print(", ", end="") + else: + print(")") + + elif cName == 'value' or cName == 'min' or cName == 'max': + if dataTypeName == 'string': + namelimit = limit - (pos - cStartPosition) + status, s, sLen, pos = ReadEncodedStringFromBuffer( + buf, pos, "String Value", namelimit) + if not status: + return False, pos + print(" Value : '" + s + + "' ({0} bytes)".format(sLen)) + elif dataTypeName == 'string_array': + namelimit = limit - (pos - cStartPosition) + status, strList, pos = ReadEncodedStringArrayFromBuffer( + buf, pos, "String Array", namelimit, lgo[0]) + if not status: + return False, pos + print(" Value : [", end="") + for j in range(len(strList)): + print("'" + strList[j] + "'", end="") + if j < len(strList) - 1: + print(", ", end="") + print("]") + + else: + if isVarCharacteristics: + cData = buf[pos:pos + cLen] + pos = pos + cLen + data = bDataToNumpyArray(cData, dataTypeName, 1) + print(" Value : {0}" + " ({1} bytes)".format(data[0], cLen)) + else: # attribute value characteristics are different + dataTypeSize = GetTypeSize(typeID) + nBytes = int(nElems * dataTypeSize) + cData = buf[pos:pos + nBytes] + pos = pos + nBytes + data = bDataToNumpyArray(cData, dataTypeName, nElems) + print(" Value : [", end="") + for j in range(nElems): + print("{0}".format(data[j]), end="") + if j < nElems - 1: + print(", ", end="") + print("]") + + elif cName == 'offset' or cName == 'payload_offset': + cData = buf[pos:pos + cLen] + pos = pos + cLen + data = bDataToNumpyArray(cData, 'unsigned_long', 1) + print(" Value : {0} ({1} bytes)".format( + data[0], cLen)) + elif cName == 'time_index' or cName == 'file_index': + cData = buf[pos:pos + cLen] + pos = pos + cLen + data = bDataToNumpyArray(cData, 'unsigned_integer', 1) + print(" Value : {0} ({1} bytes)".format( + data[0], cLen)) + elif cName == 'minmax': + nBlocks = np.frombuffer( + buf, dtype=np.uint16, count=1, offset=pos)[0] + print(" nBlocks : {0}".format(nBlocks)) + pos = pos + 2 + bminmax = bDataToNumpyArray(buf, dataTypeName, 2, pos) + pos = pos + 2 * cLen + print(" Min/max : {0} / {1}".format( + bminmax[0], bminmax[1])) + if nBlocks > 1: + method = np.frombuffer(buf, dtype=np.uint8, + count=1, offset=pos)[0] + pos = pos + 1 + print(" Division method: {0}".format(method)) + blockSize = np.frombuffer(buf, dtype=np.uint64, + count=1, offset=pos)[0] + pos = pos + 8 + print(" Block size : {0}".format(blockSize)) + div = np.frombuffer(buf, dtype=np.uint16, + count=ndim, offset=pos) + pos = pos + 2 * ndim + print(" Division vector: (", end="") + for d in range(ndim): + print("{0}".format(div[d]), end="") + if d < ndim - 1: + print(", ", end="") + else: + print(")") + minmax = bDataToNumpyArray(buf, dataTypeName, 2 * nBlocks, pos) + pos = pos + 2 * nBlocks * cLen + for i in range(nBlocks): + print(" Min/max : {0} / {1}".format( + minmax[2 * i], minmax[2 * i + 1])) + elif cName == "transform_type": + # Operator name (8 bit length) + namelimit = limit - (pos - cStartPosition) + status, s, sLen, pos = ReadEncodedStringFromBuffer( + buf, pos, "Operator Name", namelimit, lenbytes=1) + if not status: + return False, pos + print(" Operator : '" + s + + "' ({0} bytes)".format(sLen)) + + # 1 byte TYPE + typeID = buf[pos] + pos = pos + 1 + print(" Pre-type : {0} ({1}) ".format( + GetTypeName(typeID), typeID)) + + # Pre-transform dimenstions + status, pos, ndim, lgo = ReadDimensionCharacteristics(buf, pos) + if not status: + return status, pos + print(" Pre-# of dims : {0}".format(ndim)) + if ndim > 0: + print(" Pre-Dims (lgo) : (", end="") + for d in range(ndim): + p = 3 * d + nElems = int(nElems * lgo[p]) # need for value later + print("{0}:{1}:{2}".format(lgo[p], lgo[p + 1], lgo[p + 2]), + end="") + if d < ndim - 1: + print(", ", end="") + else: + print(")") + + # Operator specific metadata + omdlen = np.frombuffer(buf, dtype=np.uint16, + count=1, offset=pos)[0] + pos = pos + 2 + print(" Op. data length: {0}".format(omdlen)) + pos = pos + omdlen + else: + print(" ERROR: could not understand this " + "characteristics type '{0}' id {1}".format(cName, cID)) + return True, pos + + +def ReadPGMD(buf, idx, pos, limit, pgStartOffset): + # Read one PG index group + pgStartPosition = pos + print(" PG {0}: ".format(idx)) + print(" Starting offset : {0}".format(pgStartOffset)) + + # 2 bytes PG Length + Name length + pgLength = np.frombuffer(buf, dtype=np.uint16, count=1, offset=pos)[0] + pos = pos + 2 + print( + " PG length : {0} bytes (+2 for length)".format( + pgLength)) + if pgStartPosition + pgLength + 2 > limit: + print("ERROR: There is not enough bytes {0} left in PG index block " + "to read this single PG index ({1} bytes)").format( + limit - pgStartPosition, pgLength) + return False, pos + + pgNameLen = np.frombuffer(buf, dtype=np.uint16, count=1, offset=pos)[0] + pos = pos + 2 + if pgStartPosition + pgNameLen > limit: + print("ERROR: There is not enough bytes {0} left in PG index block " + "to read the name of this single PG index ({1} bytes)").format( + limit - pos + 2, pgNameLen) + return False, pos + + pgName = buf[pos:pos + pgNameLen].decode('ascii') + pos = pos + pgNameLen + print(" PG Name : '" + pgName + + "' ({0} bytes)".format(pgNameLen)) + + # ColumnMajor (host language Fortran) 1 byte, 'y' or 'n' + isColumnMajor = buf[pos] # this is an integer value + pos = pos + 1 + if isColumnMajor != ord('y') and isColumnMajor != ord('n'): + print( + "ERROR: Next byte for isColumnMajor must be 'y' or 'n' " + "but it isn't = {0} (={1})".format( + chr(isColumnMajor), isColumnMajor)) + return False + print(" isColumnMajor : " + chr(isColumnMajor)) + + processID = np.frombuffer(buf, dtype=np.uint32, count=1, offset=pos)[0] + pos = pos + 4 + print(" process ID : {0}".format(processID)) + + pgTimeNameLen = np.frombuffer(buf, dtype=np.uint16, count=1, offset=pos)[0] + pos = pos + 2 + if pgStartPosition + pgTimeNameLen > limit: + print("ERROR: There is not enough bytes {0} left in PG index block " + "to read the name of this single PG index ({1} bytes)").format( + limit - pos + 2, pgTimeNameLen) + return False, pos + + pgTimeName = buf[pos:pos + pgTimeNameLen].decode('ascii') + pos = pos + pgTimeNameLen + print(" Timestep Name : '" + pgTimeName + + "' ({0} bytes)".format(pgTimeNameLen)) + + step = np.frombuffer(buf, dtype=np.uint32, count=1, offset=pos)[0] + pos = pos + 4 + print(" Step : {0}".format(step)) + + ptr = np.frombuffer(buf, dtype=np.uint64, count=1, offset=pos)[0] + pos = pos + 8 + print(" Offset in data : {0}".format(ptr)) + + return True, pos + + +def ReadVarMD(buf, idx, pos, limit, varStartOffset): + # Read one VAR index group + varStartPosition = pos + print(" Var {0}: ".format(idx)) + print(" Starting offset : {0}".format(varStartOffset)) + + # 4 bytes VAR Index Length + varLength = np.frombuffer(buf, dtype=np.uint32, count=1, offset=pos)[0] + pos = pos + 4 + print(" Var idx length : {0} bytes (+4 for idx length)".format( + varLength)) + if varStartPosition + varLength + 4 > limit: + print("ERROR: There is not enough bytes in Var index block " + "to read this single Var index") + return False, pos + + memberID = np.frombuffer(buf, dtype=np.uint32, count=1, offset=pos)[0] + pos = pos + 4 + print(" MemberID : {0}".format(memberID)) + + namelimit = limit - (pos - varStartPosition) + status, grpName, grpNameLen, pos = ReadEncodedStringFromBuffer( + buf, pos, "Group Name", namelimit) + if not status: + return False, pos + print(" Group Name : '" + grpName + + "' ({0} bytes)".format(grpNameLen)) + + namelimit = limit - (pos - varStartPosition) + status, varName, varNameLen, pos = ReadEncodedStringFromBuffer( + buf, pos, "Var Name", namelimit) + if not status: + return False, pos + print(" Var Name : '" + varName + + "' ({0} bytes)".format(varNameLen)) + + # print(" Current offset : {0}".format(varStartOffset + pos)) + # namelimit = limit - (pos - varStartPosition) + # status, varPath, varPathLen, pos = ReadEncodedStringFromBuffer( + # buf, pos, "Var Path", namelimit) + # if not status: + # return False, pos + # print(" Var Path : '" + varPath + + # "' ({0} bytes)".format(varPathLen)) + + # 1 byte ORDER (K, C, F) + order = buf[pos] # this is an integer value + pos = pos + 1 + if order != ord('K') and order != ord('C') and order != ord('F'): + print( + "ERROR: Next byte for Order must be 'K', 'C', or 'F' " + "but it isn't = {0} (={1})".format( + chr(order), order)) + return False + print(" Order : " + chr(order)) + + # 1 byte UNUSED + unused = buf[pos] # this is an integer value + pos = pos + 1 + print(" Unused byte : {0}".format(unused)) + + # 1 byte TYPE + typeID = buf[pos] + pos = pos + 1 + print(" Type : {0} ({1}) ".format( + GetTypeName(typeID), typeID)) + + # 8 byte Number of Characteristics Sets + cSets = np.frombuffer(buf, dtype=np.uint64, count=1, offset=pos)[0] + pos = pos + 8 + print(" # of blocks : {0}".format(cSets)) + +# This loop only reads the number of reported blocks +# for i in range(cSets): +# # one characteristics block +# newlimit = limit - (pos - varStartPosition) +# fileOffset = varStartOffset + (pos - varStartPosition) +# status, pos = ReadCharacteristicsFromMetaData( +# buf, i, pos, newlimit, typeID, fileOffset, True) +# if not status: +# return False + +# This loop reads blocks until the reported length of variable index length + i = 0 + while pos < varStartPosition + varLength: + # one characteristics block + newlimit = limit - (pos - varStartPosition) + fileOffset = varStartOffset + (pos - varStartPosition) + status, pos = ReadCharacteristicsFromMetaData( + buf, i, pos, newlimit, typeID, fileOffset, True) + if not status: + return False + i = i + 1 + + if (i != cSets): + print( + "ERROR: reported # of blocks (={0}) != # of encoded blocks " + " (={1})".format( + cSets, i)) + + return True, pos + + +def ReadAttrMD(buf, idx, pos, limit, attrStartOffset): + # Read one ATTR index group + attrStartPosition = pos + print(" Attr {0}: ".format(idx)) + print(" Starting offset : {0}".format(attrStartOffset)) + + # 4 bytes ATTR Index Length + attrLength = np.frombuffer(buf, dtype=np.uint32, count=1, offset=pos)[0] + pos = pos + 4 + print(" Attr idx length : {0} bytes (+4 for idx length)".format( + attrLength)) + if attrStartPosition + attrLength + 4 > limit: + print("ERROR: There is not enough bytes in Attr index block " + "to read this single Attr index") + return False, pos + + memberID = np.frombuffer(buf, dtype=np.uint32, count=1, offset=pos)[0] + pos = pos + 4 + print(" MemberID : {0}".format(memberID)) + + namelimit = limit - (pos - attrStartPosition) + status, grpName, grpNameLen, pos = ReadEncodedStringFromBuffer( + buf, pos, "Group Name", namelimit) + if not status: + return False, pos + print(" Group Name : '" + grpName + + "' ({0} bytes)".format(grpNameLen)) + + namelimit = limit - (pos - attrStartPosition) + status, attrName, attrNameLen, pos = ReadEncodedStringFromBuffer( + buf, pos, "Attr Name", namelimit) + if not status: + return False, pos + print(" Attr Name : '" + attrName + + "' ({0} bytes)".format(attrNameLen)) + + # print(" Current offset : {0}".format(attrStartOffset + pos)) + namelimit = limit - (pos - attrStartPosition) + status, attrPath, attrPathLen, pos = ReadEncodedStringFromBuffer( + buf, pos, "Attr Path", namelimit) + if not status: + return False, pos + print(" Attr Path : '" + attrPath + + "' ({0} bytes)".format(attrPathLen)) + + # 1 byte TYPE + typeID = buf[pos] + pos = pos + 1 + print(" Type : {0} ({1}) ".format( + GetTypeName(typeID), typeID)) + + # 8 byte Number of Characteristics Sets + cSets = np.frombuffer(buf, dtype=np.uint64, count=1, offset=pos)[0] + pos = pos + 8 + print(" # of blocks : {0}".format(cSets)) + + for i in range(cSets): + # one characteristics block + newlimit = limit - (pos - attrStartPosition) + fileOffset = attrStartOffset + (pos - attrStartPosition) + status, pos = ReadCharacteristicsFromMetaData( + buf, i, pos, newlimit, typeID, fileOffset, False) + if not status: + return False + + return True, pos + + +def ReadMetadataStep(f, fileSize, step, WriterCount): + # Read metadata of one step + mdStartPosition = f.tell() + if step > 0: + print("========================================================") + print("Step {0}: ".format(step)) + print(" PG Index offset : {0}".format(mdStartPosition)) + + # Read the PG Index + + # 8 bytes PG Count + Index Length + totalsize = np.fromfile(f, dtype=np.uint64, count=1) + print(" TotalMetadata size : {0}".format(totalsize)) + metadatasizearray = np.fromfile(f, dtype=np.uint64, count=WriterCount) + attrsizearray = np.fromfile(f, dtype=np.uint64, count=WriterCount) + + for i in range(0, WriterCount): + print(" Writer " + str(i) + ": MDsize=" + str(metadatasizearray[i]) + + ", AttrSize=" + str(attrsizearray)) + f.read(metadatasizearray[i]) + f.read(attrsizearray[i]) + + return True + + +def DumpMetaData(fileName): + print("========================================================") + print(" Metadata File: " + fileName) + print("========================================================") + with open(fileName, "rb") as f: + fileSize = fstat(f.fileno()).st_size + status = ReadHeader(f, fileSize, "Metadata") + if isinstance(status, list): + WriterCount = status[1] + status = status[0] + step = 0 + while (f.tell() < fileSize - 12 and status): + status = ReadMetadataStep(f, fileSize, step, WriterCount) + step = step + 1 + return status + + +if __name__ == "__main__": + print("ERROR: Utility main program is bp5dbg.py") diff --git a/source/utils/bp5dbg/adios2/bp5dbg/utils.py b/source/utils/bp5dbg/adios2/bp5dbg/utils.py new file mode 100644 index 0000000000..bcc207d490 --- /dev/null +++ b/source/utils/bp5dbg/adios2/bp5dbg/utils.py @@ -0,0 +1,167 @@ + +LocalValueDim = 18446744073709551613 + +dataTypes = { + -1: 'unknown', + 0: 'byte', + 1: 'short', + 2: 'integer', + 4: 'long', + + 50: 'unsigned_byte', + 51: 'unsigned_short', + 52: 'unsigned_integer', + 54: 'unsigned_long', + + 5: 'real', + 6: 'double', + 7: 'long_double', + + 9: 'string', + 10: 'complex', + 11: 'double_complex', + 12: 'string_array', + + 55: 'char' +} + +dataTypeSize = { + -1: 0, + 0: 1, + 1: 2, + 2: 4, + 4: 8, + + 50: 1, + 51: 2, + 52: 4, + 54: 8, + + 5: 4, + 6: 8, + 7: 16, + + 9: 0, + 10: 8, + 11: 16, + 12: 0, + + 55: 1 +} + + +def GetTypeName(typeID): + name = dataTypes.get(typeID) + if name is None: + name = "unknown type" + return name + + +def GetTypeSize(typeID): + size = dataTypeSize.get(typeID) + if size is None: + size = 0 + return size + + +CharacteristicNames = { + 0: 'value', + 1: 'min', + 2: 'max', + 3: 'offset', + 4: 'dimensions', + 5: 'var_id', + 6: 'payload_offset', + 7: 'file_index', + 8: 'time_index', + 9: 'bitmap', + 10: 'stat', + 11: 'transform_type', + 12: 'minmax' +} + + +def GetCharacteristicName(cID): + name = CharacteristicNames.get(cID) + if name is None: + name = "unknown characteristic" + return name + + +def GetCharacteristicDataLength(cID, typeID): + name = CharacteristicNames.get(cID) + if (name == 'value' or name == 'min' or + name == 'max' or name == 'minmax'): + return dataTypeSize[typeID] + elif (name == 'offset' or name == 'payload_offset'): + return 8 + elif (name == 'file_index' or name == 'time_index'): + return 4 + else: + return 0 + + +# Read Header info 64 bytes +# fileType: Data, Metadata, Index Table +def ReadHeader(f, fileSize, fileType): + status = True + if fileSize < 64: + print("ERROR: Invalid " + fileType + ". File is smaller " + "than the header (64 bytes)") + return False + header = f.read(64) + hStr = header.decode('ascii') + + versionStr = hStr[0:32].replace('\0', ' ') + major = hStr[32] + minor = hStr[33] + micro = hStr[34] +# unused = hStr[35] + + endianValue = header[36] + if endianValue == 0: + endian = 'yes' + elif endianValue == 1: + endian = ' no' + else: + print("ERROR: byte 28 must be 0 or 1 to indicate endianness of " + "the data. It is however {0} in this file".format( + endianValue)) + status = False + + bpversion = int(header[37]) + active = int(header[38]) + if active == 0: + activeStr = ' no' + else: + activeStr = 'yes' + + # unused = hStr[39] + + WriterCount = int(header[40]) + aggregatorcount = int(header[44]) + iscolumnmajor = header[49] + # 45..63 unused + + print("-----------------------------------------------------------" + "-----------------------------------------------------------") + print("| Version string | Major | Minor | Patch " + "| unused | Endian | BP version | Active | WriterCount | AggCount" + + " | ColumnMajor | unused |") + print("| 32 bytes | 1B | 1B | 1B " + "| 1B | 1B | 1B | 1B | 4b | 4b " + + "| 1b | 16B |") + print("+----------------------------------------------------------" + "----------------------------------------------------------+") + print("| {0} | {1} | {2} | {3} | | {4} " + "| {5} | {6} | {7:d} | {8:d} | " + + "{9} | |".format( + versionStr, major, minor, micro, endian, bpversion, activeStr, + WriterCount, aggregatorcount, iscolumnmajor)) + print("-----------------------------------------------------------" + "-----------------------------------------------------------") + return [status, WriterCount] + + +if __name__ == "__main__": + print("ERROR: Utility main program is bp5dbg.py") diff --git a/source/utils/bp5dbg/bp5dbg.py b/source/utils/bp5dbg/bp5dbg.py new file mode 100755 index 0000000000..6926b1b192 --- /dev/null +++ b/source/utils/bp5dbg/bp5dbg.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +import argparse +from os.path import basename, exists, isdir +import glob +from adios2.bp5dbg import * + +WriterCount = -1 + +def SetupArgs(): + parser = argparse.ArgumentParser() + parser.add_argument( + "FILE", help="Name of the input file (.bp, .bp/md.idx, " + + ".bp/md.0 or .bp/data.XXX)") + # parser.add_argument("--printdata", "-p", + # help="Dump data of this variable as well", default="") + parser.add_argument("--verbose", "-v", + help="More verbosity", action="count") + parser.add_argument("--no-indextable", "-x", + help="Do not print index table md.idx", + action="store_true") + parser.add_argument("--no-metadata", "-m", + help="Do not print metadata md.0", action="store_true") + parser.add_argument("--no-data", "-d", + help="Do not print data data.*", action="store_true") + args = parser.parse_args() + + # default values + args.idxFileName = "" + args.dumpIdx = False + args.metadataFileName = "" + args.dumpMetadata = False + args.dataFileName = "" + args.dumpData = False + + # print("Verbosity = {0}".format(args.verbose)) + return args + + +def CheckFileName(args): + if not exists(args.FILE): + print("ERROR: File " + args.FILE + " does not exist", flush=True) + exit(1) + if isdir(args.FILE): + if not args.no_indextable: + args.idxFileName = args.FILE + "/" + "md.idx" + args.dumpIdx = True + if not args.no_metadata: + args.metadataFileName = args.FILE + "/" + "md.[0-9]*" + args.dumpMetadata = True + if not args.no_data: + args.dataFileName = args.FILE + "/" + "data.[0-9]*" + args.dumpData = True + return + + name = basename(args.FILE) + if name.startswith("data."): + args.dataFileName = args.FILE + args.dumpData = True + + elif name == "md.idx": + args.idxFileName = args.FILE + args.dumpIdx = True + + elif name.startswith("md."): + args.metadataFileName = args.FILE + args.dumpMetadata = True + + +def DumpIndexTableFile(args): + indexFileList = glob.glob(args.idxFileName) + if len(indexFileList) > 0: + DumpIndexTable(indexFileList[0]) + else: + print("There is no BP% Index Table file as " + args.idxFileName) + + +def DumpMetadataFiles(args): + mdFileList = glob.glob(args.metadataFileName) + if len(mdFileList) > 0: + for fname in mdFileList: + DumpMetaData(fname) + else: + print("There are no BP% Metadata files in " + args.metadataFileName) + +# def DumpDataFiles(args): +# dataFileList = glob.glob(args.dataFileName) +# if len(dataFileList) > 0: +# for fname in dataFileList: +# DumpData(fname) +# else: +# print("There are no BP5 Data files in " + args.dataFileName) + + +WriterCount = -1 + +if __name__ == "__main__": + + args = SetupArgs() + CheckFileName(args) + # print(args) + + if args.dumpIdx: + DumpIndexTableFile(args) + + if args.dumpMetadata: + DumpMetadataFiles(args) + +# if args.dumpData: +# DumpDataFiles(args) diff --git a/source/utils/bpls/bpls.cpp b/source/utils/bpls/bpls.cpp index 61c070a7be..d2398d0658 100644 --- a/source/utils/bpls/bpls.cpp +++ b/source/utils/bpls/bpls.cpp @@ -83,20 +83,21 @@ std::string format; // format string for one data element (e.g. %6.2f) // Flags from arguments or defaults bool dump; // dump data not just list info(flag == 1) bool output_xml; -bool use_regexp; // use varmasks as regular expressions -bool sortnames; // sort names before listing -bool listattrs; // do list attributes too -bool listmeshes; // do list meshes too -bool attrsonly; // do list attributes only -bool longopt; // -l is turned on -bool timestep; // read step by step -bool noindex; // do no print array indices with data -bool printByteAsChar; // print 8 bit integer arrays as string -bool plot; // dump histogram related information -bool hidden_attrs; // show hidden attrs in BP file -int hidden_attrs_flag; // to be passed on in option struct -bool show_decomp; // show decomposition of arrays -bool show_version; // print binary version info of file before work +bool use_regexp; // use varmasks as regular expressions +bool sortnames; // sort names before listing +bool listattrs; // do list attributes too +bool listmeshes; // do list meshes too +bool attrsonly; // do list attributes only +bool longopt; // -l is turned on +bool timestep; // read step by step +bool filestream = false; // are we using an engine through FileStream? +bool noindex; // do no print array indices with data +bool printByteAsChar; // print 8 bit integer arrays as string +bool plot; // dump histogram related information +bool hidden_attrs; // show hidden attrs in BP file +int hidden_attrs_flag; // to be passed on in option struct +bool show_decomp; // show decomposition of arrays +bool show_version; // print binary version info of file before work // other global variables char *prgname; /* argv[0] */ @@ -915,7 +916,7 @@ int doList_vars(core::Engine *fp, core::IO *io) { Entry e(vpair.second->m_Type, vpair.second.get()); bool valid = true; - if (timestep) + if (timestep && !filestream) { valid = e.var->IsValidStep(fp->CurrentStep() + 1); // fprintf(stdout, "Entry: ptr = %p valid = %d\n", e.var, @@ -1445,15 +1446,39 @@ std::vector getEnginesList(const std::string path) if (slen >= 3 && path.compare(slen - 3, 3, ".h5") == 0) { list.push_back("HDF5"); - list.push_back("BPFile"); + if (timestep) + { + list.push_back("FileStream"); + list.push_back("BP3"); + } + else + { + list.push_back("BPFile"); + } } else { - list.push_back("BPFile"); + if (timestep) + { + list.push_back("FileStream"); + list.push_back("BP3"); + } + else + { + list.push_back("BPFile"); + } list.push_back("HDF5"); } #else - list.push_back("BPFile"); + if (timestep) + { + list.push_back("FileStream"); + list.push_back("BP3"); + } + else + { + list.push_back("BPFile"); + } #endif return list; } @@ -1497,7 +1522,18 @@ int doList(const char *path) io.SetEngine(engineName); try { - fp = &io.Open(path, Mode::Read); + if (timestep) + { + fp = &io.Open(path, Mode::Read); + } + else + { + fp = &io.Open(path, Mode::ReadRandomAccess); + } + if (engineName == "FileStream") + { + filestream = true; + } } catch (std::exception &e) { @@ -1521,8 +1557,11 @@ int doList(const char *path) if (verbose) { printf("File info:\n"); - printf(" of variables: %zu\n", io.GetVariables().size()); - printf(" of attributes: %zu\n", io.GetAttributes().size()); + if (!timestep) + { + printf(" of variables: %zu\n", io.GetVariables().size()); + printf(" of attributes: %zu\n", io.GetAttributes().size()); + } // printf(" of meshes: %d\n", fp->nmeshes); // print_file_size(fp->file_size); // printf(" bp version: %d\n", fp->version); @@ -2045,12 +2084,9 @@ int readVar(core::Engine *fp, core::IO *io, core::Variable *variable) printf("\n"); } - if (!variable->m_SingleValue) + if (variable->m_ShapeID == ShapeID::GlobalArray) { - if (variable->m_ShapeID == ShapeID::GlobalArray) - { - variable->SetSelection({startv, countv}); - } + variable->SetSelection({startv, countv}); } if (tidx) @@ -2387,7 +2423,8 @@ bool matchesAMask(const char *name) if (excode == 0 && // matches (pmatch[0].rm_so == 0 || pmatch[0].rm_so == startpos) && // from the beginning - pmatch[0].rm_eo == strlen(name) // to the very end of the name + static_cast(pmatch[0].rm_eo) == + strlen(name) // to the very end of the name ) #else bool matches = std::regex_match(name, varregex[i]); diff --git a/testing/adios2/bindings/C/CMakeLists.txt b/testing/adios2/bindings/C/CMakeLists.txt index 42c7871723..3990e4d5ea 100644 --- a/testing/adios2/bindings/C/CMakeLists.txt +++ b/testing/adios2/bindings/C/CMakeLists.txt @@ -7,3 +7,4 @@ gtest_add_tests_helper(WriteTypes MPI_ALLOW BP Bindings.C. "") gtest_add_tests_helper(WriteReadMultiblock MPI_ALLOW BP Bindings.C. "") gtest_add_tests_helper(NullWriteRead MPI_ALLOW "" Bindings.C. "") gtest_add_tests_helper(WriteAggregateReadLocal MPI_ONLY BP Bindings.C. "") +gtest_add_tests_helper(AvailableVariablesAttribites MPI_ONLY BP Bindings.C. "") diff --git a/testing/adios2/bindings/C/TestBPAvailableVariablesAttribites.cpp b/testing/adios2/bindings/C/TestBPAvailableVariablesAttribites.cpp new file mode 100644 index 0000000000..00dc83605a --- /dev/null +++ b/testing/adios2/bindings/C/TestBPAvailableVariablesAttribites.cpp @@ -0,0 +1,283 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * TestBPAvailableVariablesAttributes.cpp + * + * Created on: 7/9/21. + * Author: Dmitry Ganyushin ganyushindi@ornl.gov + */ + +#include + +#if ADIOS2_USE_MPI +#include +#endif + +#include + +#include "SmallTestData_c.h" +#include +#include //std::iota +#include + +class BPAvailableVariablesAttributes : public ::testing::Test +{ +public: + BPAvailableVariablesAttributes() = default; +}; + +TEST_F(BPAvailableVariablesAttributes, AvailableVariablesAttributes) +{ + int rank = 0; + int size = 1; + size_t steps = 5; + +#if ADIOS2_USE_MPI + adios2_adios *adiosH = adios2_init(MPI_COMM_WORLD, adios2_debug_mode_on); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); +#else + adios2_adios *adiosH = adios2_init(adios2_debug_mode_on); +#endif + + // count dims are allocated in stack + size_t shape[1]; + shape[0] = data_Nx * static_cast(size); + + size_t start[1]; + start[0] = data_Nx * static_cast(rank); + + size_t count[1]; + count[0] = data_Nx; + + adios2_step_status status; + const std::vector startNull = {0}; + const std::vector countNull = {0}; + const std::vector startValid = {start[0] + data_Nx / 2}; + const std::vector countValid = {data_Nx / 2}; + void *nullPointer = nullptr; + + // write + { + // IO + adios2_io *ioH = adios2_declare_io(adiosH, "CArrayTypes"); + // Set engine parameters + adios2_set_engine(ioH, "BPFile"); + adios2_set_parameter(ioH, "ProfileUnits", "Microseconds"); + + adios2_define_attribute(ioH, "strvalue", adios2_type_string, + "Testing zero size blocks with null pointer"); + int32_t attr = 137; + adios2_define_attribute(ioH, "intvalue", adios2_type_int32_t, &attr); + + // Define variables in ioH + + adios2_variable *varI8 = + adios2_define_variable(ioH, "varI8", adios2_type_int8_t, 1, shape, + start, count, adios2_constant_dims_false); + adios2_variable *varI16 = + adios2_define_variable(ioH, "varI16", adios2_type_int16_t, 1, shape, + start, count, adios2_constant_dims_false); + adios2_variable *varI32 = + adios2_define_variable(ioH, "varI32", adios2_type_int32_t, 1, shape, + start, count, adios2_constant_dims_false); + adios2_variable *varI64 = + adios2_define_variable(ioH, "varI64", adios2_type_int64_t, 1, shape, + start, count, adios2_constant_dims_false); + + adios2_variable *varU8 = + adios2_define_variable(ioH, "varU8", adios2_type_uint8_t, 1, shape, + start, count, adios2_constant_dims_false); + adios2_variable *varU16 = adios2_define_variable( + ioH, "varU16", adios2_type_uint16_t, 1, shape, start, count, + adios2_constant_dims_false); + adios2_variable *varU32 = adios2_define_variable( + ioH, "varU32", adios2_type_uint32_t, 1, shape, start, count, + adios2_constant_dims_false); + adios2_variable *varU64 = adios2_define_variable( + ioH, "varU64", adios2_type_uint64_t, 1, shape, start, count, + adios2_constant_dims_false); + + adios2_variable *varR32 = + adios2_define_variable(ioH, "varR32", adios2_type_float, 1, shape, + start, count, adios2_constant_dims_false); + adios2_variable *varR64 = + adios2_define_variable(ioH, "varR64", adios2_type_double, 1, shape, + start, count, adios2_constant_dims_false); + + adios2_engine *engineH = + adios2_open(ioH, "available.bp", adios2_mode_write); + + for (size_t i = 0; i < steps; ++i) + { + adios2_begin_step(engineH, adios2_step_mode_append, -1., &status); + + adios2_set_selection(varI8, 1, startNull.data(), countNull.data()); + adios2_put(engineH, varI8, nullptr, adios2_mode_sync); + adios2_set_selection(varI8, 1, startValid.data(), + countValid.data()); + adios2_put(engineH, varI8, &data_I8[data_Nx / 2], + adios2_mode_deferred); + + adios2_set_selection(varI16, 1, startNull.data(), countNull.data()); + adios2_put(engineH, varI16, nullptr, adios2_mode_sync); + adios2_set_selection(varI16, 1, startValid.data(), + countValid.data()); + adios2_put(engineH, varI16, &data_I16[data_Nx / 2], + adios2_mode_deferred); + + adios2_set_selection(varI32, 1, startNull.data(), countNull.data()); + adios2_put(engineH, varI32, nullptr, adios2_mode_sync); + adios2_set_selection(varI32, 1, startValid.data(), + countValid.data()); + adios2_put(engineH, varI32, &data_I32[data_Nx / 2], + adios2_mode_deferred); + + adios2_set_selection(varI64, 1, startNull.data(), countNull.data()); + adios2_put(engineH, varI64, nullptr, adios2_mode_sync); + adios2_set_selection(varI64, 1, startValid.data(), + countValid.data()); + adios2_put(engineH, varI64, &data_I64[data_Nx / 2], + adios2_mode_deferred); + + adios2_set_selection(varU8, 1, startNull.data(), countNull.data()); + adios2_put(engineH, varU8, nullPointer, adios2_mode_deferred); + adios2_set_selection(varU8, 1, startValid.data(), + countValid.data()); + adios2_put(engineH, varU8, &data_U8[data_Nx / 2], + adios2_mode_deferred); + + adios2_set_selection(varU16, 1, startNull.data(), countNull.data()); + adios2_put(engineH, varU16, nullPointer, adios2_mode_deferred); + adios2_set_selection(varU16, 1, startValid.data(), + countValid.data()); + adios2_put(engineH, varU16, &data_U16[data_Nx / 2], + adios2_mode_deferred); + + adios2_set_selection(varU32, 1, startNull.data(), countNull.data()); + adios2_put(engineH, varU32, nullPointer, adios2_mode_deferred); + adios2_set_selection(varU32, 1, startValid.data(), + countValid.data()); + adios2_put(engineH, varU32, &data_U32[data_Nx / 2], + adios2_mode_deferred); + + adios2_set_selection(varU64, 1, startNull.data(), countNull.data()); + adios2_put(engineH, varU64, nullPointer, adios2_mode_deferred); + adios2_set_selection(varU64, 1, startValid.data(), + countValid.data()); + adios2_put(engineH, varU64, &data_U64[data_Nx / 2], + adios2_mode_deferred); + + adios2_set_selection(varR32, 1, startNull.data(), countNull.data()); + adios2_put(engineH, varR32, nullPointer, adios2_mode_deferred); + adios2_set_selection(varR32, 1, startValid.data(), + countValid.data()); + adios2_put(engineH, varR32, &data_R32[data_Nx / 2], + adios2_mode_deferred); + + adios2_set_selection(varR64, 1, startNull.data(), countNull.data()); + adios2_put(engineH, varR64, nullPointer, adios2_mode_deferred); + adios2_set_selection(varR64, 1, startValid.data(), + countValid.data()); + adios2_put(engineH, varR64, &data_R64[data_Nx / 2], + adios2_mode_deferred); + + adios2_end_step(engineH); + } + adios2_close(engineH); + } +#if ADIOS2_USE_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif + { + std::vector inI8(data_Nx / 2); + std::vector inI16(data_Nx / 2); + std::vector inI32(data_Nx / 2); + std::vector inI64(data_Nx / 2); + + std::vector inU8(data_Nx / 2); + std::vector inU16(data_Nx / 2); + std::vector inU32(data_Nx / 2); + std::vector inU64(data_Nx / 2); + + std::vector inR32(data_Nx / 2); + std::vector inR64(data_Nx / 2); + + adios2_io *ioH = adios2_declare_io(adiosH, "Reader"); + adios2_engine *engineH = + adios2_open(ioH, "available.bp", adios2_mode_read); + + size_t nsteps; + adios2_steps(&nsteps, engineH); + EXPECT_EQ(nsteps, steps); + + while (adios2_begin_step(engineH, adios2_step_mode_read, -1., + &status) == adios2_error_none) + { + if (status == adios2_step_status_end_of_stream) + { + break; + } + + std::vector correct_vars = { + "varI16", "varI32", "varI64", "varI8", "varR32", + "varR64", "varU16", "varU32", "varU64", "varU8"}; + std::vector correct_attrs = {"strvalue", "intvalue"}; + + std::vector vars; + size_t var_size; + char **var_names = adios2_available_variables(ioH, &var_size); + for (size_t i = 0; i < var_size; i++) + { + vars.push_back(var_names[i]); + free(var_names[i]); + } + free(var_names); + std::sort(correct_vars.begin(), correct_vars.end()); + std::sort(vars.begin(), vars.end()); + EXPECT_EQ(correct_vars, vars); + + size_t attr_size; + std::vector attrs; + char **attr_names = adios2_available_attributes(ioH, &attr_size); + for (size_t i = 0; i < attr_size; i++) + { + attrs.push_back(attr_names[i]); + free(attr_names[i]); + } + // remove memory + free(attr_names); + std::sort(attrs.begin(), attrs.end()); + std::sort(correct_attrs.begin(), correct_attrs.end()); + EXPECT_EQ(correct_attrs, attrs); + + adios2_end_step(engineH); + } + + adios2_close(engineH); + } + // deallocate adiosH + adios2_finalize(adiosH); +} + +//****************************************************************************** +// main +//****************************************************************************** + +int main(int argc, char **argv) +{ +#if ADIOS2_USE_MPI + MPI_Init(nullptr, nullptr); +#endif + + int result; + ::testing::InitGoogleTest(&argc, argv); + result = RUN_ALL_TESTS(); + +#if ADIOS2_USE_MPI + MPI_Finalize(); +#endif + + return result; +} diff --git a/testing/adios2/bindings/C/TestBPWriteReadMultiblock.cpp b/testing/adios2/bindings/C/TestBPWriteReadMultiblock.cpp index d94107716e..e8b22442ef 100644 --- a/testing/adios2/bindings/C/TestBPWriteReadMultiblock.cpp +++ b/testing/adios2/bindings/C/TestBPWriteReadMultiblock.cpp @@ -214,7 +214,6 @@ TEST_F(BPWriteReadMultiblockCC, ZeroSizeBlocks) { break; } - adios2_variable *varI8 = adios2_inquire_variable(ioH, "varI8"); adios2_set_selection(varI8, 1, startValid.data(), countValid.data()); diff --git a/testing/adios2/bindings/fortran/SmallTestData_mod.F90 b/testing/adios2/bindings/fortran/SmallTestData_mod.F90 index f459010535..4b0031463b 100644 --- a/testing/adios2/bindings/fortran/SmallTestData_mod.F90 +++ b/testing/adios2/bindings/fortran/SmallTestData_mod.F90 @@ -14,10 +14,10 @@ module small_test_data (/'Attribute oneXX', 'Attribute twoXX', 'Attribute three'/) integer(kind=1), parameter, dimension(10) :: data_I8 = & - (/0, 1, -2, 3, -4, 5, -6, 7, -8, 9/) + INT((/0, 1, -2, 3, -4, 5, -6, 7, -8, 9/), 1) integer(kind=2), parameter, dimension(10) :: data_I16 = & - (/512, 513, -510, 515, -508, 517, -506, 519, -504, 521/) + INT((/512, 513, -510, 515, -508, 517, -506, 519, -504, 521/), 2) integer(kind=4), parameter, dimension(10) :: data_I32 = & (/131072, 131073, -131070, 131075, -131068, 131077, -131066, 131079, & diff --git a/testing/adios2/bindings/fortran/TestAdios2BindingsFortranIO.F90 b/testing/adios2/bindings/fortran/TestAdios2BindingsFortranIO.F90 index 23da9feef8..29fddacf43 100644 --- a/testing/adios2/bindings/fortran/TestAdios2BindingsFortranIO.F90 +++ b/testing/adios2/bindings/fortran/TestAdios2BindingsFortranIO.F90 @@ -7,16 +7,15 @@ module testing_adios implicit none type(adios2_adios) :: adios - contains ! ------------------------------------------------------------ subroutine testing_adios_init() ! ------------------------------------------------------------ integer :: ierr - + call adios2_init(adios, MPI_COMM_WORLD, adios2_debug_mode_on, ierr) - + end subroutine testing_adios_init ! ------------------------------------------------------------ @@ -24,7 +23,7 @@ subroutine testing_adios_finalize() ! ------------------------------------------------------------ integer :: ierr - + call adios2_finalize(adios, ierr) end subroutine testing_adios_finalize @@ -149,6 +148,8 @@ program main implicit none integer :: ierr + external testing_adios_io_engine + external testing_adios_io_engine_default call MPI_Init(ierr) diff --git a/testing/adios2/bindings/fortran/TestBPReadGlobalsByName.F90 b/testing/adios2/bindings/fortran/TestBPReadGlobalsByName.F90 index 012d099557..9c4773b567 100644 --- a/testing/adios2/bindings/fortran/TestBPReadGlobalsByName.F90 +++ b/testing/adios2/bindings/fortran/TestBPReadGlobalsByName.F90 @@ -3,7 +3,7 @@ program TestBPReadGlobalsByName use adios2 implicit none - integer :: i, j, inx, iny, irank, isize, ierr + integer :: irank, isize, ierr integer(kind=4) :: diag_1d_nsp real(kind=8) :: sml_outpsi diff --git a/testing/adios2/bindings/fortran/TestBPWriteMemorySelectionRead2D.F90 b/testing/adios2/bindings/fortran/TestBPWriteMemorySelectionRead2D.F90 index 7b5a2a9741..83fa4ebf1f 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteMemorySelectionRead2D.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteMemorySelectionRead2D.F90 @@ -101,12 +101,12 @@ program TestBPWriteMemorySelectionRead2D call adios2_open(bpWriter, ioPut, 'MemSel2D_f.bp', adios2_mode_write, ierr) do s=0,nsteps-1 - data_i1(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1) = s - data_i2(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1) = s - data_i4(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1) = s - data_i8(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1) = s - data_r4(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1) = s - data_r8(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1) = s + data_i1(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1) = INT(s, 1) + data_i2(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1) = INT(s, 2) + data_i4(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1) = INT(s, 4) + data_i8(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1) = INT(s, 8) + data_r4(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1) = INT(s, 4) + data_r8(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1) = INT(s, 8) call adios2_begin_step(bpWriter, ierr) call adios2_put(bpWriter, vars(1), data_i1, ierr) @@ -225,7 +225,7 @@ program TestBPWriteMemorySelectionRead2D if(in_data_i2(i,j) /= current_step) stop 'i2 read error' if(in_data_i4(i,j) /= current_step) stop 'i4 read error' if(in_data_i8(i,j) /= current_step) stop 'i8 read rerror' - if(in_data_r4(i,j) /= current_step) stop 'r4 read error' + if(in_data_r4(i,j) /= REAL(current_step, 4)) stop 'r4 read error' if(in_data_r8(i,j) /= current_step) stop 'r8 read rerror' end do end do diff --git a/testing/adios2/bindings/fortran/TestBPWriteMemorySelectionRead3D.F90 b/testing/adios2/bindings/fortran/TestBPWriteMemorySelectionRead3D.F90 index ea9f2e4662..25ae6bf665 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteMemorySelectionRead3D.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteMemorySelectionRead3D.F90 @@ -102,12 +102,12 @@ program TestBPWriteMemorySelectionRead3D call adios2_open(bpWriter, ioPut, 'MemSel3D_f.bp', adios2_mode_write, ierr) do s=0,nsteps-1 - data_i1(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1,ghost_z+1:nz+ghost_z+1) = s - data_i2(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1,ghost_z+1:nz+ghost_z+1) = s - data_i4(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1,ghost_z+1:nz+ghost_z+1) = s - data_i8(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1,ghost_z+1:nz+ghost_z+1) = s - data_r4(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1,ghost_z+1:nz+ghost_z+1) = s - data_r8(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1,ghost_z+1:nz+ghost_z+1) = s + data_i1(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1,ghost_z+1:nz+ghost_z+1) = INT(s, 1) + data_i2(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1,ghost_z+1:nz+ghost_z+1) = INT(s, 2) + data_i4(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1,ghost_z+1:nz+ghost_z+1) = INT(s, 4) + data_i8(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1,ghost_z+1:nz+ghost_z+1) = INT(s, 8) + data_r4(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1,ghost_z+1:nz+ghost_z+1) = INT(s, 4) + data_r8(ghost_x+1:nx+ghost_x+1,ghost_y+1:ny+ghost_y+1,ghost_z+1:nz+ghost_z+1) = INT(s, 8) call adios2_begin_step(bpWriter, ierr) call adios2_put(bpWriter, vars(1), data_i1, ierr) @@ -226,7 +226,7 @@ program TestBPWriteMemorySelectionRead3D if(in_data_i2(i,j,k) /= current_step) stop 'i2 read error' if(in_data_i4(i,j,k) /= current_step) stop 'i4 read error' if(in_data_i8(i,j,k) /= current_step) stop 'i8 read error' - if(in_data_r4(i,j,k) /= current_step) stop 'r4 read error' + if(in_data_r4(i,j,k) /= REAL(current_step, 4)) stop 'r4 read error' if(in_data_r8(i,j,k) /= current_step) stop 'r8 read error' end do end do diff --git a/testing/adios2/bindings/fortran/TestBPWriteReadAttributes.F90 b/testing/adios2/bindings/fortran/TestBPWriteReadAttributes.F90 index 592e0807f8..6209c60f0b 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteReadAttributes.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteReadAttributes.F90 @@ -27,6 +27,9 @@ program TestBPWriteAttributes real, dimension(3) :: r32_array real(kind=8), dimension(3):: r64_array + type(adios2_namestruct) :: namestruct + character(len=4096), dimension(:), allocatable :: attrnamelist + ! Launch MPI call MPI_Init(ierr) @@ -107,6 +110,28 @@ program TestBPWriteAttributes call adios2_open(bpReader, ioRead, 'fattr_types.bp', adios2_mode_read, ierr) + + ! Test getting list of attribute names + call adios2_available_attributes(ioRead, namestruct, ierr) + if (ierr /= 0) stop 'adios2_available_attributes returned with error' + if (.not.namestruct%valid) stop 'adios2_available_attributes returned invalid struct' + write(*,*) 'Number of attributes = ', namestruct%count + write(*,*) 'Max name length = ', namestruct%max_name_len + if (namestruct%count /= 14) stop 'adios2_available_attributes returned not the expected 14' + + allocate(attrnamelist(namestruct%count)) + + call adios2_retrieve_names(namestruct, attrnamelist, ierr) + if (ierr /= 0) stop 'adios2_retrieve_names returned with error' + do i=1,namestruct%count + write(*,'("Attr[",i2,"] = ",a20)') i, attrnamelist(i) + end do + deallocate(attrnamelist) + + if (namestruct%f2c /= 0_8) stop 'namestruct f2c pointer is not null after adios2_retrieve_names()' + if (namestruct%valid) stop 'namestruct is not invalidated after adios2_retrieve_names()' + + call adios2_inquire_attribute(attributes_in(1), ioRead, 'att_String', ierr) call adios2_inquire_attribute(attributes_in(2), ioRead, 'att_i8', ierr) call adios2_inquire_attribute(attributes_in(3), ioRead, 'att_i16', ierr) diff --git a/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap2D.F90 b/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap2D.F90 index 1692e7b6c0..99ec972914 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap2D.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap2D.F90 @@ -175,8 +175,8 @@ program TestBPWriteReadHeatMap2D sum_i1 = 0 sum_i2 = 0 - do i2 = 1, sel_count(2) - do i1 = 1, sel_count(1) + do i2 = 1, INT(sel_count(2), 4) + do i1 = 1, INT(sel_count(1), 4) sum_i1 = sum_i1 + sel_temperatures_i1(i1, i2) sum_i2 = sum_i2 + sel_temperatures_i2(i1, i2) end do diff --git a/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap3D.F90 b/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap3D.F90 index 0efd1470b8..fb9a81f690 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap3D.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap3D.F90 @@ -176,9 +176,9 @@ program TestBPWriteReadHeatMap3D sum_i1 = 0 sum_i2 = 0 - do i3 = 1, sel_count(3) - do i2 = 1, sel_count(2) - do i1 = 1, sel_count(1) + do i3 = 1, INT(sel_count(3), 4) + do i2 = 1, INT(sel_count(2), 4) + do i1 = 1, INT(sel_count(1), 4) sum_i1 = sum_i1 + sel_temperatures_i1(i1, i2, i3) sum_i2 = sum_i2 + sel_temperatures_i2(i1, i2, i3) end do diff --git a/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap4D.F90 b/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap4D.F90 index c821cc95b5..988552248e 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap4D.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap4D.F90 @@ -178,10 +178,10 @@ program TestBPWriteReadHeatMap4D sum_i1 = 0 sum_i2 = 0 - do i4 = 1, sel_count(4) - do i3 = 1, sel_count(3) - do i2 = 1, sel_count(2) - do i1 = 1, sel_count(1) + do i4 = 1, INT(sel_count(4), 4) + do i3 = 1, INT(sel_count(3), 4) + do i2 = 1, INT(sel_count(2), 4) + do i1 = 1, INT(sel_count(1), 4) sum_i1 = sum_i1 + sel_temperatures_i1(i1, i2, i3, i4) sum_i2 = sum_i2 + sel_temperatures_i2(i1, i2, i3, i4) end do diff --git a/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap5D.F90 b/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap5D.F90 index 997a823daa..e8c5207b77 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap5D.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap5D.F90 @@ -185,11 +185,11 @@ program TestBPWriteReadHeatMap5D sum_i1 = 0 sum_i2 = 0 - do i5 = 1, sel_count(5) - do i4 = 1, sel_count(4) - do i3 = 1, sel_count(3) - do i2 = 1, sel_count(2) - do i1 = 1, sel_count(1) + do i5 = 1, INT(sel_count(5), 4) + do i4 = 1, INT(sel_count(4), 4) + do i3 = 1, INT(sel_count(3), 4) + do i2 = 1, INT(sel_count(2), 4) + do i1 = 1, INT(sel_count(1), 4) sum_i1 = sum_i1 + sel_temperatures_i1(i1, i2, i3, i4, i5) sum_i2 = sum_i2 + sel_temperatures_i2(i1, i2, i3, i4, i5) end do diff --git a/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap6D.F90 b/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap6D.F90 index 0522a0b726..6256cef607 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap6D.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteReadHeatMap6D.F90 @@ -187,12 +187,12 @@ program TestBPWriteReadHeatMap6D sum_i1 = 0 sum_i2 = 0 - do i6 = 1, sel_count(6) - do i5 = 1, sel_count(5) - do i4 = 1, sel_count(4) - do i3 = 1, sel_count(3) - do i2 = 1, sel_count(2) - do i1 = 1, sel_count(1) + do i6 = 1, INT(sel_count(6), 4) + do i5 = 1, INT(sel_count(5), 4) + do i4 = 1, INT(sel_count(4), 4) + do i3 = 1, INT(sel_count(3), 4) + do i2 = 1, INT(sel_count(2), 4) + do i1 = 1, INT(sel_count(1), 4) sum_i1 = sum_i1 + sel_temperatures_i1(i1, i2, i3, i4, i5, i6) sum_i2 = sum_i2 + sel_temperatures_i2(i1, i2, i3, i4, i5, i6) end do diff --git a/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 b/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 index 4aaa8a21bc..4187762225 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 @@ -26,6 +26,8 @@ program TestBPWriteTypes integer(kind=4) :: ndims integer(kind=8), dimension(:), allocatable :: shape_in + character(len=4096), dimension(:), allocatable :: varnamelist + type(adios2_namestruct) :: namestruct #if ADIOS2_USE_MPI ! Launch MPI @@ -233,6 +235,26 @@ program TestBPWriteTypes call adios2_steps(nsteps, bpReader, ierr) if(nsteps /= 3) stop 'ftypes.bp must have 3 steps' + call adios2_available_variables(ioRead, namestruct, ierr) + if (ierr /= 0) stop 'adios2_available_variables returned with error' + if (.not.namestruct%valid) stop 'adios2_available_variables returned invalid struct' + write(*,*) 'Number of variables = ', namestruct%count + write(*,*) 'Max name length = ', namestruct%max_name_len + if (namestruct%count /= 14) stop 'adios2_available_variables returned not the expected 14' + + allocate(varnamelist(namestruct%count)) + + call adios2_retrieve_names(namestruct, varnamelist, ierr) + if (ierr /= 0) stop 'adios2_retrieve_names returned with error' + do i=1,namestruct%count + write(*,'("Var[",i2,"] = ",a12)') i, varnamelist(i) + end do + deallocate(varnamelist) + + if (namestruct%f2c /= 0_8) stop 'namestruct f2c pointer is not null after adios2_retrieve_names()' + if (namestruct%valid) stop 'namestruct is not invalidated after adios2_retrieve_names()' + + call adios2_inquire_variable(variables(1), ioRead, "var_I8", ierr) if (variables(1)%name /= 'var_I8') stop 'var_I8 not recognized' if (variables(1)%type /= adios2_type_integer1) stop 'var_I8 type not recognized' diff --git a/testing/adios2/bindings/fortran/TestBPWriteTypesByName.F90 b/testing/adios2/bindings/fortran/TestBPWriteTypesByName.F90 index 70622b40f1..3cdce0c604 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteTypesByName.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteTypesByName.F90 @@ -5,7 +5,7 @@ program TestBPWriteTypes implicit none integer(kind=8), dimension(1) :: shape_dims, start_dims, count_dims - integer :: inx, irank, isize, ierr, i, length, step_status + integer :: inx, irank, isize, ierr, i, step_status type(adios2_adios) :: adios type(adios2_io) :: ioWrite, ioRead @@ -13,8 +13,7 @@ program TestBPWriteTypes type(adios2_engine) :: bpWriter, bpReader ! read handlers - character(len=:), allocatable :: variable_name - integer :: variable_type, ndims + integer :: ndims integer(kind=8), dimension(:), allocatable :: shape_in ! Launch MPI diff --git a/testing/adios2/bindings/fortran/TestBPWriteTypesLocal.F90 b/testing/adios2/bindings/fortran/TestBPWriteTypesLocal.F90 index 2b9ca03a9b..5508137319 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteTypesLocal.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteTypesLocal.F90 @@ -13,10 +13,7 @@ program TestBPWriteTypes type(adios2_engine) :: bpWriter, bpReader ! read handlers - character(len=:), allocatable :: variable_name, engine_type - integer :: variable_type, ndims, length - integer(kind=8), dimension(:), allocatable :: count_in - integer(kind=8) steps_start, steps_count, current_step, block_id + integer(kind=8) steps_count, current_step, block_id ! data integer(kind=1), dimension(10) :: I8, inI8 @@ -136,12 +133,12 @@ program TestBPWriteTypes end if do i = 1, inx - I8(i) = data_I8(i) + current_step - I16(i) = data_I16(i) + irank + current_step - I32(i) = data_I32(i) + irank + current_step - I64(i) = data_I64(i) + irank + current_step - R32(i) = data_R32(i) + irank + current_step - R64(i) = data_R64(i) + irank + current_step + I8(i) = data_I8(i) + INT(current_step, 1) + I16(i) = data_I16(i) + INT(irank + current_step, 2) + I32(i) = data_I32(i) + INT(irank + current_step, 4) + I64(i) = data_I64(i) + INT(irank + current_step, 8) + R32(i) = data_R32(i) + INT(irank + current_step, 4) + R64(i) = data_R64(i) + INT(irank + current_step, 8) end do ! changing count @@ -279,12 +276,12 @@ program TestBPWriteTypes call adios2_perform_gets(bpReader, ierr) do i = 1, inx - I8(i) = data_I8(i) + current_step - I16(i) = data_I16(i) + block_id + current_step - I32(i) = data_I32(i) + block_id + current_step - I64(i) = data_I64(i) + block_id + current_step - R32(i) = data_R32(i) + block_id + current_step - R64(i) = data_R64(i) + block_id + current_step + I8(i) = data_I8(i) + INT(current_step, 1) + I16(i) = data_I16(i) + INT(block_id + current_step, 2) + I32(i) = data_I32(i) + INT(block_id + current_step, 4) + I64(i) = data_I64(i) + INT(block_id + current_step, 8) + R32(i) = data_R32(i) + INT(block_id + current_step, 4) + R64(i) = data_R64(i) + INT(block_id + current_step, 8) end do do i = 1, inx diff --git a/testing/adios2/bindings/fortran/TestBPWriteVariableAttributes.F90 b/testing/adios2/bindings/fortran/TestBPWriteVariableAttributes.F90 index d894e66a72..693026dcb4 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteVariableAttributes.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteVariableAttributes.F90 @@ -6,11 +6,11 @@ program TestBPWriteVariableAttributes type(adios2_adios) :: adios - type(adios2_io) :: ioWrite, ioRead - type(adios2_engine) :: bpWriter, bpReader + type(adios2_io) :: ioWrite + type(adios2_engine) :: bpWriter type(adios2_variable) :: var type(adios2_attribute), dimension(14) :: attributes - type(adios2_attribute) :: failed_att +! type(adios2_attribute) :: failed_att integer :: ierr, i diff --git a/testing/adios2/bindings/fortran/TestF2C_BPReadFBlocks.cpp b/testing/adios2/bindings/fortran/TestF2C_BPReadFBlocks.cpp index 8bae607ae8..8caffacb2f 100644 --- a/testing/adios2/bindings/fortran/TestF2C_BPReadFBlocks.cpp +++ b/testing/adios2/bindings/fortran/TestF2C_BPReadFBlocks.cpp @@ -65,6 +65,7 @@ TEST_F(BPReadFBlocks, FHeatMap2D) EXPECT_TRUE(r32Blocks[i].IsReverseDims); } } + bpReader.EndStep(); } } } @@ -117,6 +118,7 @@ TEST_F(BPReadFBlocks, FHeatMap3D) EXPECT_EQ(r32Blocks[i].IsReverseDims, true); } } + bpReader.EndStep(); } } } diff --git a/testing/adios2/bindings/fortran/TestRemove.F90 b/testing/adios2/bindings/fortran/TestRemove.F90 index 1fe9829620..e5f4161dd9 100644 --- a/testing/adios2/bindings/fortran/TestRemove.F90 +++ b/testing/adios2/bindings/fortran/TestRemove.F90 @@ -7,14 +7,13 @@ program TestRemove implicit none integer(kind=8), dimension(1) :: shape_dims, start_dims, count_dims - integer :: inx, irank, isize, ierr, i + integer :: inx, irank, isize, ierr logical :: res ! low-level type(adios2_adios) :: adios - type(adios2_io) :: ioWrite, ioRead + type(adios2_io) :: ioWrite type(adios2_variable), dimension(12) :: variables - type(adios2_engine) :: bpWriter, bpReader #if ADIOS2_USE_MPI ! Launch MPI diff --git a/testing/adios2/bindings/fortran/operation/TestBPWriteReadSZ2D.F90 b/testing/adios2/bindings/fortran/operation/TestBPWriteReadSZ2D.F90 index 72119a9a9a..edcff011e8 100644 --- a/testing/adios2/bindings/fortran/operation/TestBPWriteReadSZ2D.F90 +++ b/testing/adios2/bindings/fortran/operation/TestBPWriteReadSZ2D.F90 @@ -34,7 +34,7 @@ program TestBPWriteReadHeatMapSZ2D integer(kind=8), dimension(2) :: sel_start, sel_count integer :: ierr, irank, isize integer :: in1, in2 - integer :: i1, i2, i + integer :: i1, i2 call MPI_INIT(ierr) call MPI_COMM_RANK(MPI_COMM_WORLD, irank, ierr) @@ -193,8 +193,8 @@ program TestBPWriteReadHeatMapSZ2D sum_i1 = 0 sum_i2 = 0 - do i2 = 1, sel_count(2) - do i1 = 1, sel_count(1) + do i2 = 1, INT(sel_count(2), 4) + do i1 = 1, INT(sel_count(1), 4) sum_i1 = sum_i1 + sel_temperatures_i1(i1, i2) sum_i2 = sum_i2 + sel_temperatures_i2(i1, i2) end do diff --git a/testing/adios2/bindings/fortran/operation/TestBPWriteReadSZ3D.F90 b/testing/adios2/bindings/fortran/operation/TestBPWriteReadSZ3D.F90 index cd15fa941a..76e9360537 100644 --- a/testing/adios2/bindings/fortran/operation/TestBPWriteReadSZ3D.F90 +++ b/testing/adios2/bindings/fortran/operation/TestBPWriteReadSZ3D.F90 @@ -191,9 +191,9 @@ program TestBPWriteReadHeatMapSZ3D sum_i1 = 0 sum_i2 = 0 - do i3 = 1, sel_count(3) - do i2 = 1, sel_count(2) - do i1 = 1, sel_count(1) + do i3 = 1, INT(sel_count(3), 4) + do i2 = 1, INT(sel_count(2), 4) + do i1 = 1, INT(sel_count(1), 4) sum_i1 = sum_i1 + sel_temperatures_i1(i1, i2, i3) sum_i2 = sum_i2 + sel_temperatures_i2(i1, i2, i3) end do diff --git a/testing/adios2/bindings/fortran/operation/TestBPWriteReadZfp2D.F90 b/testing/adios2/bindings/fortran/operation/TestBPWriteReadZfp2D.F90 index ff5a2d2a94..2a56286fd8 100644 --- a/testing/adios2/bindings/fortran/operation/TestBPWriteReadZfp2D.F90 +++ b/testing/adios2/bindings/fortran/operation/TestBPWriteReadZfp2D.F90 @@ -34,7 +34,7 @@ program TestBPWriteReadHeatMapZfp2D integer(kind=8), dimension(2) :: sel_start, sel_count integer :: ierr, irank, isize integer :: in1, in2 - integer :: i1, i2, i + integer :: i1, i2 call MPI_INIT(ierr) call MPI_COMM_RANK(MPI_COMM_WORLD, irank, ierr) @@ -192,8 +192,8 @@ program TestBPWriteReadHeatMapZfp2D sum_i1 = 0 sum_i2 = 0 - do i2 = 1, sel_count(2) - do i1 = 1, sel_count(1) + do i2 = 1, INT(sel_count(2), 4) + do i1 = 1, INT(sel_count(1), 4) sum_i1 = sum_i1 + sel_temperatures_i1(i1, i2) sum_i2 = sum_i2 + sel_temperatures_i2(i1, i2) end do diff --git a/testing/adios2/bindings/python/TestQuery.py b/testing/adios2/bindings/python/TestQuery.py new file mode 100644 index 0000000000..dfac87e9d2 --- /dev/null +++ b/testing/adios2/bindings/python/TestQuery.py @@ -0,0 +1,106 @@ +from mpi4py import MPI +import numpy as np +import adios2 +import sys +import os +import math +import matplotlib.pyplot as plt + +# MPI +comm = MPI.COMM_WORLD +rank = comm.Get_rank() +size = comm.Get_size() + +configFile = './defaultConfig.xml' +queryFile = './sampleQuery.xml' +dataPath = './heat.bp' + +def doAnalysis(reader, touched_blocks, varList): + print(" Step: ", reader.CurrentStep(), + " num touched blocks: ", len(touched_blocks)) + values = [] + data = {} + + for var in varList: + data[var] = [] + + if (len(touched_blocks) > 0): + for n in touched_blocks: + for var in varList: + values = np.zeros(n[1], dtype=np.double) + var.SetSelection(n) + reader.Get(var, values, adios2.Mode.Sync) + data[var].extend(values) + # do analysis with data here + +def runQuery(): + adios = adios2.ADIOS(configFile, comm, True) + queryIO = adios.DeclareIO("query") + reader = queryIO.Open(dataPath, adios2.Mode.Read, comm) + w = adios2.Query(queryFile, reader) + + touched_blocks = [] + + var = [queryIO.InquireVariable("T")] + + print("Num steps: ", reader.Steps()) + + while (reader.BeginStep() == adios2.StepStatus.OK): + # say only rank 0 wants to process result + if (rank == 0): + touched_blocks = w.GetResult() + doAnalysis(reader, touched_blocks, var) + + reader.EndStep() + reader.Close() + +def createConfigFile(): + print(".. Writing config file to: ", configFile) + file1 = open(configFile, 'w') + + xmlContent = ["\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", "\n"] + + file1.writelines(xmlContent) + file1.close() + +def createQueryFile(): + print(".. Writing query file to: ", queryFile) + + file1 = open(queryFile, 'w') + queryContent = [ + "\n", "\n", + " \n" + " \n", + " \n", + " \n", + " \n", " \n", + " \n", " \n", "\n" + ] + file1.writelines(queryContent) + file1.close() + + +if (os.path.exists(dataPath) is False): + print("Please generate data file:", dataPath, + " from heat transfer example first.") +else: + # configFile created + createConfigFile() + + # queryFile Generated + createQueryFile() + + print(".. Running query against: ", dataPath) + runQuery() + + print("Now clean up.") + os.remove(queryFile) + os.remove(configFile) diff --git a/testing/adios2/bindings/python/heat.bp/data.0 b/testing/adios2/bindings/python/heat.bp/data.0 new file mode 100644 index 0000000000..50cf2f5d9b Binary files /dev/null and b/testing/adios2/bindings/python/heat.bp/data.0 differ diff --git a/testing/adios2/bindings/python/heat.bp/md.0 b/testing/adios2/bindings/python/heat.bp/md.0 new file mode 100644 index 0000000000..f1d86af17c Binary files /dev/null and b/testing/adios2/bindings/python/heat.bp/md.0 differ diff --git a/testing/adios2/bindings/python/heat.bp/md.idx b/testing/adios2/bindings/python/heat.bp/md.idx new file mode 100644 index 0000000000..f6d6a9d5dc Binary files /dev/null and b/testing/adios2/bindings/python/heat.bp/md.idx differ diff --git a/testing/adios2/bindings/python/heat.bp/profiling.json b/testing/adios2/bindings/python/heat.bp/profiling.json new file mode 100644 index 0000000000..3509068d03 --- /dev/null +++ b/testing/adios2/bindings/python/heat.bp/profiling.json @@ -0,0 +1,6 @@ +[ +{ "rank": 0, "start": "Thu_Aug_19_14:58:09_2021", "threads": 1, "bytes": 3964, "mkdir_mus": 186, "memcpy_mus": 0, "aggregation_mus": 0, "meta_sort_merge_mus": 542, "minmax_mus": 48, "buffering_mus": 1136, "transport_0": { "type": "File_stdio", "close_mus": 142, "write_mus": 182, "open_mus": 0 },"transport_1": { "type": "File_stdio", "close_mus": 0, "write_mus": 48, "open_mus": 0 } }, +{ "rank": 1, "start": "Thu_Aug_19_14:58:09_2021", "threads": 1, "bytes": 3888, "mkdir_mus": 193, "memcpy_mus": 0, "aggregation_mus": 0, "meta_sort_merge_mus": 65, "minmax_mus": 61, "buffering_mus": 774, }, +{ "rank": 2, "start": "Thu_Aug_19_14:58:09_2021", "threads": 1, "bytes": 3888, "mkdir_mus": 196, "memcpy_mus": 0, "aggregation_mus": 0, "meta_sort_merge_mus": 110, "minmax_mus": 47, "buffering_mus": 768, }, +{ "rank": 3, "start": "Thu_Aug_19_14:58:09_2021", "threads": 1, "bytes": 3888, "mkdir_mus": 197, "memcpy_mus": 0, "aggregation_mus": 0, "meta_sort_merge_mus": 100, "minmax_mus": 47, "buffering_mus": 671, } +] diff --git a/testing/adios2/engine/CMakeLists.txt b/testing/adios2/engine/CMakeLists.txt index 7a0c885dc1..645ffd769a 100644 --- a/testing/adios2/engine/CMakeLists.txt +++ b/testing/adios2/engine/CMakeLists.txt @@ -17,8 +17,8 @@ if(ADIOS2_HAVE_DataMan) add_subdirectory(dataman) endif() -if(ADIOS2_HAVE_Table) - add_subdirectory(table) +if(ADIOS2_HAVE_MHS) + add_subdirectory(mhs) endif() if(ADIOS2_HAVE_SSC) diff --git a/testing/adios2/engine/bp/CMakeLists.txt b/testing/adios2/engine/bp/CMakeLists.txt index 970933116a..8b63a067f9 100644 --- a/testing/adios2/engine/bp/CMakeLists.txt +++ b/testing/adios2/engine/bp/CMakeLists.txt @@ -19,35 +19,46 @@ macro(bp3_bp4_gtest_add_tests_helper testname mpi) gtest_add_tests_helper(${testname} ${mpi} BP Engine.BP. .BP4 WORKING_DIRECTORY ${BP4_DIR} EXTRA_ARGS "BP4" ) -# gtest_add_tests_helper(${testname} ${mpi} BP Engine.BP. .BP5 -# WORKING_DIRECTORY ${BP5_DIR} EXTRA_ARGS "BP5" -# ) +endmacro() + +macro(bp_gtest_add_tests_helper testname mpi) + gtest_add_tests_helper(${testname} ${mpi} BP Engine.BP. .BP3 + WORKING_DIRECTORY ${BP3_DIR} EXTRA_ARGS "BP3" + ) + gtest_add_tests_helper(${testname} ${mpi} BP Engine.BP. .BP4 + WORKING_DIRECTORY ${BP4_DIR} EXTRA_ARGS "BP4" + ) + if(ADIOS2_HAVE_BP5) + gtest_add_tests_helper(${testname} ${mpi} BP Engine.BP. .BP5 + WORKING_DIRECTORY ${BP5_DIR} EXTRA_ARGS "BP5" + ) + endif() endmacro() add_subdirectory(operations) -bp3_bp4_gtest_add_tests_helper(WriteReadADIOS2 MPI_ALLOW) -bp3_bp4_gtest_add_tests_helper(WriteReadADIOS2fstream MPI_ALLOW) +bp_gtest_add_tests_helper(WriteReadADIOS2 MPI_ALLOW) +bp_gtest_add_tests_helper(WriteReadADIOS2fstream MPI_ALLOW) bp3_bp4_gtest_add_tests_helper(WriteReadADIOS2stdio MPI_ALLOW) -bp3_bp4_gtest_add_tests_helper(WriteReadAsStreamADIOS2 MPI_ALLOW) -bp3_bp4_gtest_add_tests_helper(WriteReadAsStreamADIOS2_Threads MPI_ALLOW) -bp3_bp4_gtest_add_tests_helper(WriteReadAttributes MPI_ALLOW) -bp3_bp4_gtest_add_tests_helper(FStreamWriteReadHighLevelAPI MPI_ALLOW) +bp_gtest_add_tests_helper(WriteReadAsStreamADIOS2 MPI_ALLOW) +bp_gtest_add_tests_helper(WriteReadAsStreamADIOS2_Threads MPI_ALLOW) +bp_gtest_add_tests_helper(WriteReadAttributes MPI_ALLOW) +bp_gtest_add_tests_helper(FStreamWriteReadHighLevelAPI MPI_ALLOW) bp3_bp4_gtest_add_tests_helper(WriteFlushRead MPI_ALLOW) bp3_bp4_gtest_add_tests_helper(WriteMultiblockRead MPI_ALLOW) bp3_bp4_gtest_add_tests_helper(WriteReadMultiblock MPI_ALLOW) -bp3_bp4_gtest_add_tests_helper(WriteReadVector MPI_ALLOW) +bp_gtest_add_tests_helper(WriteReadVector MPI_ALLOW) bp3_bp4_gtest_add_tests_helper(WriteReadAttributesMultirank MPI_ALLOW) -bp3_bp4_gtest_add_tests_helper(LargeMetadata MPI_ALLOW) -bp3_bp4_gtest_add_tests_helper(WriteMemorySelectionRead MPI_ALLOW) +bp_gtest_add_tests_helper(LargeMetadata MPI_ALLOW) +bp_gtest_add_tests_helper(WriteMemorySelectionRead MPI_ALLOW) bp3_bp4_gtest_add_tests_helper(WriteReadLocalVariables MPI_ALLOW) bp3_bp4_gtest_add_tests_helper(WriteReadLocalVariablesSel MPI_ALLOW) bp3_bp4_gtest_add_tests_helper(WriteReadLocalVariablesSelHighLevel MPI_ALLOW) -bp3_bp4_gtest_add_tests_helper(ChangingShape MPI_ALLOW) +bp_gtest_add_tests_helper(ChangingShape MPI_ALLOW) bp3_bp4_gtest_add_tests_helper(WriteReadBlockInfo MPI_ALLOW) -bp3_bp4_gtest_add_tests_helper(WriteReadVariableSpan MPI_ALLOW) +bp_gtest_add_tests_helper(WriteReadVariableSpan MPI_ALLOW) bp3_bp4_gtest_add_tests_helper(TimeAggregation MPI_ALLOW) -bp3_bp4_gtest_add_tests_helper(NoXMLRecovery MPI_ALLOW) +bp_gtest_add_tests_helper(NoXMLRecovery MPI_ALLOW) bp3_bp4_gtest_add_tests_helper(StepsFileGlobalArray MPI_ALLOW) bp3_bp4_gtest_add_tests_helper(StepsFileLocalArray MPI_ALLOW) bp3_bp4_gtest_add_tests_helper(SelectSteps MPI_ALLOW) @@ -60,7 +71,7 @@ if(NOT MSVC) endif() if(ADIOS2_HAVE_MPI) - bp3_bp4_gtest_add_tests_helper(WriteAggregateRead MPI_ONLY) + bp_gtest_add_tests_helper(WriteAggregateRead MPI_ONLY) endif() # BP3 only for now diff --git a/testing/adios2/engine/bp/TestBPChangingShape.cpp b/testing/adios2/engine/bp/TestBPChangingShape.cpp index 2f9c8446fa..daae74e86e 100644 --- a/testing/adios2/engine/bp/TestBPChangingShape.cpp +++ b/testing/adios2/engine/bp/TestBPChangingShape.cpp @@ -151,7 +151,8 @@ TEST_F(BPChangingShape, BPWriteReadShape2D) { inIO.SetEngine(engineName); } - adios2::Engine reader = inIO.Open(fname, adios2::Mode::Read); + adios2::Engine reader = + inIO.Open(fname, adios2::Mode::ReadRandomAccess); if (!rank) { @@ -327,7 +328,8 @@ TEST_F(BPChangingShape, MultiBlock) { inIO.SetEngine(engineName); } - adios2::Engine reader = inIO.Open(fname, adios2::Mode::Read); + adios2::Engine reader = + inIO.Open(fname, adios2::Mode::ReadRandomAccess); if (!rank) { @@ -351,28 +353,11 @@ TEST_F(BPChangingShape, MultiBlock) var.SetSelection( {{0, 0}, {static_cast(nproc), expected_shape}}); - // Check data on rank 0 - if (!rank) - { - std::vector data(nproc * expected_shape); - reader.Get(var, data.data()); - - reader.EndStep(); + std::vector data(nproc * expected_shape); + reader.Get(var, data.data()); - for (int i = 0; i < nproc; i++) - { - double value = static_cast(i) + - static_cast(step + 1) / 10.0; - - for (int j = 0; j < nblocks[step]; j++) - { - EXPECT_EQ(data[i * nblocks[step] + j], value); - value += 0.01; - } - } - } + EXPECT_THROW(reader.EndStep(), std::logic_error); } - reader.Close(); } } diff --git a/testing/adios2/engine/bp/TestBPInquireVariableException.cpp b/testing/adios2/engine/bp/TestBPInquireVariableException.cpp index f47f7647c7..3dca82bbb0 100644 --- a/testing/adios2/engine/bp/TestBPInquireVariableException.cpp +++ b/testing/adios2/engine/bp/TestBPInquireVariableException.cpp @@ -47,9 +47,9 @@ TEST_F(ADIOSInquireVariableException, Read) auto var1 = io_w.DefineVariable("variable1", shape, start, count); - for (int32_t step = 0; step < NSteps; ++step) + for (size_t step = 0; step < NSteps; ++step) { - std::vector Ints(10, step); + std::vector Ints(10, static_cast(step)); writer.BeginStep(); writer.Put(var1, Ints.data()); writer.EndStep(); @@ -61,7 +61,7 @@ TEST_F(ADIOSInquireVariableException, Read) auto reader = io_r.Open(filename, adios2::Mode::Read); auto var = io_r.InquireVariable("variable1"); - for (int32_t step = 0; step < NSteps; step++) + for (size_t step = 0; step < NSteps; step++) { reader.BeginStep(); std::vector myInts; diff --git a/testing/adios2/engine/bp/TestBPLargeMetadata.cpp b/testing/adios2/engine/bp/TestBPLargeMetadata.cpp index 48e6785552..683e3efd30 100644 --- a/testing/adios2/engine/bp/TestBPLargeMetadata.cpp +++ b/testing/adios2/engine/bp/TestBPLargeMetadata.cpp @@ -139,7 +139,7 @@ TEST_F(BPLargeMetadata, ManyLongStrings) // reader { io.RemoveAllVariables(); - adios2::Engine reader = io.Open(fname, adios2::Mode::Read); + adios2::Engine reader = io.Open(fname, adios2::Mode::ReadRandomAccess); std::string inString; diff --git a/testing/adios2/engine/bp/TestBPWriteAggregateRead.cpp b/testing/adios2/engine/bp/TestBPWriteAggregateRead.cpp index eb9ba1f9d6..44ecba0b65 100644 --- a/testing/adios2/engine/bp/TestBPWriteAggregateRead.cpp +++ b/testing/adios2/engine/bp/TestBPWriteAggregateRead.cpp @@ -70,6 +70,18 @@ void WriteAggRead1D8(const std::string substreams) auto var_r64 = io.DefineVariable("r64", shape, start, count); + (void)var_iString; + (void)var_i8; + (void)var_i16; + (void)var_i32; + (void)var_i64; + (void)var_u8; + (void)var_u16; + (void)var_u32; + (void)var_u64; + (void)var_r32; + (void)var_r64; + /* add operations adios2::Operator ZFPOp = adios.DefineOperator("ZFPCompressor", adios2::ops::LossyZFP); @@ -162,7 +174,8 @@ void WriteAggRead1D8(const std::string substreams) io.SetEngine(engineName); } - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); auto var_iString = io.InquireVariable("iString"); EXPECT_TRUE(var_iString); @@ -383,6 +396,18 @@ void WriteAggRead2D4x2(const std::string substreams) auto var_r32 = io.DefineVariable("r32", shape, start, count); auto var_r64 = io.DefineVariable("r64", shape, start, count); + + (void)var_iString; + (void)var_i8; + (void)var_i16; + (void)var_i32; + (void)var_i64; + (void)var_u8; + (void)var_u16; + (void)var_u32; + (void)var_u64; + (void)var_r32; + (void)var_r64; } if (!engineName.empty()) @@ -467,7 +492,8 @@ void WriteAggRead2D4x2(const std::string substreams) io.SetEngine(engineName); } - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); auto var_iString = io.InquireVariable("iString"); EXPECT_TRUE(var_iString); @@ -692,6 +718,17 @@ void WriteAggRead2D2x4(const std::string substreams) auto var_r32 = io.DefineVariable("r32", shape, start, count); auto var_r64 = io.DefineVariable("r64", shape, start, count); + + (void)var_i8; + (void)var_i16; + (void)var_i32; + (void)var_i64; + (void)var_u8; + (void)var_u16; + (void)var_u32; + (void)var_u64; + (void)var_r32; + (void)var_r64; } if (!engineName.empty()) @@ -773,7 +810,8 @@ void WriteAggRead2D2x4(const std::string substreams) { io.SetEngine(engineName); } - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); auto var_i8 = io.InquireVariable("i8"); EXPECT_TRUE(var_i8); diff --git a/testing/adios2/engine/bp/TestBPWriteAppendReadADIOS2.cpp b/testing/adios2/engine/bp/TestBPWriteAppendReadADIOS2.cpp index ef7288f05a..688595aaae 100644 --- a/testing/adios2/engine/bp/TestBPWriteAppendReadADIOS2.cpp +++ b/testing/adios2/engine/bp/TestBPWriteAppendReadADIOS2.cpp @@ -110,6 +110,18 @@ TEST_F(BPWriteAppendReadTestADIOS2, ADIOS2BPWriteAppendRead2D2x4) auto var_r32 = io.DefineVariable("r32", shape, start, count); auto var_r64 = io.DefineVariable("r64", shape, start, count); + + (void)var_iString; + (void)var_i8; + (void)var_i16; + (void)var_i32; + (void)var_i64; + (void)var_u8; + (void)var_u16; + (void)var_u32; + (void)var_u64; + (void)var_r32; + (void)var_r64; } { @@ -262,6 +274,18 @@ TEST_F(BPWriteAppendReadTestADIOS2, ADIOS2BPWriteAppendRead2D2x4) auto var_r32 = io.DefineVariable("r32", shape, start, count); auto var_r64 = io.DefineVariable("r64", shape, start, count); + + (void)var_iString; + (void)var_i8; + (void)var_i16; + (void)var_i32; + (void)var_i64; + (void)var_u8; + (void)var_u16; + (void)var_u32; + (void)var_u64; + (void)var_r32; + (void)var_r64; } { diff --git a/testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp b/testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp index a5a353904b..0b1c0a58ad 100644 --- a/testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp +++ b/testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp @@ -284,57 +284,48 @@ void BPSteps1D(const size_t ghostCells) } adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); - - auto var_i8 = io.InquireVariable("i8"); - EXPECT_TRUE(var_i8); - ASSERT_EQ(var_i8.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i8.Steps(), NSteps); - ASSERT_EQ(var_i8.Shape()[0], mpiSize * Nx); - - auto var_i16 = io.InquireVariable("i16"); - EXPECT_TRUE(var_i16); - ASSERT_EQ(var_i16.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i16.Steps(), NSteps); - ASSERT_EQ(var_i16.Shape()[0], mpiSize * Nx); - - auto var_i32 = io.InquireVariable("i32"); - EXPECT_TRUE(var_i32); - ASSERT_EQ(var_i32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i32.Steps(), NSteps); - ASSERT_EQ(var_i32.Shape()[0], mpiSize * Nx); - - auto var_i64 = io.InquireVariable("i64"); - EXPECT_TRUE(var_i64); - ASSERT_EQ(var_i64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i64.Steps(), NSteps); - ASSERT_EQ(var_i64.Shape()[0], mpiSize * Nx); - - auto var_r32 = io.InquireVariable("r32"); - EXPECT_TRUE(var_r32); - ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_r32.Steps(), NSteps); - ASSERT_EQ(var_r32.Shape()[0], mpiSize * Nx); - - auto var_r64 = io.InquireVariable("r64"); - EXPECT_TRUE(var_r64); - ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_r64.Steps(), NSteps); - ASSERT_EQ(var_r64.Shape()[0], mpiSize * Nx); - - auto var_cr32 = io.InquireVariable>("cr32"); - EXPECT_TRUE(var_cr32); - ASSERT_EQ(var_cr32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_cr32.Steps(), NSteps); - ASSERT_EQ(var_cr32.Shape()[0], mpiSize * Nx); - - auto var_cr64 = io.InquireVariable>("cr64"); - EXPECT_TRUE(var_cr64); - ASSERT_EQ(var_cr64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_cr64.Steps(), NSteps); - ASSERT_EQ(var_cr64.Shape()[0], mpiSize * Nx); - while (bpReader.BeginStep() == adios2::StepStatus::OK) { + + auto var_i8 = io.InquireVariable("i8"); + EXPECT_TRUE(var_i8); + ASSERT_EQ(var_i8.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i8.Shape()[0], mpiSize * Nx); + + auto var_i16 = io.InquireVariable("i16"); + EXPECT_TRUE(var_i16); + ASSERT_EQ(var_i16.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i16.Shape()[0], mpiSize * Nx); + + auto var_i32 = io.InquireVariable("i32"); + EXPECT_TRUE(var_i32); + ASSERT_EQ(var_i32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i32.Shape()[0], mpiSize * Nx); + + auto var_i64 = io.InquireVariable("i64"); + EXPECT_TRUE(var_i64); + ASSERT_EQ(var_i64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i64.Shape()[0], mpiSize * Nx); + + auto var_r32 = io.InquireVariable("r32"); + EXPECT_TRUE(var_r32); + ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32.Shape()[0], mpiSize * Nx); + + auto var_r64 = io.InquireVariable("r64"); + EXPECT_TRUE(var_r64); + ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64.Shape()[0], mpiSize * Nx); + + auto var_cr32 = io.InquireVariable>("cr32"); + EXPECT_TRUE(var_cr32); + ASSERT_EQ(var_cr32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_cr32.Shape()[0], mpiSize * Nx); + + auto var_cr64 = io.InquireVariable>("cr64"); + EXPECT_TRUE(var_cr64); + ASSERT_EQ(var_cr64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_cr64.Shape()[0], mpiSize * Nx); std::vector I8; std::vector I16; std::vector I32; @@ -521,64 +512,57 @@ void BPSteps2D4x2(const size_t ghostCells) adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); - auto var_i8 = io.InquireVariable("i8"); - EXPECT_TRUE(var_i8); - ASSERT_EQ(var_i8.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i8.Steps(), NSteps); - ASSERT_EQ(var_i8.Shape()[0], mpiSize * Ny); - ASSERT_EQ(var_i8.Shape()[1], Nx); - - auto var_i16 = io.InquireVariable("i16"); - EXPECT_TRUE(var_i16); - ASSERT_EQ(var_i16.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i16.Steps(), NSteps); - ASSERT_EQ(var_i16.Shape()[0], mpiSize * Ny); - ASSERT_EQ(var_i16.Shape()[1], Nx); - - auto var_i32 = io.InquireVariable("i32"); - EXPECT_TRUE(var_i32); - ASSERT_EQ(var_i32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i32.Steps(), NSteps); - ASSERT_EQ(var_i32.Shape()[0], mpiSize * Ny); - ASSERT_EQ(var_i32.Shape()[1], Nx); - - auto var_i64 = io.InquireVariable("i64"); - EXPECT_TRUE(var_i64); - ASSERT_EQ(var_i64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i64.Steps(), NSteps); - ASSERT_EQ(var_i64.Shape()[0], mpiSize * Ny); - ASSERT_EQ(var_i64.Shape()[1], Nx); - - auto var_r32 = io.InquireVariable("r32"); - EXPECT_TRUE(var_r32); - ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_r32.Steps(), NSteps); - ASSERT_EQ(var_r32.Shape()[0], mpiSize * Ny); - ASSERT_EQ(var_r32.Shape()[1], Nx); - - auto var_r64 = io.InquireVariable("r64"); - EXPECT_TRUE(var_r64); - ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_r64.Steps(), NSteps); - ASSERT_EQ(var_r64.Shape()[0], mpiSize * Ny); - ASSERT_EQ(var_r64.Shape()[1], Nx); - - auto var_cr32 = io.InquireVariable>("cr32"); - EXPECT_TRUE(var_cr32); - ASSERT_EQ(var_cr32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_cr32.Steps(), NSteps); - ASSERT_EQ(var_cr32.Shape()[0], mpiSize * Ny); - ASSERT_EQ(var_cr32.Shape()[1], Nx); - - auto var_cr64 = io.InquireVariable>("cr64"); - EXPECT_TRUE(var_cr64); - ASSERT_EQ(var_cr64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_cr64.Steps(), NSteps); - ASSERT_EQ(var_cr64.Shape()[0], mpiSize * Ny); - ASSERT_EQ(var_cr64.Shape()[1], Nx); - while (bpReader.BeginStep() == adios2::StepStatus::OK) { + + auto var_i8 = io.InquireVariable("i8"); + EXPECT_TRUE(var_i8); + ASSERT_EQ(var_i8.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i8.Shape()[0], mpiSize * Ny); + ASSERT_EQ(var_i8.Shape()[1], Nx); + + auto var_i16 = io.InquireVariable("i16"); + EXPECT_TRUE(var_i16); + ASSERT_EQ(var_i16.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i16.Shape()[0], mpiSize * Ny); + ASSERT_EQ(var_i16.Shape()[1], Nx); + + auto var_i32 = io.InquireVariable("i32"); + EXPECT_TRUE(var_i32); + ASSERT_EQ(var_i32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i32.Shape()[0], mpiSize * Ny); + ASSERT_EQ(var_i32.Shape()[1], Nx); + + auto var_i64 = io.InquireVariable("i64"); + EXPECT_TRUE(var_i64); + ASSERT_EQ(var_i64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i64.Shape()[0], mpiSize * Ny); + ASSERT_EQ(var_i64.Shape()[1], Nx); + + auto var_r32 = io.InquireVariable("r32"); + EXPECT_TRUE(var_r32); + ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32.Shape()[0], mpiSize * Ny); + ASSERT_EQ(var_r32.Shape()[1], Nx); + + auto var_r64 = io.InquireVariable("r64"); + EXPECT_TRUE(var_r64); + ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64.Shape()[0], mpiSize * Ny); + ASSERT_EQ(var_r64.Shape()[1], Nx); + + auto var_cr32 = io.InquireVariable>("cr32"); + EXPECT_TRUE(var_cr32); + ASSERT_EQ(var_cr32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_cr32.Shape()[0], mpiSize * Ny); + ASSERT_EQ(var_cr32.Shape()[1], Nx); + + auto var_cr64 = io.InquireVariable>("cr64"); + EXPECT_TRUE(var_cr64); + ASSERT_EQ(var_cr64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_cr64.Shape()[0], mpiSize * Ny); + ASSERT_EQ(var_cr64.Shape()[1], Nx); + std::vector I8; std::vector I16; std::vector I32; @@ -780,95 +764,87 @@ void BPSteps3D8x2x4(const size_t ghostCells) while (bpReader.BeginStep() == adios2::StepStatus::OK) { - const size_t currentStep = bpReader.CurrentStep(); + // const size_t currentStep = bpReader.CurrentStep(); auto var_i8 = io.InquireVariable("i8"); EXPECT_TRUE(var_i8); EXPECT_EQ(var_i8.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_i8.Steps(), NSteps); EXPECT_EQ(var_i8.Shape()[0], mpiSize * Nz); EXPECT_EQ(var_i8.Shape()[1], Ny); EXPECT_EQ(var_i8.Shape()[2], Nx); - EXPECT_EQ(var_i8.Min(), static_cast(currentStep)); - EXPECT_EQ(var_i8.Max(), static_cast(currentStep)); + // EXPECT_EQ(var_i8.Min(), static_cast(currentStep)); + // EXPECT_EQ(var_i8.Max(), static_cast(currentStep)); auto var_i16 = io.InquireVariable("i16"); EXPECT_TRUE(var_i16); EXPECT_EQ(var_i16.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_i16.Steps(), NSteps); EXPECT_EQ(var_i16.Shape()[0], mpiSize * Nz); EXPECT_EQ(var_i16.Shape()[1], Ny); EXPECT_EQ(var_i16.Shape()[2], Nx); - EXPECT_EQ(var_i16.Min(), static_cast(currentStep)); - EXPECT_EQ(var_i16.Max(), static_cast(currentStep)); + // EXPECT_EQ(var_i16.Min(), static_cast(currentStep)); + // EXPECT_EQ(var_i16.Max(), static_cast(currentStep)); auto var_i32 = io.InquireVariable("i32"); EXPECT_TRUE(var_i32); EXPECT_EQ(var_i32.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_i32.Steps(), NSteps); EXPECT_EQ(var_i32.Shape()[0], mpiSize * Nz); EXPECT_EQ(var_i32.Shape()[1], Ny); EXPECT_EQ(var_i32.Shape()[2], Nx); - EXPECT_EQ(var_i32.Min(), static_cast(currentStep)); - EXPECT_EQ(var_i32.Max(), static_cast(currentStep)); + // EXPECT_EQ(var_i32.Min(), static_cast(currentStep)); + // EXPECT_EQ(var_i32.Max(), static_cast(currentStep)); auto var_i64 = io.InquireVariable("i64"); EXPECT_TRUE(var_i64); EXPECT_EQ(var_i64.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_i64.Steps(), NSteps); EXPECT_EQ(var_i64.Shape()[0], mpiSize * Nz); EXPECT_EQ(var_i64.Shape()[1], Ny); EXPECT_EQ(var_i64.Shape()[2], Nx); - EXPECT_EQ(var_i64.Min(), static_cast(currentStep)); - EXPECT_EQ(var_i64.Max(), static_cast(currentStep)); + // EXPECT_EQ(var_i64.Min(), static_cast(currentStep)); + // EXPECT_EQ(var_i64.Max(), static_cast(currentStep)); auto var_r32 = io.InquireVariable("r32"); EXPECT_TRUE(var_r32); EXPECT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_r32.Steps(), NSteps); EXPECT_EQ(var_r32.Shape()[0], mpiSize * Nz); EXPECT_EQ(var_r32.Shape()[1], Ny); EXPECT_EQ(var_r32.Shape()[2], Nx); - EXPECT_EQ(var_r32.Min(), static_cast(currentStep)); - EXPECT_EQ(var_r32.Max(), static_cast(currentStep)); + // EXPECT_EQ(var_r32.Min(), static_cast(currentStep)); + // EXPECT_EQ(var_r32.Max(), static_cast(currentStep)); auto var_r64 = io.InquireVariable("r64"); EXPECT_TRUE(var_r64); EXPECT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_r64.Steps(), NSteps); EXPECT_EQ(var_r64.Shape()[0], mpiSize * Nz); EXPECT_EQ(var_r64.Shape()[1], Ny); EXPECT_EQ(var_r64.Shape()[2], Nx); - EXPECT_EQ(var_r64.Min(), static_cast(currentStep)); - EXPECT_EQ(var_r64.Max(), static_cast(currentStep)); + // EXPECT_EQ(var_r64.Min(), static_cast(currentStep)); + // EXPECT_EQ(var_r64.Max(), static_cast(currentStep)); auto var_cr32 = io.InquireVariable>("cr32"); EXPECT_TRUE(var_cr32); EXPECT_EQ(var_cr32.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_cr32.Steps(), NSteps); EXPECT_EQ(var_cr32.Shape()[0], mpiSize * Nz); EXPECT_EQ(var_cr32.Shape()[1], Ny); EXPECT_EQ(var_cr32.Shape()[2], Nx); - EXPECT_EQ(var_cr32.Min(), - std::complex(static_cast(currentStep), - static_cast(currentStep))); - EXPECT_EQ(var_cr32.Max(), - std::complex(static_cast(currentStep), - static_cast(currentStep))); + // EXPECT_EQ(var_cr32.Min(), + // std::complex(static_cast(currentStep), + // static_cast(currentStep))); + // EXPECT_EQ(var_cr32.Max(), + // std::complex(static_cast(currentStep), + // static_cast(currentStep))); auto var_cr64 = io.InquireVariable>("cr64"); EXPECT_TRUE(var_cr64); EXPECT_EQ(var_cr64.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_cr64.Steps(), NSteps); EXPECT_EQ(var_cr64.Shape()[0], mpiSize * Nz); EXPECT_EQ(var_cr64.Shape()[1], Ny); EXPECT_EQ(var_cr64.Shape()[2], Nx); - EXPECT_EQ(var_cr64.Min(), - std::complex(static_cast(currentStep), - static_cast(currentStep))); - EXPECT_EQ(var_cr64.Max(), - std::complex(static_cast(currentStep), - static_cast(currentStep))); + // EXPECT_EQ(var_cr64.Min(), + // std::complex(static_cast(currentStep), + // static_cast(currentStep))); + // EXPECT_EQ(var_cr64.Max(), + // std::complex(static_cast(currentStep), + // static_cast(currentStep))); std::vector I8; std::vector I16; diff --git a/testing/adios2/engine/bp/TestBPWriteProfilingJSON.cpp b/testing/adios2/engine/bp/TestBPWriteProfilingJSON.cpp index c50f2f1df9..389fb1d06a 100644 --- a/testing/adios2/engine/bp/TestBPWriteProfilingJSON.cpp +++ b/testing/adios2/engine/bp/TestBPWriteProfilingJSON.cpp @@ -19,7 +19,7 @@ #include #include -#include +#include #include "../SmallTestData.h" diff --git a/testing/adios2/engine/bp/TestBPWriteRead.cpp b/testing/adios2/engine/bp/TestBPWriteRead.cpp deleted file mode 100644 index 7dd189bfb3..0000000000 --- a/testing/adios2/engine/bp/TestBPWriteRead.cpp +++ /dev/null @@ -1,938 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - */ -#include -#include - -#include -#include - -#include -#include - -#include - -#include "../SmallTestData.h" - -std::string engineName; // comes from command line - -class BPWriteReadTest : public ::testing::Test -{ -public: - BPWriteReadTest() = default; - - SmallTestData m_TestData; -}; - -//****************************************************************************** -// 1D 1x8 test data -//****************************************************************************** - -// ADIOS2 BP write, native ADIOS1 read -TEST_F(BPWriteReadTest, ADIOS2BPWriteADIOS1Read1D8) -{ - // Each process would write a 1x8 array and all processes would - // form a mpiSize * Nx 1D array - std::string fname = "ADIOS2BPWriteADIOS1Read1D8.bp"; - - int mpiRank = 0, mpiSize = 1; - // Number of rows - const std::size_t Nx = 8; - - // Number of steps - const std::size_t NSteps = 3; - -#if ADIOS2_USE_MPI - MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); - MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); -#endif - - // Write test data using BP - { -#if ADIOS2_USE_MPI - adios2::ADIOS adios(MPI_COMM_WORLD); -#else - adios2::ADIOS adios; -#endif - adios2::IO io = adios.DeclareIO("TestIO"); - - // Declare 1D variables (NumOfProcesses * Nx) - // The local process' part (start, count) can be defined now or later - // before Write(). - { - adios2::Dims shape{static_cast(Nx * mpiSize)}; - adios2::Dims start{static_cast(Nx * mpiRank)}; - adios2::Dims count{static_cast(Nx)}; - - auto var_iString = io.DefineVariable("iString"); - auto var_i8 = io.DefineVariable("i8", shape, start, count); - auto var_i16 = - io.DefineVariable("i16", shape, start, count); - auto var_i32 = - io.DefineVariable("i32", shape, start, count); - auto var_i64 = - io.DefineVariable("i64", shape, start, count); - auto var_u8 = io.DefineVariable("u8", shape, start, count); - auto var_u16 = - io.DefineVariable("u16", shape, start, count); - auto var_u32 = - io.DefineVariable("u32", shape, start, count); - auto var_u64 = - io.DefineVariable("u64", shape, start, count); - auto var_r32 = io.DefineVariable("r32", shape, start, count); - auto var_r64 = - io.DefineVariable("r64", shape, start, count); - } - - if (!engineName.empty()) - { - io.SetEngine(engineName); - } - else - { - // Create the BP Engine - io.SetEngine("BPFile"); - } - - io.AddTransport("file"); - - // QUESTION: It seems that BPFilterWriter cannot overwrite existing - // files - // Ex. if you tune Nx and NSteps, the test would fail. But if you clear - // the cache in - // ${adios2Build}/testing/adios2/engine/bp/ADIOS2BPWriteADIOS1Read1D8.bp.dir, - // then it works - adios2::Engine engine = io.Open(fname, adios2::Mode::Write); - - for (size_t step = 0; step < NSteps; ++step) - { - // Generate test data for each process uniquely - SmallTestData currentTestData = - generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize); - - // Retrieve the variables that previously went out of scope - auto var_iString = io.InquireVariable("iString"); - auto var_i8 = io.InquireVariable("i8"); - auto var_i16 = io.InquireVariable("i16"); - auto var_i32 = io.InquireVariable("i32"); - auto var_i64 = io.InquireVariable("i64"); - auto var_u8 = io.InquireVariable("u8"); - auto var_u16 = io.InquireVariable("u16"); - auto var_u32 = io.InquireVariable("u32"); - auto var_u64 = io.InquireVariable("u64"); - auto var_r32 = io.InquireVariable("r32"); - auto var_r64 = io.InquireVariable("r64"); - - // Make a 1D selection to describe the local dimensions of the - // variable we write and its offsets in the global spaces - adios2::Box sel({mpiRank * Nx}, {Nx}); - - EXPECT_THROW(var_iString.SetSelection(sel), std::invalid_argument); - var_i8.SetSelection(sel); - var_i16.SetSelection(sel); - var_i32.SetSelection(sel); - var_i64.SetSelection(sel); - var_u8.SetSelection(sel); - var_u16.SetSelection(sel); - var_u32.SetSelection(sel); - var_u64.SetSelection(sel); - var_r32.SetSelection(sel); - var_r64.SetSelection(sel); - - // Write each one - // fill in the variable with values from starting index to - // starting index + count - engine.BeginStep(); - engine.Put(var_iString, currentTestData.S1); - engine.Put(var_i8, currentTestData.I8.data()); - engine.Put(var_i16, currentTestData.I16.data()); - engine.Put(var_i32, currentTestData.I32.data()); - engine.Put(var_i64, currentTestData.I64.data()); - engine.Put(var_u8, currentTestData.U8.data()); - engine.Put(var_u16, currentTestData.U16.data()); - engine.Put(var_u32, currentTestData.U32.data()); - engine.Put(var_u64, currentTestData.U64.data()); - engine.Put(var_r32, currentTestData.R32.data()); - engine.Put(var_r64, currentTestData.R64.data()); - engine.EndStep(); - } - - // Close the file - engine.Close(); - } - - { - adios_read_init_method(ADIOS_READ_METHOD_BP, MPI_COMM_SELF, - "verbose=3"); - - // Open the file for reading - // Note: Since collective metadata generation is not implemented yet, - // SO for now we read each subfile instead of a single bp file with all - // metadata. - // Meanwhile if we open file with MPI_COMM_WORLD, then the selection - // bounding box should be [0, Nx] - std::string index = std::to_string(mpiRank); - ADIOS_FILE *f = adios_read_open_file( - (fname + ".dir/" + fname + "." + index).c_str(), - ADIOS_READ_METHOD_BP, MPI_COMM_SELF); - ASSERT_NE(f, nullptr); - - // Check the variables exist - ADIOS_VARINFO *var_iString = adios_inq_var(f, "iString"); - ASSERT_NE(var_iString, nullptr); - ASSERT_EQ(var_iString->ndim, 0); - ASSERT_EQ(var_iString->global, 0); - ASSERT_EQ(var_iString->nsteps, NSteps); - - ADIOS_VARINFO *var_i8 = adios_inq_var(f, "i8"); - EXPECT_TRUE(var_i8); - ASSERT_EQ(var_i8->ndim, 1); - ASSERT_EQ(var_i8->global, 1); - ASSERT_EQ(var_i8->nsteps, NSteps); - ASSERT_EQ(var_i8->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_i16 = adios_inq_var(f, "i16"); - EXPECT_TRUE(var_i16); - ASSERT_EQ(var_i16->ndim, 1); - ASSERT_EQ(var_i16->global, 1); - ASSERT_EQ(var_i16->nsteps, NSteps); - ASSERT_EQ(var_i16->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_i32 = adios_inq_var(f, "i32"); - EXPECT_TRUE(var_i32); - ASSERT_EQ(var_i32->ndim, 1); - ASSERT_EQ(var_i32->global, 1); - ASSERT_EQ(var_i32->nsteps, NSteps); - ASSERT_EQ(var_i32->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_i64 = adios_inq_var(f, "i64"); - EXPECT_TRUE(var_i64); - ASSERT_EQ(var_i64->ndim, 1); - ASSERT_EQ(var_i64->global, 1); - ASSERT_EQ(var_i64->nsteps, NSteps); - ASSERT_EQ(var_i64->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_u8 = adios_inq_var(f, "u8"); - EXPECT_TRUE(var_u8); - ASSERT_EQ(var_u8->ndim, 1); - ASSERT_EQ(var_u8->global, 1); - ASSERT_EQ(var_u8->nsteps, NSteps); - ASSERT_EQ(var_u8->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_u16 = adios_inq_var(f, "u16"); - EXPECT_TRUE(var_u16); - ASSERT_EQ(var_u16->ndim, 1); - ASSERT_EQ(var_u16->global, 1); - ASSERT_EQ(var_u16->nsteps, NSteps); - ASSERT_EQ(var_u16->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_u32 = adios_inq_var(f, "u32"); - EXPECT_TRUE(var_u32); - ASSERT_EQ(var_u32->ndim, 1); - ASSERT_EQ(var_u32->global, 1); - ASSERT_EQ(var_u32->nsteps, NSteps); - ASSERT_EQ(var_u32->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_u64 = adios_inq_var(f, "u64"); - EXPECT_TRUE(var_u64); - ASSERT_EQ(var_u64->ndim, 1); - ASSERT_EQ(var_u64->global, 1); - ASSERT_EQ(var_u64->nsteps, NSteps); - ASSERT_EQ(var_u64->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_r32 = adios_inq_var(f, "r32"); - EXPECT_TRUE(var_r32); - ASSERT_EQ(var_r32->ndim, 1); - ASSERT_EQ(var_r32->global, 1); - ASSERT_EQ(var_r32->nsteps, NSteps); - ASSERT_EQ(var_r32->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_r64 = adios_inq_var(f, "r64"); - EXPECT_TRUE(var_r64); - ASSERT_EQ(var_r64->ndim, 1); - ASSERT_EQ(var_r64->global, 1); - ASSERT_EQ(var_r64->nsteps, NSteps); - ASSERT_EQ(var_r64->dims[0], mpiSize * Nx); - - SmallTestData testData; - std::vector IString(testData.S1.size()); - std::array I8; - std::array I16; - std::array I32; - std::array I64; - std::array U8; - std::array U16; - std::array U32; - std::array U64; - std::array R32; - std::array R64; - - uint64_t start[1] = {mpiRank * Nx}; - uint64_t count[1] = {Nx}; - ADIOS_SELECTION *sel = adios_selection_boundingbox(1, start, count); - - // Read stuff - for (size_t t = 0; t < NSteps; ++t) - { - // Generate test data for each rank uniquely - SmallTestData currentTestData = - generateNewSmallTestData(m_TestData, t, mpiRank, mpiSize); - // Read the current step - // TODO adios_schedule_read_byid(f, sel, var_iString->varid, - // t, 1, - // IString.data()); - adios_schedule_read_byid(f, sel, var_i8->varid, t, 1, I8.data()); - adios_schedule_read_byid(f, sel, var_i16->varid, t, 1, I16.data()); - adios_schedule_read_byid(f, sel, var_i32->varid, t, 1, I32.data()); - adios_schedule_read_byid(f, sel, var_i64->varid, t, 1, I64.data()); - adios_schedule_read_byid(f, sel, var_u8->varid, t, 1, U8.data()); - adios_schedule_read_byid(f, sel, var_u16->varid, t, 1, U16.data()); - adios_schedule_read_byid(f, sel, var_u32->varid, t, 1, U32.data()); - adios_schedule_read_byid(f, sel, var_u64->varid, t, 1, U64.data()); - adios_schedule_read_byid(f, sel, var_r32->varid, t, 1, R32.data()); - adios_schedule_read_byid(f, sel, var_r64->varid, t, 1, R64.data()); - adios_perform_reads(f, 1); - - for (size_t i = 0; i < Nx; ++i) - { - std::stringstream ss; - ss << "t=" << t << " i=" << i << " rank=" << mpiRank; - std::string msg = ss.str(); - - EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg; - EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg; - EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg; - EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg; - EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg; - EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg; - EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg; - EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg; - EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; - EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; - } - } - - adios_selection_delete(sel); - - // Cleanup variable structures - adios_free_varinfo(var_i8); - adios_free_varinfo(var_i16); - adios_free_varinfo(var_i32); - adios_free_varinfo(var_i64); - adios_free_varinfo(var_u8); - adios_free_varinfo(var_u16); - adios_free_varinfo(var_u32); - adios_free_varinfo(var_u64); - adios_free_varinfo(var_r32); - adios_free_varinfo(var_r64); - - // Cleanup file - adios_read_close(f); - - adios_read_finalize_method(ADIOS_READ_METHOD_BP); - } -} - -// ADIOS2 BP write, native ADIOS1 read -TEST_F(BPWriteReadTest, ADIOS2BPWriteADIOS1Read2D2x4) -{ - // Each process would write a 2x4 array and all processes would - // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here - std::string fname = "ADIOS2BPWriteADIOS1Read2D2x4Test.bp"; - - int mpiRank = 0, mpiSize = 1; - // Number of rows - const std::size_t Nx = 4; - - // Number of rows - const std::size_t Ny = 2; - - // Number of steps - const std::size_t NSteps = 3; - -#if ADIOS2_USE_MPI - MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); - MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); -#endif - - // Write test data using ADIOS2 - { -#if ADIOS2_USE_MPI - adios2::ADIOS adios(MPI_COMM_WORLD); -#else - adios2::ADIOS adios; -#endif - adios2::IO io = adios.DeclareIO("TestIO"); - - // Declare 2D variables (Ny * (NumOfProcesses * Nx)) - // The local process' part (start, count) can be defined now or later - // before Write(). - { - adios2::Dims shape{static_cast(Ny), - static_cast(Nx * mpiSize)}; - adios2::Dims start{static_cast(0), - static_cast(mpiRank * Nx)}; - adios2::Dims count{static_cast(Ny), - static_cast(Nx)}; - auto var_i8 = io.DefineVariable("i8", shape, start, count); - auto var_i16 = - io.DefineVariable("i16", shape, start, count); - auto var_i32 = - io.DefineVariable("i32", shape, start, count); - auto var_i64 = - io.DefineVariable("i64", shape, start, count); - auto var_u8 = io.DefineVariable("u8", shape, start, count); - auto var_u16 = - io.DefineVariable("u16", shape, start, count); - auto var_u32 = - io.DefineVariable("u32", shape, start, count); - auto var_u64 = - io.DefineVariable("u64", shape, start, count); - auto var_r32 = io.DefineVariable("r32", shape, start, count); - auto var_r64 = - io.DefineVariable("r64", shape, start, count); - } - - if (!engineName.empty()) - { - io.SetEngine(engineName); - } - else - { - // Create the BP Engine - io.SetEngine("BPFile"); - } - io.AddTransport("file"); - - adios2::Engine engine = io.Open(fname, adios2::Mode::Write); - - for (size_t step = 0; step < NSteps; ++step) - { - // Generate test data for each process uniquely - SmallTestData currentTestData = - generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize); - - // Retrieve the variables that previously went out of scope - auto var_i8 = io.InquireVariable("i8"); - auto var_i16 = io.InquireVariable("i16"); - auto var_i32 = io.InquireVariable("i32"); - auto var_i64 = io.InquireVariable("i64"); - auto var_u8 = io.InquireVariable("u8"); - auto var_u16 = io.InquireVariable("u16"); - auto var_u32 = io.InquireVariable("u32"); - auto var_u64 = io.InquireVariable("u64"); - auto var_r32 = io.InquireVariable("r32"); - auto var_r64 = io.InquireVariable("r64"); - - // Make a 2D selection to describe the local dimensions of the - // variable we write and its offsets in the global spaces - adios2::Box sel( - {0, static_cast(mpiRank * Nx)}, {Ny, Nx}); - var_i8.SetSelection(sel); - var_i16.SetSelection(sel); - var_i32.SetSelection(sel); - var_i64.SetSelection(sel); - var_u8.SetSelection(sel); - var_u16.SetSelection(sel); - var_u32.SetSelection(sel); - var_u64.SetSelection(sel); - var_r32.SetSelection(sel); - var_r64.SetSelection(sel); - - // Write each one - // fill in the variable with values from starting index to - // starting index + count - engine.BeginStep(); - engine.Put(var_i8, currentTestData.I8.data()); - engine.Put(var_i16, currentTestData.I16.data()); - engine.Put(var_i32, currentTestData.I32.data()); - engine.Put(var_i64, currentTestData.I64.data()); - engine.Put(var_u8, currentTestData.U8.data()); - engine.Put(var_u16, currentTestData.U16.data()); - engine.Put(var_u32, currentTestData.U32.data()); - engine.Put(var_u64, currentTestData.U64.data()); - engine.Put(var_r32, currentTestData.R32.data()); - engine.Put(var_r64, currentTestData.R64.data()); - engine.EndStep(); - } - - // Close the file - engine.Close(); - } - - { - adios_read_init_method(ADIOS_READ_METHOD_BP, MPI_COMM_SELF, - "verbose=3"); - - // Open the file for reading - // Note: Since collective metadata generation is not implemented yet, - // SO for now we read each subfile instead of a single bp file with all - // metadata. - // Meanwhile if we open file with MPI_COMM_WORLD, then the selection - // bounding box should be [0, Nx] - std::string index = std::to_string(mpiRank); - ADIOS_FILE *f = adios_read_open_file( - (fname + ".dir/" + fname + "." + index).c_str(), - ADIOS_READ_METHOD_BP, MPI_COMM_SELF); - ASSERT_NE(f, nullptr); - - // Check the variables exist - ADIOS_VARINFO *var_i8 = adios_inq_var(f, "i8"); - EXPECT_TRUE(var_i8); - ASSERT_EQ(var_i8->ndim, 2); - ASSERT_EQ(var_i8->global, 1); - ASSERT_EQ(var_i8->nsteps, NSteps); - ASSERT_EQ(var_i8->dims[0], Ny); - ASSERT_EQ(var_i8->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_i16 = adios_inq_var(f, "i16"); - EXPECT_TRUE(var_i16); - ASSERT_EQ(var_i16->ndim, 2); - ASSERT_EQ(var_i16->global, 1); - ASSERT_EQ(var_i16->nsteps, NSteps); - ASSERT_EQ(var_i16->dims[0], Ny); - ASSERT_EQ(var_i16->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_i32 = adios_inq_var(f, "i32"); - EXPECT_TRUE(var_i32); - ASSERT_EQ(var_i32->ndim, 2); - ASSERT_EQ(var_i32->global, 1); - ASSERT_EQ(var_i32->nsteps, NSteps); - ASSERT_EQ(var_i32->dims[0], Ny); - ASSERT_EQ(var_i32->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_i64 = adios_inq_var(f, "i64"); - EXPECT_TRUE(var_i64); - ASSERT_EQ(var_i64->ndim, 2); - ASSERT_EQ(var_i64->global, 1); - ASSERT_EQ(var_i64->nsteps, NSteps); - ASSERT_EQ(var_i64->dims[0], Ny); - ASSERT_EQ(var_i64->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u8 = adios_inq_var(f, "u8"); - EXPECT_TRUE(var_u8); - ASSERT_EQ(var_u8->ndim, 2); - ASSERT_EQ(var_u8->global, 1); - ASSERT_EQ(var_u8->nsteps, NSteps); - ASSERT_EQ(var_u8->dims[0], Ny); - ASSERT_EQ(var_u8->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u16 = adios_inq_var(f, "u16"); - EXPECT_TRUE(var_u16); - ASSERT_EQ(var_u16->ndim, 2); - ASSERT_EQ(var_u16->global, 1); - ASSERT_EQ(var_u16->nsteps, NSteps); - ASSERT_EQ(var_u16->dims[0], Ny); - ASSERT_EQ(var_u16->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u32 = adios_inq_var(f, "u32"); - EXPECT_TRUE(var_u32); - ASSERT_EQ(var_u32->ndim, 2); - ASSERT_EQ(var_u32->global, 1); - ASSERT_EQ(var_u32->nsteps, NSteps); - ASSERT_EQ(var_u32->dims[0], Ny); - ASSERT_EQ(var_u32->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u64 = adios_inq_var(f, "u64"); - EXPECT_TRUE(var_u64); - ASSERT_EQ(var_u64->ndim, 2); - ASSERT_EQ(var_u64->global, 1); - ASSERT_EQ(var_u64->nsteps, NSteps); - ASSERT_EQ(var_u64->dims[0], Ny); - ASSERT_EQ(var_u64->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_r32 = adios_inq_var(f, "r32"); - EXPECT_TRUE(var_r32); - ASSERT_EQ(var_r32->ndim, 2); - ASSERT_EQ(var_r32->global, 1); - ASSERT_EQ(var_r32->nsteps, NSteps); - ASSERT_EQ(var_r32->dims[0], Ny); - ASSERT_EQ(var_r32->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_r64 = adios_inq_var(f, "r64"); - EXPECT_TRUE(var_r64); - ASSERT_EQ(var_r64->ndim, 2); - ASSERT_EQ(var_r64->global, 1); - ASSERT_EQ(var_r64->nsteps, NSteps); - ASSERT_EQ(var_r64->dims[0], Ny); - ASSERT_EQ(var_r64->dims[1], mpiSize * Nx); - - // If the size of the array is smaller than the data - // the result is weird... double and uint64_t would get completely - // garbage data - std::array I8; - std::array I16; - std::array I32; - std::array I64; - std::array U8; - std::array U16; - std::array U32; - std::array U64; - std::array R32; - std::array R64; - - uint64_t start[2] = {0, mpiRank * Nx}; - uint64_t count[2] = {Ny, Nx}; - ADIOS_SELECTION *sel = adios_selection_boundingbox(2, start, count); - - // Read stuff - for (size_t t = 0; t < NSteps; ++t) - { - // Generate test data for each rank uniquely - SmallTestData currentTestData = - generateNewSmallTestData(m_TestData, t, mpiRank, mpiSize); - // Read the current step - adios_schedule_read_byid(f, sel, var_i8->varid, t, 1, I8.data()); - adios_schedule_read_byid(f, sel, var_i16->varid, t, 1, I16.data()); - adios_schedule_read_byid(f, sel, var_i32->varid, t, 1, I32.data()); - adios_schedule_read_byid(f, sel, var_i64->varid, t, 1, I64.data()); - adios_schedule_read_byid(f, sel, var_u8->varid, t, 1, U8.data()); - adios_schedule_read_byid(f, sel, var_u16->varid, t, 1, U16.data()); - adios_schedule_read_byid(f, sel, var_u32->varid, t, 1, U32.data()); - adios_schedule_read_byid(f, sel, var_u64->varid, t, 1, U64.data()); - adios_schedule_read_byid(f, sel, var_r32->varid, t, 1, R32.data()); - adios_schedule_read_byid(f, sel, var_r64->varid, t, 1, R64.data()); - adios_perform_reads(f, 1); - - // Check if it's correct - for (size_t i = 0; i < Nx; ++i) - { - std::stringstream ss; - ss << "t=" << t << " i=" << i << " rank=" << mpiRank; - std::string msg = ss.str(); - - EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg; - EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg; - EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg; - EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg; - EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg; - EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg; - EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg; - EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg; - EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; - EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; - } - } - - adios_selection_delete(sel); - - // Cleanup variable structures - adios_free_varinfo(var_i8); - adios_free_varinfo(var_i16); - adios_free_varinfo(var_i32); - adios_free_varinfo(var_i64); - adios_free_varinfo(var_u8); - adios_free_varinfo(var_u16); - adios_free_varinfo(var_u32); - adios_free_varinfo(var_u64); - adios_free_varinfo(var_r32); - adios_free_varinfo(var_r64); - - // Cleanup file - adios_read_close(f); - - adios_read_finalize_method(ADIOS_READ_METHOD_BP); - } -} - -// ADIOS2 write, native ADIOS1 read -TEST_F(BPWriteReadTest, ADIOS2BPWriteADIOS1Read2D4x2) -{ - // Each process would write a 4x2 array and all processes would - // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - std::string fname = "ADIOS2BPWriteADIOS1Read2D4x2Test.bp"; - - int mpiRank = 0, mpiSize = 1; - // Number of rows - const std::size_t Nx = 2; - // Number of cols - const std::size_t Ny = 4; - - // Number of steps - const std::size_t NSteps = 3; - -#if ADIOS2_USE_MPI - MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); - MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); -#endif - - // Write test data using ADIOS2 - { -#if ADIOS2_USE_MPI - adios2::ADIOS adios(MPI_COMM_WORLD); -#else - adios2::ADIOS adios; -#endif - adios2::IO io = adios.DeclareIO("TestIO"); - - // Declare 2D variables (4 * (NumberOfProcess * Nx)) - // The local process' part (start, count) can be defined now or later - // before Write(). - { - adios2::Dims shape{static_cast(Ny), - static_cast(mpiSize * Nx)}; - adios2::Dims start{static_cast(0), - static_cast(mpiRank * Nx)}; - adios2::Dims count{static_cast(Ny), - static_cast(Nx)}; - auto var_i8 = io.DefineVariable("i8", shape, start, count); - auto var_i16 = - io.DefineVariable("i16", shape, start, count); - auto var_i32 = - io.DefineVariable("i32", shape, start, count); - auto var_i64 = - io.DefineVariable("i64", shape, start, count); - auto var_u8 = io.DefineVariable("u8", shape, start, count); - auto var_u16 = - io.DefineVariable("u16", shape, start, count); - auto var_u32 = - io.DefineVariable("u32", shape, start, count); - auto var_u64 = - io.DefineVariable("u64", shape, start, count); - auto var_r32 = io.DefineVariable("r32", shape, start, count); - auto var_r64 = - io.DefineVariable("r64", shape, start, count); - } - - if (!engineName.empty()) - { - io.SetEngine(engineName); - } - else - { - // Create the BP Engine - io.SetEngine("BPFile"); - } - - io.AddTransport("file"); - - adios2::Engine engine = io.Open(fname, adios2::Mode::Write); - - for (size_t step = 0; step < NSteps; ++step) - { - // Generate test data for each process uniquely - SmallTestData currentTestData = - generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize); - - // Retrieve the variables that previously went out of scope - auto var_i8 = io.InquireVariable("i8"); - auto var_i16 = io.InquireVariable("i16"); - auto var_i32 = io.InquireVariable("i32"); - auto var_i64 = io.InquireVariable("i64"); - auto var_u8 = io.InquireVariable("u8"); - auto var_u16 = io.InquireVariable("u16"); - auto var_u32 = io.InquireVariable("u32"); - auto var_u64 = io.InquireVariable("u64"); - auto var_r32 = io.InquireVariable("r32"); - auto var_r64 = io.InquireVariable("r64"); - - // Make a 2D selection to describe the local dimensions of the - // variable we write and its offsets in the global spaces - adios2::Box sel( - {0, static_cast(mpiRank * Nx)}, {Ny, Nx}); - var_i8.SetSelection(sel); - var_i16.SetSelection(sel); - var_i32.SetSelection(sel); - var_i64.SetSelection(sel); - var_u8.SetSelection(sel); - var_u16.SetSelection(sel); - var_u32.SetSelection(sel); - var_u64.SetSelection(sel); - var_r32.SetSelection(sel); - var_r64.SetSelection(sel); - - // Write each one - // fill in the variable with values from starting index to - // starting index + count - engine.BeginStep(); - engine.Put(var_i8, currentTestData.I8.data()); - engine.Put(var_i16, currentTestData.I16.data()); - engine.Put(var_i32, currentTestData.I32.data()); - engine.Put(var_i64, currentTestData.I64.data()); - engine.Put(var_u8, currentTestData.U8.data()); - engine.Put(var_u16, currentTestData.U16.data()); - engine.Put(var_u32, currentTestData.U32.data()); - engine.Put(var_u64, currentTestData.U64.data()); - engine.Put(var_r32, currentTestData.R32.data()); - engine.Put(var_r64, currentTestData.R64.data()); - engine.EndStep(); - } - - // Close the file - engine.Close(); - } - - { - adios_read_init_method(ADIOS_READ_METHOD_BP, MPI_COMM_SELF, - "verbose=3"); - - // Open the file for reading - // Note: Since collective metadata generation is not implemented yet, - // SO for now we read each subfile instead of a single bp file with all - // metadata. - // Meanwhile if we open file with MPI_COMM_WORLD, then the selection - // bounding box should be [0, Nx] - std::string index = std::to_string(mpiRank); - ADIOS_FILE *f = adios_read_open_file( - (fname + ".dir/" + fname + "." + index).c_str(), - ADIOS_READ_METHOD_BP, MPI_COMM_SELF); - ASSERT_NE(f, nullptr); - - // Check the variables exist - ADIOS_VARINFO *var_i8 = adios_inq_var(f, "i8"); - EXPECT_TRUE(var_i8); - ASSERT_EQ(var_i8->ndim, 2); - ASSERT_EQ(var_i8->global, 1); - ASSERT_EQ(var_i8->nsteps, NSteps); - ASSERT_EQ(var_i8->dims[0], Ny); - ASSERT_EQ(var_i8->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_i16 = adios_inq_var(f, "i16"); - EXPECT_TRUE(var_i16); - ASSERT_EQ(var_i16->ndim, 2); - ASSERT_EQ(var_i16->global, 1); - ASSERT_EQ(var_i16->nsteps, NSteps); - ASSERT_EQ(var_i16->dims[0], Ny); - ASSERT_EQ(var_i16->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_i32 = adios_inq_var(f, "i32"); - EXPECT_TRUE(var_i32); - ASSERT_EQ(var_i32->ndim, 2); - ASSERT_EQ(var_i32->global, 1); - ASSERT_EQ(var_i32->nsteps, NSteps); - ASSERT_EQ(var_i32->dims[0], Ny); - ASSERT_EQ(var_i32->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_i64 = adios_inq_var(f, "i64"); - EXPECT_TRUE(var_i64); - ASSERT_EQ(var_i64->ndim, 2); - ASSERT_EQ(var_i64->global, 1); - ASSERT_EQ(var_i64->nsteps, NSteps); - ASSERT_EQ(var_i64->dims[0], Ny); - ASSERT_EQ(var_i64->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u8 = adios_inq_var(f, "u8"); - EXPECT_TRUE(var_u8); - ASSERT_EQ(var_u8->ndim, 2); - ASSERT_EQ(var_u8->global, 1); - ASSERT_EQ(var_u8->nsteps, NSteps); - ASSERT_EQ(var_u8->dims[0], Ny); - ASSERT_EQ(var_u8->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u16 = adios_inq_var(f, "u16"); - EXPECT_TRUE(var_u16); - ASSERT_EQ(var_u16->ndim, 2); - ASSERT_EQ(var_u16->global, 1); - ASSERT_EQ(var_u16->nsteps, NSteps); - ASSERT_EQ(var_u16->dims[0], Ny); - ASSERT_EQ(var_u16->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u32 = adios_inq_var(f, "u32"); - EXPECT_TRUE(var_u32); - ASSERT_EQ(var_u32->ndim, 2); - ASSERT_EQ(var_u32->global, 1); - ASSERT_EQ(var_u32->nsteps, NSteps); - ASSERT_EQ(var_u32->dims[0], Ny); - ASSERT_EQ(var_u32->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u64 = adios_inq_var(f, "u64"); - EXPECT_TRUE(var_u64); - ASSERT_EQ(var_u64->ndim, 2); - ASSERT_EQ(var_u64->global, 1); - ASSERT_EQ(var_u64->nsteps, NSteps); - ASSERT_EQ(var_u64->dims[0], Ny); - ASSERT_EQ(var_u64->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_r32 = adios_inq_var(f, "r32"); - EXPECT_TRUE(var_r32); - ASSERT_EQ(var_r32->ndim, 2); - ASSERT_EQ(var_r32->global, 1); - ASSERT_EQ(var_r32->nsteps, NSteps); - ASSERT_EQ(var_r32->dims[0], Ny); - ASSERT_EQ(var_r32->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_r64 = adios_inq_var(f, "r64"); - EXPECT_TRUE(var_r64); - ASSERT_EQ(var_r64->ndim, 2); - ASSERT_EQ(var_r64->global, 1); - ASSERT_EQ(var_r64->nsteps, NSteps); - ASSERT_EQ(var_r64->dims[0], Ny); - ASSERT_EQ(var_r64->dims[1], mpiSize * Nx); - - // If the size of the array is smaller than the data - // the result is weird... double and uint64_t would get completely - // garbage data - std::array I8; - std::array I16; - std::array I32; - std::array I64; - std::array U8; - std::array U16; - std::array U32; - std::array U64; - std::array R32; - std::array R64; - - uint64_t start[2] = {0, mpiRank * Nx}; - uint64_t count[2] = {Ny, Nx}; - ADIOS_SELECTION *sel = adios_selection_boundingbox(2, start, count); - - // Read stuff - for (size_t t = 0; t < NSteps; ++t) - { - // Generate test data for each rank uniquely - SmallTestData currentTestData = - generateNewSmallTestData(m_TestData, t, mpiRank, mpiSize); - // Read the current step - adios_schedule_read_byid(f, sel, var_i8->varid, t, 1, I8.data()); - adios_schedule_read_byid(f, sel, var_i16->varid, t, 1, I16.data()); - adios_schedule_read_byid(f, sel, var_i32->varid, t, 1, I32.data()); - adios_schedule_read_byid(f, sel, var_i64->varid, t, 1, I64.data()); - adios_schedule_read_byid(f, sel, var_u8->varid, t, 1, U8.data()); - adios_schedule_read_byid(f, sel, var_u16->varid, t, 1, U16.data()); - adios_schedule_read_byid(f, sel, var_u32->varid, t, 1, U32.data()); - adios_schedule_read_byid(f, sel, var_u64->varid, t, 1, U64.data()); - adios_schedule_read_byid(f, sel, var_r32->varid, t, 1, R32.data()); - adios_schedule_read_byid(f, sel, var_r64->varid, t, 1, R64.data()); - adios_perform_reads(f, 1); - - // Check if it's correct - for (size_t i = 0; i < Nx; ++i) - { - std::stringstream ss; - ss << "t=" << t << " i=" << i << " rank=" << mpiRank; - std::string msg = ss.str(); - - EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg; - EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg; - EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg; - EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg; - EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg; - EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg; - EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg; - EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg; - EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; - EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; - } - } - - adios_selection_delete(sel); - - // Cleanup variable structures - adios_free_varinfo(var_i8); - adios_free_varinfo(var_i16); - adios_free_varinfo(var_i32); - adios_free_varinfo(var_i64); - adios_free_varinfo(var_u8); - adios_free_varinfo(var_u16); - adios_free_varinfo(var_u32); - adios_free_varinfo(var_u64); - adios_free_varinfo(var_r32); - adios_free_varinfo(var_r64); - - // Cleanup file - adios_read_close(f); - - adios_read_finalize_method(ADIOS_READ_METHOD_BP); - } -} - -//****************************************************************************** -// main -//****************************************************************************** - -int main(int argc, char **argv) -{ -#if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); -#endif - - int result; - ::testing::InitGoogleTest(&argc, argv); - - if (argc > 1) - { - engineName = std::string(argv[1]); - } - result = RUN_ALL_TESTS(); - -#if ADIOS2_USE_MPI - MPI_Finalize(); -#endif - - return result; -} diff --git a/testing/adios2/engine/bp/TestBPWriteReadADIOS2.cpp b/testing/adios2/engine/bp/TestBPWriteReadADIOS2.cpp index 5ba96201cd..e43cb3e919 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadADIOS2.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadADIOS2.cpp @@ -224,10 +224,10 @@ TEST_F(BPWriteReadTestADIOS2, ADIOS2BPWriteRead1D8) io.SetEngine(engineName); } - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); EXPECT_EQ(bpReader.Steps(), NSteps); - EXPECT_EQ(bpReader.OpenMode(), adios2::Mode::Read); // auto var_bool = io.InquireVariable("bool"); // EXPECT_TRUE(var_bool); @@ -566,7 +566,8 @@ TEST_F(BPWriteReadTestADIOS2, ADIOS2BPWriteRead2D2x4) io.SetEngine(engineName); } - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); EXPECT_EQ(bpReader.Steps(), NSteps); auto var_iString = io.InquireVariable("iString"); @@ -883,7 +884,8 @@ TEST_F(BPWriteReadTestADIOS2, ADIOS2BPWriteRead2D4x2) io.SetEngine(engineName); } - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); EXPECT_EQ(bpReader.Steps(), NSteps); @@ -1148,7 +1150,8 @@ TEST_F(BPWriteReadTestADIOS2, ADIOS2BPWriteRead10D2x2) io.SetEngine(engineName); } - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); EXPECT_EQ(bpReader.Steps(), NSteps); @@ -1366,7 +1369,8 @@ TEST_F(BPWriteReadTestADIOS2, ADIOS2BPWriteRead2D4x2_ReadMultiSteps) io.SetEngine(engineName); } - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); EXPECT_EQ(bpReader.Steps(), NSteps); @@ -1673,7 +1677,8 @@ TEST_F(BPWriteReadTestADIOS2, ADIOS2BPWriteRead2D4x2_MultiStepsOverflow) io.SetEngine(engineName); } - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); auto var_i8 = io.InquireVariable("i8"); auto var_i16 = io.InquireVariable("i16"); @@ -1779,7 +1784,8 @@ TEST_F(BPWriteReadTestADIOS2, OpenEngineTwice) bpWriter.Close(); EXPECT_NO_THROW(io.Open(fname, adios2::Mode::Write)); - EXPECT_THROW(io.Open(fname, adios2::Mode::Read), std::invalid_argument); + EXPECT_THROW(io.Open(fname, adios2::Mode::ReadRandomAccess), + std::invalid_argument); } } @@ -1830,7 +1836,8 @@ TEST_F(BPWriteReadTestADIOS2, ReadStartCount) io.SetEngine(engineName); } - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); adios2::Variable varRange = io.InquireVariable("range"); @@ -1858,6 +1865,155 @@ TEST_F(BPWriteReadTestADIOS2, ReadStartCount) } } +//*************************************************** +// 1D test where some process does not write anything +//*************************************************** + +// ADIOS2 BP write and read 1D arrays +TEST_F(BPWriteReadTestADIOS2, ADIOS2BPWriteReadEmptyProcess) +{ +#if ADIOS2_USE_MPI + // Each process, except rank 0 would write a 1x8 array and all + // processes would form a (mpiSize-1) * Nx 1D array + const std::string fname("ADIOS2BPWriteReadEmptyProces.bp"); + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const size_t Nx = 8; + + // Number of steps + const size_t NSteps = 3; + + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + + /* This is a parallel test, do not run in serial */ + adios2::ADIOS adios(MPI_COMM_WORLD); + { + adios2::IO io = adios.DeclareIO("TestIO"); + // Declare 1D variables (NumOfProcesses * Nx) + // The local process' part (start, count) can be defined now or later + // before Write(). + + adios2::Dims shape{static_cast(Nx * (mpiSize - 1))}; + adios2::Dims start{static_cast(Nx * (mpiRank - 1))}; + adios2::Dims count{Nx}; + if (!mpiRank) + { + count[0] = 0; + start[0] = 0; + } + + auto var_r32 = io.DefineVariable("r32", shape, start, count); + EXPECT_TRUE(var_r32); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + else + { + // Create the BP Engine + io.SetEngine("BPFile"); + } + + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + // Generate test data for each process uniquely + SmallTestData currentTestData = generateNewSmallTestData( + m_TestData, static_cast(step), mpiRank, mpiSize); + + bpWriter.BeginStep(); + if (!mpiRank) + { + // in first step, rank 0 does not call Put + // in second step, it calls with a zero sized array + // in third step, it calls with nullptr + if (step == 1) + { + std::vector zero; + bpWriter.Put(var_r32, zero.data()); + } + else if (step == 2) + { + bpWriter.Put(var_r32, (float *)0); + } + } + else + { + bpWriter.Put(var_r32, currentTestData.R32.data()); + } + + bpWriter.EndStep(); + } + + // Close the file + bpWriter.Close(); + } + + { + adios2::IO io = adios.DeclareIO("ReadIO"); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + + adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + + for (size_t step = 0; step < NSteps; ++step) + { + // Generate test data for each process uniquely + SmallTestData currentTestData = generateNewSmallTestData( + m_TestData, static_cast(step), mpiRank + 1, mpiSize); + + bpReader.BeginStep(); + + auto var_r32 = io.InquireVariable("r32"); + EXPECT_TRUE(var_r32); + ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32.Shape()[0], (mpiSize - 1) * Nx); + + SmallTestData testData; + std::array R32; + + // last process does not read + // readers 0..N-2, while data was produced by 1..N-1 + adios2::Dims start{mpiRank * Nx}; + adios2::Dims count{Nx}; + + if (mpiRank == mpiSize - 1) + { + count[0] = 0; + start[0] = 0; + } + + const adios2::Box sel(start, count); + var_r32.SetSelection(sel); + + if (mpiRank < mpiSize - 1) + { + bpReader.Get(var_r32, R32.data(), adios2::Mode::Sync); + for (size_t i = 0; i < Nx; ++i) + { + std::stringstream ss; + ss << "t=" << step << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; + } + } + + bpReader.EndStep(); + } + bpReader.Close(); + } +#else + return; +#endif +} + //****************************************************************************** // main //****************************************************************************** diff --git a/testing/adios2/engine/bp/TestBPWriteReadADIOS2fstream.cpp b/testing/adios2/engine/bp/TestBPWriteReadADIOS2fstream.cpp index 35657be2fb..3d36128d2f 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadADIOS2fstream.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadADIOS2fstream.cpp @@ -184,7 +184,8 @@ TEST_F(BPWriteReadTestADIOS2fstream, ADIOS2BPWriteRead1D8) } io.AddTransport("file", {{"Library", "fstream"}}); - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); auto var_iString = io.InquireVariable("iString"); EXPECT_TRUE(var_iString); @@ -531,7 +532,8 @@ TEST_F(BPWriteReadTestADIOS2fstream, ADIOS2BPWriteRead2D2x4) } io.AddTransport("file", {{"Library", "fstream"}}); - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); auto var_iString = io.InquireVariable("iString"); EXPECT_TRUE(var_iString); @@ -878,7 +880,8 @@ TEST_F(BPWriteReadTestADIOS2fstream, ADIOS2BPWriteRead2D4x2) } io.AddTransport("file", {{"Library", "fstream"}}); - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); auto var_i8 = io.InquireVariable("i8"); EXPECT_TRUE(var_i8); @@ -1214,7 +1217,8 @@ TEST_F(BPWriteReadTestADIOS2fstream, ADIOS2BPWriteRead2D4x2_ReadMultiSteps) } io.AddTransport("file", {{"Library", "fstream"}}); - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); auto var_i8 = io.InquireVariable("i8"); EXPECT_TRUE(var_i8); @@ -1539,7 +1543,8 @@ TEST_F(BPWriteReadTestADIOS2fstream, ADIOS2BPWriteRead2D4x2_MultiStepsOverflow) } io.AddTransport("file", {{"Library", "fstream"}}); - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); auto var_i8 = io.InquireVariable("i8"); auto var_i16 = io.InquireVariable("i16"); @@ -1640,7 +1645,8 @@ TEST_F(BPWriteReadTestADIOS2fstream, OpenEngineTwice) bpWriter.Close(); EXPECT_NO_THROW(io.Open(fname, adios2::Mode::Write)); - EXPECT_THROW(io.Open(fname, adios2::Mode::Read), std::invalid_argument); + EXPECT_THROW(io.Open(fname, adios2::Mode::ReadRandomAccess), + std::invalid_argument); } } diff --git a/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2.cpp b/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2.cpp index a0b868afcf..d9d41f6b29 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2.cpp @@ -211,7 +211,6 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ADIOS2BPWriteRead1D8) EXPECT_TRUE(var_iString); EXPECT_TRUE(var_iString); ASSERT_EQ(var_iString.Shape().size(), 0); - ASSERT_EQ(var_iString.Steps(), 1); } else { @@ -230,21 +229,17 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ADIOS2BPWriteRead1D8) EXPECT_FALSE(var_u64); ASSERT_EQ(var_i8.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i8.Steps(), NSteps / 2 + NSteps % 2); ASSERT_EQ(var_i8.Shape()[0], static_cast(mpiSize * Nx)); ASSERT_EQ(var_i16.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i16.Steps(), NSteps / 2 + NSteps % 2); ASSERT_EQ(var_i16.Shape()[0], static_cast(mpiSize * Nx)); ASSERT_EQ(var_i32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i32.Steps(), NSteps / 2 + NSteps % 2); ASSERT_EQ(var_i32.Shape()[0], static_cast(mpiSize * Nx)); ASSERT_EQ(var_i64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i64.Steps(), NSteps / 2 + NSteps % 2); ASSERT_EQ(var_i64.Shape()[0], static_cast(mpiSize * Nx)); @@ -271,19 +266,15 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ADIOS2BPWriteRead1D8) EXPECT_TRUE(var_u64); ASSERT_EQ(var_u8.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u8.Steps(), NSteps / 2); ASSERT_EQ(var_u8.Shape()[0], mpiSize * Nx); ASSERT_EQ(var_u16.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u16.Steps(), NSteps / 2); ASSERT_EQ(var_u16.Shape()[0], mpiSize * Nx); ASSERT_EQ(var_u32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u32.Steps(), NSteps / 2); ASSERT_EQ(var_u32.Shape()[0], mpiSize * Nx); ASSERT_EQ(var_u64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u64.Steps(), NSteps / 2); ASSERT_EQ(var_u64.Shape()[0], mpiSize * Nx); var_u8.SetSelection(sel); @@ -308,11 +299,9 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ADIOS2BPWriteRead1D8) EXPECT_TRUE(var_r64); ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_r32.Steps(), NSteps - 1); ASSERT_EQ(var_r32.Shape()[0], mpiSize * Nx); ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_r64.Steps(), NSteps - 1); ASSERT_EQ(var_r64.Shape()[0], mpiSize * Nx); var_r32.SetSelection(sel); @@ -326,11 +315,9 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ADIOS2BPWriteRead1D8) EXPECT_TRUE(var_cr64); ASSERT_EQ(var_cr32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_cr32.Steps(), NSteps); ASSERT_EQ(var_cr32.Shape()[0], mpiSize * Nx); ASSERT_EQ(var_cr64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_cr64.Steps(), NSteps); ASSERT_EQ(var_cr64.Shape()[0], mpiSize * Nx); var_cr32.SetSelection(sel); @@ -525,70 +512,60 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ADIOS2BPWriteRead2D2x4) auto var_i8 = io.InquireVariable("i8"); EXPECT_TRUE(var_i8); ASSERT_EQ(var_i8.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i8.Steps(), NSteps); ASSERT_EQ(var_i8.Shape()[0], Ny); ASSERT_EQ(var_i8.Shape()[1], static_cast(mpiSize * Nx)); auto var_i16 = io.InquireVariable("i16"); EXPECT_TRUE(var_i16); ASSERT_EQ(var_i16.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i16.Steps(), NSteps); ASSERT_EQ(var_i16.Shape()[0], Ny); ASSERT_EQ(var_i16.Shape()[1], static_cast(mpiSize * Nx)); auto var_i32 = io.InquireVariable("i32"); EXPECT_TRUE(var_i32); ASSERT_EQ(var_i32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i32.Steps(), NSteps); ASSERT_EQ(var_i32.Shape()[0], Ny); ASSERT_EQ(var_i32.Shape()[1], static_cast(mpiSize * Nx)); auto var_i64 = io.InquireVariable("i64"); EXPECT_TRUE(var_i64); ASSERT_EQ(var_i64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i64.Steps(), NSteps); ASSERT_EQ(var_i64.Shape()[0], Ny); ASSERT_EQ(var_i64.Shape()[1], static_cast(mpiSize * Nx)); auto var_u8 = io.InquireVariable("u8"); EXPECT_TRUE(var_u8); ASSERT_EQ(var_u8.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u8.Steps(), NSteps); ASSERT_EQ(var_u8.Shape()[0], Ny); ASSERT_EQ(var_u8.Shape()[1], static_cast(mpiSize * Nx)); auto var_u16 = io.InquireVariable("u16"); EXPECT_TRUE(var_u16); ASSERT_EQ(var_u16.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u16.Steps(), NSteps); ASSERT_EQ(var_u16.Shape()[0], Ny); ASSERT_EQ(var_u16.Shape()[1], static_cast(mpiSize * Nx)); auto var_u32 = io.InquireVariable("u32"); EXPECT_TRUE(var_u32); ASSERT_EQ(var_u32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u32.Steps(), NSteps); ASSERT_EQ(var_u32.Shape()[0], Ny); ASSERT_EQ(var_u32.Shape()[1], static_cast(mpiSize * Nx)); auto var_u64 = io.InquireVariable("u64"); EXPECT_TRUE(var_u64); ASSERT_EQ(var_u64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u64.Steps(), NSteps); ASSERT_EQ(var_u64.Shape()[0], Ny); ASSERT_EQ(var_u64.Shape()[1], static_cast(mpiSize * Nx)); auto var_r32 = io.InquireVariable("r32"); EXPECT_TRUE(var_r32); ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_r32.Steps(), NSteps); ASSERT_EQ(var_r32.Shape()[0], Ny); ASSERT_EQ(var_r32.Shape()[1], static_cast(mpiSize * Nx)); auto var_r64 = io.InquireVariable("r64"); EXPECT_TRUE(var_r64); ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_r64.Steps(), NSteps); ASSERT_EQ(var_r64.Shape()[0], Ny); ASSERT_EQ(var_r64.Shape()[1], static_cast(mpiSize * Nx)); @@ -790,70 +767,60 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ADIOS2BPWriteRead2D4x2) auto var_i8 = io.InquireVariable("i8"); EXPECT_TRUE(var_i8); ASSERT_EQ(var_i8.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i8.Steps(), NSteps); ASSERT_EQ(var_i8.Shape()[0], Ny); ASSERT_EQ(var_i8.Shape()[1], static_cast(mpiSize * Nx)); auto var_i16 = io.InquireVariable("i16"); EXPECT_TRUE(var_i16); ASSERT_EQ(var_i16.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i16.Steps(), NSteps); ASSERT_EQ(var_i16.Shape()[0], Ny); ASSERT_EQ(var_i16.Shape()[1], static_cast(mpiSize * Nx)); auto var_i32 = io.InquireVariable("i32"); EXPECT_TRUE(var_i32); ASSERT_EQ(var_i32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i32.Steps(), NSteps); ASSERT_EQ(var_i32.Shape()[0], Ny); ASSERT_EQ(var_i32.Shape()[1], static_cast(mpiSize * Nx)); auto var_i64 = io.InquireVariable("i64"); EXPECT_TRUE(var_i64); ASSERT_EQ(var_i64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i64.Steps(), NSteps); ASSERT_EQ(var_i64.Shape()[0], Ny); ASSERT_EQ(var_i64.Shape()[1], static_cast(mpiSize * Nx)); auto var_u8 = io.InquireVariable("u8"); EXPECT_TRUE(var_u8); ASSERT_EQ(var_u8.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u8.Steps(), NSteps); ASSERT_EQ(var_u8.Shape()[0], Ny); ASSERT_EQ(var_u8.Shape()[1], static_cast(mpiSize * Nx)); auto var_u16 = io.InquireVariable("u16"); EXPECT_TRUE(var_u16); ASSERT_EQ(var_u16.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u16.Steps(), NSteps); ASSERT_EQ(var_u16.Shape()[0], Ny); ASSERT_EQ(var_u16.Shape()[1], static_cast(mpiSize * Nx)); auto var_u32 = io.InquireVariable("u32"); EXPECT_TRUE(var_u32); ASSERT_EQ(var_u32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u32.Steps(), NSteps); ASSERT_EQ(var_u32.Shape()[0], Ny); ASSERT_EQ(var_u32.Shape()[1], static_cast(mpiSize * Nx)); auto var_u64 = io.InquireVariable("u64"); EXPECT_TRUE(var_u64); ASSERT_EQ(var_u64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u64.Steps(), NSteps); ASSERT_EQ(var_u64.Shape()[0], Ny); ASSERT_EQ(var_u64.Shape()[1], static_cast(mpiSize * Nx)); auto var_r32 = io.InquireVariable("r32"); EXPECT_TRUE(var_r32); ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_r32.Steps(), NSteps); ASSERT_EQ(var_r32.Shape()[0], Ny); ASSERT_EQ(var_r32.Shape()[1], static_cast(mpiSize * Nx)); auto var_r64 = io.InquireVariable("r64"); EXPECT_TRUE(var_r64); ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_r64.Steps(), NSteps); ASSERT_EQ(var_r64.Shape()[0], Ny); ASSERT_EQ(var_r64.Shape()[1], static_cast(mpiSize * Nx)); @@ -1042,7 +1009,7 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ReaderWriterDefineVariable) adios2::Variable varR32 = io.InquireVariable("r32"); EXPECT_TRUE(varR32); adios2::Variable varR64 = io.InquireVariable("r64"); - EXPECT_TRUE(varR32); + EXPECT_TRUE(varR64); reader.EndStep(); } reader.Close(); diff --git a/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2_Threads.cpp b/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2_Threads.cpp index 4630d387a9..397fac4600 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2_Threads.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2_Threads.cpp @@ -137,66 +137,6 @@ TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, ADIOS2BPWriteRead1D8) io.SetParameter("Threads", "2"); adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); - auto var_i8 = io.InquireVariable("i8"); - EXPECT_TRUE(var_i8); - ASSERT_EQ(var_i8.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i8.Steps(), NSteps); - ASSERT_EQ(var_i8.Shape()[0], mpiSize * Nx); - - auto var_i16 = io.InquireVariable("i16"); - EXPECT_TRUE(var_i16); - ASSERT_EQ(var_i16.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i16.Steps(), NSteps); - ASSERT_EQ(var_i16.Shape()[0], mpiSize * Nx); - - auto var_i32 = io.InquireVariable("i32"); - EXPECT_TRUE(var_i32); - ASSERT_EQ(var_i32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i32.Steps(), NSteps); - ASSERT_EQ(var_i32.Shape()[0], mpiSize * Nx); - - auto var_i64 = io.InquireVariable("i64"); - EXPECT_TRUE(var_i64); - ASSERT_EQ(var_i64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i64.Steps(), NSteps); - ASSERT_EQ(var_i64.Shape()[0], mpiSize * Nx); - - auto var_u8 = io.InquireVariable("u8"); - EXPECT_TRUE(var_u8); - ASSERT_EQ(var_u8.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u8.Steps(), NSteps); - ASSERT_EQ(var_u8.Shape()[0], mpiSize * Nx); - - auto var_u16 = io.InquireVariable("u16"); - EXPECT_TRUE(var_u16); - ASSERT_EQ(var_u16.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u16.Steps(), NSteps); - ASSERT_EQ(var_u16.Shape()[0], mpiSize * Nx); - - auto var_u32 = io.InquireVariable("u32"); - EXPECT_TRUE(var_u32); - ASSERT_EQ(var_u32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u32.Steps(), NSteps); - ASSERT_EQ(var_u32.Shape()[0], mpiSize * Nx); - - auto var_u64 = io.InquireVariable("u64"); - EXPECT_TRUE(var_u64); - ASSERT_EQ(var_u64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u64.Steps(), NSteps); - ASSERT_EQ(var_u64.Shape()[0], mpiSize * Nx); - - auto var_r32 = io.InquireVariable("r32"); - EXPECT_TRUE(var_r32); - ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_r32.Steps(), NSteps); - ASSERT_EQ(var_r32.Shape()[0], mpiSize * Nx); - - auto var_r64 = io.InquireVariable("r64"); - EXPECT_TRUE(var_r64); - ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_r64.Steps(), NSteps); - ASSERT_EQ(var_r64.Shape()[0], mpiSize * Nx); - std::string IString; std::array I8; std::array I16; @@ -214,19 +154,6 @@ TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, ADIOS2BPWriteRead1D8) const adios2::Box sel(start, count); - var_i8.SetSelection(sel); - var_i16.SetSelection(sel); - var_i32.SetSelection(sel); - var_i64.SetSelection(sel); - - var_u8.SetSelection(sel); - var_u16.SetSelection(sel); - var_u32.SetSelection(sel); - var_u64.SetSelection(sel); - - var_r32.SetSelection(sel); - var_r64.SetSelection(sel); - unsigned int t = 0; while (bpReader.BeginStep() == adios2::StepStatus::OK) @@ -234,6 +161,69 @@ TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, ADIOS2BPWriteRead1D8) const size_t currentStep = bpReader.CurrentStep(); EXPECT_EQ(currentStep, static_cast(t)); + auto var_i8 = io.InquireVariable("i8"); + EXPECT_TRUE(var_i8); + ASSERT_EQ(var_i8.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i8.Shape()[0], mpiSize * Nx); + + auto var_i16 = io.InquireVariable("i16"); + EXPECT_TRUE(var_i16); + ASSERT_EQ(var_i16.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i16.Shape()[0], mpiSize * Nx); + + auto var_i32 = io.InquireVariable("i32"); + EXPECT_TRUE(var_i32); + ASSERT_EQ(var_i32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i32.Shape()[0], mpiSize * Nx); + + auto var_i64 = io.InquireVariable("i64"); + EXPECT_TRUE(var_i64); + ASSERT_EQ(var_i64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i64.Shape()[0], mpiSize * Nx); + + auto var_u8 = io.InquireVariable("u8"); + EXPECT_TRUE(var_u8); + ASSERT_EQ(var_u8.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u8.Shape()[0], mpiSize * Nx); + + auto var_u16 = io.InquireVariable("u16"); + EXPECT_TRUE(var_u16); + ASSERT_EQ(var_u16.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u16.Shape()[0], mpiSize * Nx); + + auto var_u32 = io.InquireVariable("u32"); + EXPECT_TRUE(var_u32); + ASSERT_EQ(var_u32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u32.Shape()[0], mpiSize * Nx); + + auto var_u64 = io.InquireVariable("u64"); + EXPECT_TRUE(var_u64); + ASSERT_EQ(var_u64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u64.Shape()[0], mpiSize * Nx); + + auto var_r32 = io.InquireVariable("r32"); + EXPECT_TRUE(var_r32); + ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32.Shape()[0], mpiSize * Nx); + + auto var_r64 = io.InquireVariable("r64"); + EXPECT_TRUE(var_r64); + ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64.Shape()[0], mpiSize * Nx); + + var_i8.SetSelection(sel); + var_i16.SetSelection(sel); + var_i32.SetSelection(sel); + var_i64.SetSelection(sel); + + var_u8.SetSelection(sel); + var_u16.SetSelection(sel); + var_u32.SetSelection(sel); + var_u64.SetSelection(sel); + + var_r32.SetSelection(sel); + var_r64.SetSelection(sel); + SmallTestData currentTestData = generateNewSmallTestData( m_TestData, static_cast(currentStep), mpiRank, mpiSize); @@ -395,76 +385,6 @@ TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, ADIOS2BPWriteRead2D2x4) io.SetParameter("Threads", "2"); adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); - auto var_i8 = io.InquireVariable("i8"); - EXPECT_TRUE(var_i8); - ASSERT_EQ(var_i8.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i8.Steps(), NSteps); - ASSERT_EQ(var_i8.Shape()[0], Ny); - ASSERT_EQ(var_i8.Shape()[1], static_cast(mpiSize * Nx)); - - auto var_i16 = io.InquireVariable("i16"); - EXPECT_TRUE(var_i16); - ASSERT_EQ(var_i16.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i16.Steps(), NSteps); - ASSERT_EQ(var_i16.Shape()[0], Ny); - ASSERT_EQ(var_i16.Shape()[1], static_cast(mpiSize * Nx)); - - auto var_i32 = io.InquireVariable("i32"); - EXPECT_TRUE(var_i32); - ASSERT_EQ(var_i32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i32.Steps(), NSteps); - ASSERT_EQ(var_i32.Shape()[0], Ny); - ASSERT_EQ(var_i32.Shape()[1], static_cast(mpiSize * Nx)); - - auto var_i64 = io.InquireVariable("i64"); - EXPECT_TRUE(var_i64); - ASSERT_EQ(var_i64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i64.Steps(), NSteps); - ASSERT_EQ(var_i64.Shape()[0], Ny); - ASSERT_EQ(var_i64.Shape()[1], static_cast(mpiSize * Nx)); - - auto var_u8 = io.InquireVariable("u8"); - EXPECT_TRUE(var_u8); - ASSERT_EQ(var_u8.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u8.Steps(), NSteps); - ASSERT_EQ(var_u8.Shape()[0], Ny); - ASSERT_EQ(var_u8.Shape()[1], static_cast(mpiSize * Nx)); - - auto var_u16 = io.InquireVariable("u16"); - EXPECT_TRUE(var_u16); - ASSERT_EQ(var_u16.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u16.Steps(), NSteps); - ASSERT_EQ(var_u16.Shape()[0], Ny); - ASSERT_EQ(var_u16.Shape()[1], static_cast(mpiSize * Nx)); - - auto var_u32 = io.InquireVariable("u32"); - EXPECT_TRUE(var_u32); - ASSERT_EQ(var_u32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u32.Steps(), NSteps); - ASSERT_EQ(var_u32.Shape()[0], Ny); - ASSERT_EQ(var_u32.Shape()[1], static_cast(mpiSize * Nx)); - - auto var_u64 = io.InquireVariable("u64"); - EXPECT_TRUE(var_u64); - ASSERT_EQ(var_u64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u64.Steps(), NSteps); - ASSERT_EQ(var_u64.Shape()[0], Ny); - ASSERT_EQ(var_u64.Shape()[1], static_cast(mpiSize * Nx)); - - auto var_r32 = io.InquireVariable("r32"); - EXPECT_TRUE(var_r32); - ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_r32.Steps(), NSteps); - ASSERT_EQ(var_r32.Shape()[0], Ny); - ASSERT_EQ(var_r32.Shape()[1], static_cast(mpiSize * Nx)); - - auto var_r64 = io.InquireVariable("r64"); - EXPECT_TRUE(var_r64); - ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_r64.Steps(), NSteps); - ASSERT_EQ(var_r64.Shape()[0], Ny); - ASSERT_EQ(var_r64.Shape()[1], static_cast(mpiSize * Nx)); - std::array I8; std::array I16; std::array I32; @@ -481,19 +401,6 @@ TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, ADIOS2BPWriteRead2D2x4) const adios2::Box sel(start, count); - var_i8.SetSelection(sel); - var_i16.SetSelection(sel); - var_i32.SetSelection(sel); - var_i64.SetSelection(sel); - - var_u8.SetSelection(sel); - var_u16.SetSelection(sel); - var_u32.SetSelection(sel); - var_u64.SetSelection(sel); - - var_r32.SetSelection(sel); - var_r64.SetSelection(sel); - unsigned int t = 0; while (bpReader.BeginStep() == adios2::StepStatus::OK) @@ -501,6 +408,79 @@ TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, ADIOS2BPWriteRead2D2x4) const size_t currentStep = bpReader.CurrentStep(); EXPECT_EQ(currentStep, static_cast(t)); + auto var_i8 = io.InquireVariable("i8"); + EXPECT_TRUE(var_i8); + ASSERT_EQ(var_i8.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i8.Shape()[0], Ny); + ASSERT_EQ(var_i8.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_i16 = io.InquireVariable("i16"); + EXPECT_TRUE(var_i16); + ASSERT_EQ(var_i16.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i16.Shape()[0], Ny); + ASSERT_EQ(var_i16.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_i32 = io.InquireVariable("i32"); + EXPECT_TRUE(var_i32); + ASSERT_EQ(var_i32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i32.Shape()[0], Ny); + ASSERT_EQ(var_i32.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_i64 = io.InquireVariable("i64"); + EXPECT_TRUE(var_i64); + ASSERT_EQ(var_i64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i64.Shape()[0], Ny); + ASSERT_EQ(var_i64.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_u8 = io.InquireVariable("u8"); + EXPECT_TRUE(var_u8); + ASSERT_EQ(var_u8.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u8.Shape()[0], Ny); + ASSERT_EQ(var_u8.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_u16 = io.InquireVariable("u16"); + EXPECT_TRUE(var_u16); + ASSERT_EQ(var_u16.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u16.Shape()[0], Ny); + ASSERT_EQ(var_u16.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_u32 = io.InquireVariable("u32"); + EXPECT_TRUE(var_u32); + ASSERT_EQ(var_u32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u32.Shape()[0], Ny); + ASSERT_EQ(var_u32.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_u64 = io.InquireVariable("u64"); + EXPECT_TRUE(var_u64); + ASSERT_EQ(var_u64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u64.Shape()[0], Ny); + ASSERT_EQ(var_u64.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_r32 = io.InquireVariable("r32"); + EXPECT_TRUE(var_r32); + ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32.Shape()[0], Ny); + ASSERT_EQ(var_r32.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_r64 = io.InquireVariable("r64"); + EXPECT_TRUE(var_r64); + ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64.Shape()[0], Ny); + ASSERT_EQ(var_r64.Shape()[1], static_cast(mpiSize * Nx)); + + var_i8.SetSelection(sel); + var_i16.SetSelection(sel); + var_i32.SetSelection(sel); + var_i64.SetSelection(sel); + + var_u8.SetSelection(sel); + var_u16.SetSelection(sel); + var_u32.SetSelection(sel); + var_u64.SetSelection(sel); + + var_r32.SetSelection(sel); + var_r64.SetSelection(sel); + SmallTestData currentTestData = generateNewSmallTestData( m_TestData, static_cast(currentStep), mpiRank, mpiSize); @@ -664,76 +644,6 @@ TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, ADIOS2BPWriteRead2D4x2) io.SetParameter("Threads", "2"); adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); - auto var_i8 = io.InquireVariable("i8"); - EXPECT_TRUE(var_i8); - ASSERT_EQ(var_i8.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i8.Steps(), NSteps); - ASSERT_EQ(var_i8.Shape()[0], Ny); - ASSERT_EQ(var_i8.Shape()[1], static_cast(mpiSize * Nx)); - - auto var_i16 = io.InquireVariable("i16"); - EXPECT_TRUE(var_i16); - ASSERT_EQ(var_i16.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i16.Steps(), NSteps); - ASSERT_EQ(var_i16.Shape()[0], Ny); - ASSERT_EQ(var_i16.Shape()[1], static_cast(mpiSize * Nx)); - - auto var_i32 = io.InquireVariable("i32"); - EXPECT_TRUE(var_i32); - ASSERT_EQ(var_i32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i32.Steps(), NSteps); - ASSERT_EQ(var_i32.Shape()[0], Ny); - ASSERT_EQ(var_i32.Shape()[1], static_cast(mpiSize * Nx)); - - auto var_i64 = io.InquireVariable("i64"); - EXPECT_TRUE(var_i64); - ASSERT_EQ(var_i64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_i64.Steps(), NSteps); - ASSERT_EQ(var_i64.Shape()[0], Ny); - ASSERT_EQ(var_i64.Shape()[1], static_cast(mpiSize * Nx)); - - auto var_u8 = io.InquireVariable("u8"); - EXPECT_TRUE(var_u8); - ASSERT_EQ(var_u8.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u8.Steps(), NSteps); - ASSERT_EQ(var_u8.Shape()[0], Ny); - ASSERT_EQ(var_u8.Shape()[1], static_cast(mpiSize * Nx)); - - auto var_u16 = io.InquireVariable("u16"); - EXPECT_TRUE(var_u16); - ASSERT_EQ(var_u16.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u16.Steps(), NSteps); - ASSERT_EQ(var_u16.Shape()[0], Ny); - ASSERT_EQ(var_u16.Shape()[1], static_cast(mpiSize * Nx)); - - auto var_u32 = io.InquireVariable("u32"); - EXPECT_TRUE(var_u32); - ASSERT_EQ(var_u32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u32.Steps(), NSteps); - ASSERT_EQ(var_u32.Shape()[0], Ny); - ASSERT_EQ(var_u32.Shape()[1], static_cast(mpiSize * Nx)); - - auto var_u64 = io.InquireVariable("u64"); - EXPECT_TRUE(var_u64); - ASSERT_EQ(var_u64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_u64.Steps(), NSteps); - ASSERT_EQ(var_u64.Shape()[0], Ny); - ASSERT_EQ(var_u64.Shape()[1], static_cast(mpiSize * Nx)); - - auto var_r32 = io.InquireVariable("r32"); - EXPECT_TRUE(var_r32); - ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_r32.Steps(), NSteps); - ASSERT_EQ(var_r32.Shape()[0], Ny); - ASSERT_EQ(var_r32.Shape()[1], static_cast(mpiSize * Nx)); - - auto var_r64 = io.InquireVariable("r64"); - EXPECT_TRUE(var_r64); - ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); - ASSERT_EQ(var_r64.Steps(), NSteps); - ASSERT_EQ(var_r64.Shape()[0], Ny); - ASSERT_EQ(var_r64.Shape()[1], static_cast(mpiSize * Nx)); - // If the size of the array is smaller than the data // the result is weird... double and uint64_t would get // completely garbage data @@ -753,19 +663,6 @@ TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, ADIOS2BPWriteRead2D4x2) const adios2::Box sel(start, count); - var_i8.SetSelection(sel); - var_i16.SetSelection(sel); - var_i32.SetSelection(sel); - var_i64.SetSelection(sel); - - var_u8.SetSelection(sel); - var_u16.SetSelection(sel); - var_u32.SetSelection(sel); - var_u64.SetSelection(sel); - - var_r32.SetSelection(sel); - var_r64.SetSelection(sel); - unsigned int t = 0; while (bpReader.BeginStep() == adios2::StepStatus::OK) @@ -773,6 +670,79 @@ TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, ADIOS2BPWriteRead2D4x2) const size_t currentStep = bpReader.CurrentStep(); EXPECT_EQ(currentStep, static_cast(t)); + auto var_i8 = io.InquireVariable("i8"); + EXPECT_TRUE(var_i8); + ASSERT_EQ(var_i8.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i8.Shape()[0], Ny); + ASSERT_EQ(var_i8.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_i16 = io.InquireVariable("i16"); + EXPECT_TRUE(var_i16); + ASSERT_EQ(var_i16.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i16.Shape()[0], Ny); + ASSERT_EQ(var_i16.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_i32 = io.InquireVariable("i32"); + EXPECT_TRUE(var_i32); + ASSERT_EQ(var_i32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i32.Shape()[0], Ny); + ASSERT_EQ(var_i32.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_i64 = io.InquireVariable("i64"); + EXPECT_TRUE(var_i64); + ASSERT_EQ(var_i64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i64.Shape()[0], Ny); + ASSERT_EQ(var_i64.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_u8 = io.InquireVariable("u8"); + EXPECT_TRUE(var_u8); + ASSERT_EQ(var_u8.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u8.Shape()[0], Ny); + ASSERT_EQ(var_u8.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_u16 = io.InquireVariable("u16"); + EXPECT_TRUE(var_u16); + ASSERT_EQ(var_u16.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u16.Shape()[0], Ny); + ASSERT_EQ(var_u16.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_u32 = io.InquireVariable("u32"); + EXPECT_TRUE(var_u32); + ASSERT_EQ(var_u32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u32.Shape()[0], Ny); + ASSERT_EQ(var_u32.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_u64 = io.InquireVariable("u64"); + EXPECT_TRUE(var_u64); + ASSERT_EQ(var_u64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u64.Shape()[0], Ny); + ASSERT_EQ(var_u64.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_r32 = io.InquireVariable("r32"); + EXPECT_TRUE(var_r32); + ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32.Shape()[0], Ny); + ASSERT_EQ(var_r32.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_r64 = io.InquireVariable("r64"); + EXPECT_TRUE(var_r64); + ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64.Shape()[0], Ny); + ASSERT_EQ(var_r64.Shape()[1], static_cast(mpiSize * Nx)); + + var_i8.SetSelection(sel); + var_i16.SetSelection(sel); + var_i32.SetSelection(sel); + var_i64.SetSelection(sel); + + var_u8.SetSelection(sel); + var_u16.SetSelection(sel); + var_u32.SetSelection(sel); + var_u64.SetSelection(sel); + + var_r32.SetSelection(sel); + var_r64.SetSelection(sel); + SmallTestData currentTestData = generateNewSmallTestData( m_TestData, static_cast(currentStep), mpiRank, mpiSize); @@ -932,7 +902,8 @@ TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, io.SetParameter("Threads", "2"); - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); auto var_i8 = io.InquireVariable("i8"); EXPECT_TRUE(var_i8); @@ -1190,7 +1161,8 @@ TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, } io.SetParameter("Threads", "2"); - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); auto var_i8 = io.InquireVariable("i8"); EXPECT_TRUE(var_i8); @@ -1459,7 +1431,8 @@ TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, } io.SetParameter("Threads", "2"); - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); auto var_i8 = io.InquireVariable("i8"); EXPECT_TRUE(var_i8); diff --git a/testing/adios2/engine/bp/TestBPWriteReadAttributes.cpp b/testing/adios2/engine/bp/TestBPWriteReadAttributes.cpp index 59db39eb47..84f7e5f00c 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadAttributes.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadAttributes.cpp @@ -114,7 +114,8 @@ TEST_F(BPWriteReadAttributes, WriteReadSingleTypes) ioRead.SetEngine(engineName); } - adios2::Engine bpRead = ioRead.Open(fName, adios2::Mode::Read); + adios2::Engine bpRead = + ioRead.Open(fName, adios2::Mode::ReadRandomAccess); auto attr_s1 = ioRead.InquireAttribute(s1_Single); auto attr_s1a = ioRead.InquireAttribute(s1_Array); @@ -334,7 +335,8 @@ TEST_F(BPWriteReadAttributes, WriteReadArrayTypes) ioRead.SetEngine(engineName); } - adios2::Engine bpRead = ioRead.Open(fName, adios2::Mode::Read); + adios2::Engine bpRead = + ioRead.Open(fName, adios2::Mode::ReadRandomAccess); auto attr_s1 = ioRead.InquireAttribute(s1_Array); @@ -544,7 +546,8 @@ TEST_F(BPWriteReadAttributes, BPWriteReadSingleTypesVar) ioRead.SetEngine(engineName); } - adios2::Engine bpRead = ioRead.Open(fName, adios2::Mode::Read); + adios2::Engine bpRead = + ioRead.Open(fName, adios2::Mode::ReadRandomAccess); auto var = ioRead.InquireVariable("myVar"); @@ -767,7 +770,8 @@ TEST_F(BPWriteReadAttributes, WriteReadArrayTypesVar) ioRead.SetEngine(engineName); } - adios2::Engine bpRead = ioRead.Open(fName, adios2::Mode::Read); + adios2::Engine bpRead = + ioRead.Open(fName, adios2::Mode::ReadRandomAccess); auto var = ioRead.InquireVariable("myVar"); @@ -930,6 +934,15 @@ TEST_F(BPWriteReadAttributes, WriteReadStreamVar) adios2::IO io = adios.DeclareIO("TestIO"); + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + else + { + io.SetEngine("FileStream"); + } + auto var1 = io.DefineVariable("var1"); auto var2 = io.DefineVariable("var2", shape, start, count); @@ -1021,6 +1034,14 @@ TEST_F(BPWriteReadAttributes, WriteReadStreamVar) }; adios2::IO io = adios.DeclareIO("ReaderIO"); + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + else + { + io.SetEngine("FileStream"); + } adios2::Engine bpReader = io.Open(fName, adios2::Mode::Read); while (bpReader.BeginStep() == adios2::StepStatus::OK) @@ -1035,7 +1056,7 @@ TEST_F(BPWriteReadAttributes, WriteReadStreamVar) auto var2 = io.InquireVariable("var2"); if (var2) { - lf_VerifyAttributes("var1", separator, io, false); + lf_VerifyAttributes("var2", separator, io, false); lf_VerifyAttributes("var2", separator, io, true); } @@ -1043,6 +1064,168 @@ TEST_F(BPWriteReadAttributes, WriteReadStreamVar) } } } + +TEST_F(BPWriteReadAttributes, WriteReadStreamModifiable) +{ + const std::string fName = "foo" + std::string(&adios2::PathSeparator, 1) + + "AttributesWriteReadModifiable.bp"; + + const std::string separator = "\\"; + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const size_t Nx = 8; + + // Number of steps + const size_t NSteps = 3; + +#if ADIOS2_USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#endif + + const double d3[3] = {-1.1, -1.2, -1.3}; + SmallTestData currentTestData = + generateNewSmallTestData(m_TestData, 0, 0, 0); + +// Write test data using BP +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif + { + const adios2::Dims shape{static_cast(Nx * mpiSize)}; + const adios2::Dims start{static_cast(Nx * mpiRank)}; + const adios2::Dims count{Nx}; + + adios2::IO io = adios.DeclareIO("TestIO"); + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + else + { + io.SetEngine("FileStream"); + } + + auto var1 = io.DefineVariable("var1"); + auto var2 = io.DefineVariable("var2", shape, start, count); + + io.DefineAttribute("dArray", d3, 3, var1.Name(), separator, + true); + io.DefineAttribute("dArray", d3, 3, var2.Name(), separator, + true); + + io.DefineAttribute("i32Value", -1, var1.Name(), separator, + true); + io.DefineAttribute("i32Value", -1, var2.Name(), separator, + true); + + adios2::Engine bpWriter = io.Open(fName, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + // Generate test data for each process uniquely + SmallTestData currentTestData = generateNewSmallTestData( + m_TestData, static_cast(step), mpiRank, mpiSize); + + const int32_t step32 = static_cast(step); + const double stepD = static_cast(step); + double d[3] = {stepD + 0.1, stepD + 0.2, stepD + 0.3}; + + bpWriter.BeginStep(); + + io.DefineAttribute("dArray", d, 3, var1.Name(), separator, + true); + io.DefineAttribute("i32Value", step32, var1.Name(), + separator, true); + bpWriter.Put(var1, step32); + + if (step % 2 == 0) + { + bpWriter.Put(var2, currentTestData.I32.data()); + io.DefineAttribute("dArray", d, 3, var2.Name(), + separator, true); + io.DefineAttribute("i32Value", step32, var2.Name(), + separator, true); + } + + bpWriter.EndStep(); + } + bpWriter.Close(); + } + + // reader + { + auto lf_VerifyAttributes = [](const int32_t step, + const std::string &variableName, + const std::string separator, + adios2::IO &io) { + const std::map attributesInfo = + io.AvailableAttributes(variableName, separator, false); + + const double stepD = static_cast(step); + const double d[3] = {stepD + 0.1, stepD + 0.2, stepD + 0.3}; + + auto itDArray = attributesInfo.find("dArray"); + EXPECT_NE(itDArray, attributesInfo.end()); + EXPECT_EQ(itDArray->second.at("Type"), "double"); + EXPECT_EQ(itDArray->second.at("Elements"), "3"); + + auto a = + io.InquireAttribute("dArray", variableName, separator); + auto adata = a.Data(); + for (int i = 0; i < 3; ++i) + { + EXPECT_EQ(adata[i], d[i]); + } + + const std::string stepS = std::to_string(step); + auto iti32Value = attributesInfo.find("i32Value"); + EXPECT_NE(iti32Value, attributesInfo.end()); + EXPECT_EQ(iti32Value->second.at("Type"), "int32_t"); + EXPECT_EQ(iti32Value->second.at("Elements"), "1"); + EXPECT_EQ(iti32Value->second.at("Value"), stepS); + }; + + adios2::IO io = adios.DeclareIO("ReaderIO"); + if (!engineName.empty()) + { + io.SetEngine(engineName); + io.SetParameter("StreamReader", "ON"); + } + else + { + io.SetEngine("FileStream"); + } + adios2::Engine bpReader = io.Open(fName, adios2::Mode::Read); + + while (bpReader.BeginStep() == adios2::StepStatus::OK) + { + int32_t step = static_cast(bpReader.CurrentStep()); + if (engineName == "BP3") + { + // BP3 does not support changing attributes + step = 0; + } + auto var1 = io.InquireVariable("var1"); + if (var1) + { + lf_VerifyAttributes(step, "var1", separator, io); + } + + auto var2 = io.InquireVariable("var2"); + if (var2) + { + lf_VerifyAttributes(step, "var2", separator, io); + } + + bpReader.EndStep(); + } + } +} + //****************************************************************************** // main //****************************************************************************** diff --git a/testing/adios2/engine/bp/TestBPWriteReadAttributesMultirank.cpp b/testing/adios2/engine/bp/TestBPWriteReadAttributesMultirank.cpp index baddebc521..0feb3c1790 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadAttributesMultirank.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadAttributesMultirank.cpp @@ -57,6 +57,8 @@ TEST_F(BPWriteReadAttributeTestMultirank, ADIOS2BPWriteReadArrayTypes) auto var = io.DefineVariable(varpath); auto attr = io.DefineAttribute(attrpath, desc); + (void)var; + (void)attr; std::cout << "Rank " << mpiRank << " create variable " << varpath << " = " << mpiRank << " and attribute " << attrpath diff --git a/testing/adios2/engine/bp/TestBPWriteReadBlockInfo.cpp b/testing/adios2/engine/bp/TestBPWriteReadBlockInfo.cpp index d957915a60..c1921cce7c 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadBlockInfo.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadBlockInfo.cpp @@ -10,6 +10,7 @@ #include #include +#include #include @@ -50,6 +51,59 @@ void CheckAllStepsBlockInfo1D( } } +void CheckStepsBlockInfo1D_C(adios2_variable *var, adios2_varinfo *vi, + const size_t NSteps, const size_t nproc, + const size_t Nx) +{ + EXPECT_EQ(vi->nblocks, nproc); + adios2_shapeid shapeid; + adios2_variable_shapeid(&shapeid, var); + size_t namelen; + char name[128]; + adios2_variable_name(name, &namelen, var); + name[namelen] = '\0'; + // std::cout << "Check info on variable " << name << " shape = " << shapeid + // << std::endl; + if (shapeid == adios2_shapeid_global_value) + { + // std::cout << "Global Value " << std::endl; + EXPECT_EQ(vi->IsValue, 1); + EXPECT_EQ(vi->Dims, 0); + } + else if (shapeid == adios2_shapeid_global_array) + { + // std::cout << "Global Array " << std::endl; + EXPECT_EQ(vi->Dims, 1); + } + else + { + // std::cout << "Unexpected shape ID " << std::endl; + throw std::invalid_argument( + "Variable " + std::string(name) + + " is expected to be a global value or array "); + } + + EXPECT_FALSE(vi->IsReverseDims); + + if (vi->Dims > 0) + { + size_t shape; + adios2_variable_shape(&shape, var); + EXPECT_EQ(vi->Shape[0], shape); + } + + for (size_t b = 0; b < vi->nblocks; ++b) + { + EXPECT_EQ(vi->BlocksInfo[b].BlockID, b); + if (vi->Dims > 0) + { + EXPECT_EQ(vi->BlocksInfo[b].Start[0], b * Nx); + EXPECT_EQ(vi->BlocksInfo[b].Count[0], Nx); + } + EXPECT_EQ(vi->BlocksInfo[b].WriterID, b); + } +} + template void CheckAllStepsBlockInfo2D( const std::vector::Info>> @@ -182,7 +236,8 @@ TEST_F(BPWriteReadBlockInfo, BPWriteReadBlockInfo1D8) { io.SetEngine("BPFile"); } - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); auto var_local = io.InquireVariable("local"); auto var_localStr = io.InquireVariable("localStr"); @@ -502,7 +557,8 @@ TEST_F(BPWriteReadBlockInfo, BPWriteReadBlockInfo2D2x4) // Create the BP Engine io.SetEngine("BPFile"); } - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); auto var_iString = io.InquireVariable("iString"); auto var_i8 = io.InquireVariable("i8"); @@ -675,6 +731,186 @@ TEST_F(BPWriteReadBlockInfo, BPWriteReadBlockInfo2D2x4) } } +TEST_F(BPWriteReadBlockInfo, BPWriteReadBlockInfo1D8_C) +{ + // Each process would write a 1x8 array and all processes would + // form a mpiSize * Nx 1D array + const std::string fname("BPWriteReadblockInfo1D8.bp"); + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const size_t Nx = 8; + + // Number of steps + const size_t NSteps = 3; + +#if ADIOS2_USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#endif + + // Write test data using BP + { +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif + adios2::IO io = adios.DeclareIO("TestIO"); + + const adios2::Dims shape{static_cast(Nx * mpiSize)}; + const adios2::Dims start{static_cast(Nx * mpiRank)}; + const adios2::Dims count{Nx}; + + auto var_local = + io.DefineVariable("local", {adios2::LocalValueDim}); + auto var_localStr = + io.DefineVariable("localStr", {adios2::LocalValueDim}); + + auto var_iString = io.DefineVariable("iString"); + auto var_i8 = io.DefineVariable("i8", shape, start, count); + auto var_i16 = io.DefineVariable("i16", shape, start, count); + auto var_i32 = io.DefineVariable("i32", shape, start, count); + auto var_i64 = io.DefineVariable("i64", shape, start, count); + auto var_u8 = io.DefineVariable("u8", shape, start, count); + auto var_u16 = io.DefineVariable("u16", shape, start, count); + auto var_u32 = io.DefineVariable("u32", shape, start, count); + auto var_u64 = io.DefineVariable("u64", shape, start, count); + auto var_r32 = io.DefineVariable("r32", shape, start, count); + auto var_r64 = io.DefineVariable("r64", shape, start, count); + auto var_cr32 = + io.DefineVariable>("cr32", shape, start, count); + auto var_cr64 = io.DefineVariable>("cr64", shape, + start, count); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + else + { + // Create the BP Engine + io.SetEngine("BPFile"); + } + io.SetParameter("AggregatorRatio", "1"); + + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + // Generate test data for each process uniquely + SmallTestData currentTestData = generateNewSmallTestData( + m_TestData, static_cast(step), mpiRank, mpiSize); + + bpWriter.BeginStep(); + + const int32_t localNumber = static_cast(mpiRank + step); + bpWriter.Put(var_local, localNumber); + bpWriter.Put(var_localStr, std::to_string(localNumber)); + + bpWriter.Put(var_iString, currentTestData.S1); + bpWriter.Put(var_i8, currentTestData.I8.data()); + bpWriter.Put(var_i16, currentTestData.I16.data()); + bpWriter.Put(var_i32, currentTestData.I32.data()); + bpWriter.Put(var_i64, currentTestData.I64.data()); + bpWriter.Put(var_u8, currentTestData.U8.data()); + bpWriter.Put(var_u16, currentTestData.U16.data()); + bpWriter.Put(var_u32, currentTestData.U32.data()); + bpWriter.Put(var_u64, currentTestData.U64.data()); + bpWriter.Put(var_r32, currentTestData.R32.data()); + bpWriter.Put(var_r64, currentTestData.R64.data()); + bpWriter.Put(var_cr32, currentTestData.CR32.data()); + bpWriter.Put(var_cr64, currentTestData.CR64.data()); + bpWriter.EndStep(); + } + + bpWriter.Close(); + } + + { +#if ADIOS2_USE_MPI + adios2_adios *adiosH = + adios2_init(MPI_COMM_WORLD, adios2_debug_mode_on); +#else + adios2_adios *adiosH = adios2_init(adios2_debug_mode_on); +#endif + adios2_io *ioR = adios2_declare_io(adiosH, "ReadIO"); + if (!engineName.empty()) + { + adios2_set_engine(ioR, engineName.data()); + } + else + { + adios2_set_engine(ioR, "BPFile"); + } + + adios2_engine *engineR = + adios2_open(ioR, fname.data(), adios2_mode_read); + + for (size_t t = 0; t < NSteps; ++t) + { + adios2_step_status status; + adios2_begin_step(engineR, adios2_step_mode_read, -1., &status); + + EXPECT_EQ(status, adios2_step_status_ok); + + { + auto *var_local = adios2_inquire_variable(ioR, "local"); + EXPECT_NE(var_local, nullptr); + adios2_varinfo *vi_local = + adios2_inquire_blockinfo(engineR, var_local, 0); + CheckStepsBlockInfo1D_C(var_local, vi_local, t, mpiSize, 1); + } + { + auto *var_localStr = adios2_inquire_variable(ioR, "localStr"); + + EXPECT_NE(var_localStr, nullptr); + adios2_varinfo *vi_localStr = + adios2_inquire_blockinfo(engineR, var_localStr, 0); + CheckStepsBlockInfo1D_C(var_localStr, vi_localStr, t, mpiSize, + 1); + } + { + auto *var_iString = adios2_inquire_variable(ioR, "iString"); + EXPECT_NE(var_iString, nullptr); + adios2_varinfo *vi_iString = + adios2_inquire_blockinfo(engineR, var_iString, 0); + CheckStepsBlockInfo1D_C(var_iString, vi_iString, t, mpiSize, 1); + } + { + auto *var_i8 = adios2_inquire_variable(ioR, "i8"); + EXPECT_NE(var_i8, nullptr); + adios2_varinfo *vi_i8 = + adios2_inquire_blockinfo(engineR, var_i8, 0); + CheckStepsBlockInfo1D_C(var_i8, vi_i8, t, mpiSize, Nx); + } + { + auto *var_r64 = adios2_inquire_variable(ioR, "r64"); + EXPECT_NE(var_r64, nullptr); + adios2_varinfo *vi_r64 = + adios2_inquire_blockinfo(engineR, var_r64, 0); + CheckStepsBlockInfo1D_C(var_r64, vi_r64, t, mpiSize, Nx); + } + /* + auto *var_i16 = adios2_inquire_variable(ioR, "i16"); + auto *var_i32 = adios2_inquire_variable(ioR, "i32"); + auto *var_i64 = adios2_inquire_variable(ioR, "i64"); + auto *var_u8 = adios2_inquire_variable(ioR, "u8"); + auto *var_u16 = adios2_inquire_variable(ioR, "u16"); + auto *var_u32 = adios2_inquire_variable(ioR, "u32"); + auto *var_u64 = adios2_inquire_variable(ioR, "u64"); + auto *var_r32 = adios2_inquire_variable(ioR, "r32"); + + auto *var_cr32 = adios2_inquire_variable(ioR, "cr32"); + auto *var_cr64 = adios2_inquire_variable(ioR, "cr64"); + */ + + adios2_end_step(engineR); + } + adios2_close(engineR); + } +} + int main(int argc, char **argv) { #if ADIOS2_USE_MPI diff --git a/testing/adios2/engine/bp/TestBPWriteReadLocalVariables.cpp b/testing/adios2/engine/bp/TestBPWriteReadLocalVariables.cpp index 1feaeb246e..b3d5865b30 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadLocalVariables.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadLocalVariables.cpp @@ -1697,7 +1697,8 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal1DBlockInfo) io.SetEngine(engineName); } - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); auto var_StepsGlobalValue = io.InquireVariable("stepsGlobalValue"); @@ -1721,6 +1722,23 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal1DBlockInfo) auto var_cr32 = io.InquireVariable>("cr32"); auto var_cr64 = io.InquireVariable>("cr64"); + (void)var_StepsGlobalValue; + (void)var_StepsGlobalValueString; + (void)var_RanksLocalValue; + (void)var_RanksLocalValueString; + (void)var_i8; + (void)var_i16; + (void)var_i32; + (void)var_i64; + (void)var_u8; + (void)var_u16; + (void)var_u32; + (void)var_u64; + (void)var_r32; + (void)var_r64; + (void)var_cr32; + (void)var_cr64; + for (size_t s = 0; s < NSteps; ++s) { const std::vector::Info> @@ -1811,9 +1829,10 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal1DSubFile) // each subfile independently #if ADIOS2_USE_MPI adios2::Engine bpReader = - io.Open(subFileName, adios2::Mode::Read, MPI_COMM_SELF); + io.Open(subFileName, adios2::Mode::ReadRandomAccess, MPI_COMM_SELF); #else - adios2::Engine bpReader = io.Open(subFileName, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(subFileName, adios2::Mode::ReadRandomAccess); #endif auto var_i32 = io.InquireVariable("i32"); @@ -1903,9 +1922,10 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal2DChangeCount) adios2::IO io = adios.DeclareIO("ReaderIO"); #if ADIOS2_USE_MPI adios2::Engine bpReader = - io.Open(fname, adios2::Mode::Read, MPI_COMM_SELF); + io.Open(fname, adios2::Mode::ReadRandomAccess, MPI_COMM_SELF); #else - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); #endif auto var_r32 = io.InquireVariable("r32"); diff --git a/testing/adios2/engine/bp/TestBPWriteReadVariableSpan.cpp b/testing/adios2/engine/bp/TestBPWriteReadVariableSpan.cpp index f59d273186..08cca10a3c 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadVariableSpan.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadVariableSpan.cpp @@ -64,7 +64,7 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8) const adios2::Dims count{Nx}; auto var_Step = io.DefineVariable("step"); - auto var_String = io.DefineVariable("iString"); + /* Why is there no Span for string variable? */ auto var_i8 = io.DefineVariable("i8", shape, start, count, adios2::ConstantDims); auto var_i16 = io.DefineVariable("i16", shape, start, count, @@ -90,6 +90,19 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8) auto var_cr64 = io.DefineVariable>( "cr64", shape, start, count, adios2::ConstantDims); + (void)var_i8; + (void)var_i16; + (void)var_i32; + (void)var_i64; + (void)var_u8; + (void)var_u16; + (void)var_u32; + (void)var_u64; + (void)var_r32; + (void)var_r64; + (void)var_cr32; + (void)var_cr64; + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); for (size_t step = 0; step < NSteps; ++step) @@ -97,12 +110,12 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8) SmallTestData currentTestData = generateNewSmallTestData( m_TestData, static_cast(step), mpiRank, mpiSize); - EXPECT_EQ(bpWriter.CurrentStep(), step); - bpWriter.BeginStep(); + EXPECT_EQ(bpWriter.CurrentStep(), step); + bpWriter.Put(var_Step, step); - bpWriter.Put("iString", currentTestData.S1); + // bpWriter.Put("iString", currentTestData.S1); adios2::Variable::Span i8Span = bpWriter.Put(var_i8); adios2::Variable::Span i16Span = bpWriter.Put(var_i16); @@ -119,6 +132,9 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8) adios2::Variable>::Span cr64Span = bpWriter.Put(var_cr64); + auto ptr = i64Span.data(); + (void)ptr; + // Testing Data() std::copy(currentTestData.I8.begin(), currentTestData.I8.begin() + Nx, i8Span.begin()); @@ -193,7 +209,7 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8) m_TestData, static_cast(currentStep), mpiRank, mpiSize); auto var_iStep = io.InquireVariable("step"); - auto var_iString = io.InquireVariable("iString"); + // auto var_iString = io.InquireVariable("iString"); auto var_i8 = io.InquireVariable("i8"); auto var_i16 = io.InquireVariable("i16"); auto var_i32 = io.InquireVariable("i32"); @@ -208,54 +224,41 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8) auto var_cr64 = io.InquireVariable>("cr64"); EXPECT_EQ(var_iStep.ShapeID(), adios2::ShapeID::GlobalValue); - EXPECT_EQ(var_iStep.Steps(), NSteps); EXPECT_EQ(var_i8.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_i8.Steps(), NSteps); EXPECT_EQ(var_i8.Shape()[0], static_cast(mpiSize * Nx)); EXPECT_EQ(var_i16.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_i16.Steps(), NSteps); EXPECT_EQ(var_i16.Shape()[0], static_cast(mpiSize * Nx)); EXPECT_EQ(var_i32.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_i32.Steps(), NSteps); EXPECT_EQ(var_i32.Shape()[0], static_cast(mpiSize * Nx)); EXPECT_EQ(var_i64.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_i64.Steps(), NSteps); EXPECT_EQ(var_i64.Shape()[0], static_cast(mpiSize * Nx)); EXPECT_EQ(var_u8.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_u8.Steps(), NSteps); EXPECT_EQ(var_u8.Shape()[0], static_cast(mpiSize * Nx)); EXPECT_EQ(var_u16.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_u16.Steps(), NSteps); EXPECT_EQ(var_u16.Shape()[0], static_cast(mpiSize * Nx)); EXPECT_EQ(var_u32.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_u32.Steps(), NSteps); EXPECT_EQ(var_u32.Shape()[0], static_cast(mpiSize * Nx)); EXPECT_EQ(var_u64.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_u64.Steps(), NSteps); EXPECT_EQ(var_u64.Shape()[0], static_cast(mpiSize * Nx)); EXPECT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_r32.Steps(), NSteps); EXPECT_EQ(var_r32.Shape()[0], static_cast(mpiSize * Nx)); EXPECT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_r64.Steps(), NSteps); EXPECT_EQ(var_r64.Shape()[0], static_cast(mpiSize * Nx)); EXPECT_EQ(var_cr32.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_cr32.Steps(), NSteps); EXPECT_EQ(var_cr32.Shape()[0], static_cast(mpiSize * Nx)); EXPECT_EQ(var_cr64.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_cr64.Steps(), NSteps); EXPECT_EQ(var_cr64.Shape()[0], static_cast(mpiSize * Nx)); var_i8.SetSelection(sel); @@ -272,7 +275,7 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8) var_cr64.SetSelection(sel); bpReader.Get(var_iStep, IStep); - bpReader.Get(var_iString, IString); + // bpReader.Get(var_iString, IString); bpReader.Get(var_i8, I8.data()); bpReader.Get(var_i16, I16.data()); bpReader.Get(var_i32, I32.data()); @@ -289,7 +292,7 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8) bpReader.EndStep(); EXPECT_EQ(IStep, currentStep); - EXPECT_EQ(IString, currentTestData.S1); + // EXPECT_EQ(IString, currentTestData.S1); for (size_t i = 0; i < Nx; ++i) { @@ -367,7 +370,6 @@ TEST_F(BPWriteReadSpan, BPWriteRead2D2x4) const adios2::Dims start{0, static_cast(mpiRank * Nx)}; const adios2::Dims count{Ny, Nx}; - auto var_String = io.DefineVariable("iString"); auto var_i8 = io.DefineVariable("i8", shape, start, count, adios2::ConstantDims); auto var_i16 = io.DefineVariable("i16", shape, start, count, @@ -393,6 +395,19 @@ TEST_F(BPWriteReadSpan, BPWriteRead2D2x4) auto var_cr64 = io.DefineVariable>( "cr64", shape, start, count, adios2::ConstantDims); + (void)var_i8; + (void)var_i16; + (void)var_i32; + (void)var_i64; + (void)var_u8; + (void)var_u16; + (void)var_u32; + (void)var_u64; + (void)var_r32; + (void)var_r64; + (void)var_cr32; + (void)var_cr64; + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); for (size_t step = 0; step < NSteps; ++step) @@ -400,9 +415,8 @@ TEST_F(BPWriteReadSpan, BPWriteRead2D2x4) SmallTestData currentTestData = generateNewSmallTestData( m_TestData, static_cast(step), mpiRank, mpiSize); - EXPECT_EQ(bpWriter.CurrentStep(), step); - bpWriter.BeginStep(); + EXPECT_EQ(bpWriter.CurrentStep(), step); adios2::Variable::Span i8Span = bpWriter.Put(var_i8); adios2::Variable::Span i16Span = bpWriter.Put(var_i16); adios2::Variable::Span i32Span = bpWriter.Put(var_i32); @@ -478,84 +492,72 @@ TEST_F(BPWriteReadSpan, BPWriteRead2D2x4) auto var_i8 = io.InquireVariable("i8"); EXPECT_TRUE(var_i8); EXPECT_EQ(var_i8.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_i8.Steps(), NSteps); EXPECT_EQ(var_i8.Shape()[0], Ny); EXPECT_EQ(var_i8.Shape()[1], static_cast(mpiSize * Nx)); auto var_i16 = io.InquireVariable("i16"); EXPECT_TRUE(var_i16); EXPECT_EQ(var_i16.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_i16.Steps(), NSteps); EXPECT_EQ(var_i16.Shape()[0], Ny); EXPECT_EQ(var_i16.Shape()[1], static_cast(mpiSize * Nx)); auto var_i32 = io.InquireVariable("i32"); EXPECT_TRUE(var_i32); EXPECT_EQ(var_i32.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_i32.Steps(), NSteps); EXPECT_EQ(var_i32.Shape()[0], Ny); EXPECT_EQ(var_i32.Shape()[1], static_cast(mpiSize * Nx)); auto var_i64 = io.InquireVariable("i64"); EXPECT_TRUE(var_i64); EXPECT_EQ(var_i64.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_i64.Steps(), NSteps); EXPECT_EQ(var_i64.Shape()[0], Ny); EXPECT_EQ(var_i64.Shape()[1], static_cast(mpiSize * Nx)); auto var_u8 = io.InquireVariable("u8"); EXPECT_TRUE(var_u8); EXPECT_EQ(var_u8.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_u8.Steps(), NSteps); EXPECT_EQ(var_u8.Shape()[0], Ny); EXPECT_EQ(var_u8.Shape()[1], static_cast(mpiSize * Nx)); auto var_u16 = io.InquireVariable("u16"); EXPECT_TRUE(var_u16); EXPECT_EQ(var_u16.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_u16.Steps(), NSteps); EXPECT_EQ(var_u16.Shape()[0], Ny); EXPECT_EQ(var_u16.Shape()[1], static_cast(mpiSize * Nx)); auto var_u32 = io.InquireVariable("u32"); EXPECT_TRUE(var_u32); EXPECT_EQ(var_u32.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_u32.Steps(), NSteps); EXPECT_EQ(var_u32.Shape()[0], Ny); EXPECT_EQ(var_u32.Shape()[1], static_cast(mpiSize * Nx)); auto var_u64 = io.InquireVariable("u64"); EXPECT_TRUE(var_u64); EXPECT_EQ(var_u64.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_u64.Steps(), NSteps); EXPECT_EQ(var_u64.Shape()[0], Ny); EXPECT_EQ(var_u64.Shape()[1], static_cast(mpiSize * Nx)); auto var_r32 = io.InquireVariable("r32"); EXPECT_TRUE(var_r32); EXPECT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_r32.Steps(), NSteps); EXPECT_EQ(var_r32.Shape()[0], Ny); EXPECT_EQ(var_r32.Shape()[1], static_cast(mpiSize * Nx)); auto var_r64 = io.InquireVariable("r64"); EXPECT_TRUE(var_r64); EXPECT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_r64.Steps(), NSteps); EXPECT_EQ(var_r64.Shape()[0], Ny); EXPECT_EQ(var_r64.Shape()[1], static_cast(mpiSize * Nx)); auto var_cr32 = io.InquireVariable>("cr32"); EXPECT_TRUE(var_cr32); EXPECT_EQ(var_cr32.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_cr32.Steps(), NSteps); EXPECT_EQ(var_cr32.Shape()[0], Ny); EXPECT_EQ(var_cr32.Shape()[1], static_cast(mpiSize * Nx)); auto var_cr64 = io.InquireVariable>("cr64"); EXPECT_TRUE(var_cr64); EXPECT_EQ(var_cr64.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_cr64.Steps(), NSteps); EXPECT_EQ(var_cr64.Shape()[0], Ny); EXPECT_EQ(var_cr64.Shape()[1], static_cast(mpiSize * Nx)); @@ -681,7 +683,6 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8Local) const adios2::Dims start{}; const adios2::Dims count{Nx}; - auto var_String = io.DefineVariable("iString"); auto var_i8 = io.DefineVariable("i8", shape, start, count, adios2::ConstantDims); auto var_i16 = io.DefineVariable("i16", shape, start, count, @@ -707,6 +708,19 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8Local) auto var_cr64 = io.DefineVariable>( "cr64", shape, start, count, adios2::ConstantDims); + (void)var_i8; + (void)var_i16; + (void)var_i32; + (void)var_i64; + (void)var_u8; + (void)var_u16; + (void)var_u32; + (void)var_u64; + (void)var_r32; + (void)var_r64; + (void)var_cr32; + (void)var_cr64; + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); for (size_t step = 0; step < NSteps; ++step) @@ -714,10 +728,9 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8Local) SmallTestData currentTestData = generateNewSmallTestData( m_TestData, static_cast(step), mpiRank, mpiSize); - EXPECT_EQ(bpWriter.CurrentStep(), step); - bpWriter.BeginStep(); - bpWriter.Put("iString", currentTestData.S1); + EXPECT_EQ(bpWriter.CurrentStep(), step); + // bpWriter.Put("iString", currentTestData.S1); adios2::Variable::Span i8Span = bpWriter.Put(var_i8); adios2::Variable::Span i16Span = bpWriter.Put(var_i16); @@ -801,7 +814,7 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8Local) SmallTestData currentTestData = generateNewSmallTestData( m_TestData, static_cast(currentStep), mpiRank, mpiSize); - auto var_iString = io.InquireVariable("iString"); + // auto var_iString = io.InquireVariable("iString"); auto var_i8 = io.InquireVariable("i8"); auto var_i16 = io.InquireVariable("i16"); auto var_i32 = io.InquireVariable("i32"); @@ -816,40 +829,17 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8Local) auto var_cr64 = io.InquireVariable>("cr64"); EXPECT_EQ(var_i8.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_i8.Steps(), NSteps); - EXPECT_EQ(var_i16.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_i16.Steps(), NSteps); - EXPECT_EQ(var_i32.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_i32.Steps(), NSteps); - EXPECT_EQ(var_i64.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_i64.Steps(), NSteps); - EXPECT_EQ(var_u8.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_u8.Steps(), NSteps); - EXPECT_EQ(var_u16.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_u16.Steps(), NSteps); - EXPECT_EQ(var_u32.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_u32.Steps(), NSteps); - EXPECT_EQ(var_u64.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_u64.Steps(), NSteps); - EXPECT_EQ(var_r32.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_r32.Steps(), NSteps); - EXPECT_EQ(var_r64.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_r64.Steps(), NSteps); - EXPECT_EQ(var_cr32.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_cr32.Steps(), NSteps); - EXPECT_EQ(var_cr64.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_cr64.Steps(), NSteps); const size_t rankBlock = static_cast(mpiRank); var_i8.SetBlockSelection(rankBlock); @@ -865,7 +855,7 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8Local) var_cr32.SetBlockSelection(rankBlock); var_cr64.SetBlockSelection(rankBlock); - bpReader.Get(var_iString, IString); + // bpReader.Get(var_iString, IString); bpReader.Get(var_i8, I8.data()); bpReader.Get(var_i16, I16.data()); bpReader.Get(var_i32, I32.data()); @@ -881,7 +871,7 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8Local) bpReader.EndStep(); - EXPECT_EQ(IString, currentTestData.S1); + // EXPECT_EQ(IString, currentTestData.S1); for (size_t i = 0; i < Nx; ++i) { @@ -956,7 +946,6 @@ TEST_F(BPWriteReadSpan, BPWriteRead2D2x4Local) const adios2::Dims start{}; const adios2::Dims count{Ny, Nx}; - auto var_String = io.DefineVariable("iString"); auto var_i8 = io.DefineVariable("i8", shape, start, count, adios2::ConstantDims); auto var_i16 = io.DefineVariable("i16", shape, start, count, @@ -982,6 +971,19 @@ TEST_F(BPWriteReadSpan, BPWriteRead2D2x4Local) auto var_cr64 = io.DefineVariable>( "cr64", shape, start, count, adios2::ConstantDims); + (void)var_i8; + (void)var_i16; + (void)var_i32; + (void)var_i64; + (void)var_u8; + (void)var_u16; + (void)var_u32; + (void)var_u64; + (void)var_r32; + (void)var_r64; + (void)var_cr32; + (void)var_cr64; + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); for (size_t step = 0; step < NSteps; ++step) @@ -989,9 +991,8 @@ TEST_F(BPWriteReadSpan, BPWriteRead2D2x4Local) SmallTestData currentTestData = generateNewSmallTestData( m_TestData, static_cast(step), mpiRank, mpiSize); - EXPECT_EQ(bpWriter.CurrentStep(), step); - bpWriter.BeginStep(); + EXPECT_EQ(bpWriter.CurrentStep(), step); adios2::Variable::Span i8Span = bpWriter.Put(var_i8); adios2::Variable::Span i16Span = bpWriter.Put(var_i16); adios2::Variable::Span i32Span = bpWriter.Put(var_i32); @@ -1067,62 +1068,50 @@ TEST_F(BPWriteReadSpan, BPWriteRead2D2x4Local) auto var_i8 = io.InquireVariable("i8"); EXPECT_TRUE(var_i8); EXPECT_EQ(var_i8.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_i8.Steps(), NSteps); auto var_i16 = io.InquireVariable("i16"); EXPECT_TRUE(var_i16); EXPECT_EQ(var_i16.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_i16.Steps(), NSteps); auto var_i32 = io.InquireVariable("i32"); EXPECT_TRUE(var_i32); EXPECT_EQ(var_i32.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_i32.Steps(), NSteps); auto var_i64 = io.InquireVariable("i64"); EXPECT_TRUE(var_i64); EXPECT_EQ(var_i64.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_i64.Steps(), NSteps); auto var_u8 = io.InquireVariable("u8"); EXPECT_TRUE(var_u8); EXPECT_EQ(var_u8.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_u8.Steps(), NSteps); auto var_u16 = io.InquireVariable("u16"); EXPECT_TRUE(var_u16); EXPECT_EQ(var_u16.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_u16.Steps(), NSteps); auto var_u32 = io.InquireVariable("u32"); EXPECT_TRUE(var_u32); EXPECT_EQ(var_u32.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_u32.Steps(), NSteps); auto var_u64 = io.InquireVariable("u64"); EXPECT_TRUE(var_u64); EXPECT_EQ(var_u64.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_u64.Steps(), NSteps); auto var_r32 = io.InquireVariable("r32"); EXPECT_TRUE(var_r32); EXPECT_EQ(var_r32.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_r32.Steps(), NSteps); auto var_r64 = io.InquireVariable("r64"); EXPECT_TRUE(var_r64); EXPECT_EQ(var_r64.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_r64.Steps(), NSteps); auto var_cr32 = io.InquireVariable>("cr32"); EXPECT_TRUE(var_cr32); EXPECT_EQ(var_cr32.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_cr32.Steps(), NSteps); auto var_cr64 = io.InquireVariable>("cr64"); EXPECT_TRUE(var_cr64); EXPECT_EQ(var_cr64.ShapeID(), adios2::ShapeID::LocalArray); - EXPECT_EQ(var_cr64.Steps(), NSteps); std::array I8; std::array I16; @@ -1242,7 +1231,6 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8FillValue) const adios2::Dims start{static_cast(Nx * mpiRank)}; const adios2::Dims count{Nx}; - auto var_String = io.DefineVariable("iString"); auto var_i8 = io.DefineVariable("i8", shape, start, count, adios2::ConstantDims); auto var_i16 = io.DefineVariable("i16", shape, start, count, @@ -1268,46 +1256,72 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8FillValue) auto var_cr64 = io.DefineVariable>( "cr64", shape, start, count, adios2::ConstantDims); + (void)var_i8; + (void)var_i16; + (void)var_i32; + (void)var_i64; + (void)var_u8; + (void)var_u16; + (void)var_u32; + (void)var_u64; + (void)var_r32; + (void)var_r64; + (void)var_cr32; + (void)var_cr64; + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); for (size_t step = 0; step < NSteps; ++step) { bpWriter.BeginStep(); EXPECT_EQ(bpWriter.CurrentStep(), step); - bpWriter.Put("iString", std::to_string(step)); + // bpWriter.Put("iString", std::to_string(step)); adios2::Variable::Span i8Span = - bpWriter.Put(var_i8, 0, static_cast(step)); + bpWriter.Put(var_i8, true, static_cast(step)); adios2::Variable::Span i16Span = - bpWriter.Put(var_i16, 0, static_cast(step)); + bpWriter.Put(var_i16, true, static_cast(step)); adios2::Variable::Span i32Span = - bpWriter.Put(var_i32, 0, static_cast(step)); + bpWriter.Put(var_i32, true, static_cast(step)); adios2::Variable::Span i64Span = - bpWriter.Put(var_i64, 0, static_cast(step)); + bpWriter.Put(var_i64, true, static_cast(step)); adios2::Variable::Span u8Span = - bpWriter.Put(var_u8, 0, static_cast(step)); + bpWriter.Put(var_u8, true, static_cast(step)); adios2::Variable::Span u16Span = - bpWriter.Put(var_u16, 0, static_cast(step)); + bpWriter.Put(var_u16, true, static_cast(step)); adios2::Variable::Span u32Span = - bpWriter.Put(var_u32, 0, static_cast(step)); + bpWriter.Put(var_u32, true, static_cast(step)); adios2::Variable::Span u64Span = - bpWriter.Put(var_u64, 0, static_cast(step)); + bpWriter.Put(var_u64, true, static_cast(step)); adios2::Variable::Span r32Span = - bpWriter.Put(var_r32, 0, static_cast(step)); + bpWriter.Put(var_r32, true, static_cast(step)); adios2::Variable::Span r64Span = - bpWriter.Put(var_r64, 0, static_cast(step)); + bpWriter.Put(var_r64, true, static_cast(step)); adios2::Variable>::Span cr32Span = bpWriter.Put( - var_cr32, 0, + var_cr32, true, {static_cast(step), static_cast(step)}); adios2::Variable>::Span cr64Span = bpWriter.Put( - var_cr64, 0, + var_cr64, true, {static_cast(step), static_cast(step)}); + (void)i8Span; + (void)i16Span; + (void)i32Span; + (void)i64Span; + (void)u8Span; + (void)u16Span; + (void)u32Span; + (void)u64Span; + (void)r32Span; + (void)r64Span; + (void)cr32Span; + (void)cr64Span; + bpWriter.EndStep(); } @@ -1354,7 +1368,7 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8FillValue) const size_t currentStep = bpReader.CurrentStep(); EXPECT_EQ(currentStep, static_cast(t)); - auto var_iString = io.InquireVariable("iString"); + // auto var_iString = io.InquireVariable("iString"); auto var_i8 = io.InquireVariable("i8"); auto var_i16 = io.InquireVariable("i16"); auto var_i32 = io.InquireVariable("i32"); @@ -1369,84 +1383,72 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8FillValue) auto var_cr64 = io.InquireVariable>("cr64"); EXPECT_EQ(var_i8.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_i8.Steps(), NSteps); EXPECT_EQ(var_i8.Shape()[0], static_cast(mpiSize * Nx)); - EXPECT_EQ(var_i8.Min(), static_cast(currentStep)); - EXPECT_EQ(var_i8.Max(), static_cast(currentStep)); + // EXPECT_EQ(var_i8.Min(), static_cast(currentStep)); + // EXPECT_EQ(var_i8.Max(), static_cast(currentStep)); EXPECT_EQ(var_i16.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_i16.Steps(), NSteps); EXPECT_EQ(var_i16.Shape()[0], static_cast(mpiSize * Nx)); - EXPECT_EQ(var_i16.Min(), static_cast(currentStep)); - EXPECT_EQ(var_i16.Max(), static_cast(currentStep)); + // EXPECT_EQ(var_i16.Min(), static_cast(currentStep)); + // EXPECT_EQ(var_i16.Max(), static_cast(currentStep)); EXPECT_EQ(var_i32.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_i32.Steps(), NSteps); EXPECT_EQ(var_i32.Shape()[0], static_cast(mpiSize * Nx)); - EXPECT_EQ(var_i32.Min(), static_cast(currentStep)); - EXPECT_EQ(var_i32.Max(), static_cast(currentStep)); + // EXPECT_EQ(var_i32.Min(), static_cast(currentStep)); + // EXPECT_EQ(var_i32.Max(), static_cast(currentStep)); EXPECT_EQ(var_i64.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_i64.Steps(), NSteps); EXPECT_EQ(var_i64.Shape()[0], static_cast(mpiSize * Nx)); - EXPECT_EQ(var_i64.Min(), static_cast(currentStep)); - EXPECT_EQ(var_i64.Max(), static_cast(currentStep)); + // EXPECT_EQ(var_i64.Min(), static_cast(currentStep)); + // EXPECT_EQ(var_i64.Max(), static_cast(currentStep)); EXPECT_EQ(var_u8.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_u8.Steps(), NSteps); EXPECT_EQ(var_u8.Shape()[0], static_cast(mpiSize * Nx)); - EXPECT_EQ(var_u8.Min(), static_cast(currentStep)); - EXPECT_EQ(var_u8.Max(), static_cast(currentStep)); + // EXPECT_EQ(var_u8.Min(), static_cast(currentStep)); + // EXPECT_EQ(var_u8.Max(), static_cast(currentStep)); EXPECT_EQ(var_u16.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_u16.Steps(), NSteps); EXPECT_EQ(var_u16.Shape()[0], static_cast(mpiSize * Nx)); - EXPECT_EQ(var_u16.Min(), static_cast(currentStep)); - EXPECT_EQ(var_u16.Max(), static_cast(currentStep)); + // EXPECT_EQ(var_u16.Min(), static_cast(currentStep)); + // EXPECT_EQ(var_u16.Max(), static_cast(currentStep)); EXPECT_EQ(var_u32.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_u32.Steps(), NSteps); EXPECT_EQ(var_u32.Shape()[0], static_cast(mpiSize * Nx)); - EXPECT_EQ(var_u32.Min(), static_cast(currentStep)); - EXPECT_EQ(var_u32.Max(), static_cast(currentStep)); + // EXPECT_EQ(var_u32.Min(), static_cast(currentStep)); + // EXPECT_EQ(var_u32.Max(), static_cast(currentStep)); EXPECT_EQ(var_u64.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_u64.Steps(), NSteps); EXPECT_EQ(var_u64.Shape()[0], static_cast(mpiSize * Nx)); - EXPECT_EQ(var_u64.Min(), static_cast(currentStep)); - EXPECT_EQ(var_u64.Max(), static_cast(currentStep)); + // EXPECT_EQ(var_u64.Min(), static_cast(currentStep)); + // EXPECT_EQ(var_u64.Max(), static_cast(currentStep)); EXPECT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_r32.Steps(), NSteps); EXPECT_EQ(var_r32.Shape()[0], static_cast(mpiSize * Nx)); - EXPECT_EQ(var_r32.Min(), static_cast(currentStep)); - EXPECT_EQ(var_r32.Max(), static_cast(currentStep)); + // EXPECT_EQ(var_r32.Min(), static_cast(currentStep)); + // EXPECT_EQ(var_r32.Max(), static_cast(currentStep)); EXPECT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_r64.Steps(), NSteps); EXPECT_EQ(var_r64.Shape()[0], static_cast(mpiSize * Nx)); - EXPECT_EQ(var_r64.Min(), static_cast(currentStep)); - EXPECT_EQ(var_r64.Max(), static_cast(currentStep)); + // EXPECT_EQ(var_r64.Min(), static_cast(currentStep)); + // EXPECT_EQ(var_r64.Max(), static_cast(currentStep)); EXPECT_EQ(var_cr32.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_cr32.Steps(), NSteps); EXPECT_EQ(var_cr32.Shape()[0], static_cast(mpiSize * Nx)); - EXPECT_EQ(var_cr32.Min(), - std::complex(static_cast(currentStep), - static_cast(currentStep))); - EXPECT_EQ(var_cr32.Max(), - std::complex(static_cast(currentStep), - static_cast(currentStep))); + // EXPECT_EQ(var_cr32.Min(), + // std::complex(static_cast(currentStep), + // static_cast(currentStep))); + // EXPECT_EQ(var_cr32.Max(), + // std::complex(static_cast(currentStep), + // static_cast(currentStep))); EXPECT_EQ(var_cr64.ShapeID(), adios2::ShapeID::GlobalArray); - EXPECT_EQ(var_cr64.Steps(), NSteps); EXPECT_EQ(var_cr64.Shape()[0], static_cast(mpiSize * Nx)); - EXPECT_EQ(var_cr64.Min(), - std::complex(static_cast(currentStep), - static_cast(currentStep))); - EXPECT_EQ(var_cr64.Max(), - std::complex(static_cast(currentStep), - static_cast(currentStep))); + // EXPECT_EQ(var_cr64.Min(), + // std::complex(static_cast(currentStep), + // static_cast(currentStep))); + // EXPECT_EQ(var_cr64.Max(), + // std::complex(static_cast(currentStep), + // static_cast(currentStep))); var_i8.SetSelection(sel); var_i16.SetSelection(sel); @@ -1461,7 +1463,7 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8FillValue) var_cr32.SetSelection(sel); var_cr64.SetSelection(sel); - bpReader.Get(var_iString, IString); + // bpReader.Get(var_iString, IString); bpReader.Get(var_i8, I8.data()); bpReader.Get(var_i16, I16.data()); bpReader.Get(var_i32, I32.data()); @@ -1477,7 +1479,7 @@ TEST_F(BPWriteReadSpan, BPWriteRead1D8FillValue) bpReader.EndStep(); - EXPECT_EQ(IString, std::to_string(currentStep)); + // EXPECT_EQ(IString, std::to_string(currentStep)); for (size_t i = 0; i < Nx; ++i) { diff --git a/testing/adios2/engine/bp/TestBPWriteReadVector.cpp b/testing/adios2/engine/bp/TestBPWriteReadVector.cpp index 371a203145..f21b63f72c 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadVector.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadVector.cpp @@ -183,7 +183,8 @@ TEST_F(BPWriteReadVector, ADIOS2BPWriteRead1D8) io.SetEngine(engineName); } - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); auto var_iString = io.InquireVariable("iString"); EXPECT_TRUE(var_iString); @@ -497,7 +498,8 @@ TEST_F(BPWriteReadVector, ADIOS2BPWriteRead2D2x4) io.SetEngine(engineName); } - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); auto var_iString = io.InquireVariable("iString"); EXPECT_TRUE(var_iString); @@ -814,7 +816,8 @@ TEST_F(BPWriteReadVector, ADIOS2BPWriteRead2D4x2) io.SetEngine(engineName); } - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); auto var_i8 = io.InquireVariable("i8"); EXPECT_TRUE(var_i8); @@ -1118,7 +1121,8 @@ TEST_F(BPWriteReadVector, ADIOS2BPWriteReadVector2D4x2_MultiSteps) io.SetEngine(engineName); } - adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); auto var_i8 = io.InquireVariable("i8"); EXPECT_TRUE(var_i8); diff --git a/testing/adios2/engine/bp/TestBPWriteReadfstream.cpp b/testing/adios2/engine/bp/TestBPWriteReadfstream.cpp deleted file mode 100644 index 038f3fc020..0000000000 --- a/testing/adios2/engine/bp/TestBPWriteReadfstream.cpp +++ /dev/null @@ -1,922 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - */ -#include -#include - -#include -#include - -#include -#include - -#include - -#include "../SmallTestData.h" - -std::string engineName; // comes from command line - -class BPWriteReadTest : public ::testing::Test -{ -public: - BPWriteReadTest() = default; - - SmallTestData m_TestData; -}; - -//****************************************************************************** -// 1D 1x8 test data -//****************************************************************************** - -// ADIOS2 BP write, native ADIOS1 read -TEST_F(BPWriteReadTest, ADIOS2BPWriteADIOS1Read1D8fstream) -{ - // Each process would write a 1x8 array and all processes would - // form a mpiSize * Nx 1D array - std::string fname = "ADIOS2BPWriteADIOS1Read1D8fstream.bp"; - - int mpiRank = 0, mpiSize = 1; - // Number of rows - const std::size_t Nx = 8; - - // Number of steps - const std::size_t NSteps = 3; - -#if ADIOS2_USE_MPI - MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); - MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); -#endif - - // Write test data using BP - { -#if ADIOS2_USE_MPI - adios2::ADIOS adios(MPI_COMM_WORLD); -#else - adios2::ADIOS adios; -#endif - adios2::IO io = adios.DeclareIO("TestIO"); - - // Declare 1D variables (NumOfProcesses * Nx) - // The local process' part (start, count) can be defined now or later - // before Write(). - { - adios2::Dims shape{static_cast(Nx * mpiSize)}; - adios2::Dims start{static_cast(Nx * mpiRank)}; - adios2::Dims count{static_cast(Nx)}; - auto var_i8 = io.DefineVariable("i8", shape, start, count); - auto var_i16 = - io.DefineVariable("i16", shape, start, count); - auto var_i32 = - io.DefineVariable("i32", shape, start, count); - auto var_i64 = - io.DefineVariable("i64", shape, start, count); - auto var_u8 = io.DefineVariable("u8", shape, start, count); - auto var_u16 = - io.DefineVariable("u16", shape, start, count); - auto var_u32 = - io.DefineVariable("u32", shape, start, count); - auto var_u64 = - io.DefineVariable("u64", shape, start, count); - auto var_r32 = io.DefineVariable("r32", shape, start, count); - auto var_r64 = - io.DefineVariable("r64", shape, start, count); - } - - if (!engineName.empty()) - { - io.SetEngine(engineName); - } - else - { - // Create the BP Engine - io.SetEngine("BPFile"); - } - io.AddTransport("file", {{"Library", "fstream"}}); - - // QUESTION: It seems that BPFilterWriter cannot overwrite existing - // files - // Ex. if you tune Nx and NSteps, the test would fail. But if you clear - // the cache in - // ${adios2Build}/testing/adios2/engine/bp/ADIOS2BPWriteADIOS1Read1D8.bp.dir, - // then it works - adios2::Engine engine = io.Open(fname, adios2::Mode::Write); - - for (size_t step = 0; step < NSteps; ++step) - { - // Generate test data for each process uniquely - SmallTestData currentTestData = - generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize); - - // Retrieve the variables that previously went out of scope - auto var_i8 = io.InquireVariable("i8"); - auto var_i16 = io.InquireVariable("i16"); - auto var_i32 = io.InquireVariable("i32"); - auto var_i64 = io.InquireVariable("i64"); - auto var_u8 = io.InquireVariable("u8"); - auto var_u16 = io.InquireVariable("u16"); - auto var_u32 = io.InquireVariable("u32"); - auto var_u64 = io.InquireVariable("u64"); - auto var_r32 = io.InquireVariable("r32"); - auto var_r64 = io.InquireVariable("r64"); - - // Make a 1D selection to describe the local dimensions of the - // variable we write and its offsets in the global spaces - adios2::Box sel({mpiRank * Nx}, {Nx}); - var_i8.SetSelection(sel); - var_i16.SetSelection(sel); - var_i32.SetSelection(sel); - var_i64.SetSelection(sel); - var_u8.SetSelection(sel); - var_u16.SetSelection(sel); - var_u32.SetSelection(sel); - var_u64.SetSelection(sel); - var_r32.SetSelection(sel); - var_r64.SetSelection(sel); - - // Write each one - // fill in the variable with values from starting index to - // starting index + count - engine.BeginStep(); - engine.Put(var_i8, currentTestData.I8.data()); - engine.Put(var_i16, currentTestData.I16.data()); - engine.Put(var_i32, currentTestData.I32.data()); - engine.Put(var_i64, currentTestData.I64.data()); - engine.Put(var_u8, currentTestData.U8.data()); - engine.Put(var_u16, currentTestData.U16.data()); - engine.Put(var_u32, currentTestData.U32.data()); - engine.Put(var_u64, currentTestData.U64.data()); - engine.Put(var_r32, currentTestData.R32.data()); - engine.Put(var_r64, currentTestData.R64.data()); - engine.EndStep(); - } - - // Close the file - engine.Close(); - } - - { - adios_read_init_method(ADIOS_READ_METHOD_BP, MPI_COMM_SELF, - "verbose=3"); - - // Open the file for reading - // Note: Since collective metadata generation is not implemented yet, - // SO for now we read each subfile instead of a single bp file with all - // metadata. - // Meanwhile if we open file with MPI_COMM_WORLD, then the selection - // bounding box should be [0, Nx] - std::string index = std::to_string(mpiRank); - ADIOS_FILE *f = adios_read_open_file( - (fname + ".dir/" + fname + "." + index).c_str(), - ADIOS_READ_METHOD_BP, MPI_COMM_SELF); - ASSERT_NE(f, nullptr); - - // Check the variables exist - ADIOS_VARINFO *var_i8 = adios_inq_var(f, "i8"); - EXPECT_TRUE(var_i8); - ASSERT_EQ(var_i8->ndim, 1); - ASSERT_EQ(var_i8->global, 1); - ASSERT_EQ(var_i8->nsteps, NSteps); - ASSERT_EQ(var_i8->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_i16 = adios_inq_var(f, "i16"); - EXPECT_TRUE(var_i16); - ASSERT_EQ(var_i16->ndim, 1); - ASSERT_EQ(var_i16->global, 1); - ASSERT_EQ(var_i16->nsteps, NSteps); - ASSERT_EQ(var_i16->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_i32 = adios_inq_var(f, "i32"); - EXPECT_TRUE(var_i32); - ASSERT_EQ(var_i32->ndim, 1); - ASSERT_EQ(var_i32->global, 1); - ASSERT_EQ(var_i32->nsteps, NSteps); - ASSERT_EQ(var_i32->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_i64 = adios_inq_var(f, "i64"); - EXPECT_TRUE(var_i64); - ASSERT_EQ(var_i64->ndim, 1); - ASSERT_EQ(var_i64->global, 1); - ASSERT_EQ(var_i64->nsteps, NSteps); - ASSERT_EQ(var_i64->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_u8 = adios_inq_var(f, "u8"); - EXPECT_TRUE(var_u8); - ASSERT_EQ(var_u8->ndim, 1); - ASSERT_EQ(var_u8->global, 1); - ASSERT_EQ(var_u8->nsteps, NSteps); - ASSERT_EQ(var_u8->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_u16 = adios_inq_var(f, "u16"); - EXPECT_TRUE(var_u16); - ASSERT_EQ(var_u16->ndim, 1); - ASSERT_EQ(var_u16->global, 1); - ASSERT_EQ(var_u16->nsteps, NSteps); - ASSERT_EQ(var_u16->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_u32 = adios_inq_var(f, "u32"); - EXPECT_TRUE(var_u32); - ASSERT_EQ(var_u32->ndim, 1); - ASSERT_EQ(var_u32->global, 1); - ASSERT_EQ(var_u32->nsteps, NSteps); - ASSERT_EQ(var_u32->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_u64 = adios_inq_var(f, "u64"); - EXPECT_TRUE(var_u64); - ASSERT_EQ(var_u64->ndim, 1); - ASSERT_EQ(var_u64->global, 1); - ASSERT_EQ(var_u64->nsteps, NSteps); - ASSERT_EQ(var_u64->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_r32 = adios_inq_var(f, "r32"); - EXPECT_TRUE(var_r32); - ASSERT_EQ(var_r32->ndim, 1); - ASSERT_EQ(var_r32->global, 1); - ASSERT_EQ(var_r32->nsteps, NSteps); - ASSERT_EQ(var_r32->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_r64 = adios_inq_var(f, "r64"); - EXPECT_TRUE(var_r64); - ASSERT_EQ(var_r64->ndim, 1); - ASSERT_EQ(var_r64->global, 1); - ASSERT_EQ(var_r64->nsteps, NSteps); - ASSERT_EQ(var_r64->dims[0], mpiSize * Nx); - - std::array I8; - std::array I16; - std::array I32; - std::array I64; - std::array U8; - std::array U16; - std::array U32; - std::array U64; - std::array R32; - std::array R64; - - uint64_t start[1] = {mpiRank * Nx}; - uint64_t count[1] = {Nx}; - ADIOS_SELECTION *sel = adios_selection_boundingbox(1, start, count); - - // Read stuff - for (size_t t = 0; t < NSteps; ++t) - { - // Generate test data for each rank uniquely - SmallTestData currentTestData = - generateNewSmallTestData(m_TestData, t, mpiRank, mpiSize); - // Read the current step - adios_schedule_read_byid(f, sel, var_i8->varid, t, 1, I8.data()); - adios_schedule_read_byid(f, sel, var_i16->varid, t, 1, I16.data()); - adios_schedule_read_byid(f, sel, var_i32->varid, t, 1, I32.data()); - adios_schedule_read_byid(f, sel, var_i64->varid, t, 1, I64.data()); - adios_schedule_read_byid(f, sel, var_u8->varid, t, 1, U8.data()); - adios_schedule_read_byid(f, sel, var_u16->varid, t, 1, U16.data()); - adios_schedule_read_byid(f, sel, var_u32->varid, t, 1, U32.data()); - adios_schedule_read_byid(f, sel, var_u64->varid, t, 1, U64.data()); - adios_schedule_read_byid(f, sel, var_r32->varid, t, 1, R32.data()); - adios_schedule_read_byid(f, sel, var_r64->varid, t, 1, R64.data()); - adios_perform_reads(f, 1); - - // Check if it's correct - for (size_t i = 0; i < Nx; ++i) - { - std::stringstream ss; - ss << "t=" << t << " i=" << i << " rank=" << mpiRank; - std::string msg = ss.str(); - - EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg; - EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg; - EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg; - EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg; - EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg; - EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg; - EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg; - EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg; - EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; - EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; - } - } - - adios_selection_delete(sel); - - // Cleanup variable structures - adios_free_varinfo(var_i8); - adios_free_varinfo(var_i16); - adios_free_varinfo(var_i32); - adios_free_varinfo(var_i64); - adios_free_varinfo(var_u8); - adios_free_varinfo(var_u16); - adios_free_varinfo(var_u32); - adios_free_varinfo(var_u64); - adios_free_varinfo(var_r32); - adios_free_varinfo(var_r64); - - // Cleanup file - adios_read_close(f); - - adios_read_finalize_method(ADIOS_READ_METHOD_BP); - } -} - -//****************************************************************************** -// 2D 2x4 test data -//****************************************************************************** - -// ADIOS2 BP write, native ADIOS1 read -TEST_F(BPWriteReadTest, ADIOS2BPWriteADIOS1Read2D2x4fstream) -{ - // Each process would write a 2x4 array and all processes would - // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here - std::string fname = "ADIOS2BPWriteADIOS1Read2D2x4Testfstream.bp"; - - int mpiRank = 0, mpiSize = 1; - // Number of rows - const std::size_t Nx = 4; - - // Number of rows - const std::size_t Ny = 2; - - // Number of steps - const std::size_t NSteps = 3; - -#if ADIOS2_USE_MPI - MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); - MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); -#endif - - // Write test data using ADIOS2 - { -#if ADIOS2_USE_MPI - adios2::ADIOS adios(MPI_COMM_WORLD); -#else - adios2::ADIOS adios; -#endif - adios2::IO io = adios.DeclareIO("TestIO"); - - // Declare 2D variables (Ny * (NumOfProcesses * Nx)) - // The local process' part (start, count) can be defined now or later - // before Write(). - { - adios2::Dims shape{static_cast(Ny), - static_cast(Nx * mpiSize)}; - adios2::Dims start{static_cast(0), - static_cast(mpiRank * Nx)}; - adios2::Dims count{static_cast(Ny), - static_cast(Nx)}; - auto var_i8 = io.DefineVariable("i8", shape, start, count); - auto var_i16 = - io.DefineVariable("i16", shape, start, count); - auto var_i32 = - io.DefineVariable("i32", shape, start, count); - auto var_i64 = - io.DefineVariable("i64", shape, start, count); - auto var_u8 = io.DefineVariable("u8", shape, start, count); - auto var_u16 = - io.DefineVariable("u16", shape, start, count); - auto var_u32 = - io.DefineVariable("u32", shape, start, count); - auto var_u64 = - io.DefineVariable("u64", shape, start, count); - auto var_r32 = io.DefineVariable("r32", shape, start, count); - auto var_r64 = - io.DefineVariable("r64", shape, start, count); - } - - if (!engineName.empty()) - { - io.SetEngine(engineName); - } - else - { - // Create the BP Engine - io.SetEngine("BPFile"); - } - io.AddTransport("file", {{"Library", "fstream"}}); - - adios2::Engine engine = io.Open(fname, adios2::Mode::Write); - - for (size_t step = 0; step < NSteps; ++step) - { - // Generate test data for each process uniquely - SmallTestData currentTestData = - generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize); - - // Retrieve the variables that previously went out of scope - auto var_i8 = io.InquireVariable("i8"); - auto var_i16 = io.InquireVariable("i16"); - auto var_i32 = io.InquireVariable("i32"); - auto var_i64 = io.InquireVariable("i64"); - auto var_u8 = io.InquireVariable("u8"); - auto var_u16 = io.InquireVariable("u16"); - auto var_u32 = io.InquireVariable("u32"); - auto var_u64 = io.InquireVariable("u64"); - auto var_r32 = io.InquireVariable("r32"); - auto var_r64 = io.InquireVariable("r64"); - - // Make a 2D selection to describe the local dimensions of the - // variable we write and its offsets in the global spaces - adios2::Box sel( - {0, static_cast(mpiRank * Nx)}, {Ny, Nx}); - var_i8.SetSelection(sel); - var_i16.SetSelection(sel); - var_i32.SetSelection(sel); - var_i64.SetSelection(sel); - var_u8.SetSelection(sel); - var_u16.SetSelection(sel); - var_u32.SetSelection(sel); - var_u64.SetSelection(sel); - var_r32.SetSelection(sel); - var_r64.SetSelection(sel); - - // Write each one - // fill in the variable with values from starting index to - // starting index + count - engine.BeginStep(); - engine.Put(var_i8, currentTestData.I8.data()); - engine.Put(var_i16, currentTestData.I16.data()); - engine.Put(var_i32, currentTestData.I32.data()); - engine.Put(var_i64, currentTestData.I64.data()); - engine.Put(var_u8, currentTestData.U8.data()); - engine.Put(var_u16, currentTestData.U16.data()); - engine.Put(var_u32, currentTestData.U32.data()); - engine.Put(var_u64, currentTestData.U64.data()); - engine.Put(var_r32, currentTestData.R32.data()); - engine.Put(var_r64, currentTestData.R64.data()); - engine.EndStep(); - } - - engine.Close(); - } - - { - adios_read_init_method(ADIOS_READ_METHOD_BP, MPI_COMM_SELF, - "verbose=3"); - - // Open the file for reading - // Note: Since collective metadata generation is not implemented yet, - // SO for now we read each subfile instead of a single bp file with all - // metadata. - // Meanwhile if we open file with MPI_COMM_WORLD, then the selection - // bounding box should be [0, Nx] - std::string index = std::to_string(mpiRank); - ADIOS_FILE *f = adios_read_open_file( - (fname + ".dir/" + fname + "." + index).c_str(), - ADIOS_READ_METHOD_BP, MPI_COMM_SELF); - ASSERT_NE(f, nullptr); - - // Check the variables exist - ADIOS_VARINFO *var_i8 = adios_inq_var(f, "i8"); - EXPECT_TRUE(var_i8); - ASSERT_EQ(var_i8->ndim, 2); - ASSERT_EQ(var_i8->global, 1); - ASSERT_EQ(var_i8->nsteps, NSteps); - ASSERT_EQ(var_i8->dims[0], Ny); - ASSERT_EQ(var_i8->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_i16 = adios_inq_var(f, "i16"); - EXPECT_TRUE(var_i16); - ASSERT_EQ(var_i16->ndim, 2); - ASSERT_EQ(var_i16->global, 1); - ASSERT_EQ(var_i16->nsteps, NSteps); - ASSERT_EQ(var_i16->dims[0], Ny); - ASSERT_EQ(var_i16->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_i32 = adios_inq_var(f, "i32"); - EXPECT_TRUE(var_i32); - ASSERT_EQ(var_i32->ndim, 2); - ASSERT_EQ(var_i32->global, 1); - ASSERT_EQ(var_i32->nsteps, NSteps); - ASSERT_EQ(var_i32->dims[0], Ny); - ASSERT_EQ(var_i32->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_i64 = adios_inq_var(f, "i64"); - EXPECT_TRUE(var_i64); - ASSERT_EQ(var_i64->ndim, 2); - ASSERT_EQ(var_i64->global, 1); - ASSERT_EQ(var_i64->nsteps, NSteps); - ASSERT_EQ(var_i64->dims[0], Ny); - ASSERT_EQ(var_i64->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u8 = adios_inq_var(f, "u8"); - EXPECT_TRUE(var_u8); - ASSERT_EQ(var_u8->ndim, 2); - ASSERT_EQ(var_u8->global, 1); - ASSERT_EQ(var_u8->nsteps, NSteps); - ASSERT_EQ(var_u8->dims[0], Ny); - ASSERT_EQ(var_u8->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u16 = adios_inq_var(f, "u16"); - EXPECT_TRUE(var_u16); - ASSERT_EQ(var_u16->ndim, 2); - ASSERT_EQ(var_u16->global, 1); - ASSERT_EQ(var_u16->nsteps, NSteps); - ASSERT_EQ(var_u16->dims[0], Ny); - ASSERT_EQ(var_u16->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u32 = adios_inq_var(f, "u32"); - EXPECT_TRUE(var_u32); - ASSERT_EQ(var_u32->ndim, 2); - ASSERT_EQ(var_u32->global, 1); - ASSERT_EQ(var_u32->nsteps, NSteps); - ASSERT_EQ(var_u32->dims[0], Ny); - ASSERT_EQ(var_u32->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u64 = adios_inq_var(f, "u64"); - EXPECT_TRUE(var_u64); - ASSERT_EQ(var_u64->ndim, 2); - ASSERT_EQ(var_u64->global, 1); - ASSERT_EQ(var_u64->nsteps, NSteps); - ASSERT_EQ(var_u64->dims[0], Ny); - ASSERT_EQ(var_u64->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_r32 = adios_inq_var(f, "r32"); - EXPECT_TRUE(var_r32); - ASSERT_EQ(var_r32->ndim, 2); - ASSERT_EQ(var_r32->global, 1); - ASSERT_EQ(var_r32->nsteps, NSteps); - ASSERT_EQ(var_r32->dims[0], Ny); - ASSERT_EQ(var_r32->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_r64 = adios_inq_var(f, "r64"); - EXPECT_TRUE(var_r64); - ASSERT_EQ(var_r64->ndim, 2); - ASSERT_EQ(var_r64->global, 1); - ASSERT_EQ(var_r64->nsteps, NSteps); - ASSERT_EQ(var_r64->dims[0], Ny); - ASSERT_EQ(var_r64->dims[1], mpiSize * Nx); - - // If the size of the array is smaller than the data - // the result is weird... double and uint64_t would get completely - // garbage data - std::array I8; - std::array I16; - std::array I32; - std::array I64; - std::array U8; - std::array U16; - std::array U32; - std::array U64; - std::array R32; - std::array R64; - - uint64_t start[2] = {0, mpiRank * Nx}; - uint64_t count[2] = {Ny, Nx}; - ADIOS_SELECTION *sel = adios_selection_boundingbox(2, start, count); - - // Read stuff - for (size_t t = 0; t < NSteps; ++t) - { - // Generate test data for each rank uniquely - SmallTestData currentTestData = - generateNewSmallTestData(m_TestData, t, mpiRank, mpiSize); - // Read the current step - adios_schedule_read_byid(f, sel, var_i8->varid, t, 1, I8.data()); - adios_schedule_read_byid(f, sel, var_i16->varid, t, 1, I16.data()); - adios_schedule_read_byid(f, sel, var_i32->varid, t, 1, I32.data()); - adios_schedule_read_byid(f, sel, var_i64->varid, t, 1, I64.data()); - adios_schedule_read_byid(f, sel, var_u8->varid, t, 1, U8.data()); - adios_schedule_read_byid(f, sel, var_u16->varid, t, 1, U16.data()); - adios_schedule_read_byid(f, sel, var_u32->varid, t, 1, U32.data()); - adios_schedule_read_byid(f, sel, var_u64->varid, t, 1, U64.data()); - adios_schedule_read_byid(f, sel, var_r32->varid, t, 1, R32.data()); - adios_schedule_read_byid(f, sel, var_r64->varid, t, 1, R64.data()); - adios_perform_reads(f, 1); - - // Check if it's correct - for (size_t i = 0; i < Nx; ++i) - { - std::stringstream ss; - ss << "t=" << t << " i=" << i << " rank=" << mpiRank; - std::string msg = ss.str(); - - EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg; - EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg; - EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg; - EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg; - EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg; - EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg; - EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg; - EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg; - EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; - EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; - } - } - - adios_selection_delete(sel); - - // Cleanup variable structures - adios_free_varinfo(var_i8); - adios_free_varinfo(var_i16); - adios_free_varinfo(var_i32); - adios_free_varinfo(var_i64); - adios_free_varinfo(var_u8); - adios_free_varinfo(var_u16); - adios_free_varinfo(var_u32); - adios_free_varinfo(var_u64); - adios_free_varinfo(var_r32); - adios_free_varinfo(var_r64); - - // Cleanup file - adios_read_close(f); - - adios_read_finalize_method(ADIOS_READ_METHOD_BP); - } -} - -// ADIOS2 write, native ADIOS1 read -TEST_F(BPWriteReadTest, ADIOS2BPWriteADIOS1Read2D4x2fstream) -{ - // Each process would write a 4x2 array and all processes would - // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - std::string fname = "ADIOS2BPWriteADIOS1Read2D4x2Testfstream.bp"; - - int mpiRank = 0, mpiSize = 1; - // Number of rows - const std::size_t Nx = 2; - // Number of cols - const std::size_t Ny = 4; - - // Number of steps - const std::size_t NSteps = 3; - -#if ADIOS2_USE_MPI - MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); - MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); -#endif - - // Write test data using ADIOS2 - { -#if ADIOS2_USE_MPI - adios2::ADIOS adios(MPI_COMM_WORLD); -#else - adios2::ADIOS adios; -#endif - adios2::IO io = adios.DeclareIO("TestIO"); - - // Declare 2D variables (4 * (NumberOfProcess * Nx)) - // The local process' part (start, count) can be defined now or later - // before Write(). - { - adios2::Dims shape{static_cast(Ny), - static_cast(mpiSize * Nx)}; - adios2::Dims start{static_cast(0), - static_cast(mpiRank * Nx)}; - adios2::Dims count{static_cast(Ny), - static_cast(Nx)}; - auto var_i8 = io.DefineVariable("i8", shape, start, count); - auto var_i16 = - io.DefineVariable("i16", shape, start, count); - auto var_i32 = - io.DefineVariable("i32", shape, start, count); - auto var_i64 = - io.DefineVariable("i64", shape, start, count); - auto var_u8 = io.DefineVariable("u8", shape, start, count); - auto var_u16 = - io.DefineVariable("u16", shape, start, count); - auto var_u32 = - io.DefineVariable("u32", shape, start, count); - auto var_u64 = - io.DefineVariable("u64", shape, start, count); - auto var_r32 = io.DefineVariable("r32", shape, start, count); - auto var_r64 = - io.DefineVariable("r64", shape, start, count); - } - - if (!engineName.empty()) - { - io.SetEngine(engineName); - } - else - { - // Create the BP Engine - io.SetEngine("BPFile"); - } - io.AddTransport("file", {{"Library", "fstream"}}); - - adios2::Engine engine = io.Open(fname, adios2::Mode::Write); - - for (size_t step = 0; step < NSteps; ++step) - { - // Generate test data for each process uniquely - SmallTestData currentTestData = - generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize); - - // Retrieve the variables that previously went out of scope - auto var_i8 = io.InquireVariable("i8"); - auto var_i16 = io.InquireVariable("i16"); - auto var_i32 = io.InquireVariable("i32"); - auto var_i64 = io.InquireVariable("i64"); - auto var_u8 = io.InquireVariable("u8"); - auto var_u16 = io.InquireVariable("u16"); - auto var_u32 = io.InquireVariable("u32"); - auto var_u64 = io.InquireVariable("u64"); - auto var_r32 = io.InquireVariable("r32"); - auto var_r64 = io.InquireVariable("r64"); - - // Make a 2D selection to describe the local dimensions of the - // variable we write and its offsets in the global spaces - adios2::Box sel( - {0, static_cast(mpiRank * Nx)}, {Ny, Nx}); - var_i8.SetSelection(sel); - var_i16.SetSelection(sel); - var_i32.SetSelection(sel); - var_i64.SetSelection(sel); - var_u8.SetSelection(sel); - var_u16.SetSelection(sel); - var_u32.SetSelection(sel); - var_u64.SetSelection(sel); - var_r32.SetSelection(sel); - var_r64.SetSelection(sel); - - // Write each one - // fill in the variable with values from starting index to - // starting index + count - engine.BeginStep(); - engine.Put(var_i8, currentTestData.I8.data()); - engine.Put(var_i16, currentTestData.I16.data()); - engine.Put(var_i32, currentTestData.I32.data()); - engine.Put(var_i64, currentTestData.I64.data()); - engine.Put(var_u8, currentTestData.U8.data()); - engine.Put(var_u16, currentTestData.U16.data()); - engine.Put(var_u32, currentTestData.U32.data()); - engine.Put(var_u64, currentTestData.U64.data()); - engine.Put(var_r32, currentTestData.R32.data()); - engine.Put(var_r64, currentTestData.R64.data()); - engine.EndStep(); - } - - engine.Close(); - } - - { - adios_read_init_method(ADIOS_READ_METHOD_BP, MPI_COMM_SELF, - "verbose=3"); - - // Open the file for reading - // Note: Since collective metadata generation is not implemented yet, - // SO for now we read each subfile instead of a single bp file with all - // metadata. - // Meanwhile if we open file with MPI_COMM_WORLD, then the selection - // bounding box should be [0, Nx] - std::string index = std::to_string(mpiRank); - ADIOS_FILE *f = adios_read_open_file( - (fname + ".dir/" + fname + "." + index).c_str(), - ADIOS_READ_METHOD_BP, MPI_COMM_SELF); - ASSERT_NE(f, nullptr); - - // Check the variables exist - ADIOS_VARINFO *var_i8 = adios_inq_var(f, "i8"); - EXPECT_TRUE(var_i8); - ASSERT_EQ(var_i8->ndim, 2); - ASSERT_EQ(var_i8->global, 1); - ASSERT_EQ(var_i8->nsteps, NSteps); - ASSERT_EQ(var_i8->dims[0], Ny); - ASSERT_EQ(var_i8->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_i16 = adios_inq_var(f, "i16"); - EXPECT_TRUE(var_i16); - ASSERT_EQ(var_i16->ndim, 2); - ASSERT_EQ(var_i16->global, 1); - ASSERT_EQ(var_i16->nsteps, NSteps); - ASSERT_EQ(var_i16->dims[0], Ny); - ASSERT_EQ(var_i16->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_i32 = adios_inq_var(f, "i32"); - EXPECT_TRUE(var_i32); - ASSERT_EQ(var_i32->ndim, 2); - ASSERT_EQ(var_i32->global, 1); - ASSERT_EQ(var_i32->nsteps, NSteps); - ASSERT_EQ(var_i32->dims[0], Ny); - ASSERT_EQ(var_i32->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_i64 = adios_inq_var(f, "i64"); - EXPECT_TRUE(var_i64); - ASSERT_EQ(var_i64->ndim, 2); - ASSERT_EQ(var_i64->global, 1); - ASSERT_EQ(var_i64->nsteps, NSteps); - ASSERT_EQ(var_i64->dims[0], Ny); - ASSERT_EQ(var_i64->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u8 = adios_inq_var(f, "u8"); - EXPECT_TRUE(var_u8); - ASSERT_EQ(var_u8->ndim, 2); - ASSERT_EQ(var_u8->global, 1); - ASSERT_EQ(var_u8->nsteps, NSteps); - ASSERT_EQ(var_u8->dims[0], Ny); - ASSERT_EQ(var_u8->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u16 = adios_inq_var(f, "u16"); - EXPECT_TRUE(var_u16); - ASSERT_EQ(var_u16->ndim, 2); - ASSERT_EQ(var_u16->global, 1); - ASSERT_EQ(var_u16->nsteps, NSteps); - ASSERT_EQ(var_u16->dims[0], Ny); - ASSERT_EQ(var_u16->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u32 = adios_inq_var(f, "u32"); - EXPECT_TRUE(var_u32); - ASSERT_EQ(var_u32->ndim, 2); - ASSERT_EQ(var_u32->global, 1); - ASSERT_EQ(var_u32->nsteps, NSteps); - ASSERT_EQ(var_u32->dims[0], Ny); - ASSERT_EQ(var_u32->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u64 = adios_inq_var(f, "u64"); - EXPECT_TRUE(var_u64); - ASSERT_EQ(var_u64->ndim, 2); - ASSERT_EQ(var_u64->global, 1); - ASSERT_EQ(var_u64->nsteps, NSteps); - ASSERT_EQ(var_u64->dims[0], Ny); - ASSERT_EQ(var_u64->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_r32 = adios_inq_var(f, "r32"); - EXPECT_TRUE(var_r32); - ASSERT_EQ(var_r32->ndim, 2); - ASSERT_EQ(var_r32->global, 1); - ASSERT_EQ(var_r32->nsteps, NSteps); - ASSERT_EQ(var_r32->dims[0], Ny); - ASSERT_EQ(var_r32->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_r64 = adios_inq_var(f, "r64"); - EXPECT_TRUE(var_r64); - ASSERT_EQ(var_r64->ndim, 2); - ASSERT_EQ(var_r64->global, 1); - ASSERT_EQ(var_r64->nsteps, NSteps); - ASSERT_EQ(var_r64->dims[0], Ny); - ASSERT_EQ(var_r64->dims[1], mpiSize * Nx); - - // If the size of the array is smaller than the data - // the result is weird... double and uint64_t would get completely - // garbage data - std::array I8; - std::array I16; - std::array I32; - std::array I64; - std::array U8; - std::array U16; - std::array U32; - std::array U64; - std::array R32; - std::array R64; - - uint64_t start[2] = {0, mpiRank * Nx}; - uint64_t count[2] = {Ny, Nx}; - ADIOS_SELECTION *sel = adios_selection_boundingbox(2, start, count); - - // Read stuff - for (size_t t = 0; t < NSteps; ++t) - { - // Generate test data for each rank uniquely - SmallTestData currentTestData = - generateNewSmallTestData(m_TestData, t, mpiRank, mpiSize); - // Read the current step - adios_schedule_read_byid(f, sel, var_i8->varid, t, 1, I8.data()); - adios_schedule_read_byid(f, sel, var_i16->varid, t, 1, I16.data()); - adios_schedule_read_byid(f, sel, var_i32->varid, t, 1, I32.data()); - adios_schedule_read_byid(f, sel, var_i64->varid, t, 1, I64.data()); - adios_schedule_read_byid(f, sel, var_u8->varid, t, 1, U8.data()); - adios_schedule_read_byid(f, sel, var_u16->varid, t, 1, U16.data()); - adios_schedule_read_byid(f, sel, var_u32->varid, t, 1, U32.data()); - adios_schedule_read_byid(f, sel, var_u64->varid, t, 1, U64.data()); - adios_schedule_read_byid(f, sel, var_r32->varid, t, 1, R32.data()); - adios_schedule_read_byid(f, sel, var_r64->varid, t, 1, R64.data()); - adios_perform_reads(f, 1); - - // Check if it's correct - for (size_t i = 0; i < Nx; ++i) - { - std::stringstream ss; - ss << "t=" << t << " i=" << i << " rank=" << mpiRank; - std::string msg = ss.str(); - - EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg; - EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg; - EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg; - EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg; - EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg; - EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg; - EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg; - EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg; - EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; - EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; - } - } - - adios_selection_delete(sel); - - // Cleanup variable structures - adios_free_varinfo(var_i8); - adios_free_varinfo(var_i16); - adios_free_varinfo(var_i32); - adios_free_varinfo(var_i64); - adios_free_varinfo(var_u8); - adios_free_varinfo(var_u16); - adios_free_varinfo(var_u32); - adios_free_varinfo(var_u64); - adios_free_varinfo(var_r32); - adios_free_varinfo(var_r64); - - // Cleanup file - adios_read_close(f); - - adios_read_finalize_method(ADIOS_READ_METHOD_BP); - } -} - -//****************************************************************************** -// main -//****************************************************************************** - -int main(int argc, char **argv) -{ -#if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); -#endif - - int result; - ::testing::InitGoogleTest(&argc, argv); - - if (argc > 1) - { - engineName = std::string(argv[1]); - } - result = RUN_ALL_TESTS(); - -#if ADIOS2_USE_MPI - MPI_Finalize(); -#endif - - return result; -} diff --git a/testing/adios2/engine/bp/TestBPWriteReadstdio.cpp b/testing/adios2/engine/bp/TestBPWriteReadstdio.cpp deleted file mode 100644 index 674c9faae9..0000000000 --- a/testing/adios2/engine/bp/TestBPWriteReadstdio.cpp +++ /dev/null @@ -1,935 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - */ -#include -#include - -#include -#include - -#include -#include - -#include - -#include "../SmallTestData.h" - -std::string engineName; // comes from command line - -class BPWriteReadTest : public ::testing::Test -{ -public: - BPWriteReadTest() = default; - - SmallTestData m_TestData; -}; - -//****************************************************************************** -// 1D 1x8 test data -//****************************************************************************** - -// ADIOS2 BP write, native ADIOS1 read -TEST_F(BPWriteReadTest, ADIOS2BPWriteADIOS1Read1D8stdio) -{ - // Each process would write a 1x8 array and all processes would - // form a mpiSize * Nx 1D array - std::string fname = "ADIOS2BPWriteADIOS1Read1D8stdio.bp"; - - int mpiRank = 0, mpiSize = 1; - // Number of rows - const std::size_t Nx = 8; - - // Number of steps - const std::size_t NSteps = 3; - -#if ADIOS2_USE_MPI - MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); - MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); -#endif - - // Write test data using BP - { -#if ADIOS2_USE_MPI - adios2::ADIOS adios(MPI_COMM_WORLD); -#else - adios2::ADIOS adios; -#endif - adios2::IO io = adios.DeclareIO("TestIO"); - - // Declare 1D variables (NumOfProcesses * Nx) - // The local process' part (start, count) can be defined now or later - // before Write(). - { - adios2::Dims shape{static_cast(Nx * mpiSize)}; - adios2::Dims start{static_cast(Nx * mpiRank)}; - adios2::Dims count{static_cast(Nx)}; - auto var_i8 = io.DefineVariable("i8", shape, start, count); - auto var_i16 = - io.DefineVariable("i16", shape, start, count); - auto var_i32 = - io.DefineVariable("i32", shape, start, count); - auto var_i64 = - io.DefineVariable("i64", shape, start, count); - auto var_u8 = io.DefineVariable("u8", shape, start, count); - auto var_u16 = - io.DefineVariable("u16", shape, start, count); - auto var_u32 = - io.DefineVariable("u32", shape, start, count); - auto var_u64 = - io.DefineVariable("u64", shape, start, count); - auto var_r32 = io.DefineVariable("r32", shape, start, count); - auto var_r64 = - io.DefineVariable("r64", shape, start, count); - } - - if (!engineName.empty()) - { - io.SetEngine(engineName); - } - else - { - // Create the BP Engine - io.SetEngine("BPFile"); - } - -#if ADIOS2_USE_MPI - io.AddTransport("file", {{"Library", "stdio"}}); -#else - io.AddTransport("file"); -#endif - // QUESTION: It seems that BPFilterWriter cannot overwrite existing - // files - // Ex. if you tune Nx and NSteps, the test would fail. But if you clear - // the cache in - // ${adios2Build}/testing/adios2/engine/bp/ADIOS2BPWriteADIOS1Read1D8.bp.dir, - // then it works - adios2::Engine engine = io.Open(fname, adios2::Mode::Write); - - for (size_t step = 0; step < NSteps; ++step) - { - // Generate test data for each process uniquely - SmallTestData currentTestData = - generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize); - - // Retrieve the variables that previously went out of scope - auto var_i8 = io.InquireVariable("i8"); - auto var_i16 = io.InquireVariable("i16"); - auto var_i32 = io.InquireVariable("i32"); - auto var_i64 = io.InquireVariable("i64"); - auto var_u8 = io.InquireVariable("u8"); - auto var_u16 = io.InquireVariable("u16"); - auto var_u32 = io.InquireVariable("u32"); - auto var_u64 = io.InquireVariable("u64"); - auto var_r32 = io.InquireVariable("r32"); - auto var_r64 = io.InquireVariable("r64"); - - // Make a 1D selection to describe the local dimensions of the - // variable we write and its offsets in the global spaces - adios2::Box sel({mpiRank * Nx}, {Nx}); - var_i8.SetSelection(sel); - var_i16.SetSelection(sel); - var_i32.SetSelection(sel); - var_i64.SetSelection(sel); - var_u8.SetSelection(sel); - var_u16.SetSelection(sel); - var_u32.SetSelection(sel); - var_u64.SetSelection(sel); - var_r32.SetSelection(sel); - var_r64.SetSelection(sel); - - // Write each one - // fill in the variable with values from starting index to - // starting index + count - engine.Put(var_i8, currentTestData.I8.data()); - engine.Put(var_i16, currentTestData.I16.data()); - engine.Put(var_i32, currentTestData.I32.data()); - engine.Put(var_i64, currentTestData.I64.data()); - engine.Put(var_u8, currentTestData.U8.data()); - engine.Put(var_u16, currentTestData.U16.data()); - engine.Put(var_u32, currentTestData.U32.data()); - engine.Put(var_u64, currentTestData.U64.data()); - engine.Put(var_r32, currentTestData.R32.data()); - engine.Put(var_r64, currentTestData.R64.data()); - - // Advance to the next time step - engine.EndStep(); - } - - // Close the file - engine.Close(); - } - - { - adios_read_init_method(ADIOS_READ_METHOD_BP, MPI_COMM_SELF, - "verbose=3"); - - // Open the file for reading - // Note: Since collective metadata generation is not implemented yet, - // SO for now we read each subfile instead of a single bp file with all - // metadata. - // Meanwhile if we open file with MPI_COMM_WORLD, then the selection - // bounding box should be [0, Nx] - std::string index = std::to_string(mpiRank); - ADIOS_FILE *f = adios_read_open_file( - (fname + ".dir/" + fname + "." + index).c_str(), - ADIOS_READ_METHOD_BP, MPI_COMM_SELF); - ASSERT_NE(f, nullptr); - - // Check the variables exist - ADIOS_VARINFO *var_i8 = adios_inq_var(f, "i8"); - EXPECT_TRUE(var_i8); - ASSERT_EQ(var_i8->ndim, 1); - ASSERT_EQ(var_i8->global, 1); - ASSERT_EQ(var_i8->nsteps, NSteps); - ASSERT_EQ(var_i8->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_i16 = adios_inq_var(f, "i16"); - EXPECT_TRUE(var_i16); - ASSERT_EQ(var_i16->ndim, 1); - ASSERT_EQ(var_i16->global, 1); - ASSERT_EQ(var_i16->nsteps, NSteps); - ASSERT_EQ(var_i16->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_i32 = adios_inq_var(f, "i32"); - EXPECT_TRUE(var_i32); - ASSERT_EQ(var_i32->ndim, 1); - ASSERT_EQ(var_i32->global, 1); - ASSERT_EQ(var_i32->nsteps, NSteps); - ASSERT_EQ(var_i32->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_i64 = adios_inq_var(f, "i64"); - EXPECT_TRUE(var_i64); - ASSERT_EQ(var_i64->ndim, 1); - ASSERT_EQ(var_i64->global, 1); - ASSERT_EQ(var_i64->nsteps, NSteps); - ASSERT_EQ(var_i64->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_u8 = adios_inq_var(f, "u8"); - EXPECT_TRUE(var_u8); - ASSERT_EQ(var_u8->ndim, 1); - ASSERT_EQ(var_u8->global, 1); - ASSERT_EQ(var_u8->nsteps, NSteps); - ASSERT_EQ(var_u8->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_u16 = adios_inq_var(f, "u16"); - EXPECT_TRUE(var_u16); - ASSERT_EQ(var_u16->ndim, 1); - ASSERT_EQ(var_u16->global, 1); - ASSERT_EQ(var_u16->nsteps, NSteps); - ASSERT_EQ(var_u16->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_u32 = adios_inq_var(f, "u32"); - EXPECT_TRUE(var_u32); - ASSERT_EQ(var_u32->ndim, 1); - ASSERT_EQ(var_u32->global, 1); - ASSERT_EQ(var_u32->nsteps, NSteps); - ASSERT_EQ(var_u32->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_u64 = adios_inq_var(f, "u64"); - EXPECT_TRUE(var_u64); - ASSERT_EQ(var_u64->ndim, 1); - ASSERT_EQ(var_u64->global, 1); - ASSERT_EQ(var_u64->nsteps, NSteps); - ASSERT_EQ(var_u64->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_r32 = adios_inq_var(f, "r32"); - EXPECT_TRUE(var_r32); - ASSERT_EQ(var_r32->ndim, 1); - ASSERT_EQ(var_r32->global, 1); - ASSERT_EQ(var_r32->nsteps, NSteps); - ASSERT_EQ(var_r32->dims[0], mpiSize * Nx); - ADIOS_VARINFO *var_r64 = adios_inq_var(f, "r64"); - EXPECT_TRUE(var_r64); - ASSERT_EQ(var_r64->ndim, 1); - ASSERT_EQ(var_r64->global, 1); - ASSERT_EQ(var_r64->nsteps, NSteps); - ASSERT_EQ(var_r64->dims[0], mpiSize * Nx); - - std::array I8; - std::array I16; - std::array I32; - std::array I64; - std::array U8; - std::array U16; - std::array U32; - std::array U64; - std::array R32; - std::array R64; - - uint64_t start[1] = {mpiRank * Nx}; - uint64_t count[1] = {Nx}; - ADIOS_SELECTION *sel = adios_selection_boundingbox(1, start, count); - - // Read stuff - for (size_t t = 0; t < NSteps; ++t) - { - // Generate test data for each rank uniquely - SmallTestData currentTestData = - generateNewSmallTestData(m_TestData, t, mpiRank, mpiSize); - // Read the current step - adios_schedule_read_byid(f, sel, var_i8->varid, t, 1, I8.data()); - adios_schedule_read_byid(f, sel, var_i16->varid, t, 1, I16.data()); - adios_schedule_read_byid(f, sel, var_i32->varid, t, 1, I32.data()); - adios_schedule_read_byid(f, sel, var_i64->varid, t, 1, I64.data()); - adios_schedule_read_byid(f, sel, var_u8->varid, t, 1, U8.data()); - adios_schedule_read_byid(f, sel, var_u16->varid, t, 1, U16.data()); - adios_schedule_read_byid(f, sel, var_u32->varid, t, 1, U32.data()); - adios_schedule_read_byid(f, sel, var_u64->varid, t, 1, U64.data()); - adios_schedule_read_byid(f, sel, var_r32->varid, t, 1, R32.data()); - adios_schedule_read_byid(f, sel, var_r64->varid, t, 1, R64.data()); - adios_perform_reads(f, 1); - - // Check if it's correct - for (size_t i = 0; i < Nx; ++i) - { - std::stringstream ss; - ss << "t=" << t << " i=" << i << " rank=" << mpiRank; - std::string msg = ss.str(); - - EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg; - EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg; - EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg; - EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg; - EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg; - EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg; - EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg; - EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg; - EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; - EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; - } - } - - adios_selection_delete(sel); - - // Cleanup variable structures - adios_free_varinfo(var_i8); - adios_free_varinfo(var_i16); - adios_free_varinfo(var_i32); - adios_free_varinfo(var_i64); - adios_free_varinfo(var_u8); - adios_free_varinfo(var_u16); - adios_free_varinfo(var_u32); - adios_free_varinfo(var_u64); - adios_free_varinfo(var_r32); - adios_free_varinfo(var_r64); - - // Cleanup file - adios_read_close(f); - - adios_read_finalize_method(ADIOS_READ_METHOD_BP); - } -} - -// ADIOS2 BP write, native ADIOS1 read -TEST_F(BPWriteReadTest, ADIOS2BPWriteADIOS1Read2D2x4stdio) -{ - // Each process would write a 2x4 array and all processes would - // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here - std::string fname = "ADIOS2BPWriteADIOS1Read2D2x4Teststdio.bp"; - - int mpiRank = 0, mpiSize = 1; - // Number of rows - const std::size_t Nx = 4; - - // Number of rows - const std::size_t Ny = 2; - - // Number of steps - const std::size_t NSteps = 3; - -#if ADIOS2_USE_MPI - MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); - MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); -#endif - - // Write test data using ADIOS2 - { -#if ADIOS2_USE_MPI - adios2::ADIOS adios(MPI_COMM_WORLD); -#else - adios2::ADIOS adios; -#endif - adios2::IO io = adios.DeclareIO("TestIO"); - - // Declare 2D variables (Ny * (NumOfProcesses * Nx)) - // The local process' part (start, count) can be defined now or later - // before Write(). - { - adios2::Dims shape{static_cast(Ny), - static_cast(Nx * mpiSize)}; - adios2::Dims start{static_cast(0), - static_cast(mpiRank * Nx)}; - adios2::Dims count{static_cast(Ny), - static_cast(Nx)}; - auto var_i8 = io.DefineVariable("i8", shape, start, count); - auto var_i16 = - io.DefineVariable("i16", shape, start, count); - auto var_i32 = - io.DefineVariable("i32", shape, start, count); - auto var_i64 = - io.DefineVariable("i64", shape, start, count); - auto var_u8 = io.DefineVariable("u8", shape, start, count); - auto var_u16 = - io.DefineVariable("u16", shape, start, count); - auto var_u32 = - io.DefineVariable("u32", shape, start, count); - auto var_u64 = - io.DefineVariable("u64", shape, start, count); - auto var_r32 = io.DefineVariable("r32", shape, start, count); - auto var_r64 = - io.DefineVariable("r64", shape, start, count); - } - - if (!engineName.empty()) - { - io.SetEngine(engineName); - } - else - { - // Create the BP Engine - io.SetEngine("BPFile"); - } -#if ADIOS2_USE_MPI - io.AddTransport("file", {{"Library", "stdio"}}); -#else - io.AddTransport("file"); -#endif - - adios2::Engine engine = io.Open(fname, adios2::Mode::Write); - - for (size_t step = 0; step < NSteps; ++step) - { - // Generate test data for each process uniquely - SmallTestData currentTestData = - generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize); - - // Retrieve the variables that previously went out of scope - auto var_i8 = io.InquireVariable("i8"); - auto var_i16 = io.InquireVariable("i16"); - auto var_i32 = io.InquireVariable("i32"); - auto var_i64 = io.InquireVariable("i64"); - auto var_u8 = io.InquireVariable("u8"); - auto var_u16 = io.InquireVariable("u16"); - auto var_u32 = io.InquireVariable("u32"); - auto var_u64 = io.InquireVariable("u64"); - auto var_r32 = io.InquireVariable("r32"); - auto var_r64 = io.InquireVariable("r64"); - - // Make a 2D selection to describe the local dimensions of the - // variable we write and its offsets in the global spaces - adios2::Box sel( - {0, static_cast(mpiRank * Nx)}, {Ny, Nx}); - var_i8.SetSelection(sel); - var_i16.SetSelection(sel); - var_i32.SetSelection(sel); - var_i64.SetSelection(sel); - var_u8.SetSelection(sel); - var_u16.SetSelection(sel); - var_u32.SetSelection(sel); - var_u64.SetSelection(sel); - var_r32.SetSelection(sel); - var_r64.SetSelection(sel); - - // Write each one - // fill in the variable with values from starting index to - // starting index + count - engine.Put(var_i8, currentTestData.I8.data()); - engine.Put(var_i16, currentTestData.I16.data()); - engine.Put(var_i32, currentTestData.I32.data()); - engine.Put(var_i64, currentTestData.I64.data()); - engine.Put(var_u8, currentTestData.U8.data()); - engine.Put(var_u16, currentTestData.U16.data()); - engine.Put(var_u32, currentTestData.U32.data()); - engine.Put(var_u64, currentTestData.U64.data()); - engine.Put(var_r32, currentTestData.R32.data()); - engine.Put(var_r64, currentTestData.R64.data()); - - // Advance to the next time step - engine.EndStep(); - } - - // Close the file - engine.Close(); - } - - { - adios_read_init_method(ADIOS_READ_METHOD_BP, MPI_COMM_SELF, - "verbose=3"); - - // Open the file for reading - // Note: Since collective metadata generation is not implemented yet, - // SO for now we read each subfile instead of a single bp file with all - // metadata. - // Meanwhile if we open file with MPI_COMM_WORLD, then the selection - // bounding box should be [0, Nx] - std::string index = std::to_string(mpiRank); - ADIOS_FILE *f = adios_read_open_file( - (fname + ".dir/" + fname + "." + index).c_str(), - ADIOS_READ_METHOD_BP, MPI_COMM_SELF); - ASSERT_NE(f, nullptr); - - // Check the variables exist - ADIOS_VARINFO *var_i8 = adios_inq_var(f, "i8"); - EXPECT_TRUE(var_i8); - ASSERT_EQ(var_i8->ndim, 2); - ASSERT_EQ(var_i8->global, 1); - ASSERT_EQ(var_i8->nsteps, NSteps); - ASSERT_EQ(var_i8->dims[0], Ny); - ASSERT_EQ(var_i8->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_i16 = adios_inq_var(f, "i16"); - EXPECT_TRUE(var_i16); - ASSERT_EQ(var_i16->ndim, 2); - ASSERT_EQ(var_i16->global, 1); - ASSERT_EQ(var_i16->nsteps, NSteps); - ASSERT_EQ(var_i16->dims[0], Ny); - ASSERT_EQ(var_i16->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_i32 = adios_inq_var(f, "i32"); - EXPECT_TRUE(var_i32); - ASSERT_EQ(var_i32->ndim, 2); - ASSERT_EQ(var_i32->global, 1); - ASSERT_EQ(var_i32->nsteps, NSteps); - ASSERT_EQ(var_i32->dims[0], Ny); - ASSERT_EQ(var_i32->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_i64 = adios_inq_var(f, "i64"); - EXPECT_TRUE(var_i64); - ASSERT_EQ(var_i64->ndim, 2); - ASSERT_EQ(var_i64->global, 1); - ASSERT_EQ(var_i64->nsteps, NSteps); - ASSERT_EQ(var_i64->dims[0], Ny); - ASSERT_EQ(var_i64->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u8 = adios_inq_var(f, "u8"); - EXPECT_TRUE(var_u8); - ASSERT_EQ(var_u8->ndim, 2); - ASSERT_EQ(var_u8->global, 1); - ASSERT_EQ(var_u8->nsteps, NSteps); - ASSERT_EQ(var_u8->dims[0], Ny); - ASSERT_EQ(var_u8->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u16 = adios_inq_var(f, "u16"); - EXPECT_TRUE(var_u16); - ASSERT_EQ(var_u16->ndim, 2); - ASSERT_EQ(var_u16->global, 1); - ASSERT_EQ(var_u16->nsteps, NSteps); - ASSERT_EQ(var_u16->dims[0], Ny); - ASSERT_EQ(var_u16->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u32 = adios_inq_var(f, "u32"); - EXPECT_TRUE(var_u32); - ASSERT_EQ(var_u32->ndim, 2); - ASSERT_EQ(var_u32->global, 1); - ASSERT_EQ(var_u32->nsteps, NSteps); - ASSERT_EQ(var_u32->dims[0], Ny); - ASSERT_EQ(var_u32->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u64 = adios_inq_var(f, "u64"); - EXPECT_TRUE(var_u64); - ASSERT_EQ(var_u64->ndim, 2); - ASSERT_EQ(var_u64->global, 1); - ASSERT_EQ(var_u64->nsteps, NSteps); - ASSERT_EQ(var_u64->dims[0], Ny); - ASSERT_EQ(var_u64->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_r32 = adios_inq_var(f, "r32"); - EXPECT_TRUE(var_r32); - ASSERT_EQ(var_r32->ndim, 2); - ASSERT_EQ(var_r32->global, 1); - ASSERT_EQ(var_r32->nsteps, NSteps); - ASSERT_EQ(var_r32->dims[0], Ny); - ASSERT_EQ(var_r32->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_r64 = adios_inq_var(f, "r64"); - EXPECT_TRUE(var_r64); - ASSERT_EQ(var_r64->ndim, 2); - ASSERT_EQ(var_r64->global, 1); - ASSERT_EQ(var_r64->nsteps, NSteps); - ASSERT_EQ(var_r64->dims[0], Ny); - ASSERT_EQ(var_r64->dims[1], mpiSize * Nx); - - // If the size of the array is smaller than the data - // the result is weird... double and uint64_t would get completely - // garbage data - std::array I8; - std::array I16; - std::array I32; - std::array I64; - std::array U8; - std::array U16; - std::array U32; - std::array U64; - std::array R32; - std::array R64; - - uint64_t start[2] = {0, mpiRank * Nx}; - uint64_t count[2] = {Ny, Nx}; - ADIOS_SELECTION *sel = adios_selection_boundingbox(2, start, count); - - // Read stuff - for (size_t t = 0; t < NSteps; ++t) - { - // Generate test data for each rank uniquely - SmallTestData currentTestData = - generateNewSmallTestData(m_TestData, t, mpiRank, mpiSize); - // Read the current step - adios_schedule_read_byid(f, sel, var_i8->varid, t, 1, I8.data()); - adios_schedule_read_byid(f, sel, var_i16->varid, t, 1, I16.data()); - adios_schedule_read_byid(f, sel, var_i32->varid, t, 1, I32.data()); - adios_schedule_read_byid(f, sel, var_i64->varid, t, 1, I64.data()); - adios_schedule_read_byid(f, sel, var_u8->varid, t, 1, U8.data()); - adios_schedule_read_byid(f, sel, var_u16->varid, t, 1, U16.data()); - adios_schedule_read_byid(f, sel, var_u32->varid, t, 1, U32.data()); - adios_schedule_read_byid(f, sel, var_u64->varid, t, 1, U64.data()); - adios_schedule_read_byid(f, sel, var_r32->varid, t, 1, R32.data()); - adios_schedule_read_byid(f, sel, var_r64->varid, t, 1, R64.data()); - adios_perform_reads(f, 1); - - // Check if it's correct - for (size_t i = 0; i < Nx; ++i) - { - std::stringstream ss; - ss << "t=" << t << " i=" << i << " rank=" << mpiRank; - std::string msg = ss.str(); - - EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg; - EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg; - EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg; - EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg; - EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg; - EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg; - EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg; - EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg; - EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; - EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; - } - } - - adios_selection_delete(sel); - - // Cleanup variable structures - adios_free_varinfo(var_i8); - adios_free_varinfo(var_i16); - adios_free_varinfo(var_i32); - adios_free_varinfo(var_i64); - adios_free_varinfo(var_u8); - adios_free_varinfo(var_u16); - adios_free_varinfo(var_u32); - adios_free_varinfo(var_u64); - adios_free_varinfo(var_r32); - adios_free_varinfo(var_r64); - - // Cleanup file - adios_read_close(f); - - adios_read_finalize_method(ADIOS_READ_METHOD_BP); - } -} - -// ADIOS2 write, native ADIOS1 read -TEST_F(BPWriteReadTest, ADIOS2BPWriteADIOS1Read2D4x2stdio) -{ - // Each process would write a 4x2 array and all processes would - // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here - std::string fname = "ADIOS2BPWriteADIOS1Read2D4x2Teststdio.bp"; - - int mpiRank = 0, mpiSize = 1; - // Number of rows - const std::size_t Nx = 2; - // Number of cols - const std::size_t Ny = 4; - - // Number of steps - const std::size_t NSteps = 3; - -#if ADIOS2_USE_MPI - MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); - MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); -#endif - - // Write test data using ADIOS2 - { -#if ADIOS2_USE_MPI - adios2::ADIOS adios(MPI_COMM_WORLD); -#else - adios2::ADIOS adios; -#endif - adios2::IO io = adios.DeclareIO("TestIO"); - - // Declare 2D variables (4 * (NumberOfProcess * Nx)) - // The local process' part (start, count) can be defined now or later - // before Write(). - { - adios2::Dims shape{static_cast(Ny), - static_cast(mpiSize * Nx)}; - adios2::Dims start{static_cast(0), - static_cast(mpiRank * Nx)}; - adios2::Dims count{static_cast(Ny), - static_cast(Nx)}; - auto var_i8 = io.DefineVariable("i8", shape, start, count); - auto var_i16 = - io.DefineVariable("i16", shape, start, count); - auto var_i32 = - io.DefineVariable("i32", shape, start, count); - auto var_i64 = - io.DefineVariable("i64", shape, start, count); - auto var_u8 = io.DefineVariable("u8", shape, start, count); - auto var_u16 = - io.DefineVariable("u16", shape, start, count); - auto var_u32 = - io.DefineVariable("u32", shape, start, count); - auto var_u64 = - io.DefineVariable("u64", shape, start, count); - auto var_r32 = io.DefineVariable("r32", shape, start, count); - auto var_r64 = - io.DefineVariable("r64", shape, start, count); - } - - if (!engineName.empty()) - { - io.SetEngine(engineName); - } - else - { - // Create the BP Engine - io.SetEngine("BPFile"); - } - -#if ADIOS2_USE_MPI - io.AddTransport("file", {{"Library", "stdio"}}); -#else - io.AddTransport("file"); -#endif - - adios2::Engine engine = io.Open(fname, adios2::Mode::Write); - - for (size_t step = 0; step < NSteps; ++step) - { - // Generate test data for each process uniquely - SmallTestData currentTestData = - generateNewSmallTestData(m_TestData, step, mpiRank, mpiSize); - - // Retrieve the variables that previously went out of scope - auto var_i8 = io.InquireVariable("i8"); - auto var_i16 = io.InquireVariable("i16"); - auto var_i32 = io.InquireVariable("i32"); - auto var_i64 = io.InquireVariable("i64"); - auto var_u8 = io.InquireVariable("u8"); - auto var_u16 = io.InquireVariable("u16"); - auto var_u32 = io.InquireVariable("u32"); - auto var_u64 = io.InquireVariable("u64"); - auto var_r32 = io.InquireVariable("r32"); - auto var_r64 = io.InquireVariable("r64"); - - // Make a 2D selection to describe the local dimensions of the - // variable we write and its offsets in the global spaces - adios2::Box sel( - {0, static_cast(mpiRank * Nx)}, {Ny, Nx}); - var_i8.SetSelection(sel); - var_i16.SetSelection(sel); - var_i32.SetSelection(sel); - var_i64.SetSelection(sel); - var_u8.SetSelection(sel); - var_u16.SetSelection(sel); - var_u32.SetSelection(sel); - var_u64.SetSelection(sel); - var_r32.SetSelection(sel); - var_r64.SetSelection(sel); - - // Write each one - // fill in the variable with values from starting index to - // starting index + count - engine.Put(var_i8, currentTestData.I8.data()); - engine.Put(var_i16, currentTestData.I16.data()); - engine.Put(var_i32, currentTestData.I32.data()); - engine.Put(var_i64, currentTestData.I64.data()); - engine.Put(var_u8, currentTestData.U8.data()); - engine.Put(var_u16, currentTestData.U16.data()); - engine.Put(var_u32, currentTestData.U32.data()); - engine.Put(var_u64, currentTestData.U64.data()); - engine.Put(var_r32, currentTestData.R32.data()); - engine.Put(var_r64, currentTestData.R64.data()); - - // Advance to the next time step - engine.EndStep(); - } - - // Close the file - engine.Close(); - } - - { - adios_read_init_method(ADIOS_READ_METHOD_BP, MPI_COMM_SELF, - "verbose=3"); - - // Open the file for reading - // Note: Since collective metadata generation is not implemented yet, - // SO for now we read each subfile instead of a single bp file with all - // metadata. - // Meanwhile if we open file with MPI_COMM_WORLD, then the selection - // bounding box should be [0, Nx] - std::string index = std::to_string(mpiRank); - ADIOS_FILE *f = adios_read_open_file( - (fname + ".dir/" + fname + "." + index).c_str(), - ADIOS_READ_METHOD_BP, MPI_COMM_SELF); - ASSERT_NE(f, nullptr); - - // Check the variables exist - ADIOS_VARINFO *var_i8 = adios_inq_var(f, "i8"); - EXPECT_TRUE(var_i8); - ASSERT_EQ(var_i8->ndim, 2); - ASSERT_EQ(var_i8->global, 1); - ASSERT_EQ(var_i8->nsteps, NSteps); - ASSERT_EQ(var_i8->dims[0], Ny); - ASSERT_EQ(var_i8->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_i16 = adios_inq_var(f, "i16"); - EXPECT_TRUE(var_i16); - ASSERT_EQ(var_i16->ndim, 2); - ASSERT_EQ(var_i16->global, 1); - ASSERT_EQ(var_i16->nsteps, NSteps); - ASSERT_EQ(var_i16->dims[0], Ny); - ASSERT_EQ(var_i16->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_i32 = adios_inq_var(f, "i32"); - EXPECT_TRUE(var_i32); - ASSERT_EQ(var_i32->ndim, 2); - ASSERT_EQ(var_i32->global, 1); - ASSERT_EQ(var_i32->nsteps, NSteps); - ASSERT_EQ(var_i32->dims[0], Ny); - ASSERT_EQ(var_i32->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_i64 = adios_inq_var(f, "i64"); - EXPECT_TRUE(var_i64); - ASSERT_EQ(var_i64->ndim, 2); - ASSERT_EQ(var_i64->global, 1); - ASSERT_EQ(var_i64->nsteps, NSteps); - ASSERT_EQ(var_i64->dims[0], Ny); - ASSERT_EQ(var_i64->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u8 = adios_inq_var(f, "u8"); - EXPECT_TRUE(var_u8); - ASSERT_EQ(var_u8->ndim, 2); - ASSERT_EQ(var_u8->global, 1); - ASSERT_EQ(var_u8->nsteps, NSteps); - ASSERT_EQ(var_u8->dims[0], Ny); - ASSERT_EQ(var_u8->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u16 = adios_inq_var(f, "u16"); - EXPECT_TRUE(var_u16); - ASSERT_EQ(var_u16->ndim, 2); - ASSERT_EQ(var_u16->global, 1); - ASSERT_EQ(var_u16->nsteps, NSteps); - ASSERT_EQ(var_u16->dims[0], Ny); - ASSERT_EQ(var_u16->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u32 = adios_inq_var(f, "u32"); - EXPECT_TRUE(var_u32); - ASSERT_EQ(var_u32->ndim, 2); - ASSERT_EQ(var_u32->global, 1); - ASSERT_EQ(var_u32->nsteps, NSteps); - ASSERT_EQ(var_u32->dims[0], Ny); - ASSERT_EQ(var_u32->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_u64 = adios_inq_var(f, "u64"); - EXPECT_TRUE(var_u64); - ASSERT_EQ(var_u64->ndim, 2); - ASSERT_EQ(var_u64->global, 1); - ASSERT_EQ(var_u64->nsteps, NSteps); - ASSERT_EQ(var_u64->dims[0], Ny); - ASSERT_EQ(var_u64->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_r32 = adios_inq_var(f, "r32"); - EXPECT_TRUE(var_r32); - ASSERT_EQ(var_r32->ndim, 2); - ASSERT_EQ(var_r32->global, 1); - ASSERT_EQ(var_r32->nsteps, NSteps); - ASSERT_EQ(var_r32->dims[0], Ny); - ASSERT_EQ(var_r32->dims[1], mpiSize * Nx); - ADIOS_VARINFO *var_r64 = adios_inq_var(f, "r64"); - EXPECT_TRUE(var_r64); - ASSERT_EQ(var_r64->ndim, 2); - ASSERT_EQ(var_r64->global, 1); - ASSERT_EQ(var_r64->nsteps, NSteps); - ASSERT_EQ(var_r64->dims[0], Ny); - ASSERT_EQ(var_r64->dims[1], mpiSize * Nx); - - // If the size of the array is smaller than the data - // the result is weird... double and uint64_t would get completely - // garbage data - std::array I8; - std::array I16; - std::array I32; - std::array I64; - std::array U8; - std::array U16; - std::array U32; - std::array U64; - std::array R32; - std::array R64; - - uint64_t start[2] = {0, mpiRank * Nx}; - uint64_t count[2] = {Ny, Nx}; - ADIOS_SELECTION *sel = adios_selection_boundingbox(2, start, count); - - // Read stuff - for (size_t t = 0; t < NSteps; ++t) - { - // Generate test data for each rank uniquely - SmallTestData currentTestData = - generateNewSmallTestData(m_TestData, t, mpiRank, mpiSize); - // Read the current step - adios_schedule_read_byid(f, sel, var_i8->varid, t, 1, I8.data()); - adios_schedule_read_byid(f, sel, var_i16->varid, t, 1, I16.data()); - adios_schedule_read_byid(f, sel, var_i32->varid, t, 1, I32.data()); - adios_schedule_read_byid(f, sel, var_i64->varid, t, 1, I64.data()); - adios_schedule_read_byid(f, sel, var_u8->varid, t, 1, U8.data()); - adios_schedule_read_byid(f, sel, var_u16->varid, t, 1, U16.data()); - adios_schedule_read_byid(f, sel, var_u32->varid, t, 1, U32.data()); - adios_schedule_read_byid(f, sel, var_u64->varid, t, 1, U64.data()); - adios_schedule_read_byid(f, sel, var_r32->varid, t, 1, R32.data()); - adios_schedule_read_byid(f, sel, var_r64->varid, t, 1, R64.data()); - adios_perform_reads(f, 1); - - // Check if it's correct - for (size_t i = 0; i < Nx; ++i) - { - std::stringstream ss; - ss << "t=" << t << " i=" << i << " rank=" << mpiRank; - std::string msg = ss.str(); - - EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg; - EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg; - EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg; - EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg; - EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg; - EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg; - EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg; - EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg; - EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; - EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; - } - } - - adios_selection_delete(sel); - - // Cleanup variable structures - adios_free_varinfo(var_i8); - adios_free_varinfo(var_i16); - adios_free_varinfo(var_i32); - adios_free_varinfo(var_i64); - adios_free_varinfo(var_u8); - adios_free_varinfo(var_u16); - adios_free_varinfo(var_u32); - adios_free_varinfo(var_u64); - adios_free_varinfo(var_r32); - adios_free_varinfo(var_r64); - - // Cleanup file - adios_read_close(f); - - adios_read_finalize_method(ADIOS_READ_METHOD_BP); - } -} - -//****************************************************************************** -// main -//****************************************************************************** -int main(int argc, char **argv) -{ -#if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); -#endif - - int result; - ::testing::InitGoogleTest(&argc, argv); - - if (argc > 1) - { - engineName = std::string(argv[1]); - } - result = RUN_ALL_TESTS(); - -#if ADIOS2_USE_MPI - MPI_Finalize(); -#endif - - return result; -} diff --git a/testing/adios2/engine/bp/operations/CMakeLists.txt b/testing/adios2/engine/bp/operations/CMakeLists.txt index 8563cf9cfe..51a4ff419c 100644 --- a/testing/adios2/engine/bp/operations/CMakeLists.txt +++ b/testing/adios2/engine/bp/operations/CMakeLists.txt @@ -17,6 +17,7 @@ endif() if(ADIOS2_HAVE_ZFP) bp3_bp4_gtest_add_tests_helper(WriteReadZfp MPI_ALLOW) bp3_bp4_gtest_add_tests_helper(WriteReadZfpHighLevelAPI MPI_ALLOW) + bp3_bp4_gtest_add_tests_helper(WriteReadZfpRemoveOperations MPI_ALLOW) bp3_bp4_gtest_add_tests_helper(WriteReadZfpConfig MPI_ALLOW) gtest_add_tests_helper(ZfpComplex MPI_ALLOW BPWriteRead Engine. "") diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadSzComplex.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadSzComplex.cpp index 6ef77aeaa3..49944ff309 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadSzComplex.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadSzComplex.cpp @@ -179,7 +179,7 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count, bpDComplexes.AddOperation(compressor, {{"accuracy", accuracy}}); io.DefineAttribute("AttInt", 110); adios2::Engine writerEngine = io.Open(fileName, adios2::Mode::Write); - for (int i = 0; i < steps; ++i) + for (int i = 0; i < static_cast(steps); ++i) { writerEngine.BeginStep(); GenData(myChars, i, start, count, shape); diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpComplex.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpComplex.cpp index 930a6a3e81..ffb1aad7c7 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpComplex.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpComplex.cpp @@ -179,7 +179,7 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count, bpDComplexes.AddOperation(zfpOp, {{"accuracy", accuracy}}); io.DefineAttribute("AttInt", 110); adios2::Engine writerEngine = io.Open(fileName, adios2::Mode::Write); - for (int i = 0; i < steps; ++i) + for (int i = 0; i < static_cast(steps); ++i) { writerEngine.BeginStep(); GenData(myChars, i, start, count, shape); diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpConfig.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpConfig.cpp index 5adcc94a25..8614d4c625 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpConfig.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpConfig.cpp @@ -64,6 +64,9 @@ void ZfpRate1D(const std::string configFile) auto var_r64 = io.DefineVariable("r64", shape, start, count, adios2::ConstantDims); + (void)var_r32; + (void)var_r64; + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); for (size_t step = 0; step < NSteps; ++step) @@ -180,6 +183,9 @@ void ZfpRate2D(const std::string configFile) auto var_r64 = io.DefineVariable("r64", shape, start, count, adios2::ConstantDims); + (void)var_r32; + (void)var_r64; + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); for (size_t step = 0; step < NSteps; ++step) @@ -296,6 +302,9 @@ void ZfpRate3D(const std::string configFile) auto var_r64 = io.DefineVariable("r64", shape, start, count, adios2::ConstantDims); + (void)var_r32; + (void)var_r64; + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); for (size_t step = 0; step < NSteps; ++step) @@ -412,6 +421,9 @@ void ZfpRate1DSel(const std::string configFile) auto var_r64 = io.DefineVariable("r64", shape, start, count, adios2::ConstantDims); + (void)var_r32; + (void)var_r64; + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); for (size_t step = 0; step < NSteps; ++step) @@ -533,6 +545,9 @@ void ZfpRate2DSel(const std::string configFile) auto var_r64 = io.DefineVariable("r64", shape, start, count, adios2::ConstantDims); + (void)var_r32; + (void)var_r64; + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); for (size_t step = 0; step < NSteps; ++step) @@ -653,6 +668,9 @@ void ZfpRate3DSel(const std::string configFile) auto var_r64 = io.DefineVariable("r64", shape, start, count, adios2::ConstantDims); + (void)var_r32; + (void)var_r64; + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); for (size_t step = 0; step < NSteps; ++step) @@ -778,6 +796,9 @@ void ZfpRate2DSmallSel(const std::string configFile) auto var_r64 = io.DefineVariable("r64", shape, start, count, adios2::ConstantDims); + (void)var_r32; + (void)var_r64; + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); for (size_t step = 0; step < NSteps; ++step) diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpRemoveOperations.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpRemoveOperations.cpp new file mode 100644 index 0000000000..b2255dc5e8 --- /dev/null +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpRemoveOperations.cpp @@ -0,0 +1,516 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + */ +#include +#include + +#include +#include +#include //std::iota +#include + +#include + +#include + +std::string engineName; // comes from command line + +/// Test suit that only compresses on odd steps to test +/// Variable::RemoveOperations functionality + +void ZFPRate1D(const std::string rate) +{ + // Each process would write a 1x8 array and all processes would + // form a mpiSize * Nx 1D array + const std::string fname("BPWRZFPOdd1D_" + rate + ".bp"); + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const size_t Nx = 100; + + // Number of steps + const size_t NSteps = 1; + + std::vector r32s(Nx); + std::vector r64s(Nx); + + // range 0 to 999 + std::iota(r32s.begin(), r32s.end(), 0.f); + std::iota(r64s.begin(), r64s.end(), 0.); + +#if ADIOS2_USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#endif + +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif + { + adios2::IO io = adios.DeclareIO("TestIO"); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + + const adios2::Dims shape{static_cast(Nx * mpiSize)}; + const adios2::Dims start{static_cast(Nx * mpiRank)}; + const adios2::Dims count{Nx}; + + auto var_r32 = io.DefineVariable("r32", shape, start, count, + adios2::ConstantDims); + auto var_r64 = io.DefineVariable("r64", shape, start, count, + adios2::ConstantDims); + + // add operations + adios2::Operator ZFPOp = + adios.DefineOperator("ZFPCompressor", adios2::ops::LossyZFP); + + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + bpWriter.BeginStep(); + + if (step % 2 == 0) + { + // remove current operations to avoid compression in even steps + var_r32.RemoveOperations(); + var_r64.RemoveOperations(); + + EXPECT_TRUE(var_r32.Operations().empty()); + EXPECT_TRUE(var_r64.Operations().empty()); + } + else + { + var_r32.AddOperation(ZFPOp, + {{adios2::ops::zfp::key::rate, rate}}); + var_r64.AddOperation(ZFPOp, + {{adios2::ops::zfp::key::rate, + std::to_string(2 * std::stod(rate))}}); + + EXPECT_FALSE(var_r32.Operations().empty()); + EXPECT_FALSE(var_r64.Operations().empty()); + } + + bpWriter.Put("r32", r32s.data()); + bpWriter.Put("r64", r64s.data()); + bpWriter.EndStep(); + } + + bpWriter.Close(); + } + +#if ADIOS2_USE_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif + { + adios2::IO io = adios.DeclareIO("ReadIO"); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + + adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + + auto var_r32 = io.InquireVariable("r32"); + EXPECT_TRUE(var_r32); + ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32.Steps(), NSteps); + ASSERT_EQ(var_r32.Shape()[0], mpiSize * Nx); + auto mmR32 = std::minmax_element(r32s.begin(), r32s.end()); + EXPECT_EQ(var_r32.Min(), *mmR32.first); + EXPECT_EQ(var_r32.Max(), *mmR32.second); + + auto var_r64 = io.InquireVariable("r64"); + EXPECT_TRUE(var_r64); + ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64.Steps(), NSteps); + ASSERT_EQ(var_r64.Shape()[0], mpiSize * Nx); + auto mmR64 = std::minmax_element(r64s.begin(), r64s.end()); + EXPECT_EQ(var_r64.Min(), *mmR64.first); + EXPECT_EQ(var_r64.Max(), *mmR64.second); + + const adios2::Dims start{mpiRank * Nx}; + const adios2::Dims count{Nx}; + const adios2::Box sel(start, count); + var_r32.SetSelection(sel); + var_r64.SetSelection(sel); + + unsigned int t = 0; + std::vector decompressedR32s; + std::vector decompressedR64s; + + while (bpReader.BeginStep() == adios2::StepStatus::OK) + { + bpReader.Get(var_r32, decompressedR32s); + bpReader.Get(var_r64, decompressedR64s); + bpReader.EndStep(); + + for (size_t i = 0; i < Nx; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + ASSERT_LT(std::abs(decompressedR32s[i] - r32s[i]), 1E-4) << msg; + ASSERT_LT(std::abs(decompressedR64s[i] - r64s[i]), 1E-4) << msg; + } + ++t; + } + + EXPECT_EQ(t, NSteps); + + bpReader.Close(); + } +} + +void ZFPRate2D(const std::string rate) +{ + // Each process would write a 1x8 array and all processes would + // form a mpiSize * Nx 1D array + const std::string fname("BPWRZFPOdd2D_" + rate + ".bp"); + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const size_t Nx = 100; + const size_t Ny = 50; + + // Number of steps + const size_t NSteps = 1; + + std::vector r32s(Nx * Ny); + std::vector r64s(Nx * Ny); + + // range 0 to 100*50 + std::iota(r32s.begin(), r32s.end(), 0.f); + std::iota(r64s.begin(), r64s.end(), 0.); + +#if ADIOS2_USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#endif + +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif + { + adios2::IO io = adios.DeclareIO("TestIO"); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + + const adios2::Dims shape{static_cast(Nx * mpiSize), Ny}; + const adios2::Dims start{static_cast(Nx * mpiRank), 0}; + const adios2::Dims count{Nx, Ny}; + + auto var_r32 = io.DefineVariable("r32", shape, start, count, + adios2::ConstantDims); + auto var_r64 = io.DefineVariable("r64", shape, start, count, + adios2::ConstantDims); + + // add operations + adios2::Operator zfpOp = + adios.DefineOperator("ZFPCompressor", adios2::ops::LossyZFP); + + var_r32.AddOperation(zfpOp, {{"Accuracy", rate}}); + var_r64.AddOperation(zfpOp, {{"accuracy", rate}}); + + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + bpWriter.BeginStep(); + + if (step % 2 == 0) + { + // remove current operations to avoid compression in even steps + var_r32.RemoveOperations(); + var_r64.RemoveOperations(); + + EXPECT_TRUE(var_r32.Operations().empty()); + EXPECT_TRUE(var_r64.Operations().empty()); + } + else + { + var_r32.AddOperation(zfpOp, + {{adios2::ops::zfp::key::rate, rate}}); + var_r64.AddOperation(zfpOp, + {{adios2::ops::zfp::key::rate, + std::to_string(2 * std::stod(rate))}}); + + EXPECT_FALSE(var_r32.Operations().empty()); + EXPECT_FALSE(var_r64.Operations().empty()); + } + + bpWriter.Put("r32", r32s.data()); + bpWriter.Put("r64", r64s.data()); + bpWriter.EndStep(); + } + + bpWriter.Close(); + } + + { + adios2::IO io = adios.DeclareIO("ReadIO"); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + + adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + + auto var_r32 = io.InquireVariable("r32"); + EXPECT_TRUE(var_r32); + ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32.Steps(), NSteps); + ASSERT_EQ(var_r32.Shape()[0], mpiSize * Nx); + ASSERT_EQ(var_r32.Shape()[1], Ny); + auto mmR32 = std::minmax_element(r32s.begin(), r32s.end()); + EXPECT_EQ(var_r32.Min(), *mmR32.first); + EXPECT_EQ(var_r32.Max(), *mmR32.second); + + auto var_r64 = io.InquireVariable("r64"); + EXPECT_TRUE(var_r64); + ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64.Steps(), NSteps); + ASSERT_EQ(var_r64.Shape()[0], mpiSize * Nx); + ASSERT_EQ(var_r64.Shape()[1], Ny); + auto mmR64 = std::minmax_element(r64s.begin(), r64s.end()); + EXPECT_EQ(var_r64.Min(), *mmR64.first); + EXPECT_EQ(var_r64.Max(), *mmR64.second); + + const adios2::Dims start{mpiRank * Nx, 0}; + const adios2::Dims count{Nx, Ny}; + const adios2::Box sel(start, count); + var_r32.SetSelection(sel); + var_r64.SetSelection(sel); + + unsigned int t = 0; + std::vector decompressedR32s; + std::vector decompressedR64s; + + while (bpReader.BeginStep() == adios2::StepStatus::OK) + { + bpReader.Get(var_r32, decompressedR32s); + bpReader.Get(var_r64, decompressedR64s); + bpReader.EndStep(); + + for (size_t i = 0; i < Nx * Ny; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + ASSERT_LT(std::abs(decompressedR32s[i] - r32s[i]), 1E-4) << msg; + ASSERT_LT(std::abs(decompressedR64s[i] - r64s[i]), 1E-4) << msg; + } + ++t; + } + + EXPECT_EQ(t, NSteps); + + bpReader.Close(); + } +} + +void ZFPRate3D(const std::string rate) +{ + // Each process would write a 1x8 array and all processes would + // form a mpiSize * Nx 1D array + const std::string fname("BPWRZFPOdd3D_" + rate + ".bp"); + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const size_t Nx = 10; + const size_t Ny = 20; + const size_t Nz = 15; + + // Number of steps + const size_t NSteps = 1; + + std::vector r32s(Nx * Ny * Nz); + std::vector r64s(Nx * Ny * Nz); + + // range 0 to 100*50 + std::iota(r32s.begin(), r32s.end(), 0.f); + std::iota(r64s.begin(), r64s.end(), 0.); + +#if ADIOS2_USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#endif + +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif + { + adios2::IO io = adios.DeclareIO("TestIO"); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + + const adios2::Dims shape{static_cast(Nx * mpiSize), Ny, Nz}; + const adios2::Dims start{static_cast(Nx * mpiRank), 0, 0}; + const adios2::Dims count{Nx, Ny, Nz}; + + auto var_r32 = io.DefineVariable("r32", shape, start, count, + adios2::ConstantDims); + auto var_r64 = io.DefineVariable("r64", shape, start, count, + adios2::ConstantDims); + + // add operations + adios2::Operator zfpOp = + adios.DefineOperator("ZFPCompressor", adios2::ops::LossyZFP); + + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + bpWriter.BeginStep(); + + if (step % 2 == 0) + { + // remove current operations to avoid compression in even steps + var_r32.RemoveOperations(); + var_r64.RemoveOperations(); + + EXPECT_TRUE(var_r32.Operations().empty()); + EXPECT_TRUE(var_r64.Operations().empty()); + } + else + { + var_r32.AddOperation(zfpOp, + {{adios2::ops::zfp::key::rate, rate}}); + var_r64.AddOperation(zfpOp, + {{adios2::ops::zfp::key::rate, + std::to_string(2 * std::stod(rate))}}); + + EXPECT_FALSE(var_r32.Operations().empty()); + EXPECT_FALSE(var_r64.Operations().empty()); + } + + bpWriter.Put("r32", r32s.data()); + bpWriter.Put("r64", r64s.data()); + bpWriter.EndStep(); + } + + bpWriter.Close(); + } + + { + adios2::IO io = adios.DeclareIO("ReadIO"); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + + adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read); + + auto var_r32 = io.InquireVariable("r32"); + EXPECT_TRUE(var_r32); + ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32.Steps(), NSteps); + ASSERT_EQ(var_r32.Shape()[0], mpiSize * Nx); + ASSERT_EQ(var_r32.Shape()[1], Ny); + ASSERT_EQ(var_r32.Shape()[2], Nz); + auto mmR32 = std::minmax_element(r32s.begin(), r32s.end()); + EXPECT_EQ(var_r32.Min(), *mmR32.first); + EXPECT_EQ(var_r32.Max(), *mmR32.second); + + auto var_r64 = io.InquireVariable("r64"); + EXPECT_TRUE(var_r64); + ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64.Steps(), NSteps); + ASSERT_EQ(var_r64.Shape()[0], mpiSize * Nx); + ASSERT_EQ(var_r64.Shape()[1], Ny); + ASSERT_EQ(var_r64.Shape()[2], Nz); + auto mmR64 = std::minmax_element(r64s.begin(), r64s.end()); + EXPECT_EQ(var_r64.Min(), *mmR64.first); + EXPECT_EQ(var_r64.Max(), *mmR64.second); + + const adios2::Dims start{mpiRank * Nx, 0, 0}; + const adios2::Dims count{Nx, Ny, Nz}; + const adios2::Box sel(start, count); + var_r32.SetSelection(sel); + var_r64.SetSelection(sel); + + unsigned int t = 0; + std::vector decompressedR32s; + std::vector decompressedR64s; + + while (bpReader.BeginStep() == adios2::StepStatus::OK) + { + bpReader.Get(var_r32, decompressedR32s); + bpReader.Get(var_r64, decompressedR64s); + bpReader.EndStep(); + + for (size_t i = 0; i < Nx * Ny * Nz; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + ASSERT_LT(std::abs(decompressedR32s[i] - r32s[i]), 1E-4) << msg; + ASSERT_LT(std::abs(decompressedR64s[i] - r64s[i]), 1E-4) << msg; + } + ++t; + } + + EXPECT_EQ(t, NSteps); + + bpReader.Close(); + } +} + +class BPWRZFPODD : public ::testing::TestWithParam +{ +public: + BPWRZFPODD() = default; + + virtual void SetUp() {} + virtual void TearDown() {} +}; + +TEST_P(BPWRZFPODD, ADIOS2BPWRZFPODD1D) { ZFPRate1D(GetParam()); } +TEST_P(BPWRZFPODD, ADIOS2BPWRZFPODD2D) { ZFPRate2D(GetParam()); } +TEST_P(BPWRZFPODD, ADIOS2BPWRZFPODD3D) { ZFPRate3D(GetParam()); } + +INSTANTIATE_TEST_SUITE_P(ZFPRate, BPWRZFPODD, + ::testing::Values("8", "9", "10")); + +int main(int argc, char **argv) +{ +#if ADIOS2_USE_MPI + MPI_Init(nullptr, nullptr); +#endif + + int result; + ::testing::InitGoogleTest(&argc, argv); + if (argc > 1) + { + engineName = std::string(argv[1]); + } + result = RUN_ALL_TESTS(); + +#if ADIOS2_USE_MPI + MPI_Finalize(); +#endif + + return result; +} diff --git a/testing/adios2/engine/dataman/CMakeLists.txt b/testing/adios2/engine/dataman/CMakeLists.txt index b8e7eef67e..f40d6b60a6 100644 --- a/testing/adios2/engine/dataman/CMakeLists.txt +++ b/testing/adios2/engine/dataman/CMakeLists.txt @@ -24,13 +24,6 @@ if(ADIOS2_HAVE_ZFP) ) endif() -if(ADIOS2_HAVE_BZip2) - gtest_add_tests_helper(2DBzip2 MPI_NONE DataMan Engine.DataMan. "") - set_tests_properties(${Test.Engine.DataMan.2DBzip2-TESTS} - PROPERTIES RUN_SERIAL TRUE - ) -endif() - if(ADIOS2_HAVE_SZ) gtest_add_tests_helper(2DSz MPI_NONE DataMan Engine.DataMan. "") set_tests_properties(${Test.Engine.DataMan.2DSz-TESTS} diff --git a/testing/adios2/engine/dataman/TestDataMan1D.cpp b/testing/adios2/engine/dataman/TestDataMan1D.cpp index d688e75603..b7dde64607 100644 --- a/testing/adios2/engine/dataman/TestDataMan1D.cpp +++ b/testing/adios2/engine/dataman/TestDataMan1D.cpp @@ -123,6 +123,7 @@ void DataManWriter(const Dims &shape, const Dims &start, const Dims &count, auto bpDComplexes = dataManIO.DefineVariable>( "bpDComplexes", shape, start, count); auto bpUInt64s = dataManIO.DefineVariable("bpUInt64s"); + auto scalarString = dataManIO.DefineVariable("scalarString"); dataManIO.DefineAttribute("AttInt", 110); adios2::Engine dataManWriter = dataManIO.Open("stream", adios2::Mode::Write); @@ -151,6 +152,7 @@ void DataManWriter(const Dims &shape, const Dims &start, const Dims &count, dataManWriter.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync); dataManWriter.Put(bpUInt64s, i); + dataManWriter.Put(scalarString, std::string("some text")); dataManWriter.EndStep(); } dataManWriter.Close(); @@ -186,7 +188,7 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count, { received_steps = true; const auto &vars = dataManIO.AvailableVariables(); - ASSERT_EQ(vars.size(), 11); + ASSERT_EQ(vars.size(), 12); currentStep = dataManReader.CurrentStep(); adios2::Variable bpChars = dataManIO.InquireVariable("bpChars"); @@ -210,6 +212,8 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count, dataManIO.InquireVariable>("bpDComplexes"); adios2::Variable bpUInt64s = dataManIO.InquireVariable("bpUInt64s"); + adios2::Variable scalarString = + dataManIO.InquireVariable("scalarString"); auto charsBlocksInfo = dataManReader.AllStepsBlocksInfo(bpChars); bpChars.SetSelection({start, count}); bpUChars.SetSelection({start, count}); @@ -233,6 +237,11 @@ void DataManReader(const Dims &shape, const Dims &start, const Dims &count, adios2::Mode::Sync); dataManReader.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync); + + std::string readString; + dataManReader.Get(scalarString, readString, adios2::Mode::Sync); + ASSERT_EQ(readString, "some text"); + uint64_t stepValue; dataManReader.Get(bpUInt64s, &stepValue, adios2::Mode::Sync); ASSERT_EQ(currentStep, stepValue); diff --git a/testing/adios2/engine/dataman/TestDataMan1DSuperLarge.cpp b/testing/adios2/engine/dataman/TestDataMan1DSuperLarge.cpp index fb6a6f28bd..fed32ad812 100644 --- a/testing/adios2/engine/dataman/TestDataMan1DSuperLarge.cpp +++ b/testing/adios2/engine/dataman/TestDataMan1DSuperLarge.cpp @@ -271,9 +271,9 @@ class DataManEngineTest : public ::testing::Test TEST_F(DataManEngineTest, 1DSuperLarge) { // set parameters - Dims shape = {100000}; + Dims shape = {50000}; Dims start = {0}; - Dims count = {100000}; + Dims count = {50000}; size_t steps = 10; // run workflow diff --git a/testing/adios2/engine/dataman/TestDataMan2DBzip2.cpp b/testing/adios2/engine/dataman/TestDataMan2DBzip2.cpp deleted file mode 100644 index 96ee4fc8a4..0000000000 --- a/testing/adios2/engine/dataman/TestDataMan2DBzip2.cpp +++ /dev/null @@ -1,343 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * TestDataMan2DZfp.cpp - * - * Created on: Nov 24, 2020 - * Author: Jason Wang - */ - -#include -#include -#include -#include -#include - -using namespace adios2; -size_t print_lines = 0; - -class DataManEngineTest : public ::testing::Test -{ -public: - DataManEngineTest() = default; -}; - -template -void PrintData(const T *data, const size_t step, const Dims &start, - const Dims &count) -{ - size_t size = std::accumulate(count.begin(), count.end(), 1, - std::multiplies()); - std::cout << "Step: " << step << " Size:" << size << "\n"; - size_t printsize = 128; - - if (size < printsize) - { - printsize = size; - } - int s = 0; - for (size_t i = 0; i < printsize; ++i) - { - ++s; - std::cout << data[i] << " "; - if (s == count[1]) - { - std::cout << std::endl; - s = 0; - } - } - - std::cout << "]" << std::endl; -} - -template -void GenData(std::vector &data, const size_t step, const Dims &start, - const Dims &count, const Dims &shape) -{ - if (start.size() == 2) - { - for (size_t i = 0; i < count[0]; ++i) - { - for (size_t j = 0; j < count[1]; ++j) - { - data[i * count[1] + j] = - (i + start[1]) * shape[1] + j + start[0] + 0.01; - } - } - } -} - -template -void VerifyData(const std::complex *data, size_t step, const Dims &start, - const Dims &count, const Dims &shape) -{ - size_t size = std::accumulate(count.begin(), count.end(), 1, - std::multiplies()); - std::vector> tmpdata(size); - GenData(tmpdata, step, start, count, shape); - for (size_t i = 0; i < size; ++i) - { - ASSERT_EQ(data[i], tmpdata[i]); - } -} - -template -void VerifyData(const T *data, size_t step, const Dims &start, - const Dims &count, const Dims &shape) -{ - size_t size = std::accumulate(count.begin(), count.end(), 1, - std::multiplies()); - std::vector tmpdata(size); - GenData(tmpdata, step, start, count, shape); - for (size_t i = 0; i < size; ++i) - { - ASSERT_EQ(data[i], tmpdata[i]); - } -} - -void DataManWriterP2PMemSelect(const Dims &shape, const Dims &start, - const Dims &count, const size_t steps, - const adios2::Params &engineParams) -{ - size_t datasize = std::accumulate(count.begin(), count.end(), 1, - std::multiplies()); - adios2::ADIOS adios; - adios2::IO dataManIO = adios.DeclareIO("WAN"); - dataManIO.SetEngine("DataMan"); - dataManIO.SetParameters(engineParams); - std::vector myChars(datasize); - std::vector myUChars(datasize); - std::vector myShorts(datasize); - std::vector myUShorts(datasize); - std::vector myInts(datasize); - std::vector myUInts(datasize); - std::vector myFloats(datasize); - std::vector myDoubles(datasize); - std::vector> myComplexes(datasize); - std::vector> myDComplexes(datasize); - auto bpChars = - dataManIO.DefineVariable("bpChars", shape, start, count); - auto bpUChars = dataManIO.DefineVariable("bpUChars", shape, - start, count); - auto bpShorts = - dataManIO.DefineVariable("bpShorts", shape, start, count); - auto bpUShorts = dataManIO.DefineVariable( - "bpUShorts", shape, start, count); - auto bpInts = dataManIO.DefineVariable("bpInts", shape, start, count); - auto bpUInts = - dataManIO.DefineVariable("bpUInts", shape, start, count); - auto bpFloats = - dataManIO.DefineVariable("bpFloats", shape, start, count); - adios2::Operator bzip2Op = - adios.DefineOperator("Compressor", adios2::ops::LosslessBZIP2); - bpFloats.AddOperation(bzip2Op, {}); - auto bpDoubles = - dataManIO.DefineVariable("bpDoubles", shape, start, count); - auto bpComplexes = dataManIO.DefineVariable>( - "bpComplexes", shape, start, count); - auto bpDComplexes = dataManIO.DefineVariable>( - "bpDComplexes", shape, start, count); - dataManIO.DefineAttribute("AttInt", 110); - adios2::Engine dataManWriter = - dataManIO.Open("stream", adios2::Mode::Write); - for (size_t i = 0; i < steps; ++i) - { - dataManWriter.BeginStep(); - GenData(myChars, i, start, count, shape); - GenData(myUChars, i, start, count, shape); - GenData(myShorts, i, start, count, shape); - GenData(myUShorts, i, start, count, shape); - GenData(myInts, i, start, count, shape); - GenData(myUInts, i, start, count, shape); - GenData(myFloats, i, start, count, shape); - GenData(myDoubles, i, start, count, shape); - GenData(myComplexes, i, start, count, shape); - GenData(myDComplexes, i, start, count, shape); - dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync); - dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync); - dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync); - dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync); - dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync); - dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync); - dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync); - dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync); - dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync); - dataManWriter.Put(bpDComplexes, myDComplexes.data(), - adios2::Mode::Sync); - dataManWriter.EndStep(); - } - dataManWriter.Close(); -} - -void DataManReaderP2PMemSelect(const Dims &shape, const Dims &start, - const Dims &count, const Dims &memStart, - const Dims &memCount, const size_t steps, - const adios2::Params &engineParams) -{ - adios2::ADIOS adios; - adios2::IO dataManIO = adios.DeclareIO("WAN"); - dataManIO.SetEngine("DataMan"); - dataManIO.SetParameters(engineParams); - adios2::Engine dataManReader = dataManIO.Open("stream", adios2::Mode::Read); - - size_t datasize = std::accumulate(memCount.begin(), memCount.end(), 1, - std::multiplies()); - std::vector myChars(datasize); - std::vector myUChars(datasize); - std::vector myShorts(datasize); - std::vector myUShorts(datasize); - std::vector myInts(datasize); - std::vector myUInts(datasize); - std::vector myFloats(datasize); - std::vector myDoubles(datasize); - std::vector> myComplexes(datasize); - std::vector> myDComplexes(datasize); - bool received_steps = false; - size_t currentStep; - while (true) - { - adios2::StepStatus status = dataManReader.BeginStep(); - if (status == adios2::StepStatus::OK) - { - received_steps = true; - const auto &vars = dataManIO.AvailableVariables(); - ASSERT_EQ(vars.size(), 10); - currentStep = dataManReader.CurrentStep(); - GenData(myChars, currentStep, memStart, memCount, shape); - GenData(myUChars, currentStep, memStart, memCount, shape); - GenData(myShorts, currentStep, memStart, memCount, shape); - GenData(myUShorts, currentStep, memStart, memCount, shape); - GenData(myInts, currentStep, memStart, memCount, shape); - GenData(myUInts, currentStep, memStart, memCount, shape); - GenData(myFloats, currentStep, memStart, memCount, shape); - GenData(myDoubles, currentStep, memStart, memCount, shape); - GenData(myComplexes, currentStep, memStart, memCount, shape); - GenData(myDComplexes, currentStep, memStart, memCount, shape); - adios2::Variable bpChars = - dataManIO.InquireVariable("bpChars"); - adios2::Variable bpUChars = - dataManIO.InquireVariable("bpUChars"); - adios2::Variable bpShorts = - dataManIO.InquireVariable("bpShorts"); - adios2::Variable bpUShorts = - dataManIO.InquireVariable("bpUShorts"); - adios2::Variable bpInts = - dataManIO.InquireVariable("bpInts"); - adios2::Variable bpUInts = - dataManIO.InquireVariable("bpUInts"); - adios2::Variable bpFloats = - dataManIO.InquireVariable("bpFloats"); - adios2::Variable bpDoubles = - dataManIO.InquireVariable("bpDoubles"); - adios2::Variable> bpComplexes = - dataManIO.InquireVariable>("bpComplexes"); - adios2::Variable> bpDComplexes = - dataManIO.InquireVariable>("bpDComplexes"); - auto charsBlocksInfo = dataManReader.AllStepsBlocksInfo(bpChars); - - bpChars.SetSelection({start, count}); - bpUChars.SetSelection({start, count}); - bpShorts.SetSelection({start, count}); - bpUShorts.SetSelection({start, count}); - bpInts.SetSelection({start, count}); - bpUInts.SetSelection({start, count}); - bpFloats.SetSelection({start, count}); - bpDoubles.SetSelection({start, count}); - bpComplexes.SetSelection({start, count}); - bpDComplexes.SetSelection({start, count}); - - bpChars.SetMemorySelection({memStart, memCount}); - bpUChars.SetMemorySelection({memStart, memCount}); - bpShorts.SetMemorySelection({memStart, memCount}); - bpUShorts.SetMemorySelection({memStart, memCount}); - bpInts.SetMemorySelection({memStart, memCount}); - bpUInts.SetMemorySelection({memStart, memCount}); - bpFloats.SetMemorySelection({memStart, memCount}); - bpDoubles.SetMemorySelection({memStart, memCount}); - bpComplexes.SetMemorySelection({memStart, memCount}); - bpDComplexes.SetMemorySelection({memStart, memCount}); - - dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync); - dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync); - dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync); - dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync); - dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync); - dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync); - dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync); - dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync); - dataManReader.Get(bpComplexes, myComplexes.data(), - adios2::Mode::Sync); - dataManReader.Get(bpDComplexes, myDComplexes.data(), - adios2::Mode::Sync); - VerifyData(myChars.data(), currentStep, memStart, memCount, shape); - VerifyData(myUChars.data(), currentStep, memStart, memCount, shape); - VerifyData(myShorts.data(), currentStep, memStart, memCount, shape); - VerifyData(myUShorts.data(), currentStep, memStart, memCount, - shape); - VerifyData(myInts.data(), currentStep, memStart, memCount, shape); - VerifyData(myUInts.data(), currentStep, memStart, memCount, shape); - VerifyData(myFloats.data(), currentStep, memStart, memCount, shape); - VerifyData(myDoubles.data(), currentStep, memStart, memCount, - shape); - VerifyData(myComplexes.data(), currentStep, memStart, memCount, - shape); - VerifyData(myDComplexes.data(), currentStep, memStart, memCount, - shape); - dataManReader.EndStep(); - } - else if (status == adios2::StepStatus::EndOfStream) - { - break; - } - else if (status == adios2::StepStatus::NotReady) - { - continue; - } - } - if (received_steps) - { - auto attInt = dataManIO.InquireAttribute("AttInt"); - ASSERT_EQ(110, attInt.Data()[0]); - } - dataManReader.Close(); - print_lines = 0; -} - -#ifdef ADIOS2_HAVE_ZEROMQ -TEST_F(DataManEngineTest, 2D_Bzip2) -{ - // set parameters - Dims shape = {10, 10}; - Dims start = {2, 2}; - Dims count = {5, 5}; - Dims memstart = start; - Dims memcount = count; - memstart = {1, 1}; - memcount = {7, 9}; - - size_t steps = 5000; - adios2::Params engineParams = { - {"IPAddress", "127.0.0.1"}, {"Port", "12310"}, {"Verbose", "0"}}; - - auto r = std::thread(DataManReaderP2PMemSelect, shape, start, count, - memstart, memcount, steps, engineParams); - - auto w = std::thread(DataManWriterP2PMemSelect, shape, start, count, steps, - engineParams); - - w.join(); - - r.join(); -} - -#endif // ZEROMQ - -int main(int argc, char **argv) -{ - int result; - ::testing::InitGoogleTest(&argc, argv); - result = RUN_ALL_TESTS(); - - return result; -} diff --git a/testing/adios2/engine/dataman/TestDataMan2DSz.cpp b/testing/adios2/engine/dataman/TestDataMan2DSz.cpp index 07cb00d16d..bbcef132fc 100644 --- a/testing/adios2/engine/dataman/TestDataMan2DSz.cpp +++ b/testing/adios2/engine/dataman/TestDataMan2DSz.cpp @@ -107,6 +107,7 @@ void DataManWriterP2PMemSelect(const Dims &shape, const Dims &start, adios2::IO dataManIO = adios.DeclareIO("WAN"); dataManIO.SetEngine("DataMan"); dataManIO.SetParameters(engineParams); + dataManIO.AddOperation("bpFloats", "sz", {{"accuracy", "0.01"}}); std::vector myChars(datasize); std::vector myUChars(datasize); std::vector myShorts(datasize); @@ -130,9 +131,6 @@ void DataManWriterP2PMemSelect(const Dims &shape, const Dims &start, dataManIO.DefineVariable("bpUInts", shape, start, count); auto bpFloats = dataManIO.DefineVariable("bpFloats", shape, start, count); - adios2::Operator szOp = - adios.DefineOperator("szCompressor", adios2::ops::LossySZ); - bpFloats.AddOperation(szOp, {{adios2::ops::sz::key::accuracy, "0.01"}}); auto bpDoubles = dataManIO.DefineVariable("bpDoubles", shape, start, count); auto bpComplexes = dataManIO.DefineVariable>( @@ -142,7 +140,7 @@ void DataManWriterP2PMemSelect(const Dims &shape, const Dims &start, dataManIO.DefineAttribute("AttInt", 110); adios2::Engine dataManWriter = dataManIO.Open("stream", adios2::Mode::Write); - for (int i = 0; i < steps; ++i) + for (size_t i = 0; i < steps; ++i) { dataManWriter.BeginStep(); GenData(myChars, i, start, count, shape); diff --git a/testing/adios2/engine/dataman/TestDataMan2DZfp.cpp b/testing/adios2/engine/dataman/TestDataMan2DZfp.cpp index 0ee1b26c48..2c7aabe668 100644 --- a/testing/adios2/engine/dataman/TestDataMan2DZfp.cpp +++ b/testing/adios2/engine/dataman/TestDataMan2DZfp.cpp @@ -107,6 +107,10 @@ void DataManWriterP2PMemSelect(const Dims &shape, const Dims &start, adios2::IO dataManIO = adios.DeclareIO("WAN"); dataManIO.SetEngine("DataMan"); dataManIO.SetParameters(engineParams); + dataManIO.AddOperation("bpFloats", "zfp", {{"accuracy", "0.01"}}); + dataManIO.AddOperation("bpDoubles", "zfp", {{"accuracy", "0.1"}}); + dataManIO.AddOperation("bpComplexes", "zfp", {{"accuracy", "0.1"}}); + dataManIO.AddOperation("bpDComplexes", "zfp", {{"accuracy", "0.1"}}); std::vector myChars(datasize); std::vector myUChars(datasize); std::vector myShorts(datasize); @@ -130,23 +134,16 @@ void DataManWriterP2PMemSelect(const Dims &shape, const Dims &start, dataManIO.DefineVariable("bpUInts", shape, start, count); auto bpFloats = dataManIO.DefineVariable("bpFloats", shape, start, count); - adios2::Operator zfpOp = - adios.DefineOperator("zfpCompressor", adios2::ops::LossyZFP); - bpFloats.AddOperation(zfpOp, {{adios2::ops::zfp::key::accuracy, "0.1"}}); auto bpDoubles = dataManIO.DefineVariable("bpDoubles", shape, start, count); - bpDoubles.AddOperation(zfpOp, {{adios2::ops::zfp::key::accuracy, "0.1"}}); auto bpComplexes = dataManIO.DefineVariable>( "bpComplexes", shape, start, count); - bpComplexes.AddOperation(zfpOp, {{adios2::ops::zfp::key::accuracy, "0.1"}}); auto bpDComplexes = dataManIO.DefineVariable>( "bpDComplexes", shape, start, count); - bpDComplexes.AddOperation(zfpOp, - {{adios2::ops::zfp::key::accuracy, "0.1"}}); dataManIO.DefineAttribute("AttInt", 110); adios2::Engine dataManWriter = dataManIO.Open("stream", adios2::Mode::Write); - for (int i = 0; i < steps; ++i) + for (size_t i = 0; i < steps; ++i) { dataManWriter.BeginStep(); GenData(myChars, i, start, count, shape); diff --git a/testing/adios2/engine/table/CMakeLists.txt b/testing/adios2/engine/mhs/CMakeLists.txt similarity index 59% rename from testing/adios2/engine/table/CMakeLists.txt rename to testing/adios2/engine/mhs/CMakeLists.txt index 228bc1b9e7..219cf6eab1 100644 --- a/testing/adios2/engine/table/CMakeLists.txt +++ b/testing/adios2/engine/mhs/CMakeLists.txt @@ -3,5 +3,6 @@ # accompanying file Copyright.txt for details. #------------------------------------------------------------------------------# -gtest_add_tests_helper(SingleRank MPI_NONE Table Engine.Table. "") -gtest_add_tests_helper(MultiRank MPI_ONLY Table Engine.Table. "") +gtest_add_tests_helper(SingleRank MPI_NONE Mhs Engine.MHS. "") +gtest_add_tests_helper(MultiRank MPI_ONLY Mhs Engine.MHS. "") +gtest_add_tests_helper(MultiReader MPI_ONLY Mhs Engine.MHS. "") diff --git a/testing/adios2/engine/mhs/TestMhsCommon.h b/testing/adios2/engine/mhs/TestMhsCommon.h new file mode 100644 index 0000000000..5d86169a72 --- /dev/null +++ b/testing/adios2/engine/mhs/TestMhsCommon.h @@ -0,0 +1,163 @@ +#include +#include +#include +#include + +using namespace adios2; + +int printed_lines = 0; + +template +void PrintData(const T *data, const size_t step, const Dims &start, + const Dims &count, const std::string &var, + const int to_print_lines) +{ + size_t size = + std::accumulate(count.begin(), count.end(), static_cast(1), + std::multiplies()); + std::cout << " Step: " << step << " Size:" << size << "\n"; + size_t printsize = 1024; + + if (size < printsize) + { + printsize = size; + } + size_t s = 0; + for (size_t i = 0; i < printsize; ++i) + { + ++s; + std::cout << data[i] << " "; + if (s == count[1]) + { + std::cout << " <--- Variable " << var << ", Step " << step + << std::endl; + s = 0; + } + } + + std::cout << "]" << std::endl; +} + +template +void GenDataRecursive(std::vector start, std::vector count, + std::vector shape, size_t n0, size_t y, + std::vector> &vec, const size_t step) +{ + for (size_t i = 0; i < count[0]; i++) + { + size_t i0 = n0 * count[0] + i; + size_t z = y * shape[0] + (i + start[0]); + + auto start_next = start; + auto count_next = count; + auto shape_next = shape; + start_next.erase(start_next.begin()); + count_next.erase(count_next.begin()); + shape_next.erase(shape_next.begin()); + + if (start_next.size() == 1) + { + for (size_t j = 0; j < count_next[0]; j++) + { + vec[i0 * count_next[0] + j] = { + static_cast(z * shape_next[0] + (j + start_next[0]) + + step), + 1}; + } + } + else + { + GenDataRecursive(start_next, count_next, shape_next, i0, z, vec, + step); + } + } +} + +template +void GenDataRecursive(std::vector start, std::vector count, + std::vector shape, size_t n0, size_t y, + std::vector &vec, const size_t step) +{ + for (size_t i = 0; i < count[0]; i++) + { + size_t i0 = n0 * count[0] + i; + size_t z = y * shape[0] + (i + start[0]); + + auto start_next = start; + auto count_next = count; + auto shape_next = shape; + start_next.erase(start_next.begin()); + count_next.erase(count_next.begin()); + shape_next.erase(shape_next.begin()); + + if (start_next.size() == 1) + { + for (size_t j = 0; j < count_next[0]; j++) + { + vec[i0 * count_next[0] + j] = static_cast( + z * shape_next[0] + (j + start_next[0]) + step); + } + } + else + { + GenDataRecursive(start_next, count_next, shape_next, i0, z, vec, + step); + } + } +} + +template +void GenData(std::vector &vec, const size_t step, + const std::vector &start, const std::vector &count, + const std::vector &shape) +{ + size_t total_size = + std::accumulate(count.begin(), count.end(), static_cast(1), + std::multiplies()); + vec.resize(total_size); + GenDataRecursive(start, count, shape, 0, 0, vec, step); +} + +template +void VerifyData(const std::complex *data, size_t step, const Dims &start, + const Dims &count, const Dims &shape, const std::string &var, + const int to_print_lines = 0, const int rank = 0) +{ + size_t size = + std::accumulate(count.begin(), count.end(), static_cast(1), + std::multiplies()); + std::vector> tmpdata(size); + GenData(tmpdata, step, start, count, shape); + for (size_t i = 0; i < size; ++i) + { + ASSERT_EQ(data[i], tmpdata[i]) << "Step " << step << " Variable " << var + << " at " << i << std::endl; + } + if (rank == 0 && printed_lines < to_print_lines) + { + PrintData(data, step, start, count, var, to_print_lines); + ++printed_lines; + } +} + +template +void VerifyData(const T *data, size_t step, const Dims &start, + const Dims &count, const Dims &shape, const std::string &var, + const int to_print_lines = 0, const int rank = 0) +{ + size_t size = + std::accumulate(count.begin(), count.end(), static_cast(1), + std::multiplies()); + std::vector tmpdata(size); + if (rank == 0 && printed_lines < to_print_lines) + { + PrintData(data, step, start, count, var, to_print_lines); + ++printed_lines; + } + GenData(tmpdata, step, start, count, shape); + for (size_t i = 0; i < size; ++i) + { + ASSERT_EQ(data[i], tmpdata[i]) << "Step " << step << " Variable " << var + << " at " << i << std::endl; + } +} diff --git a/testing/adios2/engine/mhs/TestMhsMultiRank.cpp b/testing/adios2/engine/mhs/TestMhsMultiRank.cpp new file mode 100644 index 0000000000..f763f89de5 --- /dev/null +++ b/testing/adios2/engine/mhs/TestMhsMultiRank.cpp @@ -0,0 +1,251 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + */ + +#include "TestMhsCommon.h" +#include +#include +#include +#include +#include + +using namespace adios2; +int mpiRank = 0; +int mpiSize = 1; + +char runMode; + +class MhsEngineTest : public ::testing::Test +{ +public: + MhsEngineTest() = default; +}; + +void Reader(const Dims &shape, const Dims &start, const Dims &count, + const size_t rows, const adios2::Params &engineParams, + const std::string &name) +{ + + if (mpiRank != 0) + { + return; + } + + adios2::ADIOS adios(MPI_COMM_SELF); + adios2::IO io = adios.DeclareIO("ms"); + io.SetEngine("mhs"); + io.SetParameters(engineParams); + adios2::Engine readerEngine = io.Open(name, adios2::Mode::Read); + size_t datasize = 1; + for (const auto &i : shape) + { + datasize *= i; + } + std::vector myChars(datasize); + std::vector myUChars(datasize); + std::vector myShorts(datasize); + std::vector myUShorts(datasize); + std::vector myInts(datasize); + std::vector myUInts(datasize); + std::vector myFloats(datasize); + std::vector myDoubles(datasize); + std::vector> myComplexes(datasize); + std::vector> myDComplexes(datasize); + + const auto &vars = io.AvailableVariables(); + std::cout << "All available variables : "; + for (const auto &var : vars) + { + std::cout << var.first << ", "; + } + std::cout << std::endl; + + ASSERT_EQ(vars.size(), 10); + + adios2::Variable bpChars = io.InquireVariable("bpChars"); + adios2::Variable bpUChars = + io.InquireVariable("bpUChars"); + adios2::Variable bpShorts = io.InquireVariable("bpShorts"); + adios2::Variable bpUShorts = + io.InquireVariable("bpUShorts"); + adios2::Variable bpInts = io.InquireVariable("bpInts"); + adios2::Variable bpUInts = + io.InquireVariable("bpUInts"); + adios2::Variable bpFloats = io.InquireVariable("bpFloats"); + adios2::Variable bpDoubles = + io.InquireVariable("bpDoubles"); + adios2::Variable> bpComplexes = + io.InquireVariable>("bpComplexes"); + adios2::Variable> bpDComplexes = + io.InquireVariable>("bpDComplexes"); + + for (size_t step = 0; step < 10; step++) + { + readerEngine.BeginStep(); + + bpChars.SetSelection({start, shape}); + bpUChars.SetSelection({start, shape}); + bpShorts.SetSelection({start, shape}); + bpUShorts.SetSelection({start, shape}); + bpInts.SetSelection({start, shape}); + bpUInts.SetSelection({start, shape}); + bpFloats.SetSelection({start, shape}); + bpDoubles.SetSelection({start, shape}); + bpComplexes.SetSelection({start, shape}); + bpDComplexes.SetSelection({start, shape}); + + readerEngine.Get(bpChars, myChars.data(), adios2::Mode::Sync); + VerifyData(myChars.data(), step, {0, 0, 0}, shape, shape, "bpChars"); + + readerEngine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync); + VerifyData(myUChars.data(), step, {0, 0, 0}, shape, shape, "bpUChars"); + + readerEngine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync); + VerifyData(myShorts.data(), step, {0, 0, 0}, shape, shape, "bpShorts"); + + readerEngine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync); + VerifyData(myUShorts.data(), step, {0, 0, 0}, shape, shape, + "bpUShorts"); + + readerEngine.Get(bpInts, myInts.data(), adios2::Mode::Sync); + VerifyData(myInts.data(), step, {0, 0, 0}, shape, shape, "bpInts"); + + readerEngine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync); + VerifyData(myUInts.data(), step, {0, 0, 0}, shape, shape, "bpUInts"); + + readerEngine.Get(bpFloats, myFloats.data(), adios2::Mode::Sync); + VerifyData(myFloats.data(), step, {0, 0, 0}, shape, shape, "bpFloats"); + + readerEngine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync); + VerifyData(myDoubles.data(), step, {0, 0, 0}, shape, shape, + "bpDoubles"); + + readerEngine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Sync); + VerifyData(myComplexes.data(), step, {0, 0, 0}, shape, shape, + "bpComplexes"); + + readerEngine.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync); + VerifyData(myDComplexes.data(), step, {0, 0, 0}, shape, shape, + "bpDComplexes"); + + readerEngine.EndStep(); + } + readerEngine.Close(); +} + +void Writer(const Dims &shape, const Dims &start, const Dims &count, + const size_t rows, const adios2::Params &engineParams, + const std::string &name) +{ + size_t datasize = 1; + for (const auto &i : count) + { + datasize *= i; + } + adios2::ADIOS adios(MPI_COMM_WORLD); + adios2::IO io = adios.DeclareIO("ms"); + io.SetEngine("mhs"); + io.SetParameters(engineParams); + io.AddTransport("sirius", {{"variable", "bpFloats"}}); + std::vector myChars(datasize); + std::vector myUChars(datasize); + std::vector myShorts(datasize); + std::vector myUShorts(datasize); + std::vector myInts(datasize); + std::vector myUInts(datasize); + std::vector myFloats(datasize); + std::vector myDoubles(datasize); + std::vector> myComplexes(datasize); + std::vector> myDComplexes(datasize); + auto bpChars = io.DefineVariable("bpChars", shape, start, count); + auto bpUChars = + io.DefineVariable("bpUChars", shape, start, count); + auto bpShorts = io.DefineVariable("bpShorts", shape, start, count); + auto bpUShorts = + io.DefineVariable("bpUShorts", shape, start, count); + auto bpInts = io.DefineVariable("bpInts", shape, start, count); + auto bpUInts = + io.DefineVariable("bpUInts", shape, start, count); + auto bpFloats = io.DefineVariable("bpFloats", shape, start, count); + auto bpDoubles = + io.DefineVariable("bpDoubles", shape, start, count); + auto bpComplexes = io.DefineVariable>( + "bpComplexes", shape, start, count); + auto bpDComplexes = io.DefineVariable>( + "bpDComplexes", shape, start, count); + adios2::Engine writerEngine = io.Open(name, adios2::Mode::Write); + + for (size_t step = 0; step < 10; step++) + { + writerEngine.BeginStep(); + for (int i = mpiRank; i < static_cast(rows); i += mpiSize) + { + Dims startRow = start; + startRow[0] = i; + bpChars.SetSelection({startRow, count}); + bpUChars.SetSelection({startRow, count}); + bpShorts.SetSelection({startRow, count}); + bpUShorts.SetSelection({startRow, count}); + bpInts.SetSelection({startRow, count}); + bpUInts.SetSelection({startRow, count}); + bpFloats.SetSelection({startRow, count}); + bpDoubles.SetSelection({startRow, count}); + bpComplexes.SetSelection({startRow, count}); + bpDComplexes.SetSelection({startRow, count}); + GenData(myChars, step, startRow, count, shape); + GenData(myUChars, step, startRow, count, shape); + GenData(myShorts, step, startRow, count, shape); + GenData(myUShorts, step, startRow, count, shape); + GenData(myInts, step, startRow, count, shape); + GenData(myUInts, step, startRow, count, shape); + GenData(myFloats, step, startRow, count, shape); + GenData(myDoubles, step, startRow, count, shape); + GenData(myComplexes, step, startRow, count, shape); + GenData(myDComplexes, step, startRow, count, shape); + writerEngine.Put(bpChars, myChars.data(), adios2::Mode::Sync); + writerEngine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync); + writerEngine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync); + writerEngine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync); + writerEngine.Put(bpInts, myInts.data(), adios2::Mode::Sync); + writerEngine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync); + writerEngine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync); + writerEngine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync); + writerEngine.Put(bpComplexes, myComplexes.data(), + adios2::Mode::Sync); + writerEngine.Put(bpDComplexes, myDComplexes.data(), + adios2::Mode::Sync); + } + writerEngine.EndStep(); + } + writerEngine.Close(); +} + +TEST_F(MhsEngineTest, TestMhsMultiRank) +{ + std::string filename = "TestMhsMultiRank"; + adios2::Params engineParams = {{"Verbose", "0"}, {"Tiers", "4"}}; + + size_t rows = 32; + Dims shape = {rows, 8, 16}; + Dims start = {0, 0, 0}; + Dims count = {1, 8, 16}; + + Writer(shape, start, count, rows, engineParams, filename); + MPI_Barrier(MPI_COMM_WORLD); + + Reader(shape, start, count, rows, engineParams, filename); + MPI_Barrier(MPI_COMM_WORLD); +} + +int main(int argc, char **argv) +{ + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + int result; + ::testing::InitGoogleTest(&argc, argv); + result = RUN_ALL_TESTS(); + MPI_Finalize(); + return result; +} diff --git a/testing/adios2/engine/mhs/TestMhsMultiReader.cpp b/testing/adios2/engine/mhs/TestMhsMultiReader.cpp new file mode 100644 index 0000000000..53ef64edf9 --- /dev/null +++ b/testing/adios2/engine/mhs/TestMhsMultiReader.cpp @@ -0,0 +1,245 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + */ + +#include "TestMhsCommon.h" +#include +#include +#include +#include +#include + +using namespace adios2; +int mpiRank = 0; +int mpiSize = 1; + +char runMode; + +class MhsEngineTest : public ::testing::Test +{ +public: + MhsEngineTest() = default; +}; + +void Reader(const Dims &shape, const Dims &start, const Dims &count, + const size_t rows, const adios2::Params &engineParams, + const std::string &name) +{ + adios2::ADIOS adios(MPI_COMM_WORLD); + adios2::IO io = adios.DeclareIO("ms"); + io.SetEngine("mhs"); + io.SetParameters(engineParams); + adios2::Engine readerEngine = io.Open(name, adios2::Mode::Read); + size_t datasize = 1; + for (const auto &i : shape) + { + datasize *= i; + } + std::vector myChars(datasize); + std::vector myUChars(datasize); + std::vector myShorts(datasize); + std::vector myUShorts(datasize); + std::vector myInts(datasize); + std::vector myUInts(datasize); + std::vector myFloats(datasize); + std::vector myDoubles(datasize); + std::vector> myComplexes(datasize); + std::vector> myDComplexes(datasize); + + const auto &vars = io.AvailableVariables(); + std::cout << "All available variables : "; + for (const auto &var : vars) + { + std::cout << var.first << ", "; + } + std::cout << std::endl; + + ASSERT_EQ(vars.size(), 10); + + adios2::Variable bpChars = io.InquireVariable("bpChars"); + adios2::Variable bpUChars = + io.InquireVariable("bpUChars"); + adios2::Variable bpShorts = io.InquireVariable("bpShorts"); + adios2::Variable bpUShorts = + io.InquireVariable("bpUShorts"); + adios2::Variable bpInts = io.InquireVariable("bpInts"); + adios2::Variable bpUInts = + io.InquireVariable("bpUInts"); + adios2::Variable bpFloats = io.InquireVariable("bpFloats"); + adios2::Variable bpDoubles = + io.InquireVariable("bpDoubles"); + adios2::Variable> bpComplexes = + io.InquireVariable>("bpComplexes"); + adios2::Variable> bpDComplexes = + io.InquireVariable>("bpDComplexes"); + + for (size_t step = 0; step < 10; step++) + { + readerEngine.BeginStep(); + + bpChars.SetSelection({start, shape}); + bpUChars.SetSelection({start, shape}); + bpShorts.SetSelection({start, shape}); + bpUShorts.SetSelection({start, shape}); + bpInts.SetSelection({start, shape}); + bpUInts.SetSelection({start, shape}); + bpFloats.SetSelection({start, shape}); + bpDoubles.SetSelection({start, shape}); + bpComplexes.SetSelection({start, shape}); + bpDComplexes.SetSelection({start, shape}); + + readerEngine.Get(bpChars, myChars.data(), adios2::Mode::Sync); + VerifyData(myChars.data(), step, {0, 0, 0}, shape, shape, "bpChars"); + + readerEngine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync); + VerifyData(myUChars.data(), step, {0, 0, 0}, shape, shape, "bpUChars"); + + readerEngine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync); + VerifyData(myShorts.data(), step, {0, 0, 0}, shape, shape, "bpShorts"); + + readerEngine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync); + VerifyData(myUShorts.data(), step, {0, 0, 0}, shape, shape, + "bpUShorts"); + + readerEngine.Get(bpInts, myInts.data(), adios2::Mode::Sync); + VerifyData(myInts.data(), step, {0, 0, 0}, shape, shape, "bpInts"); + + readerEngine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync); + VerifyData(myUInts.data(), step, {0, 0, 0}, shape, shape, "bpUInts"); + + readerEngine.Get(bpFloats, myFloats.data(), adios2::Mode::Sync); + VerifyData(myFloats.data(), step, {0, 0, 0}, shape, shape, "bpFloats"); + + readerEngine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync); + VerifyData(myDoubles.data(), step, {0, 0, 0}, shape, shape, + "bpDoubles"); + + readerEngine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Sync); + VerifyData(myComplexes.data(), step, {0, 0, 0}, shape, shape, + "bpComplexes"); + + readerEngine.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync); + VerifyData(myDComplexes.data(), step, {0, 0, 0}, shape, shape, + "bpDComplexes"); + + readerEngine.EndStep(); + } + readerEngine.Close(); +} + +void Writer(const Dims &shape, const Dims &start, const Dims &count, + const size_t rows, const adios2::Params &engineParams, + const std::string &name) +{ + size_t datasize = 1; + for (const auto &i : count) + { + datasize *= i; + } + adios2::ADIOS adios(MPI_COMM_WORLD); + adios2::IO io = adios.DeclareIO("ms"); + io.SetEngine("mhs"); + io.SetParameters(engineParams); + io.AddTransport("sirius", {{"variable", "bpFloats"}}); + std::vector myChars(datasize); + std::vector myUChars(datasize); + std::vector myShorts(datasize); + std::vector myUShorts(datasize); + std::vector myInts(datasize); + std::vector myUInts(datasize); + std::vector myFloats(datasize); + std::vector myDoubles(datasize); + std::vector> myComplexes(datasize); + std::vector> myDComplexes(datasize); + auto bpChars = io.DefineVariable("bpChars", shape, start, count); + auto bpUChars = + io.DefineVariable("bpUChars", shape, start, count); + auto bpShorts = io.DefineVariable("bpShorts", shape, start, count); + auto bpUShorts = + io.DefineVariable("bpUShorts", shape, start, count); + auto bpInts = io.DefineVariable("bpInts", shape, start, count); + auto bpUInts = + io.DefineVariable("bpUInts", shape, start, count); + auto bpFloats = io.DefineVariable("bpFloats", shape, start, count); + auto bpDoubles = + io.DefineVariable("bpDoubles", shape, start, count); + auto bpComplexes = io.DefineVariable>( + "bpComplexes", shape, start, count); + auto bpDComplexes = io.DefineVariable>( + "bpDComplexes", shape, start, count); + adios2::Engine writerEngine = io.Open(name, adios2::Mode::Write); + + for (size_t step = 0; step < 10; step++) + { + writerEngine.BeginStep(); + for (int i = mpiRank; i < static_cast(rows); i += mpiSize) + { + Dims startRow = start; + startRow[0] = i; + bpChars.SetSelection({startRow, count}); + bpUChars.SetSelection({startRow, count}); + bpShorts.SetSelection({startRow, count}); + bpUShorts.SetSelection({startRow, count}); + bpInts.SetSelection({startRow, count}); + bpUInts.SetSelection({startRow, count}); + bpFloats.SetSelection({startRow, count}); + bpDoubles.SetSelection({startRow, count}); + bpComplexes.SetSelection({startRow, count}); + bpDComplexes.SetSelection({startRow, count}); + GenData(myChars, step, startRow, count, shape); + GenData(myUChars, step, startRow, count, shape); + GenData(myShorts, step, startRow, count, shape); + GenData(myUShorts, step, startRow, count, shape); + GenData(myInts, step, startRow, count, shape); + GenData(myUInts, step, startRow, count, shape); + GenData(myFloats, step, startRow, count, shape); + GenData(myDoubles, step, startRow, count, shape); + GenData(myComplexes, step, startRow, count, shape); + GenData(myDComplexes, step, startRow, count, shape); + writerEngine.Put(bpChars, myChars.data(), adios2::Mode::Sync); + writerEngine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync); + writerEngine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync); + writerEngine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync); + writerEngine.Put(bpInts, myInts.data(), adios2::Mode::Sync); + writerEngine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync); + writerEngine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync); + writerEngine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync); + writerEngine.Put(bpComplexes, myComplexes.data(), + adios2::Mode::Sync); + writerEngine.Put(bpDComplexes, myDComplexes.data(), + adios2::Mode::Sync); + } + writerEngine.EndStep(); + } + writerEngine.Close(); +} + +TEST_F(MhsEngineTest, TestMhsMultiReader) +{ + std::string filename = "TestMhsMultiReader"; + adios2::Params engineParams = {{"Verbose", "0"}, {"Tiers", "4"}}; + + size_t rows = 32; + Dims shape = {rows, 8, 16}; + Dims start = {0, 0, 0}; + Dims count = {1, 8, 16}; + + Writer(shape, start, count, rows, engineParams, filename); + MPI_Barrier(MPI_COMM_WORLD); + + Reader(shape, start, count, rows, engineParams, filename); + MPI_Barrier(MPI_COMM_WORLD); +} + +int main(int argc, char **argv) +{ + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + int result; + ::testing::InitGoogleTest(&argc, argv); + result = RUN_ALL_TESTS(); + MPI_Finalize(); + return result; +} diff --git a/testing/adios2/engine/table/TestTableSingleRank.cpp b/testing/adios2/engine/mhs/TestMhsSingleRank.cpp similarity index 96% rename from testing/adios2/engine/table/TestTableSingleRank.cpp rename to testing/adios2/engine/mhs/TestMhsSingleRank.cpp index 3fc5b1108d..bd79559738 100644 --- a/testing/adios2/engine/table/TestTableSingleRank.cpp +++ b/testing/adios2/engine/mhs/TestMhsSingleRank.cpp @@ -15,10 +15,10 @@ using namespace adios2; char runMode; -class TableEngineTest : public ::testing::Test +class MhsEngineTest : public ::testing::Test { public: - TableEngineTest() = default; + MhsEngineTest() = default; }; template @@ -88,6 +88,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count, adios2::ADIOS adios; #endif adios2::IO io = adios.DeclareIO("ms"); + io.SetEngine("mhs"); io.SetParameters(engineParams); adios2::Engine readerEngine = io.Open(name, adios2::Mode::Read); size_t datasize = 1; @@ -106,7 +107,7 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count, std::vector> myComplexes(datasize); std::vector> myDComplexes(datasize); - adios2::StepStatus status = readerEngine.BeginStep(); + readerEngine.BeginStep(); const auto &vars = io.AvailableVariables(); std::cout << "All available variables : "; for (const auto &var : vars) @@ -184,8 +185,9 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count, adios2::ADIOS adios; #endif adios2::IO io = adios.DeclareIO("ms"); - io.SetEngine("table"); + io.SetEngine("mhs"); io.SetParameters(engineParams); + io.AddTransport("sirius", {{"variable", "bpFloats"}}); std::vector myChars(datasize); std::vector myUChars(datasize); std::vector myShorts(datasize); @@ -253,12 +255,12 @@ void Writer(const Dims &shape, const Dims &start, const Dims &count, writerEngine.Close(); } -TEST_F(TableEngineTest, TestTableSingleRank) +TEST_F(MhsEngineTest, TestMhsSingleRank) { - std::string filename = "TestTableSingleRank"; - adios2::Params engineParams = {{"Verbose", "0"}}; + std::string filename = "TestMhsSingleRank"; + adios2::Params engineParams = {{"Verbose", "0"}, {"Tiers", "1"}}; - size_t rows = 1000; + size_t rows = 100; Dims shape = {rows, 1, 128}; Dims start = {0, 0, 0}; Dims count = {1, 1, 128}; diff --git a/testing/adios2/engine/skeleton/TestSkeletonReader.cpp b/testing/adios2/engine/skeleton/TestSkeletonReader.cpp index cdf30bac5a..05136746a9 100644 --- a/testing/adios2/engine/skeleton/TestSkeletonReader.cpp +++ b/testing/adios2/engine/skeleton/TestSkeletonReader.cpp @@ -113,9 +113,11 @@ int main(int argc, char *argv[]) adios2::Variable varArray = io.DefineVariable( "myArray", {gndx}, {gndx / (size_t)nproc}, {gndx / (size_t)nproc}); + (void)varArray; adios2::Variable varSyncString = io.DefineVariable("mySyncString"); + (void)varSyncString; } else { diff --git a/testing/adios2/engine/staging-common/CMakeLists.txt b/testing/adios2/engine/staging-common/CMakeLists.txt index 7a3a14e736..3e5825d021 100644 --- a/testing/adios2/engine/staging-common/CMakeLists.txt +++ b/testing/adios2/engine/staging-common/CMakeLists.txt @@ -21,6 +21,7 @@ foreach(helper TestCommonWriteAttrs TestCommonWriteLocal TestCommonWriteShared + TestDefSyncWrite TestCommonRead TestCommonReadR64 TestCommonReadLocal @@ -103,7 +104,7 @@ if(ADIOS2_HAVE_MPI AND MPIEXEC_EXECUTABLE) endforeach() endif() -set (SIMPLE_TESTS "1x1;NoReaderNoWait;TimeoutOnOpen;1x1.NoData;1x1.Modes;1x1.Attrs;1x1.Local;1x1.SharedNothing;1x1.SharedIO;1x1.SharedVar;1x1.SharedNothingSync;1x1.SharedIOSync;1x1.SharedVarSync;1x1EarlyExit;CumulativeAttr.1x1") +set (SIMPLE_TESTS "1x1;1x1DefSync;1x1Flush") set (SIMPLE_FORTRAN_TESTS "") if(ADIOS2_HAVE_Fortran) @@ -146,12 +147,13 @@ endif() SET (BASIC_SST_TESTS "") if(ADIOS2_HAVE_SST) list (APPEND BASIC_SST_TESTS ${ALL_SIMPLE_TESTS} ${SPECIAL_TESTS} ${SST_SPECIFIC_TESTS}) + list (REMOVE_ITEM BASIC_SST_TESTS 1x1DefSync 1x1Flush) endif() -# For the moment, only test the default comm pattern (Peer) +# For the moment, only test the default comm pattern (Min) MutateTestSet( COMM_MIN_SST_TESTS "CommMin" writer "CPCommPattern=Min" "${BASIC_SST_TESTS}" ) -MutateTestSet( COMM_PEER_SST_TESTS "CommPeer" writer "CPCommPattern=Peer" "${BASIC_SST_TESTS}" ) +#MutateTestSet( COMM_PEER_SST_TESTS "CommPeer" writer "CPCommPattern=Peer" "${BASIC_SST_TESTS}" ) # temporarily remove PreciousTimestep CommPeer tests list (REMOVE_ITEM COMM_PEER_SST_TESTS "PreciousTimestep") @@ -223,6 +225,8 @@ if(NOT WIN32) # not on windows list (FILTER BP_TESTS EXCLUDE REGEX ".*SharedVar$") # The nobody-writes-data-in-a-timestep tests don't work for any BP file engine list (FILTER BP_TESTS EXCLUDE REGEX ".*NoData$") + # BP3 and BP4 semantics on flush differ from BP5 + list (FILTER BP_TESTS EXCLUDE REGEX ".*Flush.*") foreach(test ${BP_TESTS}) add_common_test(${test} BP4) endforeach() @@ -238,8 +242,6 @@ if(ADIOS2_HAVE_BP5) set (BP5_TESTS ${ALL_SIMPLE_TESTS}) # Delayed reader not worth testing on file engines list (FILTER BP5_TESTS EXCLUDE REGEX "DelayedReader") - # The nobody-writes-data-in-a-timestep tests don't work for any BP file engine -# list (FILTER BP5_TESTS EXCLUDE REGEX ".*NoData$") foreach(test ${BP5_TESTS}) add_common_test(${test} BP5) endforeach() @@ -279,6 +281,8 @@ if(NOT MSVC) # not on windows #list (FILTER BP4_STREAM_TESTS EXCLUDE REGEX ".*LocalVarying.BPS$") # The nobody-writes-data-in-a-timestep tests don't work for any BP file engine list (FILTER BP4_STREAM_TESTS EXCLUDE REGEX ".*NoData.BPS$") + list (FILTER BP4_STREAM_TESTS EXCLUDE REGEX ".*1x1DefSync.*") + list (FILTER BP4_STREAM_TESTS EXCLUDE REGEX ".*Flush.*") foreach(test ${BP4_STREAM_TESTS}) add_common_test(${test} BP4_stream) @@ -311,6 +315,8 @@ if(NOT MSVC) # not on windows list (FILTER FILESTREAM_TESTS EXCLUDE REGEX ".*NoReaderNoWait.FS$") list (FILTER FILESTREAM_TESTS EXCLUDE REGEX ".*TimeoutOnOpen.FS$") list (FILTER FILESTREAM_TESTS EXCLUDE REGEX ".*NoReaderNoWait.FS$") + list (FILTER FILESTREAM_TESTS EXCLUDE REGEX ".*1x1DefSync.*") + list (FILTER FILESTREAM_TESTS EXCLUDE REGEX ".*Flush.*") foreach(test ${FILESTREAM_TESTS}) add_common_test(${test} FileStream) diff --git a/testing/adios2/engine/staging-common/ParseArgs.h b/testing/adios2/engine/staging-common/ParseArgs.h index 8f606b2613..6063dd8735 100644 --- a/testing/adios2/engine/staging-common/ParseArgs.h +++ b/testing/adios2/engine/staging-common/ParseArgs.h @@ -36,11 +36,15 @@ bool VaryingDataSize = false; bool AdvancingAttrs = false; int NoData = 0; int NoDataNode = -1; +int Flush = 0; int EarlyExit = 0; int LocalCount = 1; +int DataSize = 5 * 1024 * 1024 / 8; /* DefaultMinDeferredSize is 4*1024*1024 + This should be more than that. */ std::string shutdown_name = "DieTest"; adios2::Mode GlobalWriteMode = adios2::Mode::Deferred; +adios2::Mode GlobalReadMode = adios2::Mode::Deferred; static std::string Trim(std::string &str) { @@ -76,7 +80,7 @@ static adios2::Params ParseEngineParams(std::string Input) return Ret; } -static void ParseArgs(int argc, char **argv) +void ParseArgs(int argc, char **argv) { int bare_arg = 0; while (argc > 1) @@ -181,6 +185,24 @@ static void ParseArgs(int argc, char **argv) argv++; argc--; } + else if (std::string(argv[1]) == "--read_mode") + { + if (strcasecmp(argv[2], "sync") == 0) + { + GlobalReadMode = adios2::Mode::Sync; + } + else if (strcasecmp(argv[2], "deferred") == 0) + { + GlobalReadMode = adios2::Mode::Deferred; + } + else + { + std::cerr << "Invalid mode for --write_mode " << argv[2] + << std::endl; + } + argv++; + argc--; + } else if (std::string(argv[1]) == "--engine_params") { engineParams = ParseEngineParams(argv[2]); @@ -266,10 +288,28 @@ static void ParseArgs(int argc, char **argv) argv++; argc--; } + else if (std::string(argv[1]) == "--data_size") + { + std::istringstream ss(argv[2]); + if (!(ss >> DataSize)) + std::cerr << "Invalid number for --data_size argument" + << argv[1] << '\n'; + argv++; + argc--; + } else if (std::string(argv[1]) == "--early_exit") { EarlyExit++; } + else if (std::string(argv[1]) == "--flush") + { + Flush++; + } + else if (std::string(argv[1]) == "--disable_mpmd") + { + // someone else should have eaten this arg, but if it gets here, + // ignore it + } else { if (bare_arg == 0) diff --git a/testing/adios2/engine/staging-common/TestCommonRead.cpp b/testing/adios2/engine/staging-common/TestCommonRead.cpp index 89c23b7123..f42be981ef 100644 --- a/testing/adios2/engine/staging-common/TestCommonRead.cpp +++ b/testing/adios2/engine/staging-common/TestCommonRead.cpp @@ -323,40 +323,53 @@ TEST_F(CommonReadTest, ADIOS2CommonRead1D8) in_R64_2d_rev.resize(myLength * 2); if (!NoData) { - engine.Get(var_i8, in_I8.data()); - engine.Get(var_i16, in_I16.data()); - engine.Get(var_i32, in_I32.data()); - engine.Get(var_i64, in_I64.data()); + engine.Get(var_i8, in_I8.data(), GlobalReadMode); + engine.Get(var_i16, in_I16.data(), GlobalReadMode); + engine.Get(var_i32, in_I32.data(), GlobalReadMode); + engine.Get(var_i64, in_I64.data(), GlobalReadMode); - engine.Get(scalar_r64, in_scalar_R64); + engine.Get(scalar_r64, in_scalar_R64, GlobalReadMode); - engine.Get(var_r32, in_R32.data()); - engine.Get(var_r64, in_R64.data()); + engine.Get(var_r32, in_R32.data(), GlobalReadMode); + engine.Get(var_r64, in_R64.data(), GlobalReadMode); if (!mpiRank) - engine.Get(var_time, (int64_t *)&write_time); + engine.Get(var_time, (int64_t *)&write_time, + GlobalReadMode); } else { if (NoDataNode != -1) { // someone wrote everything, get our part - engine.Get(var_r64, in_R64.data()); + engine.Get(var_r64, in_R64.data(), GlobalReadMode); } } if (var_c32) - engine.Get(var_c32, in_C32.data()); + engine.Get(var_c32, in_C32.data(), GlobalReadMode); if (var_c64) - engine.Get(var_c64, in_C64.data()); + engine.Get(var_c64, in_C64.data(), GlobalReadMode); if (var_r64_2d) - engine.Get(var_r64_2d, in_R64_2d.data()); + engine.Get(var_r64_2d, in_R64_2d.data(), GlobalReadMode); if (var_r64_2d_rev) - engine.Get(var_r64_2d_rev, in_R64_2d_rev.data()); + engine.Get(var_r64_2d_rev, in_R64_2d_rev.data(), + GlobalReadMode); if (LockGeometry) { // we'll never change our data decomposition engine.LockReaderSelections(); } } + if (!NoData && (GlobalReadMode == adios2::Mode::Sync)) + { + // go ahead and test data now, it should be valid + int result = validateCommonTestData(myStart, myLength, t, !var_c32); + if (result != 0) + { + std::cout << "Read Data Validation failed on node " << mpiRank + << " timestep " << t << std::endl; + } + EXPECT_EQ(result, 0); + } engine.EndStep(); if (!NoData) diff --git a/testing/adios2/engine/staging-common/TestCommonReadF.F90 b/testing/adios2/engine/staging-common/TestCommonReadF.F90 index e76575dd2e..72961d5b47 100644 --- a/testing/adios2/engine/staging-common/TestCommonReadF.F90 +++ b/testing/adios2/engine/staging-common/TestCommonReadF.F90 @@ -11,6 +11,7 @@ program TestSstRead #endif use adios2 implicit none + external usage #if defined(ADIOS2_HAVE_FORTRAN_F03_ARGS) # define ADIOS2_ARGC() command_argument_count() @@ -25,10 +26,10 @@ program TestSstRead integer :: numargs - integer(kind = 8), dimension(1)::shape_dims, start_dims, count_dims - integer(kind = 8), dimension(2)::shape_dims2, start_dims2, count_dims2 - integer(kind = 8), dimension(2)::shape_dims3, start_dims3, count_dims3 - integer:: irank, isize, ierr, i, insteps, step_status + integer(kind = 8), dimension(1)::start_dims, count_dims + integer(kind = 8), dimension(2)::start_dims2, count_dims2 + integer(kind = 8), dimension(2)::start_dims3, count_dims3 + integer:: irank, isize, ierr, insteps, step_status character(len=256) :: filename, engine, params @@ -36,15 +37,19 @@ program TestSstRead type(adios2_adios)::adios type(adios2_io)::ioRead - type(adios2_variable), dimension(20)::variables + type(adios2_variable), dimension(:), allocatable :: variables type(adios2_engine)::sstReader; !read handlers - character(len =:), allocatable::variable_name - integer::variable_type, ndims, testComm + integer::ndims integer(kind = 8), dimension(:), allocatable::shape_in - integer::key, color - + +#if ADIOS2_USE_MPI + integer::key, color, testComm +#endif + + allocate(variables(20)) + numargs = ADIOS2_ARGC() if ( numargs < 2 ) then @@ -106,7 +111,7 @@ program TestSstRead call adios2_variable_shape(shape_in, ndims, variables(1), ierr) if (ndims /= 1) stop 'i8 ndims is not 1' if (modulo(shape_in(1), int(nx, 8)) /= 0) stop 'i8 shape_in read failed' - writerSize = shape_in(1) / nx + writerSize = INT(shape_in(1)) / nx deallocate(shape_in) call adios2_inquire_variable(variables(2), ioRead, "i16", ierr) diff --git a/testing/adios2/engine/staging-common/TestCommonReadLocal.cpp b/testing/adios2/engine/staging-common/TestCommonReadLocal.cpp index b42ad8d5bd..0b3fc6767c 100644 --- a/testing/adios2/engine/staging-common/TestCommonReadLocal.cpp +++ b/testing/adios2/engine/staging-common/TestCommonReadLocal.cpp @@ -65,223 +65,69 @@ TEST_F(CommonReadTest, ADIOS2CommonRead1D8) size_t writerSize; - auto scalar_r64 = io.InquireVariable("scalar_r64"); - EXPECT_TRUE(scalar_r64); - auto var_time = io.InquireVariable("time"); EXPECT_TRUE(var_time); ASSERT_EQ(var_time.ShapeID(), adios2::ShapeID::GlobalArray); writerSize = var_time.Shape()[0]; - // std::cout << "Writer size is " << writerSize << std::endl; - int rankToRead = mpiRank; if (writerSize < static_cast(mpiSize)) { rankToRead = mpiRank % writerSize; } - auto var_i8 = io.InquireVariable("i8"); - EXPECT_TRUE(var_i8); - ASSERT_EQ(var_i8.ShapeID(), adios2::ShapeID::LocalArray); - - auto var_i16 = io.InquireVariable("i16"); - EXPECT_TRUE(var_i16); - ASSERT_EQ(var_i16.ShapeID(), adios2::ShapeID::LocalArray); - - auto var_i32 = io.InquireVariable("i32"); - EXPECT_TRUE(var_i32); - ASSERT_EQ(var_i32.ShapeID(), adios2::ShapeID::LocalArray); - - auto var_i64 = io.InquireVariable("i64"); - EXPECT_TRUE(var_i64); - ASSERT_EQ(var_i64.ShapeID(), adios2::ShapeID::LocalArray); - auto var_r32 = io.InquireVariable("r32"); EXPECT_TRUE(var_r32); ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::LocalArray); - auto var_r64 = io.InquireVariable("r64"); - EXPECT_TRUE(var_r64); - ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::LocalArray); - - auto var_c32 = io.InquireVariable>("c32"); - auto var_c64 = io.InquireVariable>("c64"); - auto var_r64_2d = io.InquireVariable("r64_2d"); - auto var_r64_2d_rev = io.InquireVariable("r64_2d_rev"); - if (var_c32) - { - EXPECT_TRUE(var_c32); - ASSERT_EQ(var_c32.ShapeID(), adios2::ShapeID::LocalArray); - - EXPECT_TRUE(var_c64); - ASSERT_EQ(var_c64.ShapeID(), adios2::ShapeID::LocalArray); - - EXPECT_TRUE(var_r64_2d); - ASSERT_EQ(var_r64_2d.ShapeID(), adios2::ShapeID::LocalArray); - ASSERT_EQ(var_r64_2d.Count()[0], Nx); - ASSERT_EQ(var_r64_2d.Count()[1], 2); - - EXPECT_TRUE(var_r64_2d_rev); - ASSERT_EQ(var_r64_2d_rev.ShapeID(), adios2::ShapeID::LocalArray); - ASSERT_EQ(var_r64_2d_rev.Count()[0], 2); - ASSERT_EQ(var_r64_2d_rev.Count()[1], Nx); - } - else - { - EXPECT_FALSE(var_c32); - EXPECT_FALSE(var_c64); - EXPECT_FALSE(var_r64_2d); - EXPECT_FALSE(var_r64_2d_rev); - } - - long unsigned int hisStart = rankToRead * (int)Nx; long unsigned int hisLength = (long unsigned int)Nx; - var_i8.SetBlockSelection(rankToRead); - if (VaryingDataSize) - { - ASSERT_EQ( - engine.BlocksInfo(var_i8, currentStep).at(rankToRead).Count[0], - hisLength - currentStep - rankToRead); - } - else - { - ASSERT_EQ( - engine.BlocksInfo(var_i8, currentStep).at(rankToRead).Count[0], - hisLength); - } - var_i16.SetBlockSelection(rankToRead); - ASSERT_EQ( - engine.BlocksInfo(var_i16, currentStep).at(rankToRead).Count[0], - hisLength); - var_i32.SetBlockSelection(rankToRead); - ASSERT_EQ( - engine.BlocksInfo(var_i32, currentStep).at(rankToRead).Count[0], - hisLength); - var_i64.SetBlockSelection(rankToRead); - ASSERT_EQ( - engine.BlocksInfo(var_i64, currentStep).at(rankToRead).Count[0], - hisLength); - - var_r32.SetBlockSelection(rankToRead); for (int index = 0; index < LocalCount; index++) { int indexToRead = rankToRead * LocalCount; - std::cout << "Testing blocks info for var_r32 at index " - << indexToRead << std::endl; ASSERT_EQ(engine.BlocksInfo(var_r32, currentStep) .at(indexToRead) .Count[0], hisLength); } - var_r64.SetBlockSelection(rankToRead); - ASSERT_EQ( - engine.BlocksInfo(var_r64, currentStep).at(rankToRead).Count[0], - hisLength); - - if (var_c32) - { - var_c32.SetBlockSelection(rankToRead); - ASSERT_EQ( - engine.BlocksInfo(var_c32, currentStep).at(rankToRead).Count[0], - hisLength); - } - if (var_c64) - { - var_c64.SetBlockSelection(rankToRead); - ASSERT_EQ( - engine.BlocksInfo(var_c64, currentStep).at(rankToRead).Count[0], - hisLength); - } - if (var_r64_2d) - { - var_r64_2d.SetBlockSelection(rankToRead); - ASSERT_EQ(engine.BlocksInfo(var_r64_2d, currentStep) - .at(rankToRead) - .Count[0], - hisLength); - ASSERT_EQ(engine.BlocksInfo(var_r64_2d, currentStep) - .at(rankToRead) - .Count[1], - 2); - } - if (var_r64_2d_rev) - { - var_r64_2d_rev.SetBlockSelection(rankToRead); - ASSERT_EQ(engine.BlocksInfo(var_r64_2d_rev, currentStep) - .at(rankToRead) - .Count[0], - 2); - ASSERT_EQ(engine.BlocksInfo(var_r64_2d_rev, currentStep) - .at(rankToRead) - .Count[1], - hisLength); - } const adios2::Dims start_time{0}; const adios2::Dims count_time{1}; const adios2::Box sel_time(start_time, count_time); var_time.SetSelection(sel_time); - - in_I8.resize(hisLength); - if (LocalCount == 1) - { - in_R32.resize(hisLength); - } - else + auto BI = engine.BlocksInfo(var_r32, currentStep); + in_R32_blocks.resize(BI.size()); { - in_R32_blocks.resize(LocalCount); - for (int index = 0; index < LocalCount; index++) + for (size_t index = 0; index < BI.size(); index++) { in_R32_blocks[index].resize(hisLength); - } - } - in_I16.resize(hisLength); - in_I32.resize(hisLength); - in_I64.resize(hisLength); - in_R64.resize(hisLength); - in_C32.resize(hisLength); - in_C64.resize(hisLength); - in_R64_2d.resize(hisLength * 2); - in_R64_2d_rev.resize(hisLength * 2); - engine.Get(var_i8, in_I8.data()); - engine.Get(var_i16, in_I16.data()); - engine.Get(var_i32, in_I32.data()); - engine.Get(var_i64, in_I64.data()); - - if (LocalCount == 1) - { - engine.Get(var_r32, in_R32.data()); - } - else - { - for (int index = 0; index < LocalCount; index++) - { - int indexToRead = rankToRead * LocalCount + index; - auto block = in_R32_blocks[index]; - var_r32.SetBlockSelection(indexToRead); + var_r32.SetBlockSelection(index); engine.Get(var_r32, in_R32_blocks[index].data()); } } - engine.Get(scalar_r64, in_scalar_R64); - - engine.Get(var_r64, in_R64.data()); - if (var_c32) - engine.Get(var_c32, in_C32.data()); - if (var_c64) - engine.Get(var_c64, in_C64.data()); - if (var_r64_2d) - engine.Get(var_r64_2d, in_R64_2d.data()); - if (var_r64_2d_rev) - engine.Get(var_r64_2d_rev, in_R64_2d_rev.data()); std::time_t write_time; engine.Get(var_time, (int64_t *)&write_time); engine.EndStep(); - int result = validateCommonTestData(hisStart, hisLength, t, !var_c32, - VaryingDataSize, rankToRead); + int result = 0; + for (size_t index = 0; index < BI.size(); index++) + { + for (size_t i = 0; i < Nx; i++) + { + int64_t j = index * Nx * 10 + t; + float expected = (float)j + 10 * i; + if (in_R32_blocks[index][i] != expected) + { + std::cout << "Expected " << expected << ", got " + << in_R32_blocks[index][i] << " for in_R32[" << i + << "][" << index << "[" << i << "], timestep " + << t << std::endl; + result++; + } + } + } + if (result != 0) { std::cout << "Read Data Validation failed on node " << mpiRank diff --git a/testing/adios2/engine/staging-common/TestCommonWriteF.F90 b/testing/adios2/engine/staging-common/TestCommonWriteF.F90 index 9739302db6..6e3fcaaab0 100644 --- a/testing/adios2/engine/staging-common/TestCommonWriteF.F90 +++ b/testing/adios2/engine/staging-common/TestCommonWriteF.F90 @@ -10,6 +10,7 @@ program TestSstWrite #endif use adios2 implicit none + external usage #if defined(ADIOS2_HAVE_FORTRAN_F03_ARGS) # define ADIOS2_ARGC() command_argument_count() @@ -28,21 +29,23 @@ program TestSstWrite integer(kind = 8), dimension(2)::shape_dims2, start_dims2, count_dims2 integer(kind = 8), dimension(2)::shape_dims3, start_dims3, count_dims3 integer(kind = 8), dimension(1)::shape_time, start_time, count_time - integer::inx, irank, isize, ierr, i, insteps, status + integer::irank, isize, ierr, i, insteps, status character(len=256) :: filename, engine, params type(adios2_adios)::adios type(adios2_io)::ioWrite - type(adios2_variable), dimension(20)::variables + type(adios2_variable), dimension(:), allocatable :: variables type(adios2_engine)::sstWriter; !read handlers - character(len =:), allocatable::variable_name - integer::variable_type, ndims - integer(kind = 8), dimension(:), allocatable::shape_in integer(kind = 8)::localtime - integer::color, key, testComm + +#if ADIOS2_USE_MPI + integer::testComm, color, key +#endif + + allocate(variables(20)) numargs = ADIOS2_ARGC() if ( numargs < 2 ) then @@ -59,14 +62,14 @@ program TestSstWrite #if ADIOS2_USE_MPI !Launch MPI - call MPI_Init(ierr) + call MPI_Init(ierr) call MPI_Comm_rank(MPI_COMM_WORLD, key, ierr); color = 1 call MPI_Comm_split(MPI_COMM_WORLD, color, key, testComm, ierr); - call MPI_Comm_rank(testComm, irank, ierr) + call MPI_Comm_rank(testComm, irank, ierr) call MPI_Comm_size(testComm, isize, ierr) #else ! No MPI @@ -74,10 +77,10 @@ program TestSstWrite isize = 1; #endif - !Application variables + !Application variables insteps = 10; - !Variable dimensions + !Variable dimensions shape_dims(1) = isize * nx start_dims(1) = irank * nx count_dims(1) = nx @@ -102,24 +105,24 @@ program TestSstWrite #endif !!!!!!!!!!!!!!!!!!!!!!!!!WRITER!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -!!!!!!!!!!!Declare an IO process configuration inside adios +!!!!!!!!!!!Declare an IO process configuration inside adios call adios2_declare_io(ioWrite, adios, "ioWrite", ierr) if (numargs > 2) then call adios2_set_parameters(ioWrite, params, ierr) - endif + endif call adios2_set_engine(ioWrite, engine, ierr) - !Defines a variable to be written + !Defines a variable to be written call adios2_define_variable(variables(12), ioWrite, "scalar_r64", & adios2_type_dp, ierr) - + call adios2_define_variable(variables(1), ioWrite, "i8", & adios2_type_integer1, 1, & shape_dims, start_dims, count_dims, & adios2_constant_dims, ierr) - + call adios2_define_variable(variables(2), ioWrite, "i16", & adios2_type_integer2, 1, & shape_dims, start_dims, count_dims, & @@ -174,7 +177,7 @@ program TestSstWrite !Put array contents to bp buffer, based on var1 metadata do i = 1, insteps - call GenerateTestData(i - 1, irank, isize) + call GenerateTestData(i - 1, irank) call adios2_begin_step(sstWriter, adios2_step_mode_append, -1.0, & status, ierr) call adios2_put(sstWriter, variables(12), data_scalar_r64, ierr) @@ -198,8 +201,8 @@ program TestSstWrite !Closes engine1 and deallocates it, becomes unreachable call adios2_close(sstWriter, ierr) - !Deallocates adios and calls its destructor - call adios2_finalize(adios, ierr) + !Deallocates adios and calls its destructor + call adios2_finalize(adios, ierr) #if ADIOS2_USE_MPI call MPI_Finalize(ierr) diff --git a/testing/adios2/engine/staging-common/TestCommonWriteLocal.cpp b/testing/adios2/engine/staging-common/TestCommonWriteLocal.cpp index fa6da9e2ef..bcefb2126c 100644 --- a/testing/adios2/engine/staging-common/TestCommonWriteLocal.cpp +++ b/testing/adios2/engine/staging-common/TestCommonWriteLocal.cpp @@ -60,43 +60,14 @@ TEST_F(CommonWriteTest, ADIOS2CommonWrite) } { - adios2::Dims shape{static_cast(Nx * mpiSize)}; - adios2::Dims start{static_cast(Nx * mpiRank)}; adios2::Dims count{static_cast(Nx)}; - adios2::Dims shape2{static_cast(Nx * mpiSize), 2}; - adios2::Dims start2{static_cast(Nx * mpiRank), 0}; - adios2::Dims count2{static_cast(Nx), 2}; - adios2::Dims shape3{2, static_cast(Nx * mpiSize)}; - adios2::Dims start3{0, static_cast(Nx * mpiRank)}; - adios2::Dims count3{2, static_cast(Nx)}; adios2::Dims time_shape{static_cast(mpiSize)}; adios2::Dims time_start{static_cast(mpiRank)}; adios2::Dims time_count{1}; - (void)io.DefineVariable("scalar_r64"); - (void)io.DefineVariable("i8", {}, {}, count); - (void)io.DefineVariable("i16", {}, {}, count); - (void)io.DefineVariable("i32", {}, {}, count); - (void)io.DefineVariable("i64", {}, {}, count); - auto local_var_r32 = io.DefineVariable("r32", {}, {}, count); - auto local_var_r64 = io.DefineVariable("r64", {}, {}, count); - (void)io.DefineVariable>("c32", {}, {}, count); - (void)io.DefineVariable>("c64", {}, {}, count); - auto local_var_r64_2d = - io.DefineVariable("r64_2d", {}, {}, count2); - auto local_var_r64_2d_rev = - io.DefineVariable("r64_2d_rev", {}, {}, count3); + (void)io.DefineVariable("r32", {}, {}, count); (void)io.DefineVariable("time", time_shape, time_start, time_count); - if (CompressZfp) - { - adios2::Operator ZfpOp = - adios.DefineOperator("zfpCompressor", "zfp"); - local_var_r32.AddOperation(ZfpOp, {{"rate", "20"}}); - local_var_r64.AddOperation(ZfpOp, {{"rate", "20"}}); - local_var_r64_2d.AddOperation(ZfpOp, {{"rate", "20"}}); - local_var_r64_2d_rev.AddOperation(ZfpOp, {{"rate", "20"}}); - } } // Create the Engine @@ -105,74 +76,25 @@ TEST_F(CommonWriteTest, ADIOS2CommonWrite) adios2::Engine engine = io.Open(fname, adios2::Mode::Write); + data_R32.resize(Nx); for (int step = 0; step < NSteps; ++step) { - // Generate test data for each process uniquely - generateCommonTestData(step, mpiRank, mpiSize, (int)Nx, (int)Nx); - engine.BeginStep(); - // Retrieve the variables that previously went out of scope - auto scalar_r64 = io.InquireVariable("scalar_r64"); - auto var_i8 = io.InquireVariable("i8"); - auto var_i16 = io.InquireVariable("i16"); - auto var_i32 = io.InquireVariable("i32"); - auto var_i64 = io.InquireVariable("i64"); auto var_r32 = io.InquireVariable("r32"); - auto var_r64 = io.InquireVariable("r64"); - auto var_c32 = io.InquireVariable>("c32"); - auto var_c64 = io.InquireVariable>("c64"); - auto var_r64_2d = io.InquireVariable("r64_2d"); - auto var_r64_2d_rev = io.InquireVariable("r64_2d_rev"); auto var_time = io.InquireVariable("time"); - // // Make a 1D selection to describe the local dimensions of the - // // variable we write - adios2::Box sel_shrinking({}, {Nx - step - mpiRank}); - // adios2::Box sel({mpiRank * Nx}, {Nx}); - // adios2::Box sel2({mpiRank * Nx, 0}, {Nx, 2}); - // adios2::Box sel3({0, mpiRank * Nx}, {2, Nx}); - // adios2::Box sel_time( - // {static_cast(mpiRank)}, {1}); - if (VaryingDataSize) - { - var_i8.SetSelection(sel_shrinking); - } - // var_i16.SetSelection(sel); - // var_i32.SetSelection(sel); - // var_i64.SetSelection(sel); - // var_r32.SetSelection(sel); - // var_r64.SetSelection(sel); - // var_c32.SetSelection(sel); - // var_c64.SetSelection(sel); - // var_r64_2d.SetSelection(sel2); - // var_r64_2d_rev.SetSelection(sel3); - // var_time.SetSelection(sel_time); - - // Write each one - // fill in the variable with values from starting index to - // starting index + count const adios2::Mode sync = adios2::Mode::Deferred; - if (mpiRank == 0) - engine.Put(scalar_r64, data_scalar_R64); - engine.Put(var_i8, data_I8.data(), sync); - engine.Put(var_i16, data_I16.data(), sync); - engine.Put(var_i32, data_I32.data(), sync); - engine.Put(var_i64, data_I64.data(), sync); - engine.Put(var_r32, data_R32.data(), sync); - for (int index = 1; index < LocalCount; index++) + for (int index = 0; index < LocalCount; index++) { - for (size_t i = 0; i < data_R32.size(); i++) + for (size_t i = 0; i < Nx; i++) { - data_R32[i] += 1000.0; + int64_t j = (mpiRank * LocalCount + index) * Nx * 10 + step; + data_R32[i] = (float)j + 10 * i; } + engine.Put(var_r32, data_R32.data(), sync); } - engine.Put(var_r64, data_R64.data(), sync); - engine.Put(var_c32, data_C32.data(), sync); - engine.Put(var_c64, data_C64.data(), sync); - engine.Put(var_r64_2d, &data_R64_2d[0], sync); - engine.Put(var_r64_2d_rev, &data_R64_2d_rev[0], sync); // Advance to the next time step std::time_t localtime = std::time(NULL); engine.Put(var_time, (int64_t *)&localtime); diff --git a/testing/adios2/engine/staging-common/TestData.h b/testing/adios2/engine/staging-common/TestData.h index 11a423a19c..6a30c5b17b 100644 --- a/testing/adios2/engine/staging-common/TestData.h +++ b/testing/adios2/engine/staging-common/TestData.h @@ -193,8 +193,9 @@ int validateCommonTestData(int start, int length, size_t step, { std::cout << "Expected 0x" << std::hex << (int16_t)((i + start) * 10 + step) << ", got 0x" - << std::hex << (int16_t)in_I8[i] << " for in_I8[" << i - << "](global[" << i + start << "])" << std::endl; + << std::hex << (int16_t)in_I8[i] << std::dec + << " for in_I8[" << i << "](global[" << i + start + << "]), timestep " << step << std::endl; failures++; } } @@ -203,7 +204,8 @@ int validateCommonTestData(int start, int length, size_t step, std::cout << "Expected 0x" << std::hex << (int16_t)((i + start) * 10 + step) << ", got 0x" << std::hex << in_I16[i] << " for in_I16[" << i - << "](global[" << i + start << "])" << std::endl; + << "](global[" << i + start << "]), timestep " << step + << std::endl; failures++; } if (in_I32[i] != (int32_t)((i + start) * 10 + step)) @@ -211,7 +213,8 @@ int validateCommonTestData(int start, int length, size_t step, std::cout << "Expected 0x" << std::hex << (int32_t)((i + start) * 10 + step) << ", got 0x" << std::hex << in_I32[i] << " for in_I32[" << i - << "](global[" << i + start << "])" << std::endl; + << "](global[" << i + start << "]), timestep " << step + << std::endl; failures++; } if (in_I64[i] != (int64_t)((i + start) * 10 + step)) @@ -219,7 +222,8 @@ int validateCommonTestData(int start, int length, size_t step, std::cout << "Expected 0x" << std::hex << (int64_t)((i + start) * 10 + step) << ", got 0x" << std::hex << in_I64[i] << " for in_I64[" << i - << "](global[" << i + start << "])" << std::endl; + << "](global[" << i + start << "]), timestep " << step + << std::endl; failures++; } @@ -229,22 +233,26 @@ int validateCommonTestData(int start, int length, size_t step, { std::cout << "Expected " << (float)((i + start) * 10 + step) << ", got " << in_R32[i] << " for in_R32[" << i - << "](global[" << i + start << "])" << std::endl; + << "](global[" << i + start << "]), timestep " << step + << std::endl; failures++; } } else { + std::cout << "Blocks size is " << in_R32_blocks.size() << std::endl; for (size_t j = 0; j < in_R32_blocks.size(); j++) { - if (in_R32_blocks[j][i] != - (float)((i + start) * 10 + step + 1000.0 * j)) + std::cout << " Verifying block " << j << " at data " + << (void *)in_R32_blocks[j].data() << std::endl; + float expected = (float)((i + start) * 10 + step + 1000.0 * j + + (((int)(j / LocalCount)) * 100.0)); + if (in_R32_blocks[j][i] != expected) { - std::cout << "Expected " - << (float)((i + start) * 10 + step + 1000.0 * j) - << ", got " << in_R32_blocks[j][i] - << " for in_R32[" << i << "][" << j << "(global[" - << i + start << "])" << std::endl; + std::cout << "Expected " << expected << ", got " + << in_R32_blocks[j][i] << " for in_R32[" << i + << "][" << j << "(global[" << i + start + << "]), timestep " << step << std::endl; failures++; } } @@ -254,7 +262,8 @@ int validateCommonTestData(int start, int length, size_t step, { std::cout << "Expected " << (double)((i + start) * 10 + step) << ", got " << in_R64[i] << " for in_R64[" << i - << "](global[" << i + start << "])" << std::endl; + << "](global[" << i + start << "]), timestep " << step + << std::endl; failures++; } if (!missing_end_data) @@ -265,7 +274,8 @@ int validateCommonTestData(int start, int length, size_t step, std::cout << "Expected [" << (float)((i + start) * 10 + step) << ", " << -(float)((i + start) * 10 + step) << "], got " << in_C32[i] << " for in_C32[" << i - << "](global[" << i + start << "])" << std::endl; + << "](global[" << i + start << "]), timestep " << step + << std::endl; failures++; } if ((in_C64[i].imag() != (double)((i + start) * 10 + step)) || @@ -274,15 +284,16 @@ int validateCommonTestData(int start, int length, size_t step, std::cout << "Expected [" << (double)((i + start) * 10 + step) << ", " << -(double)((i + start) * 10 + step) << "], got " << in_C64[i] << " for in_C64[" << i - << "](global[" << i + start << "])" << std::endl; + << "](global[" << i + start << "]), timestep " << step + << std::endl; failures++; } if (in_R64_2d[2 * i] != (double)((i + start) * 10 + step)) { std::cout << "Expected " << (double)((i + start) * 10 + step) << ", got " << in_R64_2d[i] << " for in_R64_2d[" << i - << "][0](global[" << i + start << "][0])" - << std::endl; + << "][0](global[" << i + start << "][0]), timestep " + << step << std::endl; failures++; } if (in_R64_2d[2 * i + 1] != @@ -291,8 +302,8 @@ int validateCommonTestData(int start, int length, size_t step, std::cout << "Expected " << (double)(10000 + (i + start) * 10 + step) << ", got " << in_R64_2d[i] << " for in_R64_2d[" << i - << "][1](global[" << i + start << "][1])" - << std::endl; + << "][1](global[" << i + start << "][1]), timestep " + << step << std::endl; failures++; } if (in_R64_2d_rev[i] != (double)((i + start) * 10 + step)) @@ -300,7 +311,7 @@ int validateCommonTestData(int start, int length, size_t step, std::cout << "Expected " << (double)((i + start) * 10 + step) << ", got " << in_R64_2d_rev[i] << " for in_R64_2d_rev[0][" << i << "](global[0][" - << i + start << "])" << std::endl; + << i + start << "]), timestep " << step << std::endl; failures++; } if (in_R64_2d_rev[i + length] != @@ -310,7 +321,7 @@ int validateCommonTestData(int start, int length, size_t step, << (double)(10000 + (i + start) * 10 + step) << ", got " << in_R64_2d_rev[i + length] << " for in_R64_2d_rev[1][" << i << "](global[1][" - << i + start << "])" << std::endl; + << i + start << "]), timestep " << step << std::endl; failures++; } } @@ -329,7 +340,8 @@ int validateCommonTestDataR64(int start, int length, size_t step, { std::cout << "Expected " << (double)((i + start) * 10 + step) << ", got " << in_R64[i] << " for in_R64[" << i - << "](global[" << i + start << "])" << std::endl; + << "](global[" << i + start << "]), timestep " << step + << std::endl; failures++; } } diff --git a/testing/adios2/engine/staging-common/TestData_mod.F90 b/testing/adios2/engine/staging-common/TestData_mod.F90 index 559b9a7442..e122311c06 100644 --- a/testing/adios2/engine/staging-common/TestData_mod.F90 +++ b/testing/adios2/engine/staging-common/TestData_mod.F90 @@ -29,28 +29,28 @@ module sst_test_data integer(kind=4), dimension(:), allocatable :: in_I32 integer(kind=8), dimension(:), allocatable :: in_I64 real(kind=4), dimension(:), allocatable :: in_R32 - real(kind=8), dimension(:), allocatable :: in_R64 + real(kind=8), dimension(:), allocatable :: in_R64 complex(kind=4), dimension(:), allocatable :: in_C32 - complex(kind=8), dimension(:), allocatable :: in_C64 - real (kind=8), dimension(:,:), allocatable :: in_R64_2d - real (kind=8), dimension(:,:), allocatable :: in_R64_2d_rev + complex(kind=8), dimension(:), allocatable :: in_C64 + real (kind=8), dimension(:,:), allocatable :: in_R64_2d + real (kind=8), dimension(:,:), allocatable :: in_R64_2d_rev real (kind=8) :: in_scalar_R64 contains - subroutine GenerateTestData(step, rank, size) - INTEGER, INTENT(IN) :: step, rank, size + subroutine GenerateTestData(step, rank) + INTEGER, INTENT(IN) :: step, rank + - integer (kind=8) :: i, j j = rank * Nx * 10 + step; data_scalar_r64 = (step + 1) * 1.5D0; do i = 1, Nx - data_I8(i) = (j + 10 * (i-1)); - data_I16(i) = (j + 10 * (i-1)); - data_I32(i) = (j + 10 * (i-1)); - data_I64(i) = (j + 10 * (i-1)); - data_R32(i) = (j + 10 * (i-1)); - data_R64(i) = (j + 10 * (i-1)); + data_I8(i) = INT(j + 10 * (i-1), 1); + data_I16(i) = INT(j + 10 * (i-1), 2); + data_I32(i) = INT(j + 10 * (i-1), 4); + data_I64(i) = INT(j + 10 * (i-1), 8); + data_R32(i) = INT(j + 10 * (i-1), 4); + data_R64(i) = INT(j + 10 * (i-1), 8); data_C32(i) = cmplx(-(j + 10 * (i-1)), (j + 10 * (i-1))); data_C64(i) = cmplx(-(j + 10 * (i-1)), (j + 10 * (i-1))); data_R64_2d(1,i) = (j + 10 * (i-1)); @@ -63,13 +63,13 @@ end subroutine GenerateTestData subroutine ValidateTestData(start, length, step) INTEGER, INTENT(IN) :: start, length, step - + integer (kind=8) :: i do i = 1, length if (in_scalar_R64 /= (step + 1) * 1.5D0) then stop 'scalar_r64 value failed' end if - if (in_I8(i) /= INT(((i - 1 + start)* 10 + step), kind=1)) then + if (in_I8(i) /= INT(((i - 1 + start)* 10 + step), 1)) then stop 'data_I8 value failed' end if if (in_I16(i) /= (i - 1 + start)* 10 + step) then @@ -81,7 +81,7 @@ subroutine ValidateTestData(start, length, step) if (in_I64(i) /= (i - 1 + start)* 10 + step) then stop 'data_I64 value failed' end if - if (in_R32(i) /= (i - 1 + start)* 10 + step) then + if (in_R32(i) /= INT((i - 1 + start)* 10 + step, 4)) then stop 'data_R32 value failed' end if if (in_R64(i) /= (i - 1 + start)* 10 + step) then @@ -97,7 +97,6 @@ subroutine ValidateTestData(start, length, step) stop 'data_R64_rev value failed' end if if (in_R64_2d_rev(i, 2) /= 10000 + (i - 1 + start)* 10 + step) then - stop 'data_R64_rev value failed' end if end do diff --git a/testing/adios2/engine/staging-common/TestDefSyncWrite.cpp b/testing/adios2/engine/staging-common/TestDefSyncWrite.cpp new file mode 100644 index 0000000000..acb3a69b49 --- /dev/null +++ b/testing/adios2/engine/staging-common/TestDefSyncWrite.cpp @@ -0,0 +1,228 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + */ +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include + +#include "TestData.h" + +#include "ParseArgs.h" +int StartStep = 0; +int EndStep = 4 * 4 * 4 * 4 * 4; // all 4 possibilities for all 5 variables +int SmallSize = 100; + +// ADIOS2 write +TEST(CommonWriteTest, ADIOS2CommonWrite) +{ + adios2::ADIOS adios; + + adios2::IO io = adios.DeclareIO("TestIO"); + + adios2::Dims big_shape{static_cast(DataSize)}; + adios2::Dims big_start{static_cast(0)}; + adios2::Dims big_count{static_cast(DataSize)}; + + adios2::Dims small_shape{static_cast(SmallSize)}; + adios2::Dims small_start{static_cast(0)}; + adios2::Dims small_count{static_cast(SmallSize)}; + + std::vector> vars(5); + vars[0] = + io.DefineVariable("big1", big_shape, big_start, big_count); + vars[1] = io.DefineVariable("small1", small_shape, small_start, + small_count); + vars[2] = + io.DefineVariable("big2", big_shape, big_start, big_count); + vars[3] = io.DefineVariable("small2", small_shape, small_start, + small_count); + vars[4] = + io.DefineVariable("big3", big_shape, big_start, big_count); + + std::vector> data(5); + for (int i = 0; i < 5; i++) + { + int size = DataSize; + if ((i == 1) || (i == 3)) + { + size = SmallSize; + } + data[i] = std::vector(size); + } + + // Create the Engine + io.SetEngine(engine); + io.SetParameters(engineParams); + + adios2::Engine engine = io.Open(fname, adios2::Mode::Write); + + /* + * write possibilities: + * Don't write + * Sync - always destroy data afterwards + * Deferred + * Deferred with immediate PerformPuts() or Flush() - Destroy all + *prior data + * + */ + for (int step = StartStep; step < EndStep; ++step) + { + int mask = step; + engine.BeginStep(); + + std::cout << "Begin Write Step " << step << " writing vars : "; + for (int j = 0; j < 5; j++) + { + std::fill(data[j].begin(), data[j].end(), (double)j + 1.0); + } + for (int j = 0; j < 5; j++) + { + adios2::Mode write_mode; + char c; + int this_var_mask = (mask & 0x3); + mask >>= 2; + switch (this_var_mask) + { + case 0: + continue; + case 1: + write_mode = adios2::Mode::Sync; + c = 's'; + break; + case 2: + case 3: + write_mode = adios2::Mode::Deferred; + c = 'd'; + break; + } + std::cout << j << c << " "; + engine.Put(vars[j], data[j].data(), write_mode); + if (this_var_mask == 1) + { + std::fill(data[j].begin(), data[j].end(), -100.0); + } + else if (this_var_mask == 3) + { + if (Flush) + { + std::cout << "F "; + engine.Flush(); + } + else + { + std::cout << "P "; + engine.PerformPuts(); + } + for (int k = 0; k <= j; k++) + std::fill(data[k].begin(), data[k].end(), -100.0); + } + } + std::cout << std::endl; + engine.EndStep(); + } + + // Close the file + engine.Close(); +} + +// ADIOS2 write +TEST(CommonWriteTest, ADIOS2CommonRead) +{ + adios2::ADIOS adios; + + adios2::IO io = adios.DeclareIO("TestIO"); + + std::vector> data(5); + for (int i = 0; i < 5; i++) + { + int size = DataSize; + if ((i == 1) || (i == 3)) + { + size = SmallSize; + } + data[i] = (std::vector(size)); + } + + // Create the Engine + io.SetEngine(engine); + io.SetParameters(engineParams); + + adios2::Engine engine = io.Open(fname, adios2::Mode::Read); + EXPECT_TRUE(engine); + + /* + * write possibilities: + * Don't write + * Sync - always destroy data afterwards + * Deferred + * Deferred with immediate PerformPuts() - Destroy all prior data + * + */ + for (int step = StartStep; step < EndStep; ++step) + { + EXPECT_TRUE(engine.BeginStep() == adios2::StepStatus::OK); + + std::vector> vars(5); + vars[0] = io.InquireVariable("big1"); + vars[1] = io.InquireVariable("small1"); + vars[2] = io.InquireVariable("big2"); + vars[3] = io.InquireVariable("small2"); + vars[4] = io.InquireVariable("big3"); + + std::vector var_present(vars.size()); + for (size_t i = 0; i < data.size(); i++) + std::fill(data[i].begin(), data[i].end(), -200.0); + std::cout << "Variables Read in TS " << step << ": "; + for (int j = 0; j < 5; j++) + { + var_present[j] = (bool)vars[j]; + if (vars[j]) + { + std::cout << " " << j; + engine.Get(vars[j], data[j].data()); + } + } + std::cout << std::endl; + engine.EndStep(); + for (int j = 0; j < 5; j++) + { + if (var_present[j]) + { + for (std::size_t index = 0; index < data[j].size(); ++index) + { + EXPECT_EQ(data[j][index], j + 1.0) + << "Data isn't correct, for " << vars[j].Name() << "[" + << index << "]"; + } + } + } + } + + // Close the file + engine.Close(); +} + +int main(int argc, char **argv) +{ + + int result; + ::testing::InitGoogleTest(&argc, argv); + + DelayMS = 0; // zero for common writer + + ParseArgs(argc, argv); + + result = RUN_ALL_TESTS(); + + return result; +} diff --git a/testing/adios2/engine/staging-common/TestSupp.cmake b/testing/adios2/engine/staging-common/TestSupp.cmake index c1865e18c2..183224e845 100644 --- a/testing/adios2/engine/staging-common/TestSupp.cmake +++ b/testing/adios2/engine/staging-common/TestSupp.cmake @@ -63,6 +63,9 @@ find_package(PythonInterp REQUIRED) set (STAGING_COMMON_TEST_SUPP_VERBOSE OFF) set (1x1_CMD "run_test.py.$ -nw 1 -nr 1") +set (1x1GetSync_CMD "run_test.py.$ -nw 1 -nr 1 --rarg=--read_mode --rarg=sync") +set (1x1DefSync_CMD "TestDefSyncWrite --data_size 200 --engine_params ChunkSize=500,MinDeferredSize=150") +set (1x1Flush_CMD "TestDefSyncWrite --flush --data_size 200 --engine_params ChunkSize=500,MinDeferredSize=150") set (1x1.NoPreload_CMD "run_test.py.$ -nw 1 -nr 1 --rarg=PreloadMode=SstPreloadNone,RENGINE_PARAMS") set (1x1.SstRUDP_CMD "run_test.py.$ -nw 1 -nr 1 --rarg=DataTransport=WAN,WANDataTransport=enet,RENGINE_PARAMS --warg=DataTransport=WAN,WANDataTransport=enet,WENGINE_PARAMS") set (1x1.NoData_CMD "run_test.py.$ -nw 1 -nr 1 --warg=--no_data --rarg=--no_data") @@ -234,10 +237,17 @@ function(add_common_test basename engine) if ("${${basename}_CMD}" STREQUAL "") message(SEND_ERROR "Staging-Common test ${basename} has no defined ${basename}_CMD") endif() - set(command "${PYTHON_EXECUTABLE} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${${basename}_CMD}") + string(FIND ${${basename}_CMD} "run_test.py" pos) + if (NOT ${pos} EQUAL -1 ) + set(command "${PYTHON_EXECUTABLE} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${${basename}_CMD}") + set(place 2) + else() + set(command "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${${basename}_CMD}") + set(place 1) + endif() remove_engine_params_placeholder(command "${command}") separate_arguments(command) - list(INSERT command 2 "${engine}" "${testname}") + list(INSERT command ${place} "${engine}" "${testname}") if(NOT ADIOS2_RUN_MPI_MPMD_TESTS) list(APPEND command "--disable_mpmd") endif() diff --git a/testing/adios2/engine/staging-common/run_test.py.gen.in b/testing/adios2/engine/staging-common/run_test.py.gen.in index cb77305225..0b99343daf 100755 --- a/testing/adios2/engine/staging-common/run_test.py.gen.in +++ b/testing/adios2/engine/staging-common/run_test.py.gen.in @@ -321,6 +321,13 @@ reader_command_line.extend([reader_executable, canonical_engine, args.filename]) if args.rarg is not None: reader_command_line.extend(args.rarg) +if args.num_writers == 0: + writer_command_line = None + mpmd_possible = False + print("TestDriver: No writers, setting MPMD false\n") +else: + print("TestDriver: Writer command line : " + " ".join(writer_command_line)) + if args.num_readers == 0: reader_command_line = None mpmd_possible = False @@ -329,13 +336,6 @@ if args.num_readers == 0: else: print("TestDriver: Reader command line : " + " ".join(reader_command_line)) -if args.num_writers == 0: - writer_command_line = None - mpmd_possible = False - print("TestDriver: No writers, setting MPMD false\n") -else: - print("TestDriver: Writer command line : " + " ".join(writer_command_line)) - if is_file_engine[args.engine.lower()]: print("TestDriver: Is file engine, setting MPMD false\n") mpmd_possible = False diff --git a/testing/adios2/engine/table/TestTableMultiRank.cpp b/testing/adios2/engine/table/TestTableMultiRank.cpp deleted file mode 100644 index 0c07706766..0000000000 --- a/testing/adios2/engine/table/TestTableMultiRank.cpp +++ /dev/null @@ -1,277 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - */ - -#include -#include -#include -#include -#include - -using namespace adios2; -int mpiRank = 0; -int mpiSize = 1; - -char runMode; - -class TableEngineTest : public ::testing::Test -{ -public: - TableEngineTest() = default; -}; - -template -void GenData(std::complex *data, const size_t row, const Dims &count) -{ - for (size_t i = 0; i < count[1]; ++i) - { - for (size_t j = 0; j < count[2]; ++j) - { - data[i * count[2] + j] = {static_cast(i * count[2] + j + row), - static_cast(i * count[2])}; - } - } -} - -template -void GenData(T *data, const size_t row, const Dims &count) -{ - for (size_t i = 0; i < count[1]; ++i) - { - for (size_t j = 0; j < count[2]; ++j) - { - data[i * count[2] + j] = static_cast(i * count[2] + j + row); - } - } -} - -template -void GenData(std::vector &data, const size_t row, const Dims &count) -{ - GenData(data.data(), row, count); -} - -template -void VerifyData(const T *data, const size_t rows, const Dims &shape) -{ - size_t columnSize = 1; - for (const auto &i : shape) - { - columnSize *= i; - } - size_t rowSize = 1; - for (size_t i = 1; i < shape.size(); ++i) - { - rowSize *= shape[i]; - } - - std::vector tmpdata(columnSize); - size_t position = 0; - for (size_t i = 0; i < rows; ++i) - { - GenData(tmpdata.data() + position, i, shape); - position += rowSize; - } - for (size_t i = 0; i < columnSize; ++i) - { - ASSERT_EQ(data[i], tmpdata[i]); - } -} - -void Reader(const Dims &shape, const Dims &start, const Dims &count, - const size_t rows, const adios2::Params &engineParams, - const std::string &name) -{ - adios2::ADIOS adios(MPI_COMM_WORLD); - adios2::IO io = adios.DeclareIO("ms"); - io.SetParameters(engineParams); - adios2::Engine readerEngine = io.Open(name, adios2::Mode::Read); - size_t datasize = 1; - for (const auto &i : shape) - { - datasize *= i; - } - std::vector myChars(datasize); - std::vector myUChars(datasize); - std::vector myShorts(datasize); - std::vector myUShorts(datasize); - std::vector myInts(datasize); - std::vector myUInts(datasize); - std::vector myFloats(datasize); - std::vector myDoubles(datasize); - std::vector> myComplexes(datasize); - std::vector> myDComplexes(datasize); - - adios2::StepStatus status = readerEngine.BeginStep(); - const auto &vars = io.AvailableVariables(); - std::cout << "All available variables : "; - for (const auto &var : vars) - { - std::cout << var.first << ", "; - } - std::cout << std::endl; - ASSERT_EQ(vars.size(), 10); - - adios2::Variable bpChars = io.InquireVariable("bpChars"); - adios2::Variable bpUChars = - io.InquireVariable("bpUChars"); - adios2::Variable bpShorts = io.InquireVariable("bpShorts"); - adios2::Variable bpUShorts = - io.InquireVariable("bpUShorts"); - adios2::Variable bpInts = io.InquireVariable("bpInts"); - adios2::Variable bpUInts = - io.InquireVariable("bpUInts"); - adios2::Variable bpFloats = io.InquireVariable("bpFloats"); - adios2::Variable bpDoubles = - io.InquireVariable("bpDoubles"); - adios2::Variable> bpComplexes = - io.InquireVariable>("bpComplexes"); - adios2::Variable> bpDComplexes = - io.InquireVariable>("bpDComplexes"); - - bpChars.SetSelection({start, shape}); - bpUChars.SetSelection({start, shape}); - bpShorts.SetSelection({start, shape}); - bpUShorts.SetSelection({start, shape}); - bpInts.SetSelection({start, shape}); - bpUInts.SetSelection({start, shape}); - bpFloats.SetSelection({start, shape}); - bpDoubles.SetSelection({start, shape}); - bpComplexes.SetSelection({start, shape}); - bpDComplexes.SetSelection({start, shape}); - - readerEngine.Get(bpChars, myChars.data(), adios2::Mode::Sync); - readerEngine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync); - readerEngine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync); - readerEngine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync); - readerEngine.Get(bpInts, myInts.data(), adios2::Mode::Sync); - readerEngine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync); - readerEngine.Get(bpFloats, myFloats.data(), adios2::Mode::Sync); - readerEngine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync); - readerEngine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Sync); - readerEngine.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync); - - VerifyData(myChars.data(), rows, shape); - VerifyData(myUChars.data(), rows, shape); - VerifyData(myShorts.data(), rows, shape); - VerifyData(myUShorts.data(), rows, shape); - VerifyData(myInts.data(), rows, shape); - VerifyData(myUInts.data(), rows, shape); - VerifyData(myFloats.data(), rows, shape); - VerifyData(myDoubles.data(), rows, shape); - VerifyData(myComplexes.data(), rows, shape); - VerifyData(myDComplexes.data(), rows, shape); - - readerEngine.EndStep(); - readerEngine.Close(); -} - -void Writer(const Dims &shape, const Dims &start, const Dims &count, - const size_t rows, const adios2::Params &engineParams, - const std::string &name) -{ - size_t datasize = 1; - for (const auto &i : count) - { - datasize *= i; - } - adios2::ADIOS adios(MPI_COMM_WORLD); - adios2::IO io = adios.DeclareIO("ms"); - io.SetEngine("table"); - io.SetParameters(engineParams); - std::vector myChars(datasize); - std::vector myUChars(datasize); - std::vector myShorts(datasize); - std::vector myUShorts(datasize); - std::vector myInts(datasize); - std::vector myUInts(datasize); - std::vector myFloats(datasize); - std::vector myDoubles(datasize); - std::vector> myComplexes(datasize); - std::vector> myDComplexes(datasize); - auto bpChars = io.DefineVariable("bpChars", shape, start, count); - auto bpUChars = - io.DefineVariable("bpUChars", shape, start, count); - auto bpShorts = io.DefineVariable("bpShorts", shape, start, count); - auto bpUShorts = - io.DefineVariable("bpUShorts", shape, start, count); - auto bpInts = io.DefineVariable("bpInts", shape, start, count); - auto bpUInts = - io.DefineVariable("bpUInts", shape, start, count); - auto bpFloats = io.DefineVariable("bpFloats", shape, start, count); - auto bpDoubles = - io.DefineVariable("bpDoubles", shape, start, count); - auto bpComplexes = io.DefineVariable>( - "bpComplexes", shape, start, count); - auto bpDComplexes = io.DefineVariable>( - "bpDComplexes", shape, start, count); - adios2::Engine writerEngine = io.Open(name, adios2::Mode::Write); - writerEngine.BeginStep(); - for (int i = mpiRank; i < static_cast(rows); i += mpiSize) - { - Dims startRow = start; - startRow[0] = i; - bpChars.SetSelection({startRow, count}); - bpUChars.SetSelection({startRow, count}); - bpShorts.SetSelection({startRow, count}); - bpUShorts.SetSelection({startRow, count}); - bpInts.SetSelection({startRow, count}); - bpUInts.SetSelection({startRow, count}); - bpFloats.SetSelection({startRow, count}); - bpDoubles.SetSelection({startRow, count}); - bpComplexes.SetSelection({startRow, count}); - bpDComplexes.SetSelection({startRow, count}); - GenData(myChars, i, count); - GenData(myUChars, i, count); - GenData(myShorts, i, count); - GenData(myUShorts, i, count); - GenData(myInts, i, count); - GenData(myUInts, i, count); - GenData(myFloats, i, count); - GenData(myDoubles, i, count); - GenData(myComplexes, i, count); - GenData(myDComplexes, i, count); - writerEngine.Put(bpChars, myChars.data(), adios2::Mode::Sync); - writerEngine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync); - writerEngine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync); - writerEngine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync); - writerEngine.Put(bpInts, myInts.data(), adios2::Mode::Sync); - writerEngine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync); - writerEngine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync); - writerEngine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync); - writerEngine.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync); - writerEngine.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync); - } - writerEngine.EndStep(); - writerEngine.Close(); -} - -TEST_F(TableEngineTest, TestTableMultiRank) -{ - std::string filename = "TestTableMultiRank"; - adios2::Params engineParams = {{"Verbose", "0"}}; - - size_t rows = 800; - Dims shape = {rows, 8, 64}; - Dims start = {0, 0, 0}; - Dims count = {1, 8, 64}; - - Writer(shape, start, count, rows, engineParams, filename); - MPI_Barrier(MPI_COMM_WORLD); - - Reader(shape, start, count, rows, engineParams, filename); - MPI_Barrier(MPI_COMM_WORLD); -} - -int main(int argc, char **argv) -{ - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); - MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); - int result; - ::testing::InitGoogleTest(&argc, argv); - result = RUN_ALL_TESTS(); - MPI_Finalize(); - return result; -} diff --git a/testing/adios2/hierarchy/CMakeLists.txt b/testing/adios2/hierarchy/CMakeLists.txt index c699ec9ac5..a721ea0497 100644 --- a/testing/adios2/hierarchy/CMakeLists.txt +++ b/testing/adios2/hierarchy/CMakeLists.txt @@ -3,4 +3,4 @@ #accompanying file Copyright.txt for details. #------------------------------------------------------------------------------# -gtest_add_tests_helper(HierarchicalReading MPI_NONE "" "" . "") \ No newline at end of file +bp_gtest_add_tests_helper(HierarchicalReading MPI_NONE) \ No newline at end of file diff --git a/testing/adios2/hierarchy/TestHierarchicalReading.cpp b/testing/adios2/hierarchy/TestBPHierarchicalReading.cpp similarity index 86% rename from testing/adios2/hierarchy/TestHierarchicalReading.cpp rename to testing/adios2/hierarchy/TestBPHierarchicalReading.cpp index 0c18cbfbb0..1f01c37a8e 100644 --- a/testing/adios2/hierarchy/TestHierarchicalReading.cpp +++ b/testing/adios2/hierarchy/TestBPHierarchicalReading.cpp @@ -104,21 +104,8 @@ TEST_F(ADIOSHierarchicalReadVariableTest, Read) EXPECT_EQ(res.size(), 5); res = g.AvailableAttributes(); EXPECT_EQ(res.size(), 0); - engine.EndStep(); - } - for (size_t step = 0; step < NSteps; step++) - { - engine.BeginStep(); - auto g = io.InquireGroup('/'); - auto res = g.AvailableGroups(); - EXPECT_EQ(res[0], "group1"); - res = g.AvailableVariables(); - EXPECT_EQ(res[0], "variable6"); - engine.EndStep(); - } - for (size_t step = 0; step < NSteps; step++) - { - auto g = io.InquireGroup('/'); + + g = io.InquireGroup('/'); auto var = g.InquireVariable("variable6"); EXPECT_TRUE(var); if (var) @@ -128,12 +115,8 @@ TEST_F(ADIOSHierarchicalReadVariableTest, Read) engine.Get(var, myInts, adios2::Mode::Sync); EXPECT_EQ(Ints, myInts); } - } - for (size_t step = 0; step < NSteps; step++) - { - auto g = io.InquireGroup('/'); g.setPath("group1/group2/group3/group4"); - auto var = g.InquireVariable("variable1"); + var = g.InquireVariable("variable1"); EXPECT_TRUE(var); if (var) { @@ -143,6 +126,7 @@ TEST_F(ADIOSHierarchicalReadVariableTest, Read) EXPECT_EQ(Ints, myInts); } + engine.EndStep(); } engine.Close(); } diff --git a/testing/adios2/interface/TestADIOSDefineAttribute.cpp b/testing/adios2/interface/TestADIOSDefineAttribute.cpp index 4ae737af9a..3eb7aa1816 100644 --- a/testing/adios2/interface/TestADIOSDefineAttribute.cpp +++ b/testing/adios2/interface/TestADIOSDefineAttribute.cpp @@ -45,6 +45,7 @@ TEST_F(ADIOSDefineAttributeTest, DefineAttributeNameException) auto availableAttributes = io.AvailableAttributes(); EXPECT_EQ(availableAttributes.size(), 1); + // Redefinition is not allowed (non-modifiable attribute) EXPECT_THROW(io.DefineAttribute(name, "0"), std::invalid_argument); @@ -54,6 +55,20 @@ TEST_F(ADIOSDefineAttributeTest, DefineAttributeNameException) auto attributeString2 = io.InquireAttribute(name); EXPECT_TRUE(attributeString2); + + /* Modifiable attribute can change its value(s) ... */ + io.DefineAttribute("modifiable", "initial", "", "", true); + io.DefineAttribute("modifiable", "modified", "", "", true); + + auto attributeString3 = io.InquireAttribute("modifiable"); + EXPECT_TRUE(attributeString3); + auto attributeString3Value = attributeString3.Data(); + ASSERT_EQ(attributeString3Value.size() == 1, true); + EXPECT_EQ(attributeString3Value[0], "modified"); + + /* ... but not its type */ + EXPECT_THROW(io.DefineAttribute("modifiable", 1.0, "", "", true), + std::invalid_argument); } TEST_F(ADIOSDefineAttributeTest, DefineAttributeTypeByValue) @@ -749,7 +764,7 @@ TEST_F(ADIOSDefineAttributeTest, VariableException) "myVar1", separator), std::invalid_argument); - auto var = io.DefineVariable("myVar1"); + io.DefineVariable("myVar1"); EXPECT_NO_THROW( io.DefineAttribute("Hello Value", "Value", "myVar1")); diff --git a/testing/adios2/interface/TestADIOSInterface.cpp b/testing/adios2/interface/TestADIOSInterface.cpp index d5192cc9a3..8569ae66b0 100644 --- a/testing/adios2/interface/TestADIOSInterface.cpp +++ b/testing/adios2/interface/TestADIOSInterface.cpp @@ -19,6 +19,7 @@ TEST(ADIOSInterface, MPICommRemoved) MPI_Comm_free(&myComm); adios2::Engine engine = io.Open("test.bp", adios2::Mode::Write); + (void)engine; } #endif @@ -28,9 +29,7 @@ TEST(ADIOSInterface, BADConfigFile) EXPECT_THROW(adios2::ADIOS adios("notthere.xml"); adios2::IO io = adios.DeclareIO("TestIO"); - adios2::Engine engine = - io.Open("test.bp", adios2::Mode::Write); - , std::logic_error); + io.Open("test.bp", adios2::Mode::Write);, std::logic_error); } /** ADIOS2_CXX11_API diff --git a/testing/adios2/interface/TestADIOSInterfaceWrite.cpp b/testing/adios2/interface/TestADIOSInterfaceWrite.cpp index 631f592393..20efe62899 100644 --- a/testing/adios2/interface/TestADIOSInterfaceWrite.cpp +++ b/testing/adios2/interface/TestADIOSInterfaceWrite.cpp @@ -49,8 +49,8 @@ TEST_F(ADIOSInterfaceWriteTest, DefineVar_int8_t_1x10) // Verify exceptions are thrown upon duplicate variable names EXPECT_THROW(auto foo = - io.DefineVariable(name, {}, {}, adios2::Dims{10}), - std::invalid_argument); + io.DefineVariable(name, {}, {}, adios2::Dims{10}); + (void)foo, std::invalid_argument); // Verify the dimensions, name, and type are correct ASSERT_EQ(var_int8_t.Shape().size(), 0); @@ -82,8 +82,8 @@ TEST_F(ADIOSInterfaceWriteTest, DefineVar_int16_t_1x10) // Verify exceptions are thrown upon duplicate variable names EXPECT_THROW(auto foo = - io.DefineVariable(name, {}, {}, adios2::Dims{10}), - std::invalid_argument); + io.DefineVariable(name, {}, {}, adios2::Dims{10}); + (void)foo, std::invalid_argument); // Verify the dimensions, name, and type are correct ASSERT_EQ(var_int16_t.Shape().size(), 0); @@ -115,8 +115,8 @@ TEST_F(ADIOSInterfaceWriteTest, DefineVar_int32_t_1x10) // Verify exceptions are thrown upon duplicate variable names EXPECT_THROW(auto foo = - io.DefineVariable(name, {}, {}, adios2::Dims{10}), - std::invalid_argument); + io.DefineVariable(name, {}, {}, adios2::Dims{10}); + (void)foo, std::invalid_argument); // Verify the dimensions, name, and type are correct ASSERT_EQ(var_int32_t.Shape().size(), 0); @@ -149,8 +149,8 @@ TEST_F(ADIOSInterfaceWriteTest, DefineVar_int64_t_1x10) // Verify exceptions are thrown upon duplicate variable names EXPECT_THROW(auto foo = - io.DefineVariable(name, {}, {}, adios2::Dims{10}), - std::invalid_argument); + io.DefineVariable(name, {}, {}, adios2::Dims{10}); + (void)foo, std::invalid_argument); // Verify the dimensions, name, and type are correct ASSERT_EQ(var_int64_t.Shape().size(), 0); @@ -184,8 +184,8 @@ TEST_F(ADIOSInterfaceWriteTest, DefineVar_uint8_t_1x10) // Verify exceptions are thrown upon duplicate variable names EXPECT_THROW(auto foo = - io.DefineVariable(name, {}, {}, adios2::Dims{10}), - std::invalid_argument); + io.DefineVariable(name, {}, {}, adios2::Dims{10}); + (void)foo, std::invalid_argument); // Verify the dimensions, name, and type are correct ASSERT_EQ(var_uint8_t.Shape().size(), 0); @@ -218,8 +218,8 @@ TEST_F(ADIOSInterfaceWriteTest, DefineVar_uint16_t_1x10) // Verify exceptions are thrown upon duplicate variable names EXPECT_THROW( - auto foo = io.DefineVariable(name, {}, {}, adios2::Dims{10}), - std::invalid_argument); + auto foo = io.DefineVariable(name, {}, {}, adios2::Dims{10}); + (void)foo, std::invalid_argument); // Verify the dimensions, name, and type are correct ASSERT_EQ(var_uint16_t.Shape().size(), 0); @@ -252,8 +252,8 @@ TEST_F(ADIOSInterfaceWriteTest, DefineVar_uint32_t_1x10) // Verify exceptions are thrown upon duplicate variable names EXPECT_THROW( - auto foo = io.DefineVariable(name, {}, {}, adios2::Dims{10}), - std::invalid_argument); + auto foo = io.DefineVariable(name, {}, {}, adios2::Dims{10}); + (void)foo, std::invalid_argument); // Verify the dimensions, name, and type are correct ASSERT_EQ(var_uint32_t.Shape().size(), 0); @@ -286,8 +286,8 @@ TEST_F(ADIOSInterfaceWriteTest, DefineVar_uint64_t_1x10) // Verify exceptions are thrown upon duplicate variable names EXPECT_THROW( - auto foo = io.DefineVariable(name, {}, {}, adios2::Dims{10}), - std::invalid_argument); + auto foo = io.DefineVariable(name, {}, {}, adios2::Dims{10}); + (void)foo, std::invalid_argument); // Verify the dimensions, name, and type are correct ASSERT_EQ(var_uint64_t.Shape().size(), 0); @@ -324,8 +324,8 @@ TEST_F(ADIOSInterfaceWriteTest, DefineVar_int8_t_2x5) // Verify exceptions are thrown upon duplicate variable names EXPECT_THROW( - auto foo = io.DefineVariable(name, {}, {}, adios2::Dims{2, 5}), - std::invalid_argument); + auto foo = io.DefineVariable(name, {}, {}, adios2::Dims{2, 5}); + (void)foo, std::invalid_argument); // Verify the dimensions, name, and type are correct ASSERT_EQ(var_int8_t.Shape().size(), 0); @@ -359,8 +359,8 @@ TEST_F(ADIOSInterfaceWriteTest, DefineVar_int16_t_2x5) // Verify exceptions are thrown upon duplicate variable names EXPECT_THROW( - auto foo = io.DefineVariable(name, {}, {}, adios2::Dims{2, 5}), - std::invalid_argument); + auto foo = io.DefineVariable(name, {}, {}, adios2::Dims{2, 5}); + (void)foo, std::invalid_argument); // Verify the dimensions, name, and type are correct ASSERT_EQ(var_int16_t.Shape().size(), 0); @@ -394,8 +394,8 @@ TEST_F(ADIOSInterfaceWriteTest, DefineVar_int32_t_2x5) // Verify exceptions are thrown upon duplicate variable names EXPECT_THROW( - auto foo = io.DefineVariable(name, {}, {}, adios2::Dims{2, 5}), - std::invalid_argument); + auto foo = io.DefineVariable(name, {}, {}, adios2::Dims{2, 5}); + (void)foo, std::invalid_argument); // Verify the dimensions, name, and type are correct ASSERT_EQ(var_int32_t.Shape().size(), 0); @@ -429,8 +429,8 @@ TEST_F(ADIOSInterfaceWriteTest, DefineVar_int64_t_2x5) // Verify exceptions are thrown upon duplicate variable names EXPECT_THROW( - auto foo = io.DefineVariable(name, {}, {}, adios2::Dims{2, 5}), - std::invalid_argument); + auto foo = io.DefineVariable(name, {}, {}, adios2::Dims{2, 5}); + (void)foo, std::invalid_argument); // Verify the dimensions, name, and type are correct ASSERT_EQ(var_int64_t.Shape().size(), 0); @@ -465,8 +465,8 @@ TEST_F(ADIOSInterfaceWriteTest, DefineVar_uint8_t_2x5) // Verify exceptions are thrown upon duplicate variable names EXPECT_THROW( - auto foo = io.DefineVariable(name, {}, {}, adios2::Dims{2, 5}), - std::invalid_argument); + auto foo = io.DefineVariable(name, {}, {}, adios2::Dims{2, 5}); + (void)foo, std::invalid_argument); // Verify the dimensions, name, and type are correct ASSERT_EQ(var_uint8_t.Shape().size(), 0); @@ -500,8 +500,8 @@ TEST_F(ADIOSInterfaceWriteTest, DefineVar_uint16_t_2x5) // Verify exceptions are thrown upon duplicate variable names EXPECT_THROW(auto foo = io.DefineVariable(name, {}, {}, - adios2::Dims{2, 5}), - std::invalid_argument); + adios2::Dims{2, 5}); + (void)foo, std::invalid_argument); // Verify the dimensions, name, and type are correct ASSERT_EQ(var_uint16_t.Shape().size(), 0); @@ -535,8 +535,8 @@ TEST_F(ADIOSInterfaceWriteTest, DefineVar_uint32_t_2x5) // Verify exceptions are thrown upon duplicate variable names EXPECT_THROW(auto foo = io.DefineVariable(name, {}, {}, - adios2::Dims{2, 5}), - std::invalid_argument); + adios2::Dims{2, 5}); + (void)foo, std::invalid_argument); // Verify the dimensions, name, and type are correct ASSERT_EQ(var_uint32_t.Shape().size(), 0); @@ -570,8 +570,8 @@ TEST_F(ADIOSInterfaceWriteTest, DefineVar_uint64_t_2x5) // Verify exceptions are thrown upon duplicate variable names EXPECT_THROW(auto foo = io.DefineVariable(name, {}, {}, - adios2::Dims{2, 5}), - std::invalid_argument); + adios2::Dims{2, 5}); + (void)foo, std::invalid_argument); // Verify the dimensions, name, and type are correct ASSERT_EQ(var_uint64_t.Shape().size(), 0); diff --git a/testing/adios2/performance/manyvars/TestManyVars.cpp b/testing/adios2/performance/manyvars/TestManyVars.cpp index cf8c1e22c3..93e0d4bdf2 100644 --- a/testing/adios2/performance/manyvars/TestManyVars.cpp +++ b/testing/adios2/performance/manyvars/TestManyVars.cpp @@ -187,7 +187,7 @@ class TestManyVars : public ::testing::TestWithParam digit++; } - char fmt[16]; + char fmt[32]; sprintf(fmt, "v%%%d.%dd", digit, digit); for (size_t i = 0; i < NVARS; i++) { diff --git a/testing/adios2/xml/TestXMLConfig.cpp b/testing/adios2/xml/TestXMLConfig.cpp index a6f5b70ed8..3667284e46 100644 --- a/testing/adios2/xml/TestXMLConfig.cpp +++ b/testing/adios2/xml/TestXMLConfig.cpp @@ -34,8 +34,8 @@ TEST_F(XMLConfigTest, TwoIOs) #endif // must be declared at least once - EXPECT_THROW(adios2::IO io = adios.AtIO("Test IO 1"), - std::invalid_argument); + EXPECT_THROW(adios2::IO io = adios.AtIO("Test IO 1"); + (void)io, std::invalid_argument); EXPECT_NO_THROW({ adios2::IO io = adios.DeclareIO("Test IO 1"); @@ -51,10 +51,10 @@ TEST_F(XMLConfigTest, TwoIOs) io.Open("Test BP Writer 1", adios2::Mode::Write); engine.Close(); }); - EXPECT_NO_THROW(adios2::IO io = adios.AtIO("Test IO 1")); + EXPECT_NO_THROW(adios2::IO io = adios.AtIO("Test IO 1"); (void)io); - EXPECT_THROW(adios2::IO io = adios.AtIO("Test IO 2"), - std::invalid_argument); + EXPECT_THROW(adios2::IO io = adios.AtIO("Test IO 2"); + (void)io, std::invalid_argument); EXPECT_NO_THROW({ adios2::IO io = adios.DeclareIO("Test IO 2"); const adios2::Params params = io.Parameters(); diff --git a/testing/adios2/yaml/TestYAMLConfig.cpp b/testing/adios2/yaml/TestYAMLConfig.cpp index 1473b33e87..0d37c16b51 100644 --- a/testing/adios2/yaml/TestYAMLConfig.cpp +++ b/testing/adios2/yaml/TestYAMLConfig.cpp @@ -30,8 +30,8 @@ TEST_F(YAMLConfigTest, TwoIOs) #endif // must be declared at least once - EXPECT_THROW(adios2::IO io = adios.AtIO("Test IO 1"), - std::invalid_argument); + EXPECT_THROW(adios2::IO io = adios.AtIO("Test IO 1"); + (void)io, std::invalid_argument); EXPECT_NO_THROW({ adios2::IO io = adios.DeclareIO("Test IO 1"); @@ -47,10 +47,10 @@ TEST_F(YAMLConfigTest, TwoIOs) io.Open("Test BP Writer 1", adios2::Mode::Write); engine.Close(); }); - EXPECT_NO_THROW(adios2::IO io = adios.AtIO("Test IO 1")); + EXPECT_NO_THROW(adios2::IO io = adios.AtIO("Test IO 1"); (void)io); - EXPECT_THROW(adios2::IO io = adios.AtIO("Test IO 2"), - std::invalid_argument); + EXPECT_THROW(adios2::IO io = adios.AtIO("Test IO 2"); + (void)io, std::invalid_argument); EXPECT_NO_THROW({ adios2::IO io = adios.DeclareIO("Test IO 2"); const adios2::Params params = io.Parameters(); diff --git a/testing/h5vol/TestH5VolWriteReadBPFile.cpp b/testing/h5vol/TestH5VolWriteReadBPFile.cpp index f760bd68d3..5282ce1957 100644 --- a/testing/h5vol/TestH5VolWriteReadBPFile.cpp +++ b/testing/h5vol/TestH5VolWriteReadBPFile.cpp @@ -209,12 +209,14 @@ void HDF5NativeWriter::CreateAndStoreScalar(std::string const &variableName, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); ret = H5Dwrite(dsetID, type, H5S_ALL, H5S_ALL, plistID, values); + EXPECT_GE(ret, 0); #ifdef DOUBLECHECK size_t typesize = H5Tget_size(type); char *val = (char *)(calloc(typesize, sizeof(char))); hid_t ret2 = H5Dread(dsetID, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, val); + EXPECT_GE(ret2, 0); std::cerr << " .... typesize=" << typesize << " val=" << val << std::endl; free val; diff --git a/testing/install/CMakeLists.txt b/testing/install/CMakeLists.txt index e039080036..cd8481d942 100644 --- a/testing/install/CMakeLists.txt +++ b/testing/install/CMakeLists.txt @@ -82,6 +82,17 @@ set_tests_properties(Install.Setup PROPERTIES FIXTURES_SETUP Install ) +# Remove -Wall -Werror in this scope +if(CMAKE_C_FLAGS) + string(REPLACE "-Wall" " " CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) + string(REPLACE "-Werror" " " CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) +endif() + +if(CMAKE_CXX_FLAGS) + string(REPLACE "-Wall" " " CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) + string(REPLACE "-Werror" " " CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) +endif() + add_install_cmake_test(C) add_install_cmake_test(CXX11) if(ADIOS2_HAVE_Fortran) diff --git a/testing/install/CXX11/CMakeLists.txt b/testing/install/CXX11/CMakeLists.txt index d7dcca6a4f..7a01b6b252 100644 --- a/testing/install/CXX11/CMakeLists.txt +++ b/testing/install/CXX11/CMakeLists.txt @@ -15,10 +15,14 @@ target_link_libraries(adios_cxx11_test adios2::cxx11) add_test(NAME adios_cxx11_test COMMAND adios_cxx11_test) if(ADIOS2_HAVE_MPI) + # Avoid using MPI::MPI_CXX + enable_language(C) + find_package(MPI REQUIRED) + set(mpilib MPI::MPI_C) set(src main_mpi.cxx) - set(mpilib MPI::MPI_CXX) set(mpiexec ${MPIEXEC_EXECUTABLE} ${MPIEXEC_EXTRA_FLAGS}) + add_compile_definitions(OMPI_SKIP_MPICXX MPICH_SKIP_MPICXX) # Test using mpi C++11 bindings through adios2::cxx11_mpi target. add_executable(adios_cxx11_mpi_test main_mpi.cxx) diff --git a/thirdparty/EVPath/EVPath/.github/workflows/build-and-test.yml b/thirdparty/EVPath/EVPath/.github/workflows/build-and-test.yml index b05345f552..6482bd757a 100644 --- a/thirdparty/EVPath/EVPath/.github/workflows/build-and-test.yml +++ b/thirdparty/EVPath/EVPath/.github/workflows/build-and-test.yml @@ -1,31 +1,126 @@ -on: - push: - branches: - - master - pull_request: - branches: - - master +name: Build and Test + +on: + push: + branches: + - master + pull_request: + branches: + - master jobs: - ci: - runs-on: ${{ matrix.os-image }} + linux: + # The jobs should run pretty quick; anything over 30m essentially means + # someting is stuck somewhere + timeout-minutes: 30 + runs-on: ubuntu-latest + container: ${{ matrix.container }} + env: + GH_YML_JOBNAME: ${{ matrix.os }}-${{ matrix.compiler }} + GH_YML_BUILDTYPE: ${{ matrix.buildtype }} + GH_YML_SHA: ${{ github.event.pull_request.head.sha || github.sha }} strategy: fail-fast: false matrix: - os-image: [ubuntu-latest, macos-latest] + buildtype: [ release, debug ] + os: [ centos7, centos8, ubuntu1604, ubuntu1804, ubuntu2004 ] + compiler: [ clang, gcc, nvhpc ] + exclude: + - { os: centos8, compiler: nvhpc } + - { os: ubuntu1604, compiler: nvhpc } + - { os: ubuntu1804, compiler: nvhpc } + include: + - os: centos7 + container: centos:7 + - os: centos8 + container: centos:8 + - os: ubuntu1604 + container: ubuntu:16.04 + - os: ubuntu1804 + container: ubuntu:18.04 + - os: ubuntu2004 + container: ubuntu:20.04 + - os: centos7 + compiler: nvhpc + container: nvcr.io/nvidia/nvhpc:21.2-devel-cuda11.2-centos7 + - os: ubuntu2004 + compiler: nvhpc + container: nvcr.io/nvidia/nvhpc:21.2-devel-cuda11.2-ubuntu20.04 steps: - uses: actions/checkout@v2 with: ref: ${{ github.event.pull_request.head.sha }} + path: source - name: Setup - run: ci/gh-actions/setup.sh + run: | + . source/scripts/ci/setup/linux.sh + source/scripts/ci/setup/install-atl.sh ${{ matrix.buildtype }} + source/scripts/ci/setup/install-dill.sh ${{ matrix.buildtype }} + source/scripts/ci/setup/install-ffs.sh ${{ matrix.buildtype }} + - name: Update + run: source/scripts/ci/gh-actions/run.sh update - name: Configure - run: ci/gh-actions/run.sh configure + run: source/scripts/ci/gh-actions/run.sh configure - name: Build - run: ci/gh-actions/run.sh build + run: source/scripts/ci/gh-actions/run.sh build - name: Test - run: ci/gh-actions/run.sh test - - name: Install - run: ci/gh-actions/run.sh install + run: source/scripts/ci/gh-actions/run.sh test + + # mac_and_windows: + # # The jobs should run pretty quick; anything over 30m essentially means + # # something is stuck somewhere + # timeout-minutes: 30 + # runs-on: ${{ matrix.vm }} + # env: + # GH_YML_JOBNAME: ${{ matrix.jobname }} + # GH_YML_BUILDTYPE: ${{ matrix.buildtype }} + # GH_YML_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + # + # strategy: + # fail-fast: false + # matrix: + # buildtype: [ release, debug ] + # jobname: [ + # windows-vs2019-msvc, + # windows-vs2019-clang, + # macos-clang ] + # include: + # - jobname: windows-vs2019-msvc + # vm: windows-latest + # - jobname: windows-vs2019-clang + # vm: windows-latest + # - jobname: macos-clang + # vm: macos-latest + + # defaults: + # run: + # shell: bash + + # steps: + # - uses: actions/checkout@v2 + # with: + # ref: ${{ github.event.pull_request.head.sha }} + # path: source + # - name: Setup + # if: ${{ runner.os == 'Windows' }} + # run: | + # . source/scripts/ci/setup/windows.sh + # source/scripts/ci/setup/install-atl.sh ${{ matrix.buildtype }} + # source/scripts/ci/setup/install-ffs.sh ${{ matrix.buildtype }} + # - name: Setup + # if: ${{ runner.os == 'macOS' }} + # run: | + # . source/scripts/ci/setup/macos.sh + # source/scripts/ci/setup/install-atl.sh ${{ matrix.buildtype }} + # source/scripts/ci/setup/install-dill.sh ${{ matrix.buildtype }} + # source/scripts/ci/setup/install-ffs.sh ${{ matrix.buildtype }} + # - name: Update + # run: source/scripts/ci/gh-actions/run.sh update + # - name: Configure + # run: source/scripts/ci/gh-actions/run.sh configure + # - name: Build + # run: source/scripts/ci/gh-actions/run.sh build + # - name: Test + # run: source/scripts/ci/gh-actions/run.sh test diff --git a/thirdparty/EVPath/EVPath/.github/workflows/triggers.yml b/thirdparty/EVPath/EVPath/.github/workflows/triggers.yml new file mode 100644 index 0000000000..3ee5556d9e --- /dev/null +++ b/thirdparty/EVPath/EVPath/.github/workflows/triggers.yml @@ -0,0 +1,15 @@ +name: Triggers + +on: + workflow_run: + workflows: ["Build and Test"] + types: [requested] + +jobs: + all_triggers: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Post CDash Status + run: scripts/ci/scripts/post-cdash-status.sh ${{ github.event.repository.full_name }} ${{ github.event.workflow_run.head_sha }} ${{ secrets.GITHUB_TOKEN }} diff --git a/thirdparty/EVPath/EVPath/CMakeLists.txt b/thirdparty/EVPath/EVPath/CMakeLists.txt index 1f43eab746..f2453e5dff 100644 --- a/thirdparty/EVPath/EVPath/CMakeLists.txt +++ b/thirdparty/EVPath/EVPath/CMakeLists.txt @@ -1,4 +1,9 @@ cmake_minimum_required(VERSION 3.2) + +# The directory label is used for CDash to treat EVPath as a subproject of +# GTKorvo +set(CMAKE_DIRECTORY_LABELS EVPath) + project(EVPath VERSION 4.5.0 LANGUAGES C CXX) # Enable _ROOT variables for dependency searching @@ -59,6 +64,11 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) set_property(CACHE CMAKE_BUILD_TYPE PROPERTY VALUE RelWithDebInfo) endif() +if(NOT MSVC) + set(CMAKE_C_STANDARD 99) + set(CMAKE_C_STANDARD_REQUIRED True) +endif() + set(CPACK_DEBIAN_PACKAGE_DEPENDS "dill, atl, ffs") set(CPACK_RPM_PACKAGE_REQUIRES "dill, atl, ffs") set(ENABLE_SOMETHING AUTO CACHE STRING "Enable SOMETHING support") # @@ -131,6 +141,9 @@ set(_pkg_config_libs) set(_pkg_config_private_libs) include(CheckCSourceCompiles) + +# Avoid recent compilers to optimizing out libm function calls +set(CMAKE_REQUIRED_FLAGS "-O0") set(LIBM_TEST_SOURCE "#include\nfloat f; int main(){sqrt(f);return 0;}") check_c_source_compiles("${LIBM_TEST_SOURCE}" HAVE_MATH) if(NOT HAVE_MATH) @@ -300,7 +313,7 @@ if(DEFINED EVPATH_USE_ENET) find_package(enet 1.3.13 REQUIRED) endif() else() - find_package(enet 1.3.13) + find_package(enet 1.3.13 QUIET) option(EVPATH_USE_ENET "Build the enet transport" ${ENET_FOUND}) endif() if(ENET_FOUND) diff --git a/thirdparty/EVPath/EVPath/CTestConfig.cmake b/thirdparty/EVPath/EVPath/CTestConfig.cmake new file mode 100644 index 0000000000..91a3947718 --- /dev/null +++ b/thirdparty/EVPath/EVPath/CTestConfig.cmake @@ -0,0 +1,9 @@ +set(CTEST_PROJECT_NAME "GTKorvo") +set(CTEST_NIGHTLY_START_TIME "01:00:00 UTC") + +set(CTEST_DROP_METHOD "http") +set(CTEST_DROP_SITE "open.cdash.org") +set(CTEST_DROP_LOCATION "/submit.php?project=GTKorvo") +set(CTEST_DROP_SITE_CDASH TRUE) + +set(CTEST_LABELS_FOR_SUBPROJECTS EVPath) diff --git a/thirdparty/EVPath/EVPath/CTestCustom.ctest.in b/thirdparty/EVPath/EVPath/CTestCustom.ctest.in index d4aaac50f3..fcc146b306 100644 --- a/thirdparty/EVPath/EVPath/CTestCustom.ctest.in +++ b/thirdparty/EVPath/EVPath/CTestCustom.ctest.in @@ -1,2 +1,18 @@ +set(CTEST_CUSTOM_MAXIMUM_NUMBER_OF_ERRORS 1000) +set(CTEST_CUSTOM_MAXIMUM_NUMBER_OF_WARNINGS 1000) +set(CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE 1048576) +set(CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE 1048576) + +list(APPEND CTEST_CUSTOM_WARNING_EXCEPTION + "warning:.*__d0" + "warning:.*__d1" +) + +list(APPEND CTEST_CUSTOM_COVERAGE_EXCLUDE +) + +list(APPEND CTEST_CUSTOM_TESTS_IGNORE +) + set (CTEST_CUSTOM_MEMCHECK_IGNORE) -LIST(APPEND CTEST_CUSTOM_MEMCHECK_IGNORE all_transports non_blocking_transports) +list(APPEND CTEST_CUSTOM_MEMCHECK_IGNORE t4 compare_t4 marshal_test2 marshal_test) diff --git a/thirdparty/EVPath/EVPath/ci/gh-actions/run.sh b/thirdparty/EVPath/EVPath/ci/gh-actions/run.sh deleted file mode 100755 index a02a085ee6..0000000000 --- a/thirdparty/EVPath/EVPath/ci/gh-actions/run.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -export CI_ROOT_DIR="${GITHUB_WORKSPACE//\\//}/.." -export CI_SOURCE_DIR="${GITHUB_WORKSPACE//\\//}" -export CI_DEP_DIR="${CI_ROOT_DIR}/dependencies" -export CI_BIN_DIR="${CI_ROOT_DIR}/build" - -export CMAKE_PREFIX_PATH=${CI_DEP_DIR}/install -export PATH=${CI_DEP_DIR}/tools/bin:${CI_DEP_DIR}/install/bin:${PATH} -case "$(uname -s)" in - Linux) - export LD_LIBRARY_PATH=${CI_DEP_DIR}/install/lib:${LD_LIBRARY_PATH} - ;; - Darwin) - export DYLD_LIBRARY_PATH=${CI_DEP_DIR}/install/lib:${DYLD_LIBRARY_PATH} - ;; -esac - - -mkdir -p ${CI_BIN_DIR} -cd ${CI_BIN_DIR} - -case "$1" in - configure) - cmake -GNinja -DCMAKE_INSTALL_PREFIX=${CI_ROOT_DIR}/install ${CI_SOURCE_DIR} - ;; - build) - ninja - ;; - test) - if [ "$(uname -s)" = "Darwin" ] - then - # Disable the firewall - sudo /usr/libexec/ApplicationFirewall/socketfilterfw --setglobalstate off - - # Force the use of the loopback interface - #export CM_IP="127.0.0.1" - export CM_HOSTNAME="localhost" - CTEST_EXCLUDES="mtests_non_blocking_bulk" - fi - ctest --timeout 300 -j2 -VV -E "${CTEST_EXCLUDES}" - ;; - install) - ninja install - ;; -esac diff --git a/thirdparty/EVPath/EVPath/ci/gh-actions/setup-linux.sh b/thirdparty/EVPath/EVPath/ci/gh-actions/setup-linux.sh deleted file mode 100755 index 0cf208475e..0000000000 --- a/thirdparty/EVPath/EVPath/ci/gh-actions/setup-linux.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -echo -echo "Installing ninja" -mkdir -p ${CI_DEP_DIR}/tools/bin -cd ${CI_DEP_DIR}/tools/bin -curl -O -L https://github.com/ninja-build/ninja/releases/download/v1.10.0/ninja-linux.zip -unzip ninja-linux.zip - -echo -echo "Installing libfabric" -mkdir -p ${CI_DEP_DIR}/libfabric -cd ${CI_DEP_DIR}/libfabric -curl -L https://github.com/ofiwg/libfabric/releases/download/v1.9.1/libfabric-1.9.1.tar.bz2 | tar -xj -cd libfabric-1.9.1 -./configure --prefix=${CI_DEP_DIR}/install --disable-static --enable-shared --enable-{shm,sockets,tcp,udp} --disable-{bgq,efa,gni,mrail,psm,psm2,rstream,rxd,rxm,usnic,verbs} -make -j2 install - -# Install cmake -echo -echo "Installing CMake" -mkdir -p ${CI_DEP_DIR}/tools -cd ${CI_DEP_DIR}/tools -curl -L https://github.com/Kitware/CMake/releases/download/v3.3.2/cmake-3.3.2-Linux-x86_64.tar.gz | tar --strip-components=1 -xz - -export PATH=${CI_DEP_DIR}/tools/bin:${PATH} diff --git a/thirdparty/EVPath/EVPath/ci/gh-actions/setup-macos.sh b/thirdparty/EVPath/EVPath/ci/gh-actions/setup-macos.sh deleted file mode 100755 index cf47c72215..0000000000 --- a/thirdparty/EVPath/EVPath/ci/gh-actions/setup-macos.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -echo -echo "Installing ninja" -brew install ninja - -echo -echo "Installing libfabric" -brew install pkg-config -brew install libfabric - -# Install cmake -echo -echo "Installing CMake" -mkdir -p ${CI_DEP_DIR}/tools -cd ${CI_DEP_DIR}/tools -curl -L https://github.com/Kitware/CMake/releases/download/v3.3.2/cmake-3.3.2-Darwin-x86_64.tar.gz | tar --strip-components=3 -xz - -export PATH=${CI_DEP_DIR}/tools/bin:${PATH} diff --git a/thirdparty/EVPath/EVPath/ci/gh-actions/setup.sh b/thirdparty/EVPath/EVPath/ci/gh-actions/setup.sh deleted file mode 100755 index 5a113a70e9..0000000000 --- a/thirdparty/EVPath/EVPath/ci/gh-actions/setup.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -export CI_ROOT_DIR="${GITHUB_WORKSPACE//\\//}/.." -export CI_SOURCE_DIR="${GITHUB_WORKSPACE//\\//}" -export CI_DEP_DIR="${CI_ROOT_DIR}/dependencies" - -# Install ninja, pkgconfig, and libfabric -case "$(uname -s)" in - Linux) - . $(dirname ${BASH_SOURCE[0]})/setup-linux.sh - ;; - Darwin) - . $(dirname ${BASH_SOURCE[0]})/setup-macos.sh - ;; -esac - -export CMAKE_PREFIX_PATH=${CI_DEP_DIR}/install - -# Install atl -echo -echo "Installing atl" -mkdir -p ${CI_DEP_DIR}/atl -cd ${CI_DEP_DIR}/atl -git clone https://github.com/GTKorvo/atl.git source -mkdir build -cd build -cmake -GNinja -DCMAKE_INSTALL_PREFIX=${CI_DEP_DIR}/install ../source -ninja install - -# Install dill -echo -echo "Installing dill" -mkdir -p ${CI_DEP_DIR}/dill -cd ${CI_DEP_DIR}/dill -git clone https://github.com/GTKorvo/dill.git source -mkdir build -cd build -cmake -GNinja -DCMAKE_INSTALL_PREFIX=${CI_DEP_DIR}/install ../source -ninja install - -# Install ffs -echo -echo "Installing ffs" -mkdir -p ${CI_DEP_DIR}/ffs -cd ${CI_DEP_DIR}/ffs -git clone https://github.com/GTKorvo/ffs.git source -mkdir build -cd build -cmake -GNinja -DCMAKE_INSTALL_PREFIX=${CI_DEP_DIR}/install ../source -ninja install - -# Install enet -echo -echo "Installing enet" -mkdir -p ${CI_DEP_DIR}/enet -cd ${CI_DEP_DIR}/enet -git clone https://github.com/GTKorvo/enet.git source -mkdir build -cd build -cmake -GNinja -DCMAKE_INSTALL_PREFIX=${CI_DEP_DIR}/install ../source -ninja install diff --git a/thirdparty/EVPath/EVPath/cmake/FindNVML.cmake b/thirdparty/EVPath/EVPath/cmake/FindNVML.cmake index 4ae72a5fbb..430c3a5de2 100644 --- a/thirdparty/EVPath/EVPath/cmake/FindNVML.cmake +++ b/thirdparty/EVPath/EVPath/cmake/FindNVML.cmake @@ -11,7 +11,7 @@ find_path(NVML_INCLUDE_DIR nvml.h ${_NVML_EXTRA_INC_ARGS}) mark_as_advanced(NVML_LIBRARY NVML_INCLUDE_DIR) include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(nvml DEFAULT_MSG +find_package_handle_standard_args(NVML DEFAULT_MSG NVML_LIBRARY NVML_INCLUDE_DIR) if(NVML_FOUND) diff --git a/thirdparty/EVPath/EVPath/ev_dfg.c b/thirdparty/EVPath/EVPath/ev_dfg.c index c746439851..4a17189663 100644 --- a/thirdparty/EVPath/EVPath/ev_dfg.c +++ b/thirdparty/EVPath/EVPath/ev_dfg.c @@ -1456,16 +1456,24 @@ extern char *INT_EVmaster_get_contact_list(EVmaster master) atom_t CM_TRANSPORT = attr_atom_from_string("CM_TRANSPORT"); atom_t CM_ENET_CONN_TIMEOUT = attr_atom_from_string("CM_ENET_CONN_TIMEOUT"); CManager cm = master->cm; - char *tmp; + char *tmp = NULL; /* use enet transport if available */ +#if defined(ENET_FOUND) || defined(ZPL_ENET_AVAILABLE) listen_list = create_attr_list(); +#if defined(ENET_FOUND) add_string_attr(listen_list, CM_TRANSPORT, strdup("enet")); +#elif defined(ZPL_ENET_AVAILABLE) + add_string_attr(listen_list, CM_TRANSPORT, strdup("zplenet")); +#endif /* and kick up the connection timeout value. We can wait 60 secs */ contact_list = INT_CMget_specific_contact_list(cm, listen_list); - add_int_attr(contact_list, CM_ENET_CONN_TIMEOUT, 60000); - + if(contact_list) { + add_int_attr(contact_list, CM_ENET_CONN_TIMEOUT, 60000); + } free_attr_list(listen_list); +#endif + if (contact_list == NULL) { contact_list = INT_CMget_contact_list(cm); if (contact_list == NULL) { @@ -1473,7 +1481,9 @@ extern char *INT_EVmaster_get_contact_list(EVmaster master) contact_list = INT_CMget_contact_list(cm); } } - tmp = attr_list_to_string(contact_list); + if(contact_list) { + tmp = attr_list_to_string(contact_list); + } free_attr_list(contact_list); return tmp; } diff --git a/thirdparty/EVPath/EVPath/scripts/ci/cmake/centos7-clang.cmake b/thirdparty/EVPath/EVPath/scripts/ci/cmake/centos7-clang.cmake new file mode 100644 index 0000000000..6f6f2e2d40 --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/cmake/centos7-clang.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} clang) +set(ENV{CXX} clang++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/EVPath/EVPath/scripts/ci/cmake/centos7-gcc.cmake b/thirdparty/EVPath/EVPath/scripts/ci/cmake/centos7-gcc.cmake new file mode 100644 index 0000000000..8a1c976d8f --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/cmake/centos7-gcc.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} gcc) +set(ENV{CXX} g++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/EVPath/EVPath/scripts/ci/cmake/centos7-nvhpc.cmake b/thirdparty/EVPath/EVPath/scripts/ci/cmake/centos7-nvhpc.cmake new file mode 100644 index 0000000000..30e1f6cd58 --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/cmake/centos7-nvhpc.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} nvc) +set(ENV{CXX} nvc++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/EVPath/EVPath/scripts/ci/cmake/centos8-clang.cmake b/thirdparty/EVPath/EVPath/scripts/ci/cmake/centos8-clang.cmake new file mode 100644 index 0000000000..6f6f2e2d40 --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/cmake/centos8-clang.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} clang) +set(ENV{CXX} clang++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/EVPath/EVPath/scripts/ci/cmake/centos8-gcc.cmake b/thirdparty/EVPath/EVPath/scripts/ci/cmake/centos8-gcc.cmake new file mode 100644 index 0000000000..8a1c976d8f --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/cmake/centos8-gcc.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} gcc) +set(ENV{CXX} g++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/EVPath/EVPath/scripts/ci/cmake/centos8-intel.cmake b/thirdparty/EVPath/EVPath/scripts/ci/cmake/centos8-intel.cmake new file mode 100644 index 0000000000..b173d984ec --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/cmake/centos8-intel.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} icc) +set(ENV{CXX} icpc) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/EVPath/EVPath/scripts/ci/cmake/centos8-inteloneapi.cmake b/thirdparty/EVPath/EVPath/scripts/ci/cmake/centos8-inteloneapi.cmake new file mode 100644 index 0000000000..09ca1203cd --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/cmake/centos8-inteloneapi.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} icx) +set(ENV{CXX} icpx) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/EVPath/EVPath/scripts/ci/cmake/common.cmake b/thirdparty/EVPath/EVPath/scripts/ci/cmake/common.cmake new file mode 100644 index 0000000000..57f4d63fef --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/cmake/common.cmake @@ -0,0 +1,63 @@ +string(APPEND dashboard_cache " +BUILD_TESTING:BOOL=ON +") + +if(POLICY CMP0057) + cmake_policy(SET CMP0057 NEW) +endif() + +if(NOT CTEST_BUILD_CONFIGURATION) + set(CTEST_BUILD_CONFIGURATION Debug) +endif() + +if(NOT DEFINED NCPUS) + include(ProcessorCount) + ProcessorCount(NCPUS) +endif() +math(EXPR N2CPUS "${NCPUS}*2") +if(NOT CTEST_BUILD_FLAGS) + if(CTEST_CMAKE_GENERATOR STREQUAL "Unix Makefiles") + set(CTEST_BUILD_FLAGS "-k -j${N2CPUS}") + elseif(CTEST_CMAKE_GENERATOR STREQUAL "Ninja") + set(CTEST_BUILD_FLAGS "-k0 -j${N2CPUS}") + endif() +endif() +if(NOT PARALLEL_LEVEL IN_LIST CTEST_TEST_ARGS) + list(APPEND CTEST_TEST_ARGS PARALLEL_LEVEL 1) +endif() + +if(NOT dashboard_model) + set(dashboard_model Experimental) +endif() +if(NOT dashboard_binary_name) + set(dashboard_binary_name "build") +endif() +if(NOT dashboard_track) + set(dashboard_track "Continuous Integration") +endif() +if(NOT "$ENV{CI_COMMIT_SHA}" STREQUAL "") + set(CTEST_UPDATE_VERSION_OVERRIDE "$ENV{CI_COMMIT_SHA}") + set(CTEST_UPDATE_VERSION_ONLY ON) +endif() +if(NOT "$ENV{CI_SITE_NAME}" STREQUAL "") + set(CTEST_SITE "$ENV{CI_SITE_NAME}") +endif() +if(NOT "$ENV{CI_BUILD_NAME}" STREQUAL "") + set(CTEST_BUILD_NAME "$ENV{CI_BUILD_NAME}") +endif() +if(NOT "$ENV{CI_ROOT_DIR}" STREQUAL "") + set(CTEST_DASHBOARD_ROOT "$ENV{CI_ROOT_DIR}") +endif() +if(NOT "$ENV{CI_SOURCE_DIR}" STREQUAL "") + set(CTEST_SOURCE_DIRECTORY "$ENV{CI_SOURCE_DIR}") +endif() +if(NOT "$ENV{CI_BIN_DIR}" STREQUAL "") + set(CTEST_BINARY_DIRECTORY "$ENV{CI_BIN_DIR}") +endif() + +find_program(CTEST_GIT_COMMAND git) +set(CTEST_UPDATE_COMMAND ${CTEST_GIT_COMMAND}) +set(CTEST_UPDATE_TYPE git) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/../../dashboard/evpath_common.cmake) diff --git a/thirdparty/EVPath/EVPath/scripts/ci/cmake/macos-clang.cmake b/thirdparty/EVPath/EVPath/scripts/ci/cmake/macos-clang.cmake new file mode 100644 index 0000000000..6f6f2e2d40 --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/cmake/macos-clang.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} clang) +set(ENV{CXX} clang++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu1604-clang.cmake b/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu1604-clang.cmake new file mode 100644 index 0000000000..6f6f2e2d40 --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu1604-clang.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} clang) +set(ENV{CXX} clang++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu1604-gcc.cmake b/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu1604-gcc.cmake new file mode 100644 index 0000000000..8a1c976d8f --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu1604-gcc.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} gcc) +set(ENV{CXX} g++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu1804-clang.cmake b/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu1804-clang.cmake new file mode 100644 index 0000000000..6f6f2e2d40 --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu1804-clang.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} clang) +set(ENV{CXX} clang++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu1804-gcc.cmake b/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu1804-gcc.cmake new file mode 100644 index 0000000000..8a1c976d8f --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu1804-gcc.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} gcc) +set(ENV{CXX} g++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu1804-intel.cmake b/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu1804-intel.cmake new file mode 100644 index 0000000000..b173d984ec --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu1804-intel.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} icc) +set(ENV{CXX} icpc) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu1804-inteloneapi.cmake b/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu1804-inteloneapi.cmake new file mode 100644 index 0000000000..09ca1203cd --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu1804-inteloneapi.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} icx) +set(ENV{CXX} icpx) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu2004-clang.cmake b/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu2004-clang.cmake new file mode 100644 index 0000000000..6f6f2e2d40 --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu2004-clang.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} clang) +set(ENV{CXX} clang++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu2004-gcc.cmake b/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu2004-gcc.cmake new file mode 100644 index 0000000000..8a1c976d8f --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu2004-gcc.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} gcc) +set(ENV{CXX} g++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu2004-nvhpc.cmake b/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu2004-nvhpc.cmake new file mode 100644 index 0000000000..30e1f6cd58 --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/cmake/ubuntu2004-nvhpc.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} nvc) +set(ENV{CXX} nvc++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/EVPath/EVPath/scripts/ci/cmake/unix-common.cmake b/thirdparty/EVPath/EVPath/scripts/ci/cmake/unix-common.cmake new file mode 100644 index 0000000000..7eb02102a2 --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/cmake/unix-common.cmake @@ -0,0 +1,23 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{atl_ROOT} "$ENV{CI_ROOT_DIR}/atl/install") +set(ENV{LD_LIBRARY_PATH} "$ENV{atl_ROOT}/lib:$ENV{LD_LIBRARY_PATH}") + +set(ENV{dill_ROOT} "$ENV{CI_ROOT_DIR}/dill/install") +set(ENV{LD_LIBRARY_PATH} "$ENV{dill_ROOT}/lib:$ENV{LD_LIBRARY_PATH}") + +set(ENV{ffs_ROOT} "$ENV{CI_ROOT_DIR}/ffs/install") +set(ENV{LD_LIBRARY_PATH} "$ENV{ffs_ROOT}/lib:$ENV{LD_LIBRARY_PATH}") + + +string(APPEND dashboard_cache " +") + +if(NOT CTEST_CMAKE_GENERATOR) + set(CTEST_CMAKE_GENERATOR "Unix Makefiles") +endif() + +set(CTEST_BUILD_WARNINGS_AS_ERRORS FALSE) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) diff --git a/thirdparty/EVPath/EVPath/scripts/ci/cmake/windows-common.cmake b/thirdparty/EVPath/EVPath/scripts/ci/cmake/windows-common.cmake new file mode 100644 index 0000000000..0b534f3e10 --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/cmake/windows-common.cmake @@ -0,0 +1,17 @@ +# Client maintainer: chuck.atkins@kitware.com + +message("DEBUG: PATH: $ENV{PATH}") + +set(ENV{atl_ROOT} "$ENV{CI_ROOT_DIR}/atl/install") +set(ENV{PATH} "$ENV{CI_ROOT_DIR}/atl/install/bin:$ENV{PATH}") + +set(ENV{ffs_ROOT} "$ENV{CI_ROOT_DIR}/ffs/install") +set(ENV{PATH} "$ENV{CI_ROOT_DIR}/ffs/install/bin:$ENV{PATH}") + +string(APPEND dashboard_cache " +") + +set(CTEST_BUILD_WARNINGS_AS_ERRORS FALSE) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) diff --git a/thirdparty/EVPath/EVPath/scripts/ci/cmake/windows-vs2019-clang.cmake b/thirdparty/EVPath/EVPath/scripts/ci/cmake/windows-vs2019-clang.cmake new file mode 100644 index 0000000000..a1907cec3f --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/cmake/windows-vs2019-clang.cmake @@ -0,0 +1,11 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(dashboard_cache " +") + +set(CTEST_CMAKE_GENERATOR "Visual Studio 16") +set(CTEST_CMAKE_GENERATOR_PLATFORM x64) +set(CTEST_CMAKE_GENERATOR_TOOLSET ClangCL) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/windows-common.cmake) diff --git a/thirdparty/EVPath/EVPath/scripts/ci/cmake/windows-vs2019-msvc.cmake b/thirdparty/EVPath/EVPath/scripts/ci/cmake/windows-vs2019-msvc.cmake new file mode 100644 index 0000000000..c6eb48d0a4 --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/cmake/windows-vs2019-msvc.cmake @@ -0,0 +1,10 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(dashboard_cache " +") + +set(CTEST_CMAKE_GENERATOR "Visual Studio 16") +set(CTEST_CMAKE_GENERATOR_PLATFORM x64) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/windows-common.cmake) diff --git a/thirdparty/EVPath/EVPath/scripts/ci/gh-actions/run.sh b/thirdparty/EVPath/EVPath/scripts/ci/gh-actions/run.sh new file mode 100755 index 0000000000..fdc910dddd --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/gh-actions/run.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +readlinkf() { perl -MCwd -e 'print Cwd::abs_path shift' "$1"; } + +set -e + +STEP=$1 +shift + +export CI_SITE_NAME="GitHub Actions" +if [ "${GITHUB_EVENT_NAME}" = "pull_request" ] +then + GH_PR_NUMBER=$(expr "${GITHUB_REF}" : 'refs/pull/\([^/]*\)') + export CI_BUILD_NAME="pr${GH_PR_NUMBER}_${GITHUB_HEAD_REF}_${GH_YML_JOBNAME}_${GH_YML_BUILDTYPE}" +else + export CI_BUILD_NAME="${GITHUB_REF#refs/heads/}_${GH_YML_JOBNAME}_${GH_YML_BUILDTYPE}" +fi +if [[ "${RUNNER_OS}" =~ "Windows" ]] +then + export CI_ROOT_DIR="${GITHUB_WORKSPACE//\\//}" + export CI_SOURCE_DIR="${GITHUB_WORKSPACE//\\//}/source" +else + export CI_ROOT_DIR="${GITHUB_WORKSPACE}" + export CI_SOURCE_DIR="${GITHUB_WORKSPACE}/source" +fi +export CI_BIN_DIR="${CI_ROOT_DIR}/build" + +export CI_COMMIT_SHA=${GH_YML_SHA} + +if command -v ctest3 >/dev/null +then + CTEST=ctest3 +else + CTEST=ctest +fi + +# Update and Test steps enable an extra step +CTEST_STEP_ARGS="" +case ${STEP} in + test) CTEST_STEP_ARGS="${CTEST_STEP_ARGS} -Ddashboard_do_end=ON" ;; +esac +CTEST_STEP_ARGS="${CTEST_STEP_ARGS} -Ddashboard_do_${STEP}=ON" + +echo "********** Environment **********" + +env | sort + +echo "********** Running CTest **********" + +${CTEST} -VV \ + -S ${CI_SOURCE_DIR}/scripts/ci/cmake/${GH_YML_JOBNAME}.cmake \ + -DCTEST_BUILD_CONFIGURATION=${GH_YML_BUILDTYPE} \ + -Ddashboard_do_submit=ON \ + -Ddashboard_full=OFF \ + ${CTEST_STEP_ARGS} \ + "$@" + diff --git a/thirdparty/EVPath/EVPath/scripts/ci/scripts/post-cdash-status.sh b/thirdparty/EVPath/EVPath/scripts/ci/scripts/post-cdash-status.sh new file mode 100755 index 0000000000..fb822f3fd2 --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/scripts/post-cdash-status.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +if [ $# -eq 0 ] +then + echo "$0 org/repo [SHA [TOKEN]]" + exit 1 +fi + +GH_REPO=$1 +if [ $# -gt 1 ] +then + GH_SHA=$2 + if [ $# -gt 2 ] + then + GH_TOKEN=$3 + if [ $# -gt 3 ] + then + echo "$0 org/repo [SHA [TOKEN]]" + exit 1 + fi + fi +fi + +if [ -z "${GH_SHA}" ] +then + echo "Error: GH_SHA is undefined" + exit 2 +fi +if [ -z "${GH_TOKEN}" ] +then + echo "Error: GH_TOKEN is undefined" + exit 3 +fi + +cdash_url() { + printf 'https://open.cdash.org/index.php?project=%s&subproject=%s&filtercount=1&showfilters=1&field1=revision&compare1=61&value1=%s' $1 $2 $3 +} + +build_status_body() { +cat < /etc/yum.repos.d/group_git-maint-git-epel-7.repo + ;; + centos8*) + curl -L https://copr.fedorainfracloud.org/coprs/g/git-maint/git/repo/epel-8/group_git-maint-git-epel-8.repo > /etc/yum.repos.d/group_git-maint-git-epel-8.repo + ;; + ubuntu*) + export DEBIAN_FRONTEND=noninteractive + add-apt-repository ppa:git-core/ppa -y + apt-get update + ;; +esac +${PKG_CMD} install -y git + +######################################## +# Compilers +######################################## +case ${GH_YML_JOBNAME} in + centos*-clang) PKGS="clang gcc gcc-c++" ;; + centos*-gcc) PKGS="gcc gcc-c++" ;; + centos*-nvhpc) PKGS="gcc gcc-c++" ;; + ubuntu*-clang) PKGS="clang gcc g++" ;; + ubuntu*-gcc) PKGS="gcc g++" ;; + ubuntu*-nvhpc) PKGS="gcc g++" ;; +esac +${PKG_CMD} install -y ${PKGS} + + +case ${GH_YML_JOBNAME} in + *-clang) export CC=clang CXX=clang++ ;; + *-gcc) export CC=gcc CXX=g++ ;; + *-nvhpc) export CC=nvc CXX=nvc++ ;; +esac + +######################################## +# CMake +######################################## +FILENAME=$(curl https://cmake.org/files/LatestRelease/cmake-latest-files-v1.json 2>/dev/null | grep 'cmake.*sh' | sed -n 's|.*"\(cmake.*x86_64.sh\).*|\1|p') +VERSION=$(echo ${FILENAME} | sed 's|cmake-\([^\-]*\).*|\1|') +curl -L https://github.com/Kitware/CMake/releases/download/v${VERSION}/${FILENAME} > cmake.sh +chmod +x cmake.sh +./cmake.sh --skip-license --exclude-subdir --prefix=/usr/local +rm -f cmake.sh diff --git a/thirdparty/EVPath/EVPath/scripts/ci/setup/macos.sh b/thirdparty/EVPath/EVPath/scripts/ci/setup/macos.sh new file mode 100755 index 0000000000..a9bf588e2f --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/setup/macos.sh @@ -0,0 +1 @@ +#!/bin/bash diff --git a/thirdparty/EVPath/EVPath/scripts/ci/setup/windows.sh b/thirdparty/EVPath/EVPath/scripts/ci/setup/windows.sh new file mode 100755 index 0000000000..a9bf588e2f --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/ci/setup/windows.sh @@ -0,0 +1 @@ +#!/bin/bash diff --git a/thirdparty/EVPath/EVPath/scripts/dashboard/common.cmake b/thirdparty/EVPath/EVPath/scripts/dashboard/common.cmake new file mode 100644 index 0000000000..4b50bf291e --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/dashboard/common.cmake @@ -0,0 +1,515 @@ +#------------------------------------------------------------------------------# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +#------------------------------------------------------------------------------# +# +# "Universal" Dashboard Script +# +# This script contains basic dashboard driver code common to all +# clients and projects. It is a combination of the universal.cmake script in +# the Kitware DashboardScriptsNG repo and cmake_common.cmake used by CMake +# dashboards. +# +# Create a project-specific common script with code of the following form, +# where the final line includes this script. +# +# set(CTEST_PROJECT_NAME "OpenChemistry") +# set(CTEST_DROP_SITE "cdash.openchemistry.org") +# +# set(dashboard_git_url "git://source.openchemistry.org/openchemistry.git") +# set(dashboard_root_name "MyTests") +# set(dashboard_source_name "openchemistry") +# +# get_filename_component(dir ${CMAKE_CURRENT_LIST_FILE} PATH) +# include(${dir}/universal.cmake) +# +# The following variables may be set before including this script +# to configure it: +# +# dashboard_model = Nightly | Experimental +# dashboard_root_name = Change name of "My Tests" directory +# dashboard_source_name = Name of source directory (CMake) +# dashboard_binary_name = Name of binary directory (CMake-build) +# dashboard_cache = Initial CMakeCache.txt file content +# dashboard_track = The name of the CDash "Track" to submit to + +# dashboard_do_checkout = True to enable source checkout via git +# dashboard_do_update = True to enable the Update step +# dashboard_do_configure = True to enable the Configure step +# dashboard_do_build = True to enable the Build step +# dashboard_do_test = True to enable the Test step +# dashboard_do_coverage = True to enable coverage (ex: gcov) +# dashboard_do_memcheck = True to enable memcheck (ex: valgrind) +# dashboard_do_submit = Submit each step (ON) +# dashboard_do_submit_only = Only submit step results, do nto run them (OFF) + +# CTEST_GIT_COMMAND = path to git command-line client +# CTEST_BUILD_FLAGS = build tool arguments (ex: -j2) +# CTEST_DASHBOARD_ROOT = Where to put source and build trees +# CTEST_TEST_CTEST = Whether to run long CTestTest* tests +# CTEST_TEST_TIMEOUT = Per-test timeout length +# CTEST_TEST_ARGS = ctest_test args (ex: PARALLEL_LEVEL 4) +# CMAKE_MAKE_PROGRAM = Path to "make" tool to use +# +# Options to configure Git: +# dashboard_git_url = Custom git clone url +# dashboard_git_branch = Custom remote branch to track +# dashboard_git_crlf = Value of core.autocrlf for repository +# +# For Makefile generators the script may be executed from an +# environment already configured to use the desired compilers. +# Alternatively the environment may be set at the top of the script: +# +# set(ENV{CC} /path/to/cc) # C compiler +# set(ENV{CXX} /path/to/cxx) # C++ compiler +# set(ENV{FC} /path/to/fc) # Fortran compiler (optional) +# set(ENV{LD_LIBRARY_PATH} /path/to/vendor/lib) # (if necessary) + +cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR) + +if(NOT DEFINED dashboard_full) + set(dashboard_full TRUE) +endif() + +# Initialize all build steps to "ON" +if(NOT DEFINED dashboard_do_update) + set(dashboard_do_update ${dashboard_full}) +endif() + +if(NOT DEFINED dashboard_do_checkout) + set(dashboard_do_checkout ${dashboard_full}) +endif() + +if(NOT DEFINED dashboard_do_configure) + set(dashboard_do_configure ${dashboard_full}) +endif() + +if(NOT DEFINED dashboard_do_build) + set(dashboard_do_build ${dashboard_full}) +endif() + +if(NOT DEFINED dashboard_do_test) + set(dashboard_do_test ${dashboard_full}) +endif() + +# Default code coverage and memtesting to off +if(NOT DEFINED dashboard_do_coverage) + set(dashboard_do_coverage FALSE) +endif() + +if(NOT DEFINED dashboard_do_memcheck) + set(dashboard_do_memcheck FALSE) +endif() + +if(NOT DEFINED dashboard_fresh) + if(dashboard_full OR dashboard_do_update) + set(dashboard_fresh TRUE) + else() + set(dashboard_fresh FALSE) + endif() +endif() + +if(NOT DEFINED dashboard_do_submit_only) + set(dashboard_do_submit_only FALSE) +endif() + +if(NOT DEFINED dashboard_do_submit) + set(dashboard_do_submit TRUE) +endif() + +if(NOT DEFINED CTEST_PROJECT_NAME) + message(FATAL_ERROR "project-specific script including 'universal.cmake' should set CTEST_PROJECT_NAME") +endif() + +if(NOT DEFINED dashboard_user_home) + set(dashboard_user_home "$ENV{HOME}") +endif() + +# Select the top dashboard directory. +if(NOT DEFINED dashboard_root_name) + set(dashboard_root_name "My Tests") +endif() +if(NOT DEFINED CTEST_DASHBOARD_ROOT) + get_filename_component(CTEST_DASHBOARD_ROOT "${CTEST_SCRIPT_DIRECTORY}/../${dashboard_root_name}" ABSOLUTE) +endif() + +# Select the model (Nightly, Experimental, Continuous). +if(NOT DEFINED dashboard_model) + set(dashboard_model Nightly) +endif() +if(NOT "${dashboard_model}" MATCHES "^(Nightly|Experimental)$") + message(FATAL_ERROR "dashboard_model must be Nightly or Experimental") +endif() + +# Default to a Debug build. +if(NOT DEFINED CTEST_BUILD_CONFIGURATION) + set(CTEST_BUILD_CONFIGURATION Debug) +endif() + +if(NOT DEFINED CTEST_CONFIGURATION_TYPE) + set(CTEST_CONFIGURATION_TYPE ${CTEST_BUILD_CONFIGURATION}) +endif() + +# Choose CTest reporting mode. +if(NOT "${CTEST_CMAKE_GENERATOR}" MATCHES "Make|Ninja") + # Launchers work only with Makefile and Ninja generators. + set(CTEST_USE_LAUNCHERS 0) +elseif(NOT DEFINED CTEST_USE_LAUNCHERS) + # The setting is ignored by CTest < 2.8 so we need no version test. + set(CTEST_USE_LAUNCHERS 1) +endif() + +# Configure testing. +if(NOT DEFINED CTEST_TEST_CTEST) + set(CTEST_TEST_CTEST 1) +endif() +if(NOT CTEST_TEST_TIMEOUT) + set(CTEST_TEST_TIMEOUT 1500) +endif() + +# Select Git source to use. +if(dashboard_do_checkout) + if(NOT DEFINED dashboard_git_url) + message(FATAL_ERROR "project-specific script including 'universal.cmake' should set dashboard_git_url") + endif() + if(NOT DEFINED dashboard_git_branch) + set(dashboard_git_branch master) + endif() + if(NOT DEFINED dashboard_git_crlf) + if(UNIX) + set(dashboard_git_crlf false) + else() + set(dashboard_git_crlf true) + endif() + endif() + + # Look for a GIT command-line client. + if(NOT DEFINED CTEST_GIT_COMMAND) + find_program(CTEST_GIT_COMMAND + NAMES git git.cmd + PATH_SUFFIXES Git/cmd Git/bin + ) + endif() + if(NOT CTEST_GIT_COMMAND) + message(FATAL_ERROR "CTEST_GIT_COMMAND not available!") + endif() +endif() + +# Select a source directory name. +if(NOT DEFINED CTEST_SOURCE_DIRECTORY) + if(DEFINED dashboard_source_name) + set(CTEST_SOURCE_DIRECTORY ${CTEST_DASHBOARD_ROOT}/${dashboard_source_name}) + else() + set(CTEST_SOURCE_DIRECTORY ${CTEST_DASHBOARD_ROOT}/${CTEST_PROJECT_NAME}) + endif() +endif() + +# Select a build directory name. +if(NOT DEFINED CTEST_BINARY_DIRECTORY) + if(DEFINED dashboard_binary_name) + set(CTEST_BINARY_DIRECTORY ${CTEST_DASHBOARD_ROOT}/${dashboard_binary_name}) + else() + set(CTEST_BINARY_DIRECTORY ${CTEST_SOURCE_DIRECTORY}-build) + endif() +endif() + +macro(dashboard_git) + execute_process( + COMMAND ${CTEST_GIT_COMMAND} ${ARGN} + WORKING_DIRECTORY "${CTEST_SOURCE_DIRECTORY}" + OUTPUT_VARIABLE dashboard_git_output + ERROR_VARIABLE dashboard_git_output + RESULT_VARIABLE dashboard_git_failed + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_STRIP_TRAILING_WHITESPACE + ) +endmacro() + +if(dashboard_do_checkout) + # Delete source tree if it is incompatible with current VCS. + if(EXISTS ${CTEST_SOURCE_DIRECTORY}) + if(NOT EXISTS "${CTEST_SOURCE_DIRECTORY}/.git") + set(vcs_refresh "because it is not managed by git.") + else() + execute_process( + COMMAND ${CTEST_GIT_COMMAND} reset --hard + WORKING_DIRECTORY "${CTEST_SOURCE_DIRECTORY}" + OUTPUT_VARIABLE output + ERROR_VARIABLE output + RESULT_VARIABLE failed + ) + if(failed) + set(vcs_refresh "because its .git may be corrupted.") + endif() + endif() + if(vcs_refresh AND "${CTEST_SOURCE_DIRECTORY}" MATCHES "/CMake[^/]*") + message("Deleting source tree\n") + message(" ${CTEST_SOURCE_DIRECTORY}\n${vcs_refresh}") + file(REMOVE_RECURSE "${CTEST_SOURCE_DIRECTORY}") + endif() + endif() + + # Support initial checkout if necessary. + if(NOT EXISTS "${CTEST_SOURCE_DIRECTORY}" + AND NOT DEFINED CTEST_CHECKOUT_COMMAND) + # Generate an initial checkout script. + get_filename_component(_name "${CTEST_SOURCE_DIRECTORY}" NAME) + set(ctest_checkout_script ${CTEST_DASHBOARD_ROOT}/${_name}-init.cmake) + file(WRITE ${ctest_checkout_script} "# git repo init script for ${_name} +execute_process( + COMMAND \"${CTEST_GIT_COMMAND}\" clone -n -- \"${dashboard_git_url}\" + \"${CTEST_SOURCE_DIRECTORY}\" + ) +if(EXISTS \"${CTEST_SOURCE_DIRECTORY}/.git\") + execute_process( + COMMAND \"${CTEST_GIT_COMMAND}\" config core.autocrlf ${dashboard_git_crlf} + WORKING_DIRECTORY \"${CTEST_SOURCE_DIRECTORY}\" + ) + execute_process( + COMMAND \"${CTEST_GIT_COMMAND}\" fetch + WORKING_DIRECTORY \"${CTEST_SOURCE_DIRECTORY}\" + ) + execute_process( + COMMAND \"${CTEST_GIT_COMMAND}\" checkout ${dashboard_git_branch} + WORKING_DIRECTORY \"${CTEST_SOURCE_DIRECTORY}\" + ) +endif()" + ) + set(CTEST_CHECKOUT_COMMAND "\"${CMAKE_COMMAND}\" -P \"${ctest_checkout_script}\"") + elseif(EXISTS "${CTEST_SOURCE_DIRECTORY}/.git") + # Upstream URL. + dashboard_git(config --get remote.origin.url) + if(NOT dashboard_git_output STREQUAL "${dashboard_git_url}") + dashboard_git(config remote.origin.url "${dashboard_git_url}") + endif() + + # Local checkout. + dashboard_git(symbolic-ref HEAD) + if(NOT dashboard_git_output STREQUAL "${dashboard_git_branch}") + dashboard_git(checkout ${dashboard_git_branch}) + if(dashboard_git_failed) + message(FATAL_ERROR "Failed to checkout branch ${dashboard_git_branch}:\n${dashboard_git_output}") + endif() + endif() + endif() +endif() + +#----------------------------------------------------------------------------- + +# Check for required variables. +foreach(req + CTEST_CMAKE_GENERATOR + CTEST_SITE + CTEST_BUILD_NAME + ) + if(NOT DEFINED ${req}) + message(FATAL_ERROR "The containing script must set ${req}") + endif() +endforeach(req) + +# Print summary information. +set(vars "") +foreach(v + CTEST_SITE + CTEST_BUILD_NAME + CTEST_SOURCE_DIRECTORY + CTEST_BINARY_DIRECTORY + CTEST_CMAKE_GENERATOR + CTEST_BUILD_CONFIGURATION + CTEST_GIT_COMMAND + CTEST_CHECKOUT_COMMAND + CTEST_CONFIGURE_COMMAND + CTEST_SCRIPT_DIRECTORY + CTEST_USE_LAUNCHERS + ) + set(vars "${vars} ${v}=[${${v}}]\n") +endforeach(v) +message("Dashboard script configuration:\n${vars}\n") + +# Avoid non-ascii characters in tool output. +set(ENV{LC_ALL} C) + +# Helper macro to write the initial cache. +macro(write_cache) + set(cache_build_type "") + set(cache_make_program "") + if(CTEST_CMAKE_GENERATOR MATCHES "Make|Ninja") + set(cache_build_type CMAKE_BUILD_TYPE:STRING=${CTEST_BUILD_CONFIGURATION}) + if(CMAKE_MAKE_PROGRAM) + set(cache_make_program CMAKE_MAKE_PROGRAM:FILEPATH=${CMAKE_MAKE_PROGRAM}) + endif() + endif() + file(WRITE ${CTEST_BINARY_DIRECTORY}/CMakeCache.txt " +SITE:STRING=${CTEST_SITE} +BUILDNAME:STRING=${CTEST_BUILD_NAME} +CTEST_TEST_CTEST:BOOL=${CTEST_TEST_CTEST} +CTEST_USE_LAUNCHERS:BOOL=${CTEST_USE_LAUNCHERS} +DART_TESTING_TIMEOUT:STRING=${CTEST_TEST_TIMEOUT} +GIT_EXECUTABLE:FILEPATH=${CTEST_GIT_COMMAND} +${cache_build_type} +${cache_make_program} +${dashboard_cache} +") +endmacro(write_cache) + +if(COMMAND dashboard_hook_init) + dashboard_hook_init() +endif() + +if(dashboard_fresh) + if(EXISTS "${CTEST_BINARY_DIRECTORY}") + message("Clearing build tree...") + ctest_empty_binary_directory("${CTEST_BINARY_DIRECTORY}") + else() + file(MAKE_DIRECTORY "${CTEST_BINARY_DIRECTORY}") + endif() + message("Starting fresh build...") + write_cache() +endif() + +# Start a new submission. +if(dashboard_track) + set(dashboard_track_arg TRACK "${dashboard_track}") +endif() +if(dashboard_fresh) + if(COMMAND dashboard_hook_start) + dashboard_hook_start() + endif() + message("Calling ctest_start") + ctest_start(${dashboard_model} ${dashboard_track_arg}) + if(dashboard_do_submit) + message("Calling ctest_submit") + ctest_submit(PARTS Start) + endif() + if(COMMAND dashboard_hook_started) + dashboard_hook_started() + endif() +else() + message("Calling ctest_start(APPEND)") + ctest_start(${dashboard_model} ${dashboard_track_arg} APPEND) +endif() + +# Look for updates. +if(dashboard_do_update) + if(NOT dashboard_do_submit_only) + if(COMMAND dashboard_hook_update) + dashboard_hook_update() + endif() + message("Calling ctest_update") + ctest_update(RETURN_VALUE count) + set(CTEST_CHECKOUT_COMMAND) # checkout on first iteration only + message("Found ${count} changed files") + endif() + + if(dashboard_do_submit) + if(CTEST_SUBMIT_NOTES) + message("Submitting dashboard scripts as Notes") + # Send the main script as a note while submitting the Update part + set(CTEST_NOTES_FILES + ${CTEST_UPDATE_NOTES_FILES} + "${CMAKE_CURRENT_LIST_FILE}") + message("Calling ctest_submit") + ctest_submit(PARTS Update Notes) + unset(CTEST_NOTES_FILES) + else() + message("Skipping notes submission for Update step") + message("Calling ctest_submit") + ctest_submit(PARTS Update) + endif() + endif() +endif() + +if(dashboard_do_configure) + if(NOT dashboard_do_submit_only) + if(COMMAND dashboard_hook_configure) + dashboard_hook_configure() + endif() + message("Calling ctest_configure") + ctest_configure(${dashboard_configure_args}) + endif() + if(dashboard_do_submit) + if(CTEST_SUBMIT_NOTES) + message("Submitting CMakeCache.txt as Notes") + set(CTEST_NOTES_FILES "${CTEST_BINARY_DIRECTORY}/CMakeCache.txt") + message("Calling ctest_submit") + ctest_submit(PARTS Configure Notes) + unset(CTEST_NOTES_FILES) + else() + message("Skipping notes submission for Configure step") + message("Calling ctest_submit") + ctest_submit(PARTS Configure) + endif() + endif() +endif() + +ctest_read_custom_files(${CTEST_BINARY_DIRECTORY}) + +if(dashboard_do_build) + if(NOT dashboard_do_submit_only) + if(COMMAND dashboard_hook_build) + dashboard_hook_build() + endif() + message("Calling ctest_build") + ctest_build( + NUMBER_WARNINGS ctest_build_num_warnings + ) + endif() + if(dashboard_do_submit) + message("Calling ctest_submit") + ctest_submit(PARTS Build) + endif() +endif() + +if(dashboard_do_test) + if(NOT dashboard_do_submit_only) + if(COMMAND dashboard_hook_test) + dashboard_hook_test() + endif() + message("Calling ctest_test") + ctest_test(${CTEST_TEST_ARGS} RETURN_VALUE TEST_RESULTS) + if(${TEST_RESULTS} EQUAL 0) + message("ctest test results return value: ${TEST_RESULTS}") + else() + message(SEND_ERROR "Some tests failed") + endif() + endif() + if(dashboard_do_submit) + message("Calling ctest_submit") + ctest_submit(PARTS Test) + endif() +endif() + +if(dashboard_do_coverage) + if(NOT dashboard_do_submit_only) + if(COMMAND dashboard_hook_coverage) + dashboard_hook_coverage() + endif() + message("Calling ctest_coverage") + ctest_coverage() + endif() + if(dashboard_do_submit) + message("Calling ctest_submit") + ctest_submit(PARTS Coverage) + endif() +endif() + +if(dashboard_do_memcheck) + if(NOT dashboard_do_submit_only) + if(COMMAND dashboard_hook_memcheck) + dashboard_hook_memcheck() + endif() + message("Calling ctest_memcheck") + ctest_memcheck(RETURN_VALUE MEMCHECK_RETURN DEFECT_COUNT MEMCHECK_DEFECTS) + if(NOT (MEMCHECK_DEFECTS EQUAL 0)) + message(SEND_ERROR "ctest memcheck defects found: ${MEMCHECK_DEFECTS}") + endif() + endif() + if(dashboard_do_submit) + message("Calling ctest_submit") + ctest_submit(PARTS MemCheck) + endif() +endif() + +if(COMMAND dashboard_hook_end) + dashboard_hook_end() +endif() diff --git a/thirdparty/EVPath/EVPath/scripts/dashboard/evpath_common.cmake b/thirdparty/EVPath/EVPath/scripts/dashboard/evpath_common.cmake new file mode 100644 index 0000000000..b68b62fedd --- /dev/null +++ b/thirdparty/EVPath/EVPath/scripts/dashboard/evpath_common.cmake @@ -0,0 +1,114 @@ +#------------------------------------------------------------------------------# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +#------------------------------------------------------------------------------# +# +# EVPath Common Dashboard Script +# +# This script contains basic dashboard driver code common to all +# clients. +# +# # Client maintainer: me@mydomain.net +# set(CTEST_SITE "machine.site") +# set(CTEST_BUILD_NAME "Platform-Compiler") +# set(CTEST_CONFIGURATION_TYPE Debug) +# set(CTEST_CMAKE_GENERATOR "Unix Makefiles") +# include(${CTEST_SCRIPT_DIRECTORY}/evpath_common.cmake) +# +# Then run a scheduled task (cron job) with a command line such as +# +# ctest -S ~/Dashboards/Scripts/my_dashboard.cmake -V +# +# By default the source and build trees will be placed in the path +# "../My Tests/" relative to your script location. +# +# The following variables may be set before including this script +# to configure it: +# +# dashboard_model = Nightly | Experimental +# dashboard_root_name = Change name of "MyTests" directory +# dashboard_source_name = Name of source directory (evpath) +# dashboard_binary_name = Name of binary directory (evpath-build) +# dashboard_cache = Initial CMakeCache.txt file content + +# dashboard_do_checkout = True to enable source checkout via git +# dashboard_do_update = True to enable source update +# dashboard_do_configure = True to enable the Configure step +# dashboard_do_build = True to enable the Build step +# dashboard_do_test = True to enable the Test step +# dashboard_do_coverage = True to enable coverage (ex: gcov) +# dashboard_do_memcheck = True to enable memcheck (ex: valgrind) + +# CTEST_GIT_COMMAND = path to git command-line client +# CTEST_BUILD_FLAGS = build tool arguments (ex: -j2) +# CTEST_DASHBOARD_ROOT = Where to put source and build trees +# CTEST_TEST_CTEST = Whether to run long CTestTest* tests +# CTEST_TEST_TIMEOUT = Per-test timeout length +# CTEST_TEST_ARGS = ctest_test args (ex: PARALLEL_LEVEL 4) +# CMAKE_MAKE_PROGRAM = Path to "make" tool to use +# +# Options to configure Git: +# dashboard_git_url = Custom git clone url +# dashboard_git_branch = Custom remote branch to track +# dashboard_git_crlf = Value of core.autocrlf for repository +# + +# For Makefile generators the script may be executed from an +# environment already configured to use the desired compilers. +# Alternatively the environment may be set at the top of the script: +# +# set(ENV{CC} /path/to/cc) # C compiler +# set(ENV{CXX} /path/to/cxx) # C++ compiler +# set(ENV{FC} /path/to/fc) # Fortran compiler (optional) +# set(ENV{LD_LIBRARY_PATH} /path/to/vendor/lib) # (if necessary) + +set(CTEST_PROJECT_NAME "EVPath") +set(CTEST_DROP_SITE "open.cdash.org") +if(NOT dashboard_git_url) + set(dashboard_git_url "https://github.com/GTkorvo/evpath.git") +endif() + +if(NOT DEFINED CTEST_TEST_TIMEOUT) + set(CTEST_TEST_TIMEOUT 300) +endif() +if(NOT DEFINED CTEST_SUBMIT_NOTES) + set(CTEST_SUBMIT_NOTES TRUE) +endif() +if(NOT DEFINED dashboard_source_name) + set(dashboard_source_name "evpath") +endif() +if(NOT DEFINED dashboard_model) + set(dashboard_model Experimental) +endif() +if(NOT DEFINED dashboard_do_submit) + set(dashboard_do_submit FALSE) +endif() + +# Setup defaults for various CTEST settings +if(NOT DEFINED CTEST_SITE) + cmake_host_system_information(RESULT CTEST_SITE QUERY FQDN) +endif() +if(NOT DEFINED CTEST_BUILD_NAME) + set(CTEST_BUILD_NAME "generic-build") +endif() +if(NOT DEFINED CTEST_SOURCE_DIRECTORY) + set(CTEST_SOURCE_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/../..) +endif() +if(NOT DEFINED CTEST_BINARY_DIRECTORY) + set(CTEST_BINARY_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${CTEST_BUILD_NAME}") +endif() +if(NOT DEFINED CTEST_CMAKE_GENERATOR) + set(CTEST_CMAKE_GENERATOR "Unix Makefiles") +endif() +if(NOT DEFINED CTEST_BUILD_WARNINGS_AS_ERRORS) + set(CTEST_BUILD_WARNINGS_AS_ERRORS TRUE) +endif() + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) + +if(CTEST_BUILD_WARNINGS_AS_ERRORS) + if(ctest_build_num_warnings GREATER 0) + message(FATAL_ERROR "Found ${ctest_build_num_warnings} build warnings.") + endif() +endif() diff --git a/thirdparty/ffs/ffs/CMakeLists.txt b/thirdparty/ffs/ffs/CMakeLists.txt index c835eb8a6a..0aa725c500 100644 --- a/thirdparty/ffs/ffs/CMakeLists.txt +++ b/thirdparty/ffs/ffs/CMakeLists.txt @@ -133,10 +133,13 @@ else() ${CMAKE_CURRENT_BINARY_DIR}/lex.yy.c) ADD_FLEX_BISON_DEPENDENCY(CODScanner CODParser) endif() + +find_package(Perl REQUIRED) + add_custom_command( - OUTPUT "cod_node.c" "structs.h" + OUTPUT "cod_node.c" COMMAND - perl + ${PERL_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/cod/struct.pl ${CMAKE_CURRENT_SOURCE_DIR}/cod/cod.structs DEPENDS diff --git a/thirdparty/ffs/ffs/cod/cg.c b/thirdparty/ffs/ffs/cod/cg.c index c8bc7ef14d..f1d55c0c5c 100644 --- a/thirdparty/ffs/ffs/cod/cg.c +++ b/thirdparty/ffs/ffs/cod/cg.c @@ -558,7 +558,6 @@ evaluate_constant_expr(dill_stream s, sm_ref expr, long*value) } *value = l; return 1; - break; } case cod_identifier: return evaluate_constant_expr(s, expr->node.identifier.sm_declaration, value); @@ -2214,7 +2213,6 @@ execute_operator_cg(dill_stream s, operator_t op, int op_type, dill_reg result, right_op.in_kernel = 0; } return right_op; - break; } case op_address: ret_op.reg = right; @@ -2288,7 +2286,6 @@ cg_subroutine_call(dill_stream s, sm_ref expr, cod_code descr) sm_list formals = func_ref->node.declaration.params; sm_list args = expr->node.subroutine_call.arguments; sm_ref arg, formal; - sm_ref last_arg = NULL; dill_reg args_array[MAX_ARG]; /* should be large enough */ int types_array[MAX_ARG]; /* should be large enough */ int start, i, direction; @@ -2366,7 +2363,6 @@ cg_subroutine_call(dill_stream s, sm_ref expr, cod_code descr) } arg_count++; args = args->next; - last_arg = arg; if (formals) formals = formals->next; if (next_formal_is_drisc_exec_ctx(formals)) { if (arg_count < MAX_ARG) { diff --git a/thirdparty/ffs/ffs/cod/cod.y b/thirdparty/ffs/ffs/cod/cod.y index 721bdc4db3..7db22328e1 100644 --- a/thirdparty/ffs/ffs/cod/cod.y +++ b/thirdparty/ffs/ffs/cod/cod.y @@ -1,5 +1,8 @@ %{ #include "config.h" +#ifdef __NVCOMPILER +#pragma diag_suppress 550, 111, 941 +#endif #if defined (__INTEL_COMPILER) # pragma warning (disable: 2215) #endif @@ -5863,8 +5866,8 @@ int *must_free_p; ret->node.reference_type_decl.cg_referenced_type = DILL_ERR; ret->node.reference_type_decl.sm_complex_referenced_type = subtype; if (must_free_flag) { - if (ret->node.array_type_decl.freeable_complex_element_type) { - cod_rfree(ret->node.array_type_decl.freeable_complex_element_type); + if (ret->node.reference_type_decl.freeable_complex_referenced_type) { + cod_rfree(ret->node.reference_type_decl.freeable_complex_referenced_type); } ret->node.reference_type_decl.freeable_complex_referenced_type = subtype; } diff --git a/thirdparty/ffs/ffs/cod/pregen_source/Windows/cod.tab.c b/thirdparty/ffs/ffs/cod/pregen_source/Windows/cod.tab.c new file mode 100644 index 0000000000..1d42833f97 --- /dev/null +++ b/thirdparty/ffs/ffs/cod/pregen_source/Windows/cod.tab.c @@ -0,0 +1,9205 @@ +/* A Bison parser, made by GNU Bison 2.3. */ + +/* Skeleton implementation for Bison's Yacc-like parsers in C + + Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006 + Free Software Foundation, Inc. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. */ + +/* As a special exception, you may create a larger work that contains + part or all of the Bison parser skeleton and distribute that work + under terms of your choice, so long as that work isn't itself a + parser generator using the skeleton or a modified version thereof + as a parser skeleton. Alternatively, if you modify or redistribute + the parser skeleton itself, you may (at your option) remove this + special exception, which will cause the skeleton and the resulting + Bison output files to be licensed under the GNU General Public + License without this special exception. + + This special exception was added by the Free Software Foundation in + version 2.2 of Bison. */ + +/* C LALR(1) parser skeleton written by Richard Stallman, by + simplifying the original so-called "semantic" parser. */ + +/* All symbols defined below should begin with yy or YY, to avoid + infringing on user name space. This should be done even for local + variables, as they might otherwise be expanded by user macros. + There are some unavoidable exceptions within include files to + define necessary library symbols; they are noted "INFRINGES ON + USER NAME SPACE" below. */ + +/* Identify Bison output. */ +#define YYBISON 1 + +/* Bison version. */ +#define YYBISON_VERSION "2.3" + +/* Skeleton name. */ +#define YYSKELETON_NAME "yacc.c" + +/* Pure parsers. */ +#define YYPURE 0 + +/* Using locations. */ +#define YYLSP_NEEDED 0 + + + +/* Tokens. */ +#ifndef YYTOKENTYPE +# define YYTOKENTYPE + /* Put the tokens into the symbol table, so that GDB and other debuggers + know about them. */ + enum yytokentype { + ARROW = 258, + LPAREN = 259, + RPAREN = 260, + LCURLY = 261, + RCURLY = 262, + COLON = 263, + QUESTION = 264, + LBRACKET = 265, + RBRACKET = 266, + DOT = 267, + STAR = 268, + AT = 269, + SLASH = 270, + MODULUS = 271, + PLUS = 272, + MINUS = 273, + TILDE = 274, + LEQ = 275, + LT = 276, + GEQ = 277, + GT = 278, + EQ = 279, + NEQ = 280, + LEFT_SHIFT = 281, + RIGHT_SHIFT = 282, + ASSIGN = 283, + MUL_ASSIGN = 284, + DIV_ASSIGN = 285, + MOD_ASSIGN = 286, + ADD_ASSIGN = 287, + SUB_ASSIGN = 288, + LEFT_ASSIGN = 289, + RIGHT_ASSIGN = 290, + AND_ASSIGN = 291, + XOR_ASSIGN = 292, + OR_ASSIGN = 293, + LOG_OR = 294, + LOG_AND = 295, + ARITH_OR = 296, + ARITH_AND = 297, + ARITH_XOR = 298, + INC_OP = 299, + DEC_OP = 300, + BANG = 301, + SEMI = 302, + IF = 303, + ELSE = 304, + FOR = 305, + DO = 306, + WHILE = 307, + CHAR = 308, + SHORT = 309, + INT = 310, + LONG = 311, + UNSIGNED = 312, + SIGNED = 313, + FLOAT = 314, + DOUBLE = 315, + VOID = 316, + STRING = 317, + STATIC = 318, + EXTERN_TOKEN = 319, + STRUCT = 320, + ENUM = 321, + UNION = 322, + CONST = 323, + SIZEOF = 324, + TYPEDEF = 325, + RETURN_TOKEN = 326, + CONTINUE = 327, + BREAK = 328, + GOTO = 329, + PRINT = 330, + COMMA = 331, + DOTDOTDOT = 332, + integer_constant = 333, + character_constant = 334, + string_constant = 335, + floating_constant = 336, + identifier_ref = 337, + type_id = 338, + enumeration_constant = 339 + }; +#endif +/* Tokens. */ +#define ARROW 258 +#define LPAREN 259 +#define RPAREN 260 +#define LCURLY 261 +#define RCURLY 262 +#define COLON 263 +#define QUESTION 264 +#define LBRACKET 265 +#define RBRACKET 266 +#define DOT 267 +#define STAR 268 +#define AT 269 +#define SLASH 270 +#define MODULUS 271 +#define PLUS 272 +#define MINUS 273 +#define TILDE 274 +#define LEQ 275 +#define LT 276 +#define GEQ 277 +#define GT 278 +#define EQ 279 +#define NEQ 280 +#define LEFT_SHIFT 281 +#define RIGHT_SHIFT 282 +#define ASSIGN 283 +#define MUL_ASSIGN 284 +#define DIV_ASSIGN 285 +#define MOD_ASSIGN 286 +#define ADD_ASSIGN 287 +#define SUB_ASSIGN 288 +#define LEFT_ASSIGN 289 +#define RIGHT_ASSIGN 290 +#define AND_ASSIGN 291 +#define XOR_ASSIGN 292 +#define OR_ASSIGN 293 +#define LOG_OR 294 +#define LOG_AND 295 +#define ARITH_OR 296 +#define ARITH_AND 297 +#define ARITH_XOR 298 +#define INC_OP 299 +#define DEC_OP 300 +#define BANG 301 +#define SEMI 302 +#define IF 303 +#define ELSE 304 +#define FOR 305 +#define DO 306 +#define WHILE 307 +#define CHAR 308 +#define SHORT 309 +#define INT 310 +#define LONG 311 +#define UNSIGNED 312 +#define SIGNED 313 +#define FLOAT 314 +#define DOUBLE 315 +#define VOID 316 +#define STRING 317 +#define STATIC 318 +#define EXTERN_TOKEN 319 +#define STRUCT 320 +#define ENUM 321 +#define UNION 322 +#define CONST 323 +#define SIZEOF 324 +#define TYPEDEF 325 +#define RETURN_TOKEN 326 +#define CONTINUE 327 +#define BREAK 328 +#define GOTO 329 +#define PRINT 330 +#define COMMA 331 +#define DOTDOTDOT 332 +#define integer_constant 333 +#define character_constant 334 +#define string_constant 335 +#define floating_constant 336 +#define identifier_ref 337 +#define type_id 338 +#define enumeration_constant 339 + + + + +/* Copy the first part of user declarations. */ +#line 1 "cod/cod.y" + +#include "config.h" +#if defined (__INTEL_COMPILER) +# pragma warning (disable: 2215) +#endif +#ifdef SEGMENTED_POINTERS +int cod_segmented_pointers = 1; +#else +int cod_segmented_pointers = 0; +#endif +#ifdef KPLUGINS_INTEGRATION +int cod_kplugins_integration = 1; +#else +int cod_kplugins_integration = 0; +#endif +#ifndef LINUX_KERNEL_MODULE +#include "stdio.h" +#endif +#ifdef LINUX_KERNEL_MODULE +#ifndef MODULE +#define MODULE +#endif +#ifndef __KERNEL__ +#define __KERNEL__ +#endif +#include +#include +#endif +#ifndef LINUX_KERNEL_MODULE +#ifdef HAVE_STDLIB_H +#include +#endif +#endif +#include "fm.h" +#include "fm_internal.h" +#include "cod.h" +#include "cod_internal.h" +#undef NDEBUG +#include "assert.h" +#ifndef LINUX_KERNEL_MODULE +#include +#ifdef HAVE_MALLOC_H +#include +#endif +#include +#else +#include +#include +#include +#endif +#include "float.h" +#ifdef DBL_DECIMAL_DIG + #define OP_DBL_Digs (DBL_DECIMAL_DIG) +#else + #ifdef DECIMAL_DIG + #define OP_DBL_Digs (DECIMAL_DIG) + #else + #define OP_DBL_Digs (DBL_DIG + 3) + #endif +#endif +#include "structs.h" +#ifdef HAVE_DILL_H +#include "dill.h" +#else +enum { + DILL_C, /* char */ + DILL_UC, /* unsigned char */ + DILL_S, /* short */ + DILL_US, /* unsigned short */ + DILL_I, /* int */ + DILL_U, /* unsigned */ + DILL_L, /* long */ + DILL_UL, /* unsigned long */ + DILL_P, /* pointer */ + DILL_F, /* floating */ + DILL_D, /* double */ + DILL_V, /* void */ + DILL_B, /* block structure */ + DILL_EC, + DILL_ERR /* no type */ +}; +#endif +#if defined(_MSC_VER) +#define strdup _strdup +#endif +#ifndef LINUX_KERNEL_MODULE +#ifdef STDC_HEADERS +#include +#else +#include +#endif +#else +#include "kecl.h" +#define malloc (void *)DAllocMM +#define free(a) DFreeMM((addrs_t)a) +#define realloc(a,b) DReallocMM((addrs_t)a,b) +#define fprintf(fmt, args...) printk(args); +#define printf printk +char *strdup(const char *s) +{ + char *p; + + p = (char *)kmalloc(strlen(s)+1, GFP_KERNEL); + if (p != NULL) + strcpy(p,s); + return p; +} +#endif +#define YY_NO_INPUT + +static char* +gen_anon() +{ + static int anon_count = 0; + char *ret = malloc(strlen("Anonymous-xxxxxxxxxxxxxxxxx")); + sprintf(ret, "Anonymous-%d", anon_count++); + return ret; +} + +#define yyparse cod_yyparse +#define yylex cod_yylex +#define yyrestart cod_yyrestart +#define yywrap cod_yywrap +#define yyerror cod_yyerror +#define yylineno cod_yylineno +#define yy_flex_debug cod_yy_flex_debug +#define yy_create_buffer cod_yy_create_buffer +#define yy_delete_buffer cod_yy_delete_buffer +#define yy_flush_buffer cod_yy_flush_buffer +#define yy_init_buffer cod_yy_init_buffer +#define yy_load_buffer_state cod_yy_load_buffer_state +#define yy_scan_buffer cod_yy_scan_buffer +#define yy_scan_bytes cod_yy_scan_bytes +#define yy_scan_string cod_yy_scan_string +#define yy_switch_to_buffer cod_yy_switch_to_buffer +#define yychar cod_yychar +#define yyin cod_yyin +#define yyleng cod_yyleng +#define yylval cod_yylval +#define yynerrs cod_yynerrs +#define yyout cod_yyout +#define yytext cod_yytext +#define yyset_out cod_yyset_out +#define yyset_lineno cod_yyset_lineno +#define yyset_in cod_yyset_in +#define yyset_debug cod_yyset_debug +#define yyrealloc cod_yyrealloc +#define yyalloc cod_yyalloc +#define yyfree cod_yyfree +#define yypush_buffer_state cod_yypush_buffer_state +#define yypop_buffer_state cod_yypop_buffer_state +#define yylex_destroy cod_yylex_destroy +#define yyget_out cod_yyget_out +#define yyget_lineno cod_yyget_lineno +#define yyget_in cod_yyget_in +#define yyget_debug cod_yyget_debug +#define yyget_text cod_yyget_text +#define yyget_leng cod_yyget_leng + +static char *create_string_from_yytext(); +extern int yylex(); +extern int yyparse(); +static sm_ref yyparse_value; +static int yyerror_count = 1; +extern void yyerror(char *str); +static int parsing_type = 0; +static int parsing_param_spec = 0; +static cod_parse_context yycontext; +static sm_ref cod_build_parsed_type_node(cod_parse_context c, char *name, sm_list l); +static sm_list +cod_dup_list(sm_list list) +{ + sm_list ret_list, new_list = NULL; + sm_list *last_p = &ret_list; + while (list != NULL) { + *last_p = new_list = malloc(sizeof(struct list_struct)); + last_p = &(new_list->next); + new_list->node = cod_copy(list->node); + list = list->next; + } + *last_p = NULL; + return ret_list; +} + + +/* Enabling traces. */ +#ifndef YYDEBUG +# define YYDEBUG 0 +#endif + +/* Enabling verbose error messages. */ +#ifdef YYERROR_VERBOSE +# undef YYERROR_VERBOSE +# define YYERROR_VERBOSE 1 +#else +# define YYERROR_VERBOSE 0 +#endif + +/* Enabling the token table. */ +#ifndef YYTOKEN_TABLE +# define YYTOKEN_TABLE 0 +#endif + +#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED +typedef union YYSTYPE +#line 187 "cod/cod.y" +{ + lx_info info; + sm_ref reference; + operator_t operator; + sm_list list; + char *string; +} +/* Line 193 of yacc.c. */ +#line 457 "/Users/eisen/prog/ffs/build/cod.tab.c" + YYSTYPE; +# define yystype YYSTYPE /* obsolescent; will be withdrawn */ +# define YYSTYPE_IS_DECLARED 1 +# define YYSTYPE_IS_TRIVIAL 1 +#endif + + + +/* Copy the second part of user declarations. */ + + +/* Line 216 of yacc.c. */ +#line 470 "/Users/eisen/prog/ffs/build/cod.tab.c" + +#ifdef short +# undef short +#endif + +#ifdef YYTYPE_UINT8 +typedef YYTYPE_UINT8 yytype_uint8; +#else +typedef unsigned char yytype_uint8; +#endif + +#ifdef YYTYPE_INT8 +typedef YYTYPE_INT8 yytype_int8; +#elif (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +typedef signed char yytype_int8; +#else +typedef short int yytype_int8; +#endif + +#ifdef YYTYPE_UINT16 +typedef YYTYPE_UINT16 yytype_uint16; +#else +typedef unsigned short int yytype_uint16; +#endif + +#ifdef YYTYPE_INT16 +typedef YYTYPE_INT16 yytype_int16; +#else +typedef short int yytype_int16; +#endif + +#ifndef YYSIZE_T +# ifdef __SIZE_TYPE__ +# define YYSIZE_T __SIZE_TYPE__ +# elif defined size_t +# define YYSIZE_T size_t +# elif ! defined YYSIZE_T && (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +# include /* INFRINGES ON USER NAME SPACE */ +# define YYSIZE_T size_t +# else +# define YYSIZE_T unsigned int +# endif +#endif + +#define YYSIZE_MAXIMUM ((YYSIZE_T) -1) + +#ifndef YY_ +# if defined YYENABLE_NLS && YYENABLE_NLS +# if ENABLE_NLS +# include /* INFRINGES ON USER NAME SPACE */ +# define YY_(msgid) dgettext ("bison-runtime", msgid) +# endif +# endif +# ifndef YY_ +# define YY_(msgid) msgid +# endif +#endif + +/* Suppress unused-variable warnings by "using" E. */ +#if ! defined lint || defined __GNUC__ +# define YYUSE(e) ((void) (e)) +#else +# define YYUSE(e) /* empty */ +#endif + +/* Identity function, used to suppress warnings about constant conditions. */ +#ifndef lint +# define YYID(n) (n) +#else +#if (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +static int +YYID (int i) +#else +static int +YYID (i) + int i; +#endif +{ + return i; +} +#endif + +#if ! defined yyoverflow || YYERROR_VERBOSE + +/* The parser invokes alloca or malloc; define the necessary symbols. */ + +# ifdef YYSTACK_USE_ALLOCA +# if YYSTACK_USE_ALLOCA +# ifdef __GNUC__ +# define YYSTACK_ALLOC __builtin_alloca +# elif defined __BUILTIN_VA_ARG_INCR +# include /* INFRINGES ON USER NAME SPACE */ +# elif defined _AIX +# define YYSTACK_ALLOC __alloca +# elif defined _MSC_VER +# include /* INFRINGES ON USER NAME SPACE */ +# define alloca _alloca +# else +# define YYSTACK_ALLOC alloca +# if ! defined _ALLOCA_H && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +# include /* INFRINGES ON USER NAME SPACE */ +# ifndef _STDLIB_H +# define _STDLIB_H 1 +# endif +# endif +# endif +# endif +# endif + +# ifdef YYSTACK_ALLOC + /* Pacify GCC's `empty if-body' warning. */ +# define YYSTACK_FREE(Ptr) do { /* empty */; } while (YYID (0)) +# ifndef YYSTACK_ALLOC_MAXIMUM + /* The OS might guarantee only one guard page at the bottom of the stack, + and a page size can be as small as 4096 bytes. So we cannot safely + invoke alloca (N) if N exceeds 4096. Use a slightly smaller number + to allow for a few compiler-allocated temporary stack slots. */ +# define YYSTACK_ALLOC_MAXIMUM 4032 /* reasonable circa 2006 */ +# endif +# else +# define YYSTACK_ALLOC YYMALLOC +# define YYSTACK_FREE YYFREE +# ifndef YYSTACK_ALLOC_MAXIMUM +# define YYSTACK_ALLOC_MAXIMUM YYSIZE_MAXIMUM +# endif +# if (defined __cplusplus && ! defined _STDLIB_H \ + && ! ((defined YYMALLOC || defined malloc) \ + && (defined YYFREE || defined free))) +# include /* INFRINGES ON USER NAME SPACE */ +# ifndef _STDLIB_H +# define _STDLIB_H 1 +# endif +# endif +# ifndef YYMALLOC +# define YYMALLOC malloc +# if ! defined malloc && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +void *malloc (YYSIZE_T); /* INFRINGES ON USER NAME SPACE */ +# endif +# endif +# ifndef YYFREE +# define YYFREE free +# if ! defined free && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +void free (void *); /* INFRINGES ON USER NAME SPACE */ +# endif +# endif +# endif +#endif /* ! defined yyoverflow || YYERROR_VERBOSE */ + + +#if (! defined yyoverflow \ + && (! defined __cplusplus \ + || (defined YYSTYPE_IS_TRIVIAL && YYSTYPE_IS_TRIVIAL))) + +/* A type that is properly aligned for any stack member. */ +union yyalloc +{ + yytype_int16 yyss; + YYSTYPE yyvs; + }; + +/* The size of the maximum gap between one aligned stack and the next. */ +# define YYSTACK_GAP_MAXIMUM (sizeof (union yyalloc) - 1) + +/* The size of an array large to enough to hold all stacks, each with + N elements. */ +# define YYSTACK_BYTES(N) \ + ((N) * (sizeof (yytype_int16) + sizeof (YYSTYPE)) \ + + YYSTACK_GAP_MAXIMUM) + +/* Copy COUNT objects from FROM to TO. The source and destination do + not overlap. */ +# ifndef YYCOPY +# if defined __GNUC__ && 1 < __GNUC__ +# define YYCOPY(To, From, Count) \ + __builtin_memcpy (To, From, (Count) * sizeof (*(From))) +# else +# define YYCOPY(To, From, Count) \ + do \ + { \ + YYSIZE_T yyi; \ + for (yyi = 0; yyi < (Count); yyi++) \ + (To)[yyi] = (From)[yyi]; \ + } \ + while (YYID (0)) +# endif +# endif + +/* Relocate STACK from its old location to the new one. The + local variables YYSIZE and YYSTACKSIZE give the old and new number of + elements in the stack, and YYPTR gives the new location of the + stack. Advance YYPTR to a properly aligned location for the next + stack. */ +# define YYSTACK_RELOCATE(Stack) \ + do \ + { \ + YYSIZE_T yynewbytes; \ + YYCOPY (&yyptr->Stack, Stack, yysize); \ + Stack = &yyptr->Stack; \ + yynewbytes = yystacksize * sizeof (*Stack) + YYSTACK_GAP_MAXIMUM; \ + yyptr += yynewbytes / sizeof (*yyptr); \ + } \ + while (YYID (0)) + +#endif + +/* YYFINAL -- State number of the termination state. */ +#define YYFINAL 88 +/* YYLAST -- Last index in YYTABLE. */ +#define YYLAST 985 + +/* YYNTOKENS -- Number of terminals. */ +#define YYNTOKENS 85 +/* YYNNTS -- Number of nonterminals. */ +#define YYNNTS 67 +/* YYNRULES -- Number of rules. */ +#define YYNRULES 206 +/* YYNRULES -- Number of states. */ +#define YYNSTATES 321 + +/* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX. */ +#define YYUNDEFTOK 2 +#define YYMAXUTOK 339 + +#define YYTRANSLATE(YYX) \ + ((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK) + +/* YYTRANSLATE[YYLEX] -- Bison symbol number corresponding to YYLEX. */ +static const yytype_uint8 yytranslate[] = +{ + 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 1, 2, 3, 4, + 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, + 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + 75, 76, 77, 78, 79, 80, 81, 82, 83, 84 +}; + +#if YYDEBUG +/* YYPRHS[YYN] -- Index of the first RHS symbol of rule number YYN in + YYRHS. */ +static const yytype_uint16 yyprhs[] = +{ + 0, 0, 3, 5, 7, 9, 11, 15, 17, 22, + 26, 31, 35, 39, 42, 45, 47, 51, 53, 56, + 59, 62, 65, 70, 72, 74, 76, 78, 80, 82, + 84, 89, 91, 95, 99, 103, 105, 109, 113, 115, + 119, 123, 125, 129, 133, 137, 141, 143, 147, 151, + 153, 157, 159, 163, 165, 169, 171, 175, 177, 181, + 183, 189, 191, 193, 195, 197, 199, 201, 203, 205, + 207, 209, 211, 213, 217, 219, 223, 225, 227, 231, + 232, 233, 239, 242, 244, 247, 249, 252, 254, 257, + 259, 263, 265, 267, 269, 271, 273, 275, 277, 279, + 281, 283, 285, 287, 289, 291, 293, 295, 301, 306, + 309, 311, 313, 315, 318, 321, 325, 327, 331, 333, + 336, 338, 341, 343, 348, 354, 360, 367, 370, 372, + 376, 380, 382, 384, 386, 389, 391, 395, 400, 404, + 409, 413, 415, 418, 421, 425, 427, 430, 433, 437, + 439, 442, 444, 448, 450, 454, 456, 459, 461, 464, + 466, 470, 475, 477, 480, 482, 487, 491, 494, 496, + 499, 503, 506, 508, 510, 513, 516, 519, 521, 523, + 525, 527, 529, 531, 535, 538, 542, 544, 547, 551, + 554, 557, 560, 564, 566, 569, 575, 583, 593, 599, + 607, 608, 610, 612, 614, 616, 618 +}; + +/* YYRHS -- A `-1'-separated list of the rules' RHS. */ +static const yytype_int16 yyrhs[] = +{ + 86, 0, -1, 145, -1, 144, -1, 82, -1, 151, + -1, 4, 106, 5, -1, 87, -1, 88, 10, 106, + 11, -1, 88, 12, 82, -1, 88, 4, 89, 5, + -1, 88, 4, 5, -1, 88, 3, 82, -1, 88, + 44, -1, 88, 45, -1, 105, -1, 89, 76, 105, + -1, 88, -1, 44, 90, -1, 45, 90, -1, 91, + 92, -1, 69, 90, -1, 69, 4, 134, 5, -1, + 42, -1, 13, -1, 17, -1, 18, -1, 19, -1, + 46, -1, 90, -1, 4, 134, 5, 92, -1, 92, + -1, 93, 13, 92, -1, 93, 15, 92, -1, 93, + 16, 92, -1, 93, -1, 94, 17, 93, -1, 94, + 18, 93, -1, 94, -1, 95, 26, 94, -1, 95, + 27, 94, -1, 95, -1, 96, 21, 95, -1, 96, + 23, 95, -1, 96, 20, 95, -1, 96, 22, 95, + -1, 96, -1, 97, 24, 96, -1, 97, 25, 96, + -1, 97, -1, 98, 42, 97, -1, 98, -1, 99, + 43, 98, -1, 99, -1, 100, 41, 99, -1, 100, + -1, 101, 40, 100, -1, 101, -1, 102, 39, 101, + -1, 102, -1, 102, 9, 106, 8, 103, -1, 28, + -1, 29, -1, 30, -1, 31, -1, 32, -1, 33, + -1, 34, -1, 35, -1, 36, -1, 37, -1, 38, + -1, 103, -1, 90, 104, 105, -1, 105, -1, 106, + 76, 105, -1, 103, -1, 113, -1, 108, 76, 113, + -1, -1, -1, 112, 110, 108, 111, 47, -1, 112, + 47, -1, 114, -1, 114, 112, -1, 115, -1, 115, + 112, -1, 126, -1, 126, 112, -1, 127, -1, 127, + 28, 136, -1, 70, -1, 63, -1, 64, -1, 53, + -1, 54, -1, 55, -1, 56, -1, 59, -1, 60, + -1, 61, -1, 58, -1, 57, -1, 62, -1, 83, + -1, 116, -1, 123, -1, 117, 82, 6, 118, 7, + -1, 117, 6, 118, 7, -1, 117, 82, -1, 65, + -1, 67, -1, 119, -1, 118, 119, -1, 122, 47, + -1, 122, 120, 47, -1, 121, -1, 120, 76, 121, + -1, 127, -1, 115, 122, -1, 115, -1, 126, 122, + -1, 126, -1, 66, 6, 124, 7, -1, 66, 6, + 124, 76, 7, -1, 66, 82, 6, 124, 7, -1, + 66, 82, 6, 124, 76, 7, -1, 66, 82, -1, + 125, -1, 124, 76, 125, -1, 82, 28, 107, -1, + 82, -1, 68, -1, 128, -1, 129, 128, -1, 82, + -1, 4, 127, 5, -1, 82, 4, 131, 5, -1, + 82, 4, 5, -1, 128, 10, 107, 11, -1, 128, + 10, 11, -1, 13, -1, 13, 130, -1, 13, 129, + -1, 13, 130, 129, -1, 14, -1, 14, 130, -1, + 14, 129, -1, 14, 130, 129, -1, 126, -1, 130, + 126, -1, 132, -1, 132, 76, 77, -1, 133, -1, + 132, 76, 133, -1, 112, -1, 112, 127, -1, 122, + -1, 122, 135, -1, 129, -1, 6, 137, 7, -1, + 6, 137, 76, 7, -1, 105, -1, 138, 136, -1, + 136, -1, 137, 76, 138, 136, -1, 137, 76, 136, + -1, 139, 28, -1, 140, -1, 139, 140, -1, 10, + 107, 11, -1, 12, 82, -1, 142, -1, 109, -1, + 1, 47, -1, 141, 142, -1, 141, 109, -1, 143, + -1, 144, -1, 147, -1, 148, -1, 149, -1, 146, + -1, 82, 8, 142, -1, 6, 7, -1, 6, 141, + 7, -1, 109, -1, 145, 109, -1, 71, 106, 47, + -1, 71, 47, -1, 72, 47, -1, 73, 47, -1, + 74, 82, 47, -1, 47, -1, 106, 47, -1, 48, + 4, 106, 5, 142, -1, 48, 4, 106, 5, 142, + 49, 142, -1, 50, 4, 150, 47, 150, 47, 150, + 5, 142, -1, 52, 4, 106, 5, 142, -1, 51, + 142, 52, 4, 106, 5, 47, -1, -1, 106, -1, + 78, -1, 81, -1, 80, -1, 79, -1, 84, -1 +}; + +/* YYRLINE[YYN] -- source line where rule number YYN was defined. */ +static const yytype_uint16 yyrline[] = +{ + 0, 344, 344, 348, 354, 360, 362, 369, 371, 378, + 385, 392, 398, 404, 411, 421, 427, 441, 442, 449, + 456, 463, 470, 485, 488, 491, 494, 497, 500, 506, + 507, 516, 518, 527, 536, 547, 549, 558, 569, 571, + 580, 591, 593, 602, 611, 620, 631, 633, 642, 653, + 655, 666, 668, 679, 681, 692, 694, 705, 707, 718, + 720, 731, 733, 735, 737, 739, 741, 743, 745, 747, + 749, 751, 756, 759, 770, 772, 782, 786, 791, 810, + 817, 809, 900, 906, 911, 917, 922, 928, 933, 941, + 943, 959, 964, 969, 978, 983, 988, 993, 998, 1003, + 1008, 1013, 1018, 1023, 1028, 1033, 1036, 1042, 1045, 1048, + 1054, 1055, 1061, 1062, 1074, 1075, 1126, 1131, 1143, 1146, + 1152, 1157, 1163, 1171, 1178, 1185, 1192, 1199, 1209, 1215, + 1225, 1231, 1239, 1247, 1249, 1263, 1272, 1275, 1284, 1293, + 1300, 1310, 1318, 1326, 1334, 1348, 1359, 1370, 1381, 1401, + 1406, 1419, 1420, 1435, 1441, 1456, 1465, 1504, 1505, 1551, + 1555, 1560, 1565, 1570, 1578, 1586, 1599, 1615, 1620, 1625, + 1638, 1644, 1653, 1659, 1662, 1665, 1671, 1676, 1677, 1678, + 1679, 1680, 1681, 1688, 1695, 1698, 1706, 1708, 1722, 1727, + 1732, 1738, 1744, 1753, 1757, 1767, 1776, 1793, 1803, 1813, + 1827, 1829, 1832, 1839, 1846, 1853, 1860 +}; +#endif + +#if YYDEBUG || YYERROR_VERBOSE || YYTOKEN_TABLE +/* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM. + First, the terminals, then, starting at YYNTOKENS, nonterminals. */ +static const char *const yytname[] = +{ + "$end", "error", "$undefined", "ARROW", "LPAREN", "RPAREN", "LCURLY", + "RCURLY", "COLON", "QUESTION", "LBRACKET", "RBRACKET", "DOT", "STAR", + "AT", "SLASH", "MODULUS", "PLUS", "MINUS", "TILDE", "LEQ", "LT", "GEQ", + "GT", "EQ", "NEQ", "LEFT_SHIFT", "RIGHT_SHIFT", "ASSIGN", "MUL_ASSIGN", + "DIV_ASSIGN", "MOD_ASSIGN", "ADD_ASSIGN", "SUB_ASSIGN", "LEFT_ASSIGN", + "RIGHT_ASSIGN", "AND_ASSIGN", "XOR_ASSIGN", "OR_ASSIGN", "LOG_OR", + "LOG_AND", "ARITH_OR", "ARITH_AND", "ARITH_XOR", "INC_OP", "DEC_OP", + "BANG", "SEMI", "IF", "ELSE", "FOR", "DO", "WHILE", "CHAR", "SHORT", + "INT", "LONG", "UNSIGNED", "SIGNED", "FLOAT", "DOUBLE", "VOID", "STRING", + "STATIC", "EXTERN_TOKEN", "STRUCT", "ENUM", "UNION", "CONST", "SIZEOF", + "TYPEDEF", "RETURN_TOKEN", "CONTINUE", "BREAK", "GOTO", "PRINT", "COMMA", + "DOTDOTDOT", "integer_constant", "character_constant", "string_constant", + "floating_constant", "identifier_ref", "type_id", "enumeration_constant", + "$accept", "start", "primary_expression", "postfix_expression", + "argument_expression_list", "unary_expression", "unary_operator", + "cast_expression", "multiplicative_expression", "additive_expression", + "shift_expression", "relational_expression", "equality_expression", + "and_expression", "exclusive_or_expression", "inclusive_or_expression", + "logical_and_expression", "logical_or_expression", + "conditional_expression", "assignment_operator", "assignment_expression", + "expression", "constant_expression", "init_declarator_list", + "declaration", "@1", "@2", "declaration_specifiers", "init_declarator", + "storage_class_specifier", "type_specifier", "struct_or_union_specifier", + "struct_or_union", "struct_declaration_list", "struct_declaration", + "struct_declarator_list", "struct_declarator", + "specifier_qualifier_list", "enum_specifier", "enumerator_list", + "enumerator", "type_qualifier", "declarator", "direct_declarator", + "pointer", "type_qualifier_list", "parameter_type_list", + "parameter_list", "parameter_declaration", "type_name", + "abstract_declarator", "initializer", "initializer_list", "designation", + "designator_list", "designator", "decls_stmts_list", "statement", + "labeled_statement", "compound_statement", "declaration_list", + "jump_statement", "expression_statement", "selection_statement", + "iteration_statement", "expression_opt", "constant", 0 +}; +#endif + +# ifdef YYPRINT +/* YYTOKNUM[YYLEX-NUM] -- Internal token number corresponding to + token YYLEX-NUM. */ +static const yytype_uint16 yytoknum[] = +{ + 0, 256, 257, 258, 259, 260, 261, 262, 263, 264, + 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, + 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, + 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, + 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, + 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, + 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, + 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, + 335, 336, 337, 338, 339 +}; +# endif + +/* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */ +static const yytype_uint8 yyr1[] = +{ + 0, 85, 86, 86, 87, 87, 87, 88, 88, 88, + 88, 88, 88, 88, 88, 89, 89, 90, 90, 90, + 90, 90, 90, 91, 91, 91, 91, 91, 91, 92, + 92, 93, 93, 93, 93, 94, 94, 94, 95, 95, + 95, 96, 96, 96, 96, 96, 97, 97, 97, 98, + 98, 99, 99, 100, 100, 101, 101, 102, 102, 103, + 103, 104, 104, 104, 104, 104, 104, 104, 104, 104, + 104, 104, 105, 105, 106, 106, 107, 108, 108, 110, + 111, 109, 109, 112, 112, 112, 112, 112, 112, 113, + 113, 114, 114, 114, 115, 115, 115, 115, 115, 115, + 115, 115, 115, 115, 115, 115, 115, 116, 116, 116, + 117, 117, 118, 118, 119, 119, 120, 120, 121, 122, + 122, 122, 122, 123, 123, 123, 123, 123, 124, 124, + 125, 125, 126, 127, 127, 128, 128, 128, 128, 128, + 128, 129, 129, 129, 129, 129, 129, 129, 129, 130, + 130, 131, 131, 132, 132, 133, 133, 134, 134, 135, + 136, 136, 136, 137, 137, 137, 137, 138, 139, 139, + 140, 140, 141, 141, 141, 141, 141, 142, 142, 142, + 142, 142, 142, 143, 144, 144, 145, 145, 146, 146, + 146, 146, 146, 147, 147, 148, 148, 149, 149, 149, + 150, 150, 151, 151, 151, 151, 151 +}; + +/* YYR2[YYN] -- Number of symbols composing right hand side of rule YYN. */ +static const yytype_uint8 yyr2[] = +{ + 0, 2, 1, 1, 1, 1, 3, 1, 4, 3, + 4, 3, 3, 2, 2, 1, 3, 1, 2, 2, + 2, 2, 4, 1, 1, 1, 1, 1, 1, 1, + 4, 1, 3, 3, 3, 1, 3, 3, 1, 3, + 3, 1, 3, 3, 3, 3, 1, 3, 3, 1, + 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, + 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 3, 1, 3, 1, 1, 3, 0, + 0, 5, 2, 1, 2, 1, 2, 1, 2, 1, + 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 5, 4, 2, + 1, 1, 1, 2, 2, 3, 1, 3, 1, 2, + 1, 2, 1, 4, 5, 5, 6, 2, 1, 3, + 3, 1, 1, 1, 2, 1, 3, 4, 3, 4, + 3, 1, 2, 2, 3, 1, 2, 2, 3, 1, + 2, 1, 3, 1, 3, 1, 2, 1, 2, 1, + 3, 4, 1, 2, 1, 4, 3, 2, 1, 2, + 3, 2, 1, 1, 2, 2, 2, 1, 1, 1, + 1, 1, 1, 3, 2, 3, 1, 2, 3, 2, + 2, 2, 3, 1, 2, 5, 7, 9, 5, 7, + 0, 1, 1, 1, 1, 1, 1 +}; + +/* YYDEFACT[STATE-NAME] -- Default rule to reduce with in state + STATE-NUM when YYTABLE doesn't specify something else to do. Zero + means the default is an error. */ +static const yytype_uint8 yydefact[] = +{ + 0, 0, 94, 95, 96, 97, 102, 101, 98, 99, + 100, 103, 92, 93, 110, 0, 111, 132, 91, 104, + 0, 186, 79, 83, 85, 105, 0, 106, 87, 3, + 2, 0, 0, 184, 24, 25, 26, 27, 23, 0, + 0, 28, 193, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 202, 205, 204, 203, 4, 206, 7, 17, + 29, 0, 31, 35, 38, 41, 46, 49, 51, 53, + 55, 57, 59, 72, 74, 0, 173, 0, 172, 177, + 178, 182, 179, 180, 181, 5, 0, 127, 1, 82, + 0, 84, 86, 0, 109, 88, 187, 174, 4, 0, + 120, 157, 122, 0, 0, 18, 19, 0, 200, 0, + 0, 0, 21, 189, 0, 190, 191, 0, 0, 0, + 0, 0, 0, 13, 14, 61, 62, 63, 64, 65, + 66, 67, 68, 69, 70, 71, 0, 29, 20, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, + 185, 176, 175, 131, 0, 128, 0, 0, 141, 145, + 135, 80, 77, 89, 133, 0, 0, 112, 0, 0, + 6, 119, 159, 158, 121, 0, 0, 201, 0, 0, + 0, 0, 188, 192, 183, 12, 11, 0, 15, 0, + 9, 73, 32, 33, 34, 36, 37, 39, 40, 44, + 42, 45, 43, 47, 48, 50, 52, 54, 56, 0, + 58, 75, 0, 123, 0, 0, 0, 149, 143, 142, + 147, 146, 0, 0, 0, 0, 0, 134, 108, 113, + 114, 0, 116, 118, 0, 30, 0, 200, 0, 0, + 22, 10, 0, 8, 0, 76, 130, 124, 129, 125, + 0, 136, 150, 144, 148, 138, 155, 0, 151, 153, + 78, 81, 0, 162, 90, 140, 0, 115, 0, 107, + 195, 0, 0, 198, 16, 60, 126, 156, 137, 0, + 0, 0, 164, 0, 0, 0, 168, 139, 117, 0, + 200, 0, 152, 154, 0, 171, 160, 0, 163, 167, + 169, 196, 0, 199, 170, 161, 166, 0, 0, 165, + 197 +}; + +/* YYDEFGOTO[NTERM-NUM]. */ +static const yytype_int16 yydefgoto[] = +{ + -1, 20, 58, 59, 197, 60, 61, 62, 63, 64, + 65, 66, 67, 68, 69, 70, 71, 72, 73, 136, + 74, 75, 256, 171, 21, 90, 234, 22, 172, 23, + 24, 25, 26, 176, 177, 241, 242, 178, 27, 164, + 165, 28, 173, 174, 175, 229, 267, 268, 269, 103, + 183, 274, 293, 294, 295, 296, 77, 78, 79, 80, + 30, 81, 82, 83, 84, 188, 85 +}; + +/* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing + STATE-NUM. */ +#define YYPACT_NINF -239 +static const yytype_int16 yypact[] = +{ + 808, 232, -239, -239, -239, -239, -239, -239, -239, -239, + -239, -239, -239, -239, -239, 46, -239, -239, -239, -239, + 64, -239, 13, 352, 352, -239, 47, -239, 352, -239, + 352, 39, 385, -239, -239, -239, -239, -239, -239, 653, + 653, -239, -239, 71, 84, 466, 146, 723, 539, 108, + 118, 85, -239, -239, -239, -239, 162, -239, -239, 170, + 194, 744, -239, 32, 25, 11, 140, 128, 129, 132, + 144, 147, 7, -239, -239, -18, -239, 313, -239, -239, + -239, -239, -239, -239, -239, -239, 107, 184, -239, -239, + 17, -239, -239, 902, 186, -239, -239, -239, -239, 21, + 902, 42, 902, 188, 744, -239, -239, 744, 744, 142, + 744, 385, -239, -239, -11, -239, -239, 148, 466, 114, + 583, 744, 116, -239, -239, -239, -239, -239, -239, -239, + -239, -239, -239, -239, -239, -239, 744, -239, -239, 744, + 744, 744, 744, 744, 744, 744, 744, 744, 744, 744, + 744, 744, 744, 744, 744, 744, 744, 744, -239, 744, + -239, -239, -239, 172, 59, -239, 107, 17, 135, 135, + 201, 130, -239, 183, 203, 20, 717, -239, -3, 902, + -239, -239, -239, -239, -239, 744, 28, 159, 169, 233, + 49, 235, -239, -239, -239, -239, -239, 54, -239, 65, + -239, -239, -239, -239, -239, 32, 32, 25, 25, 11, + 11, 11, 11, 140, 140, 128, 129, 132, 144, 27, + 147, -239, 744, -239, 0, 60, 236, -239, -239, 135, + -239, 135, 776, 17, 195, 631, 674, 203, -239, -239, + -239, -7, -239, -239, 840, -239, 466, 744, 744, 466, + -239, -239, 744, -239, 744, -239, -239, -239, -239, -239, + 18, -239, -239, -239, -239, -239, 17, 238, 168, -239, + -239, -239, 561, -239, -239, -239, 237, -239, 17, -239, + 197, 200, 56, -239, -239, -239, -239, -239, -239, 871, + 744, 171, -239, 61, 631, 29, -239, -239, -239, 466, + 744, 205, -239, -239, 244, -239, -239, 482, -239, -239, + -239, -239, 252, -239, -239, -239, -239, 631, 466, -239, + -239 +}; + +/* YYPGOTO[NTERM-NUM]. */ +static const yytype_int16 yypgoto[] = +{ + -239, -239, -239, -239, -239, -34, -239, -46, -71, 12, + 30, 33, 106, 109, 105, 110, 103, -239, -82, -239, + -108, -30, -93, -239, 19, -239, -239, -20, 34, -239, + 31, -239, -239, 82, -106, -239, -15, -13, -239, 98, + -210, -10, -144, 91, -84, 99, -239, -239, -17, 160, + -239, -126, -239, -26, -239, 14, -239, -45, -239, 307, + -239, -239, -239, -239, -239, -238, -239 +}; + +/* YYTABLE[YYPACT[STATE-NUM]]. What to do in state STATE-NUM. If + positive, shift that token. If negative, reduce the rule which + number is the opposite. If zero, do what YYDEFACT says. + If YYTABLE_NINF, syntax error. */ +#define YYTABLE_NINF -1 +static const yytype_uint16 yytable[] = +{ + 109, 167, 99, 91, 92, 105, 106, 257, 95, 281, + 168, 169, 198, 112, 258, 138, 156, 182, 114, 101, + 76, 167, 102, 226, 167, 286, 180, 137, 201, 158, + 168, 169, 162, 246, 243, 254, 192, 144, 145, 290, + 277, 291, 142, 143, 240, 139, 157, 140, 141, 96, + 258, 221, 86, 93, 249, 168, 169, 309, 159, 251, + 89, 301, 312, 100, 88, 159, 223, 259, 306, 278, + 239, 205, 206, 194, 99, 107, 253, 186, 187, 170, + 190, 99, 163, 102, 228, 230, 97, 181, 108, 184, + 102, 199, 102, 202, 203, 204, 161, 159, 101, 170, + 163, 102, 170, 159, 159, 137, 137, 137, 137, 137, + 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 137, 137, 287, 137, 100, 159, 219, 273, 87, 94, + 252, 100, 159, 100, 243, 224, 260, 307, 239, 245, + 255, 159, 100, 276, 284, 263, 292, 264, 168, 169, + 110, 137, 150, 151, 255, 115, 207, 208, 227, 227, + 146, 147, 148, 149, 273, 116, 102, 117, 308, 102, + 118, 152, 285, 119, 120, 153, 209, 210, 211, 212, + 121, 316, 122, 213, 214, 154, 273, 155, 137, 163, + 166, 319, 179, 185, 189, 193, 195, 304, 200, 273, + 222, 280, 137, 17, 283, 232, 233, 100, 255, 273, + 100, 235, 266, 236, 123, 124, 247, 187, 282, 262, + 137, 262, 125, 126, 127, 128, 129, 130, 131, 132, + 133, 134, 135, 31, 102, 159, 32, 248, 1, 33, + 250, 261, 271, 288, 289, 34, 299, 300, 297, 35, + 36, 37, 313, 305, 311, 314, 137, 318, 215, 217, + 220, 244, 216, 298, 225, 218, 237, 270, 231, 266, + 187, 191, 303, 320, 38, 100, 39, 40, 41, 42, + 43, 317, 44, 45, 46, 2, 3, 4, 5, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 47, 18, 48, 49, 50, 51, 29, 0, 310, + 52, 53, 54, 55, 56, 19, 57, 32, 0, 1, + 160, 0, 0, 0, 0, 0, 34, 0, 0, 0, + 35, 36, 37, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 38, 0, 39, 40, 41, + 42, 43, 0, 44, 45, 46, 2, 3, 4, 5, + 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 47, 18, 48, 49, 50, 51, 0, 32, + 0, 52, 53, 54, 55, 56, 19, 57, 34, 0, + 0, 0, 35, 36, 37, 2, 3, 4, 5, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 0, 18, 0, 0, 0, 0, 38, 0, 39, + 40, 41, 0, 0, 0, 19, 0, 0, 2, 3, + 4, 5, 6, 7, 8, 9, 10, 11, 0, 0, + 14, 15, 16, 17, 47, 0, 0, 0, 0, 0, + 0, 0, 0, 52, 53, 54, 55, 98, 19, 57, + 32, 0, 1, 0, 0, 0, 0, 0, 0, 34, + 0, 0, 0, 35, 36, 37, 32, 0, 272, 315, + 0, 0, 290, 0, 291, 34, 0, 0, 0, 35, + 36, 37, 0, 0, 0, 0, 0, 0, 38, 0, + 39, 40, 41, 42, 43, 0, 44, 45, 46, 0, + 0, 0, 0, 0, 38, 0, 39, 40, 41, 0, + 0, 0, 0, 0, 0, 47, 0, 48, 49, 50, + 51, 0, 0, 32, 52, 53, 54, 55, 56, 0, + 57, 47, 34, 0, 0, 0, 35, 36, 37, 0, + 52, 53, 54, 55, 98, 32, 57, 272, 0, 0, + 0, 290, 0, 291, 34, 0, 0, 0, 35, 36, + 37, 38, 0, 39, 40, 41, 113, 32, 196, 0, + 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, + 35, 36, 37, 38, 0, 39, 40, 41, 47, 0, + 0, 0, 0, 0, 0, 0, 0, 52, 53, 54, + 55, 98, 0, 57, 0, 38, 0, 39, 40, 41, + 47, 0, 0, 0, 0, 32, 0, 272, 0, 52, + 53, 54, 55, 98, 34, 57, 0, 0, 35, 36, + 37, 0, 47, 0, 0, 0, 0, 104, 0, 0, + 0, 52, 53, 54, 55, 98, 34, 57, 0, 0, + 35, 36, 37, 38, 0, 39, 40, 41, 32, 0, + 0, 0, 0, 0, 0, 275, 0, 34, 0, 0, + 0, 35, 36, 37, 0, 38, 0, 39, 40, 41, + 47, 0, 0, 0, 0, 0, 0, 0, 0, 52, + 53, 54, 55, 98, 0, 57, 38, 0, 39, 40, + 41, 0, 47, 0, 238, 0, 0, 111, 0, 0, + 0, 52, 53, 54, 55, 98, 34, 57, 0, 0, + 35, 36, 37, 47, 0, 0, 0, 0, 32, 0, + 0, 0, 52, 53, 54, 55, 98, 34, 57, 0, + 0, 35, 36, 37, 0, 38, 0, 39, 40, 41, + 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 0, 265, 14, 15, 16, 17, 38, 0, 39, 40, + 41, 0, 47, 0, 0, 0, 0, 0, 0, 0, + 19, 52, 53, 54, 55, 98, 0, 57, 0, 0, + 0, 0, 0, 47, 1, 0, 0, 0, 0, 0, + 0, 0, 52, 53, 54, 55, 98, 0, 57, 2, + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16, 17, 0, 18, 279, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, + 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 0, 18, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 19, 0, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 0, 0, 14, 15, 16, 17, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 19, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 0, 18, 0, 0, 0, 0, 0, 0, 302, 0, + 0, 0, 0, 0, 19, 2, 3, 4, 5, 6, + 7, 8, 9, 10, 11, 0, 0, 14, 15, 16, + 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 19 +}; + +static const yytype_int16 yycheck[] = +{ + 45, 4, 32, 23, 24, 39, 40, 7, 28, 247, + 13, 14, 120, 47, 224, 61, 9, 101, 48, 32, + 1, 4, 32, 167, 4, 7, 5, 61, 136, 47, + 13, 14, 77, 5, 178, 8, 47, 26, 27, 10, + 47, 12, 17, 18, 47, 13, 39, 15, 16, 30, + 260, 159, 6, 6, 5, 13, 14, 28, 76, 5, + 47, 5, 300, 32, 0, 76, 7, 7, 7, 76, + 176, 142, 143, 118, 104, 4, 11, 107, 108, 82, + 110, 111, 82, 93, 168, 169, 47, 100, 4, 102, + 100, 121, 102, 139, 140, 141, 77, 76, 111, 82, + 82, 111, 82, 76, 76, 139, 140, 141, 142, 143, + 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, + 154, 155, 266, 157, 93, 76, 156, 235, 82, 82, + 76, 100, 76, 102, 278, 76, 76, 76, 244, 185, + 222, 76, 111, 236, 252, 229, 272, 231, 13, 14, + 4, 185, 24, 25, 236, 47, 144, 145, 168, 169, + 20, 21, 22, 23, 272, 47, 176, 82, 294, 179, + 8, 42, 254, 3, 4, 43, 146, 147, 148, 149, + 10, 307, 12, 150, 151, 41, 294, 40, 222, 82, + 6, 317, 6, 5, 52, 47, 82, 290, 82, 307, + 28, 246, 236, 68, 249, 4, 76, 176, 290, 317, + 179, 28, 232, 10, 44, 45, 47, 247, 248, 229, + 254, 231, 28, 29, 30, 31, 32, 33, 34, 35, + 36, 37, 38, 1, 244, 76, 4, 4, 6, 7, + 5, 5, 47, 5, 76, 13, 49, 47, 11, 17, + 18, 19, 47, 82, 299, 11, 290, 5, 152, 154, + 157, 179, 153, 278, 166, 155, 175, 233, 169, 289, + 300, 111, 289, 318, 42, 244, 44, 45, 46, 47, + 48, 307, 50, 51, 52, 53, 54, 55, 56, 57, + 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, + 68, 69, 70, 71, 72, 73, 74, 0, -1, 295, + 78, 79, 80, 81, 82, 83, 84, 4, -1, 6, + 7, -1, -1, -1, -1, -1, 13, -1, -1, -1, + 17, 18, 19, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 42, -1, 44, 45, 46, + 47, 48, -1, 50, 51, 52, 53, 54, 55, 56, + 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, + 67, 68, 69, 70, 71, 72, 73, 74, -1, 4, + -1, 78, 79, 80, 81, 82, 83, 84, 13, -1, + -1, -1, 17, 18, 19, 53, 54, 55, 56, 57, + 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, + 68, -1, 70, -1, -1, -1, -1, 42, -1, 44, + 45, 46, -1, -1, -1, 83, -1, -1, 53, 54, + 55, 56, 57, 58, 59, 60, 61, 62, -1, -1, + 65, 66, 67, 68, 69, -1, -1, -1, -1, -1, + -1, -1, -1, 78, 79, 80, 81, 82, 83, 84, + 4, -1, 6, -1, -1, -1, -1, -1, -1, 13, + -1, -1, -1, 17, 18, 19, 4, -1, 6, 7, + -1, -1, 10, -1, 12, 13, -1, -1, -1, 17, + 18, 19, -1, -1, -1, -1, -1, -1, 42, -1, + 44, 45, 46, 47, 48, -1, 50, 51, 52, -1, + -1, -1, -1, -1, 42, -1, 44, 45, 46, -1, + -1, -1, -1, -1, -1, 69, -1, 71, 72, 73, + 74, -1, -1, 4, 78, 79, 80, 81, 82, -1, + 84, 69, 13, -1, -1, -1, 17, 18, 19, -1, + 78, 79, 80, 81, 82, 4, 84, 6, -1, -1, + -1, 10, -1, 12, 13, -1, -1, -1, 17, 18, + 19, 42, -1, 44, 45, 46, 47, 4, 5, -1, + -1, -1, -1, -1, -1, -1, 13, -1, -1, -1, + 17, 18, 19, 42, -1, 44, 45, 46, 69, -1, + -1, -1, -1, -1, -1, -1, -1, 78, 79, 80, + 81, 82, -1, 84, -1, 42, -1, 44, 45, 46, + 69, -1, -1, -1, -1, 4, -1, 6, -1, 78, + 79, 80, 81, 82, 13, 84, -1, -1, 17, 18, + 19, -1, 69, -1, -1, -1, -1, 4, -1, -1, + -1, 78, 79, 80, 81, 82, 13, 84, -1, -1, + 17, 18, 19, 42, -1, 44, 45, 46, 4, -1, + -1, -1, -1, -1, -1, 11, -1, 13, -1, -1, + -1, 17, 18, 19, -1, 42, -1, 44, 45, 46, + 69, -1, -1, -1, -1, -1, -1, -1, -1, 78, + 79, 80, 81, 82, -1, 84, 42, -1, 44, 45, + 46, -1, 69, -1, 7, -1, -1, 4, -1, -1, + -1, 78, 79, 80, 81, 82, 13, 84, -1, -1, + 17, 18, 19, 69, -1, -1, -1, -1, 4, -1, + -1, -1, 78, 79, 80, 81, 82, 13, 84, -1, + -1, 17, 18, 19, -1, 42, -1, 44, 45, 46, + 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, + -1, 5, 65, 66, 67, 68, 42, -1, 44, 45, + 46, -1, 69, -1, -1, -1, -1, -1, -1, -1, + 83, 78, 79, 80, 81, 82, -1, 84, -1, -1, + -1, -1, -1, 69, 6, -1, -1, -1, -1, -1, + -1, -1, 78, 79, 80, 81, 82, -1, 84, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, -1, 70, 7, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, 83, + -1, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 62, 63, 64, 65, 66, 67, 68, -1, 70, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 83, -1, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, -1, -1, 65, 66, 67, 68, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 83, 53, 54, 55, 56, 57, 58, + 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, + -1, 70, -1, -1, -1, -1, -1, -1, 77, -1, + -1, -1, -1, -1, 83, 53, 54, 55, 56, 57, + 58, 59, 60, 61, 62, -1, -1, 65, 66, 67, + 68, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 83 +}; + +/* YYSTOS[STATE-NUM] -- The (internal number of the) accessing + symbol of state STATE-NUM. */ +static const yytype_uint8 yystos[] = +{ + 0, 6, 53, 54, 55, 56, 57, 58, 59, 60, + 61, 62, 63, 64, 65, 66, 67, 68, 70, 83, + 86, 109, 112, 114, 115, 116, 117, 123, 126, 144, + 145, 1, 4, 7, 13, 17, 18, 19, 42, 44, + 45, 46, 47, 48, 50, 51, 52, 69, 71, 72, + 73, 74, 78, 79, 80, 81, 82, 84, 87, 88, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, + 100, 101, 102, 103, 105, 106, 109, 141, 142, 143, + 144, 146, 147, 148, 149, 151, 6, 82, 0, 47, + 110, 112, 112, 6, 82, 112, 109, 47, 82, 106, + 115, 122, 126, 134, 4, 90, 90, 4, 4, 142, + 4, 4, 90, 47, 106, 47, 47, 82, 8, 3, + 4, 10, 12, 44, 45, 28, 29, 30, 31, 32, + 33, 34, 35, 36, 37, 38, 104, 90, 92, 13, + 15, 16, 17, 18, 26, 27, 20, 21, 22, 23, + 24, 25, 42, 43, 41, 40, 9, 39, 47, 76, + 7, 109, 142, 82, 124, 125, 6, 4, 13, 14, + 82, 108, 113, 127, 128, 129, 118, 119, 122, 6, + 5, 122, 129, 135, 122, 5, 106, 106, 150, 52, + 106, 134, 47, 47, 142, 82, 5, 89, 105, 106, + 82, 105, 92, 92, 92, 93, 93, 94, 94, 95, + 95, 95, 95, 96, 96, 97, 98, 99, 100, 106, + 101, 105, 28, 7, 76, 124, 127, 126, 129, 130, + 129, 130, 4, 76, 111, 28, 10, 128, 7, 119, + 47, 120, 121, 127, 118, 92, 5, 47, 4, 5, + 5, 5, 76, 11, 8, 103, 107, 7, 125, 7, + 76, 5, 126, 129, 129, 5, 112, 131, 132, 133, + 113, 47, 6, 105, 136, 11, 107, 47, 76, 7, + 142, 150, 106, 142, 105, 103, 7, 127, 5, 76, + 10, 12, 136, 137, 138, 139, 140, 11, 121, 49, + 47, 5, 77, 133, 107, 82, 7, 76, 136, 28, + 140, 142, 150, 47, 11, 7, 136, 138, 5, 136, + 142 +}; + +#define yyerrok (yyerrstatus = 0) +#define yyclearin (yychar = YYEMPTY) +#define YYEMPTY (-2) +#define YYEOF 0 + +#define YYACCEPT goto yyacceptlab +#define YYABORT goto yyabortlab +#define YYERROR goto yyerrorlab + + +/* Like YYERROR except do call yyerror. This remains here temporarily + to ease the transition to the new meaning of YYERROR, for GCC. + Once GCC version 2 has supplanted version 1, this can go. */ + +#define YYFAIL goto yyerrlab + +#define YYRECOVERING() (!!yyerrstatus) + +#define YYBACKUP(Token, Value) \ +do \ + if (yychar == YYEMPTY && yylen == 1) \ + { \ + yychar = (Token); \ + yylval = (Value); \ + yytoken = YYTRANSLATE (yychar); \ + YYPOPSTACK (1); \ + goto yybackup; \ + } \ + else \ + { \ + yyerror (YY_("syntax error: cannot back up")); \ + YYERROR; \ + } \ +while (YYID (0)) + + +#define YYTERROR 1 +#define YYERRCODE 256 + + +/* YYLLOC_DEFAULT -- Set CURRENT to span from RHS[1] to RHS[N]. + If N is 0, then set CURRENT to the empty location which ends + the previous symbol: RHS[0] (always defined). */ + +#define YYRHSLOC(Rhs, K) ((Rhs)[K]) +#ifndef YYLLOC_DEFAULT +# define YYLLOC_DEFAULT(Current, Rhs, N) \ + do \ + if (YYID (N)) \ + { \ + (Current).first_line = YYRHSLOC (Rhs, 1).first_line; \ + (Current).first_column = YYRHSLOC (Rhs, 1).first_column; \ + (Current).last_line = YYRHSLOC (Rhs, N).last_line; \ + (Current).last_column = YYRHSLOC (Rhs, N).last_column; \ + } \ + else \ + { \ + (Current).first_line = (Current).last_line = \ + YYRHSLOC (Rhs, 0).last_line; \ + (Current).first_column = (Current).last_column = \ + YYRHSLOC (Rhs, 0).last_column; \ + } \ + while (YYID (0)) +#endif + + +/* YY_LOCATION_PRINT -- Print the location on the stream. + This macro was not mandated originally: define only if we know + we won't break user code: when these are the locations we know. */ + +#ifndef YY_LOCATION_PRINT +# if defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL +# define YY_LOCATION_PRINT(File, Loc) \ + fprintf (File, "%d.%d-%d.%d", \ + (Loc).first_line, (Loc).first_column, \ + (Loc).last_line, (Loc).last_column) +# else +# define YY_LOCATION_PRINT(File, Loc) ((void) 0) +# endif +#endif + + +/* YYLEX -- calling `yylex' with the right arguments. */ + +#ifdef YYLEX_PARAM +# define YYLEX yylex (YYLEX_PARAM) +#else +# define YYLEX yylex () +#endif + +/* Enable debugging if requested. */ +#if YYDEBUG + +# ifndef YYFPRINTF +# include /* INFRINGES ON USER NAME SPACE */ +# define YYFPRINTF fprintf +# endif + +# define YYDPRINTF(Args) \ +do { \ + if (yydebug) \ + YYFPRINTF Args; \ +} while (YYID (0)) + +# define YY_SYMBOL_PRINT(Title, Type, Value, Location) \ +do { \ + if (yydebug) \ + { \ + YYFPRINTF (stderr, "%s ", Title); \ + yy_symbol_print (stderr, \ + Type, Value); \ + YYFPRINTF (stderr, "\n"); \ + } \ +} while (YYID (0)) + + +/*--------------------------------. +| Print this symbol on YYOUTPUT. | +`--------------------------------*/ + +/*ARGSUSED*/ +#if (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +static void +yy_symbol_value_print (FILE *yyoutput, int yytype, YYSTYPE const * const yyvaluep) +#else +static void +yy_symbol_value_print (yyoutput, yytype, yyvaluep) + FILE *yyoutput; + int yytype; + YYSTYPE const * const yyvaluep; +#endif +{ + if (!yyvaluep) + return; +# ifdef YYPRINT + if (yytype < YYNTOKENS) + YYPRINT (yyoutput, yytoknum[yytype], *yyvaluep); +# else + YYUSE (yyoutput); +# endif + switch (yytype) + { + default: + break; + } +} + + +/*--------------------------------. +| Print this symbol on YYOUTPUT. | +`--------------------------------*/ + +#if (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +static void +yy_symbol_print (FILE *yyoutput, int yytype, YYSTYPE const * const yyvaluep) +#else +static void +yy_symbol_print (yyoutput, yytype, yyvaluep) + FILE *yyoutput; + int yytype; + YYSTYPE const * const yyvaluep; +#endif +{ + if (yytype < YYNTOKENS) + YYFPRINTF (yyoutput, "token %s (", yytname[yytype]); + else + YYFPRINTF (yyoutput, "nterm %s (", yytname[yytype]); + + yy_symbol_value_print (yyoutput, yytype, yyvaluep); + YYFPRINTF (yyoutput, ")"); +} + +/*------------------------------------------------------------------. +| yy_stack_print -- Print the state stack from its BOTTOM up to its | +| TOP (included). | +`------------------------------------------------------------------*/ + +#if (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +static void +yy_stack_print (yytype_int16 *bottom, yytype_int16 *top) +#else +static void +yy_stack_print (bottom, top) + yytype_int16 *bottom; + yytype_int16 *top; +#endif +{ + YYFPRINTF (stderr, "Stack now"); + for (; bottom <= top; ++bottom) + YYFPRINTF (stderr, " %d", *bottom); + YYFPRINTF (stderr, "\n"); +} + +# define YY_STACK_PRINT(Bottom, Top) \ +do { \ + if (yydebug) \ + yy_stack_print ((Bottom), (Top)); \ +} while (YYID (0)) + + +/*------------------------------------------------. +| Report that the YYRULE is going to be reduced. | +`------------------------------------------------*/ + +#if (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +static void +yy_reduce_print (YYSTYPE *yyvsp, int yyrule) +#else +static void +yy_reduce_print (yyvsp, yyrule) + YYSTYPE *yyvsp; + int yyrule; +#endif +{ + int yynrhs = yyr2[yyrule]; + int yyi; + unsigned long int yylno = yyrline[yyrule]; + YYFPRINTF (stderr, "Reducing stack by rule %d (line %lu):\n", + yyrule - 1, yylno); + /* The symbols being reduced. */ + for (yyi = 0; yyi < yynrhs; yyi++) + { + fprintf (stderr, " $%d = ", yyi + 1); + yy_symbol_print (stderr, yyrhs[yyprhs[yyrule] + yyi], + &(yyvsp[(yyi + 1) - (yynrhs)]) + ); + fprintf (stderr, "\n"); + } +} + +# define YY_REDUCE_PRINT(Rule) \ +do { \ + if (yydebug) \ + yy_reduce_print (yyvsp, Rule); \ +} while (YYID (0)) + +/* Nonzero means print parse trace. It is left uninitialized so that + multiple parsers can coexist. */ +int yydebug; +#else /* !YYDEBUG */ +# define YYDPRINTF(Args) +# define YY_SYMBOL_PRINT(Title, Type, Value, Location) +# define YY_STACK_PRINT(Bottom, Top) +# define YY_REDUCE_PRINT(Rule) +#endif /* !YYDEBUG */ + + +/* YYINITDEPTH -- initial size of the parser's stacks. */ +#ifndef YYINITDEPTH +# define YYINITDEPTH 200 +#endif + +/* YYMAXDEPTH -- maximum size the stacks can grow to (effective only + if the built-in stack extension method is used). + + Do not make this value too large; the results are undefined if + YYSTACK_ALLOC_MAXIMUM < YYSTACK_BYTES (YYMAXDEPTH) + evaluated with infinite-precision integer arithmetic. */ + +#ifndef YYMAXDEPTH +# define YYMAXDEPTH 10000 +#endif + + + +#if YYERROR_VERBOSE + +# ifndef yystrlen +# if defined __GLIBC__ && defined _STRING_H +# define yystrlen strlen +# else +/* Return the length of YYSTR. */ +#if (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +static YYSIZE_T +yystrlen (const char *yystr) +#else +static YYSIZE_T +yystrlen (yystr) + const char *yystr; +#endif +{ + YYSIZE_T yylen; + for (yylen = 0; yystr[yylen]; yylen++) + continue; + return yylen; +} +# endif +# endif + +# ifndef yystpcpy +# if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE +# define yystpcpy stpcpy +# else +/* Copy YYSRC to YYDEST, returning the address of the terminating '\0' in + YYDEST. */ +#if (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +static char * +yystpcpy (char *yydest, const char *yysrc) +#else +static char * +yystpcpy (yydest, yysrc) + char *yydest; + const char *yysrc; +#endif +{ + char *yyd = yydest; + const char *yys = yysrc; + + while ((*yyd++ = *yys++) != '\0') + continue; + + return yyd - 1; +} +# endif +# endif + +# ifndef yytnamerr +/* Copy to YYRES the contents of YYSTR after stripping away unnecessary + quotes and backslashes, so that it's suitable for yyerror. The + heuristic is that double-quoting is unnecessary unless the string + contains an apostrophe, a comma, or backslash (other than + backslash-backslash). YYSTR is taken from yytname. If YYRES is + null, do not copy; instead, return the length of what the result + would have been. */ +static YYSIZE_T +yytnamerr (char *yyres, const char *yystr) +{ + if (*yystr == '"') + { + YYSIZE_T yyn = 0; + char const *yyp = yystr; + + for (;;) + switch (*++yyp) + { + case '\'': + case ',': + goto do_not_strip_quotes; + + case '\\': + if (*++yyp != '\\') + goto do_not_strip_quotes; + /* Fall through. */ + default: + if (yyres) + yyres[yyn] = *yyp; + yyn++; + break; + + case '"': + if (yyres) + yyres[yyn] = '\0'; + return yyn; + } + do_not_strip_quotes: ; + } + + if (! yyres) + return yystrlen (yystr); + + return yystpcpy (yyres, yystr) - yyres; +} +# endif + +/* Copy into YYRESULT an error message about the unexpected token + YYCHAR while in state YYSTATE. Return the number of bytes copied, + including the terminating null byte. If YYRESULT is null, do not + copy anything; just return the number of bytes that would be + copied. As a special case, return 0 if an ordinary "syntax error" + message will do. Return YYSIZE_MAXIMUM if overflow occurs during + size calculation. */ +static YYSIZE_T +yysyntax_error (char *yyresult, int yystate, int yychar) +{ + int yyn = yypact[yystate]; + + if (! (YYPACT_NINF < yyn && yyn <= YYLAST)) + return 0; + else + { + int yytype = YYTRANSLATE (yychar); + YYSIZE_T yysize0 = yytnamerr (0, yytname[yytype]); + YYSIZE_T yysize = yysize0; + YYSIZE_T yysize1; + int yysize_overflow = 0; + enum { YYERROR_VERBOSE_ARGS_MAXIMUM = 5 }; + char const *yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM]; + int yyx; + +# if 0 + /* This is so xgettext sees the translatable formats that are + constructed on the fly. */ + YY_("syntax error, unexpected %s"); + YY_("syntax error, unexpected %s, expecting %s"); + YY_("syntax error, unexpected %s, expecting %s or %s"); + YY_("syntax error, unexpected %s, expecting %s or %s or %s"); + YY_("syntax error, unexpected %s, expecting %s or %s or %s or %s"); +# endif + char *yyfmt; + char const *yyf; + static char const yyunexpected[] = "syntax error, unexpected %s"; + static char const yyexpecting[] = ", expecting %s"; + static char const yyor[] = " or %s"; + char yyformat[sizeof yyunexpected + + sizeof yyexpecting - 1 + + ((YYERROR_VERBOSE_ARGS_MAXIMUM - 2) + * (sizeof yyor - 1))]; + char const *yyprefix = yyexpecting; + + /* Start YYX at -YYN if negative to avoid negative indexes in + YYCHECK. */ + int yyxbegin = yyn < 0 ? -yyn : 0; + + /* Stay within bounds of both yycheck and yytname. */ + int yychecklim = YYLAST - yyn + 1; + int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; + int yycount = 1; + + yyarg[0] = yytname[yytype]; + yyfmt = yystpcpy (yyformat, yyunexpected); + + for (yyx = yyxbegin; yyx < yyxend; ++yyx) + if (yycheck[yyx + yyn] == yyx && yyx != YYTERROR) + { + if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM) + { + yycount = 1; + yysize = yysize0; + yyformat[sizeof yyunexpected - 1] = '\0'; + break; + } + yyarg[yycount++] = yytname[yyx]; + yysize1 = yysize + yytnamerr (0, yytname[yyx]); + yysize_overflow |= (yysize1 < yysize); + yysize = yysize1; + yyfmt = yystpcpy (yyfmt, yyprefix); + yyprefix = yyor; + } + + yyf = YY_(yyformat); + yysize1 = yysize + yystrlen (yyf); + yysize_overflow |= (yysize1 < yysize); + yysize = yysize1; + + if (yysize_overflow) + return YYSIZE_MAXIMUM; + + if (yyresult) + { + /* Avoid sprintf, as that infringes on the user's name space. + Don't have undefined behavior even if the translation + produced a string with the wrong number of "%s"s. */ + char *yyp = yyresult; + int yyi = 0; + while ((*yyp = *yyf) != '\0') + { + if (*yyp == '%' && yyf[1] == 's' && yyi < yycount) + { + yyp += yytnamerr (yyp, yyarg[yyi++]); + yyf += 2; + } + else + { + yyp++; + yyf++; + } + } + } + return yysize; + } +} +#endif /* YYERROR_VERBOSE */ + + +/*-----------------------------------------------. +| Release the memory associated to this symbol. | +`-----------------------------------------------*/ + +/*ARGSUSED*/ +#if (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +static void +yydestruct (const char *yymsg, int yytype, YYSTYPE *yyvaluep) +#else +static void +yydestruct (yymsg, yytype, yyvaluep) + const char *yymsg; + int yytype; + YYSTYPE *yyvaluep; +#endif +{ + YYUSE (yyvaluep); + + if (!yymsg) + yymsg = "Deleting"; + YY_SYMBOL_PRINT (yymsg, yytype, yyvaluep, yylocationp); + + switch (yytype) + { + + default: + break; + } +} + + +/* Prevent warnings from -Wmissing-prototypes. */ + +#ifdef YYPARSE_PARAM +#if defined __STDC__ || defined __cplusplus +int yyparse (void *YYPARSE_PARAM); +#else +int yyparse (); +#endif +#else /* ! YYPARSE_PARAM */ +#if defined __STDC__ || defined __cplusplus +int yyparse (void); +#else +int yyparse (); +#endif +#endif /* ! YYPARSE_PARAM */ + + + +/* The look-ahead symbol. */ +int yychar; + +/* The semantic value of the look-ahead symbol. */ +YYSTYPE yylval; + +/* Number of syntax errors so far. */ +int yynerrs; + + + +/*----------. +| yyparse. | +`----------*/ + +#ifdef YYPARSE_PARAM +#if (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +int +yyparse (void *YYPARSE_PARAM) +#else +int +yyparse (YYPARSE_PARAM) + void *YYPARSE_PARAM; +#endif +#else /* ! YYPARSE_PARAM */ +#if (defined __STDC__ || defined __C99__FUNC__ \ + || defined __cplusplus || defined _MSC_VER) +int +yyparse (void) +#else +int +yyparse () + +#endif +#endif +{ + + int yystate; + int yyn; + int yyresult; + /* Number of tokens to shift before error messages enabled. */ + int yyerrstatus; + /* Look-ahead token as an internal (translated) token number. */ + int yytoken = 0; +#if YYERROR_VERBOSE + /* Buffer for error messages, and its allocated size. */ + char yymsgbuf[128]; + char *yymsg = yymsgbuf; + YYSIZE_T yymsg_alloc = sizeof yymsgbuf; +#endif + + /* Three stacks and their tools: + `yyss': related to states, + `yyvs': related to semantic values, + `yyls': related to locations. + + Refer to the stacks thru separate pointers, to allow yyoverflow + to reallocate them elsewhere. */ + + /* The state stack. */ + yytype_int16 yyssa[YYINITDEPTH]; + yytype_int16 *yyss = yyssa; + yytype_int16 *yyssp; + + /* The semantic value stack. */ + YYSTYPE yyvsa[YYINITDEPTH]; + YYSTYPE *yyvs = yyvsa; + YYSTYPE *yyvsp; + + + +#define YYPOPSTACK(N) (yyvsp -= (N), yyssp -= (N)) + + YYSIZE_T yystacksize = YYINITDEPTH; + + /* The variables used to return semantic value and location from the + action routines. */ + YYSTYPE yyval; + + + /* The number of symbols on the RHS of the reduced rule. + Keep to zero when no symbol should be popped. */ + int yylen = 0; + + YYDPRINTF ((stderr, "Starting parse\n")); + + yystate = 0; + yyerrstatus = 0; + yynerrs = 0; + yychar = YYEMPTY; /* Cause a token to be read. */ + + /* Initialize stack pointers. + Waste one element of value and location stack + so that they stay on the same level as the state stack. + The wasted elements are never initialized. */ + + yyssp = yyss; + yyvsp = yyvs; + + goto yysetstate; + +/*------------------------------------------------------------. +| yynewstate -- Push a new state, which is found in yystate. | +`------------------------------------------------------------*/ + yynewstate: + /* In all cases, when you get here, the value and location stacks + have just been pushed. So pushing a state here evens the stacks. */ + yyssp++; + + yysetstate: + *yyssp = yystate; + + if (yyss + yystacksize - 1 <= yyssp) + { + /* Get the current used size of the three stacks, in elements. */ + YYSIZE_T yysize = yyssp - yyss + 1; + +#ifdef yyoverflow + { + /* Give user a chance to reallocate the stack. Use copies of + these so that the &'s don't force the real ones into + memory. */ + YYSTYPE *yyvs1 = yyvs; + yytype_int16 *yyss1 = yyss; + + + /* Each stack pointer address is followed by the size of the + data in use in that stack, in bytes. This used to be a + conditional around just the two extra args, but that might + be undefined if yyoverflow is a macro. */ + yyoverflow (YY_("memory exhausted"), + &yyss1, yysize * sizeof (*yyssp), + &yyvs1, yysize * sizeof (*yyvsp), + + &yystacksize); + + yyss = yyss1; + yyvs = yyvs1; + } +#else /* no yyoverflow */ +# ifndef YYSTACK_RELOCATE + goto yyexhaustedlab; +# else + /* Extend the stack our own way. */ + if (YYMAXDEPTH <= yystacksize) + goto yyexhaustedlab; + yystacksize *= 2; + if (YYMAXDEPTH < yystacksize) + yystacksize = YYMAXDEPTH; + + { + yytype_int16 *yyss1 = yyss; + union yyalloc *yyptr = + (union yyalloc *) YYSTACK_ALLOC (YYSTACK_BYTES (yystacksize)); + if (! yyptr) + goto yyexhaustedlab; + YYSTACK_RELOCATE (yyss); + YYSTACK_RELOCATE (yyvs); + +# undef YYSTACK_RELOCATE + if (yyss1 != yyssa) + YYSTACK_FREE (yyss1); + } +# endif +#endif /* no yyoverflow */ + + yyssp = yyss + yysize - 1; + yyvsp = yyvs + yysize - 1; + + + YYDPRINTF ((stderr, "Stack size increased to %lu\n", + (unsigned long int) yystacksize)); + + if (yyss + yystacksize - 1 <= yyssp) + YYABORT; + } + + YYDPRINTF ((stderr, "Entering state %d\n", yystate)); + + goto yybackup; + +/*-----------. +| yybackup. | +`-----------*/ +yybackup: + + /* Do appropriate processing given the current state. Read a + look-ahead token if we need one and don't already have one. */ + + /* First try to decide what to do without reference to look-ahead token. */ + yyn = yypact[yystate]; + if (yyn == YYPACT_NINF) + goto yydefault; + + /* Not known => get a look-ahead token if don't already have one. */ + + /* YYCHAR is either YYEMPTY or YYEOF or a valid look-ahead symbol. */ + if (yychar == YYEMPTY) + { + YYDPRINTF ((stderr, "Reading a token: ")); + yychar = YYLEX; + } + + if (yychar <= YYEOF) + { + yychar = yytoken = YYEOF; + YYDPRINTF ((stderr, "Now at end of input.\n")); + } + else + { + yytoken = YYTRANSLATE (yychar); + YY_SYMBOL_PRINT ("Next token is", yytoken, &yylval, &yylloc); + } + + /* If the proper action on seeing token YYTOKEN is to reduce or to + detect an error, take that action. */ + yyn += yytoken; + if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken) + goto yydefault; + yyn = yytable[yyn]; + if (yyn <= 0) + { + if (yyn == 0 || yyn == YYTABLE_NINF) + goto yyerrlab; + yyn = -yyn; + goto yyreduce; + } + + if (yyn == YYFINAL) + YYACCEPT; + + /* Count tokens shifted since error; after three, turn off error + status. */ + if (yyerrstatus) + yyerrstatus--; + + /* Shift the look-ahead token. */ + YY_SYMBOL_PRINT ("Shifting", yytoken, &yylval, &yylloc); + + /* Discard the shifted token unless it is eof. */ + if (yychar != YYEOF) + yychar = YYEMPTY; + + yystate = yyn; + *++yyvsp = yylval; + + goto yynewstate; + + +/*-----------------------------------------------------------. +| yydefault -- do the default action for the current state. | +`-----------------------------------------------------------*/ +yydefault: + yyn = yydefact[yystate]; + if (yyn == 0) + goto yyerrlab; + goto yyreduce; + + +/*-----------------------------. +| yyreduce -- Do a reduction. | +`-----------------------------*/ +yyreduce: + /* yyn is the number of a rule to reduce with. */ + yylen = yyr2[yyn]; + + /* If YYLEN is nonzero, implement the default value of the action: + `$$ = $1'. + + Otherwise, the following line sets YYVAL to garbage. + This behavior is undocumented and Bison + users should not rely upon it. Assigning to YYVAL + unconditionally makes the parser a bit smaller, and it avoids a + GCC warning that YYVAL may be used uninitialized. */ + yyval = yyvsp[1-yylen]; + + + YY_REDUCE_PRINT (yyn); + switch (yyn) + { + case 2: +#line 344 "cod/cod.y" + { + yyparse_value = (sm_ref)(yyvsp[(1) - (1)].list); + ;} + break; + + case 3: +#line 348 "cod/cod.y" + { + yyparse_value = (yyvsp[(1) - (1)].reference); + ;} + break; + + case 4: +#line 354 "cod/cod.y" + { + (yyval.reference) = cod_new_identifier(); + (yyval.reference)->node.identifier.id = (yyvsp[(1) - (1)].info).string; + (yyval.reference)->node.identifier.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + ;} + break; + + case 6: +#line 363 "cod/cod.y" + { (yyval.reference) = (yyvsp[(2) - (3)].reference); ;} + break; + + case 8: +#line 371 "cod/cod.y" + { + (yyval.reference) = cod_new_element_ref(); + (yyval.reference)->node.element_ref.lx_srcpos = (yyvsp[(2) - (4)].info).lx_srcpos; + (yyval.reference)->node.element_ref.expression = (yyvsp[(3) - (4)].reference); + (yyval.reference)->node.element_ref.array_ref = (yyvsp[(1) - (4)].reference); + ;} + break; + + case 9: +#line 378 "cod/cod.y" + { + (yyval.reference) = cod_new_field_ref(); + (yyval.reference)->node.field_ref.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.field_ref.lx_field = (yyvsp[(3) - (3)].info).string; + (yyval.reference)->node.field_ref.struct_ref = (yyvsp[(1) - (3)].reference); + ;} + break; + + case 10: +#line 385 "cod/cod.y" + { + (yyval.reference) = cod_new_subroutine_call(); + (yyval.reference)->node.subroutine_call.lx_srcpos = (yyvsp[(2) - (4)].info).lx_srcpos; + (yyval.reference)->node.subroutine_call.arguments = (yyvsp[(3) - (4)].list); + (yyval.reference)->node.subroutine_call.sm_func_ref = (yyvsp[(1) - (4)].reference); + ;} + break; + + case 11: +#line 392 "cod/cod.y" + { + (yyval.reference) = cod_new_subroutine_call(); + (yyval.reference)->node.subroutine_call.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.subroutine_call.arguments = NULL; + (yyval.reference)->node.subroutine_call.sm_func_ref = (yyvsp[(1) - (3)].reference); + ;} + break; + + case 12: +#line 398 "cod/cod.y" + { + (yyval.reference) = cod_new_field_ref(); + (yyval.reference)->node.field_ref.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.field_ref.lx_field = (yyvsp[(3) - (3)].info).string; + (yyval.reference)->node.field_ref.struct_ref = (yyvsp[(1) - (3)].reference); + ;} + break; + + case 13: +#line 404 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(2) - (2)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_inc; + (yyval.reference)->node.operator.right = NULL; + (yyval.reference)->node.operator.left = (yyvsp[(1) - (2)].reference); + ;} + break; + + case 14: +#line 411 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(2) - (2)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_dec; + (yyval.reference)->node.operator.right = NULL; + (yyval.reference)->node.operator.left = (yyvsp[(1) - (2)].reference); + ;} + break; + + case 15: +#line 421 "cod/cod.y" + { + (yyval.list) = malloc(sizeof(struct list_struct)); + (yyval.list)->node = (yyvsp[(1) - (1)].reference); + (yyval.list)->next = NULL; + ;} + break; + + case 16: +#line 427 "cod/cod.y" + { + sm_list tmp = (yyvsp[(1) - (3)].list); + while (tmp->next != NULL) { + tmp = tmp->next; + } + tmp->next = malloc(sizeof(struct list_struct)); + tmp->next->node = (yyvsp[(3) - (3)].reference); + tmp->next->next = NULL; + (yyval.list) = (yyvsp[(1) - (3)].list); + ;} + break; + + case 18: +#line 442 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(1) - (2)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_inc; + (yyval.reference)->node.operator.right = (yyvsp[(2) - (2)].reference); + (yyval.reference)->node.operator.left = NULL; + ;} + break; + + case 19: +#line 449 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(1) - (2)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_dec; + (yyval.reference)->node.operator.right = (yyvsp[(2) - (2)].reference); + (yyval.reference)->node.operator.left = NULL; + ;} + break; + + case 20: +#line 456 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(1) - (2)].info).lx_srcpos; + (yyval.reference)->node.operator.op = (yyvsp[(1) - (2)].info).op; + (yyval.reference)->node.operator.right = (yyvsp[(2) - (2)].reference); + (yyval.reference)->node.operator.left = NULL; + ;} + break; + + case 21: +#line 463 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(1) - (2)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_sizeof; + (yyval.reference)->node.operator.right = (yyvsp[(2) - (2)].reference); + (yyval.reference)->node.operator.left = NULL; + ;} + break; + + case 22: +#line 470 "cod/cod.y" + { + /* dummy up a cast to hold the sm_list of the type */ + sm_ref cast = cod_new_cast(); + cast->node.cast.lx_srcpos = (yyvsp[(1) - (4)].info).lx_srcpos; + cast->node.cast.type_spec = (yyvsp[(3) - (4)].list); + cast->node.cast.expression = NULL; + + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(1) - (4)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_sizeof; + (yyval.reference)->node.operator.right = cast; + (yyval.reference)->node.operator.left = NULL; + ;} + break; + + case 23: +#line 485 "cod/cod.y" + { + (yyval.info).op = op_address; + ;} + break; + + case 24: +#line 488 "cod/cod.y" + { + (yyval.info).op = op_deref; + ;} + break; + + case 25: +#line 491 "cod/cod.y" + { + (yyval.info).op = op_plus; + ;} + break; + + case 26: +#line 494 "cod/cod.y" + { + (yyval.info).op = op_minus; + ;} + break; + + case 27: +#line 497 "cod/cod.y" + { + (yyval.info).op = op_not; + ;} + break; + + case 28: +#line 500 "cod/cod.y" + { + (yyval.info).op = op_log_neg; + ;} + break; + + case 30: +#line 507 "cod/cod.y" + { + (yyval.reference) = cod_new_cast(); + (yyval.reference)->node.cast.lx_srcpos = (yyvsp[(1) - (4)].info).lx_srcpos; + (yyval.reference)->node.cast.type_spec = (yyvsp[(2) - (4)].list); + (yyval.reference)->node.cast.expression = (yyvsp[(4) - (4)].reference); + ;} + break; + + case 32: +#line 519 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_mult; + (yyval.reference)->node.operator.right = (yyvsp[(3) - (3)].reference); + (yyval.reference)->node.operator.left = (yyvsp[(1) - (3)].reference); + ;} + break; + + case 33: +#line 528 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_div; + (yyval.reference)->node.operator.right = (yyvsp[(3) - (3)].reference); + (yyval.reference)->node.operator.left = (yyvsp[(1) - (3)].reference); + ;} + break; + + case 34: +#line 537 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_modulus; + (yyval.reference)->node.operator.right = (yyvsp[(3) - (3)].reference); + (yyval.reference)->node.operator.left = (yyvsp[(1) - (3)].reference); + ;} + break; + + case 36: +#line 550 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.op = op_plus; + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.operator.right = (yyvsp[(3) - (3)].reference); + (yyval.reference)->node.operator.left = (yyvsp[(1) - (3)].reference); + ;} + break; + + case 37: +#line 559 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_minus; + (yyval.reference)->node.operator.right = (yyvsp[(3) - (3)].reference); + (yyval.reference)->node.operator.left = (yyvsp[(1) - (3)].reference); + ;} + break; + + case 39: +#line 572 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_left_shift; + (yyval.reference)->node.operator.right = (yyvsp[(3) - (3)].reference); + (yyval.reference)->node.operator.left = (yyvsp[(1) - (3)].reference); + ;} + break; + + case 40: +#line 581 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_right_shift; + (yyval.reference)->node.operator.right = (yyvsp[(3) - (3)].reference); + (yyval.reference)->node.operator.left = (yyvsp[(1) - (3)].reference); + ;} + break; + + case 42: +#line 594 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_lt; + (yyval.reference)->node.operator.right = (yyvsp[(3) - (3)].reference); + (yyval.reference)->node.operator.left = (yyvsp[(1) - (3)].reference); + ;} + break; + + case 43: +#line 603 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_gt; + (yyval.reference)->node.operator.right = (yyvsp[(3) - (3)].reference); + (yyval.reference)->node.operator.left = (yyvsp[(1) - (3)].reference); + ;} + break; + + case 44: +#line 612 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_leq; + (yyval.reference)->node.operator.right = (yyvsp[(3) - (3)].reference); + (yyval.reference)->node.operator.left = (yyvsp[(1) - (3)].reference); + ;} + break; + + case 45: +#line 621 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_geq; + (yyval.reference)->node.operator.right = (yyvsp[(3) - (3)].reference); + (yyval.reference)->node.operator.left = (yyvsp[(1) - (3)].reference); + ;} + break; + + case 47: +#line 634 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_eq; + (yyval.reference)->node.operator.right = (yyvsp[(3) - (3)].reference); + (yyval.reference)->node.operator.left = (yyvsp[(1) - (3)].reference); + ;} + break; + + case 48: +#line 643 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_neq; + (yyval.reference)->node.operator.right = (yyvsp[(3) - (3)].reference); + (yyval.reference)->node.operator.left = (yyvsp[(1) - (3)].reference); + ;} + break; + + case 50: +#line 656 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_arith_and; + (yyval.reference)->node.operator.right = (yyvsp[(3) - (3)].reference); + (yyval.reference)->node.operator.left = (yyvsp[(1) - (3)].reference); + ;} + break; + + case 52: +#line 669 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_arith_xor; + (yyval.reference)->node.operator.right = (yyvsp[(3) - (3)].reference); + (yyval.reference)->node.operator.left = (yyvsp[(1) - (3)].reference); + ;} + break; + + case 54: +#line 682 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_arith_or; + (yyval.reference)->node.operator.right = (yyvsp[(3) - (3)].reference); + (yyval.reference)->node.operator.left = (yyvsp[(1) - (3)].reference); + ;} + break; + + case 56: +#line 695 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_log_and; + (yyval.reference)->node.operator.right = (yyvsp[(3) - (3)].reference); + (yyval.reference)->node.operator.left = (yyvsp[(1) - (3)].reference); + ;} + break; + + case 58: +#line 708 "cod/cod.y" + { + (yyval.reference) = cod_new_operator(); + (yyval.reference)->node.operator.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.operator.op = op_log_or; + (yyval.reference)->node.operator.right = (yyvsp[(3) - (3)].reference); + (yyval.reference)->node.operator.left = (yyvsp[(1) - (3)].reference); + ;} + break; + + case 60: +#line 721 "cod/cod.y" + { + (yyval.reference) = cod_new_conditional_operator(); + (yyval.reference)->node.conditional_operator.lx_srcpos = (yyvsp[(2) - (5)].info).lx_srcpos; + (yyval.reference)->node.conditional_operator.condition = (yyvsp[(1) - (5)].reference); + (yyval.reference)->node.conditional_operator.e1 = (yyvsp[(3) - (5)].reference); + (yyval.reference)->node.conditional_operator.e2 = (yyvsp[(5) - (5)].reference); + ;} + break; + + case 61: +#line 732 "cod/cod.y" + { (yyval.info) = (yyvsp[(1) - (1)].info); (yyval.info).op = op_eq;;} + break; + + case 62: +#line 734 "cod/cod.y" + { (yyval.info) = (yyvsp[(1) - (1)].info); (yyval.info).op = op_mult;;} + break; + + case 63: +#line 736 "cod/cod.y" + { (yyval.info) = (yyvsp[(1) - (1)].info); (yyval.info).op = op_div;;} + break; + + case 64: +#line 738 "cod/cod.y" + { (yyval.info) = (yyvsp[(1) - (1)].info); (yyval.info).op = op_modulus;;} + break; + + case 65: +#line 740 "cod/cod.y" + { (yyval.info) = (yyvsp[(1) - (1)].info); (yyval.info).op = op_plus;;} + break; + + case 66: +#line 742 "cod/cod.y" + { (yyval.info) = (yyvsp[(1) - (1)].info); (yyval.info).op = op_minus;;} + break; + + case 67: +#line 744 "cod/cod.y" + { (yyval.info) = (yyvsp[(1) - (1)].info); (yyval.info).op = op_left_shift;;} + break; + + case 68: +#line 746 "cod/cod.y" + { (yyval.info) = (yyvsp[(1) - (1)].info); (yyval.info).op = op_right_shift;;} + break; + + case 69: +#line 748 "cod/cod.y" + { (yyval.info) = (yyvsp[(1) - (1)].info); (yyval.info).op = op_arith_and;;} + break; + + case 70: +#line 750 "cod/cod.y" + { (yyval.info) = (yyvsp[(1) - (1)].info); (yyval.info).op = op_arith_xor;;} + break; + + case 71: +#line 752 "cod/cod.y" + { (yyval.info) = (yyvsp[(1) - (1)].info); (yyval.info).op = op_arith_or;;} + break; + + case 72: +#line 757 "cod/cod.y" + { (yyval.reference) = (yyvsp[(1) - (1)].reference);;} + break; + + case 73: +#line 760 "cod/cod.y" + { + (yyval.reference) = cod_new_assignment_expression(); + (yyval.reference)->node.assignment_expression.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.assignment_expression.left = (yyvsp[(1) - (3)].reference); + (yyval.reference)->node.assignment_expression.right = (yyvsp[(3) - (3)].reference); + (yyval.reference)->node.assignment_expression.op = (yyvsp[(2) - (3)].info).op; + ;} + break; + + case 74: +#line 771 "cod/cod.y" + {(yyval.reference) = (yyvsp[(1) - (1)].reference);;} + break; + + case 75: +#line 773 "cod/cod.y" + { + (yyval.reference) = cod_new_comma_expression(); + (yyval.reference)->node.comma_expression.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.comma_expression.left = (yyvsp[(1) - (3)].reference); + (yyval.reference)->node.comma_expression.right = (yyvsp[(3) - (3)].reference); + ;} + break; + + case 77: +#line 786 "cod/cod.y" + { + (yyval.list) = malloc(sizeof(struct list_struct)); + (yyval.list)->node = (yyvsp[(1) - (1)].reference); + (yyval.list)->next = NULL; + ;} + break; + + case 78: +#line 791 "cod/cod.y" + { + sm_list tmp = (yyvsp[(1) - (3)].list); + while (tmp->next != NULL) { + tmp = tmp->next; + } + tmp->next = malloc(sizeof(struct list_struct)); + tmp = tmp->next; + tmp->node = (yyvsp[(3) - (3)].reference); + tmp->next = NULL; + (yyval.list) = (yyvsp[(1) - (3)].list); + ;} + break; + + case 79: +#line 810 "cod/cod.y" + { + if (parsing_type) { + yyparse_value = (sm_ref) (yyvsp[(1) - (1)].list); + YYACCEPT; + } + ;} + break; + + case 80: +#line 817 "cod/cod.y" + { /* stop here if we're just doing a proc decl */ + if (parsing_param_spec) { + (yyval.reference) = (yyvsp[(3) - (3)].list)->node; + if ((yyval.reference)->node_type == cod_declaration) { + if ((yyval.reference)->node.declaration.type_spec == NULL) { + (yyval.reference)->node.declaration.type_spec = (yyvsp[(1) - (3)].list); + } else { + /* + * the pointer type list (with the declarator) + * goes at the end + */ + sm_list tmp = (yyvsp[(1) - (3)].list); + while (tmp->next != NULL) { + tmp = tmp->next; + } + tmp->next = (yyval.reference)->node.declaration.type_spec; + (yyval.reference)->node.declaration.type_spec = (yyvsp[(1) - (3)].list); + } + } else { + printf("unexpected node in init_declarator\n"); + cod_print((yyval.reference)); + } + yyparse_value = (yyvsp[(3) - (3)].list)->node; + free((yyvsp[(3) - (3)].list)); + YYACCEPT; + } + ;} + break; + + case 81: +#line 845 "cod/cod.y" + { + (yyval.list) = (yyvsp[(3) - (5)].list); + sm_list dtmp = (yyvsp[(3) - (5)].list); + while (dtmp) { + sm_list type_spec; + if (dtmp->next != NULL) { + type_spec = cod_dup_list((yyvsp[(1) - (5)].list)); + } else { + type_spec = (yyvsp[(1) - (5)].list); + } + sm_ref decl = dtmp->node; + if (decl->node_type == cod_declaration) { + if (decl->node.declaration.type_spec == NULL) { + decl->node.declaration.type_spec = type_spec; + } else { + /* + * the pointer type list (with the declarator) + * goes at the end + */ + sm_list tmp = type_spec; + while (tmp->next != NULL) { + tmp = tmp->next; + } + tmp->next = decl->node.declaration.type_spec; + decl->node.declaration.type_spec = type_spec; + } + } else if (decl->node_type == cod_array_type_decl) { + if (decl->node.array_type_decl.type_spec == NULL) { + decl->node.array_type_decl.type_spec = type_spec; + } else { + /* + * the pointer type list (with the declarator) + * goes at the end + */ + sm_list tmp = type_spec; + while (tmp->next != NULL) { + tmp = tmp->next; + } + tmp->next = decl->node.array_type_decl.type_spec; + decl->node.array_type_decl.type_spec = type_spec; + } + } else { + printf("Unknown decl entry\n"); + cod_print(decl); + } + while (type_spec != NULL) { + if (type_spec->node->node.type_specifier.token == TYPEDEF) { + cod_add_defined_type(decl->node.declaration.id, yycontext); + } + type_spec = type_spec->next; + } + dtmp = dtmp->next; + } + (void)(yyvsp[(4) - (5)].reference); + ;} + break; + + case 82: +#line 900 "cod/cod.y" + { + (yyval.list) = (yyvsp[(1) - (2)].list); + ;} + break; + + case 83: +#line 906 "cod/cod.y" + { + (yyval.list) = malloc(sizeof(struct list_struct)); + (yyval.list)->node = (yyvsp[(1) - (1)].reference); + (yyval.list)->next = NULL; + ;} + break; + + case 84: +#line 911 "cod/cod.y" + { + sm_list tmp = malloc(sizeof(struct list_struct)); + tmp->node = (yyvsp[(1) - (2)].reference); + tmp->next = (yyvsp[(2) - (2)].list); + (yyval.list) = tmp; + ;} + break; + + case 85: +#line 917 "cod/cod.y" + { + (yyval.list) = malloc(sizeof(struct list_struct)); + (yyval.list)->node = (yyvsp[(1) - (1)].reference); + (yyval.list)->next = NULL; + ;} + break; + + case 86: +#line 922 "cod/cod.y" + { + sm_list tmp = malloc(sizeof(struct list_struct)); + tmp->node = (yyvsp[(1) - (2)].reference); + tmp->next = (yyvsp[(2) - (2)].list); + (yyval.list) = tmp; + ;} + break; + + case 87: +#line 928 "cod/cod.y" + { + (yyval.list) = malloc(sizeof(struct list_struct)); + (yyval.list)->node = (yyvsp[(1) - (1)].reference); + (yyval.list)->next = NULL; + ;} + break; + + case 88: +#line 933 "cod/cod.y" + { + sm_list tmp = malloc(sizeof(struct list_struct)); + tmp->node = (yyvsp[(1) - (2)].reference); + tmp->next = (yyvsp[(2) - (2)].list); + (yyval.list) = tmp; + ;} + break; + + case 90: +#line 944 "cod/cod.y" + { + if ((yyvsp[(1) - (3)].reference)->node_type == cod_declaration) { + (yyvsp[(1) - (3)].reference)->node.declaration.init_value = (yyvsp[(3) - (3)].reference); + } else if ((yyvsp[(1) - (3)].reference)->node_type == cod_array_type_decl) { + sm_ref tmp = (yyvsp[(1) - (3)].reference)->node.array_type_decl.element_ref; + while (tmp->node_type == cod_array_type_decl) { + tmp = tmp->node.array_type_decl.element_ref; + } + assert(tmp->node_type == cod_declaration); + tmp->node.declaration.init_value = (yyvsp[(3) - (3)].reference); + } + ;} + break; + + case 91: +#line 959 "cod/cod.y" + { + (yyval.reference) = cod_new_type_specifier(); + (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + (yyval.reference)->node.type_specifier.token = TYPEDEF; + ;} + break; + + case 92: +#line 964 "cod/cod.y" + { + (yyval.reference) = cod_new_type_specifier(); + (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + (yyval.reference)->node.type_specifier.token = STATIC; + ;} + break; + + case 93: +#line 969 "cod/cod.y" + { + (yyval.reference) = cod_new_type_specifier(); + (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + (yyval.reference)->node.type_specifier.token = EXTERN_TOKEN; + ;} + break; + + case 94: +#line 978 "cod/cod.y" + { + (yyval.reference) = cod_new_type_specifier(); + (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + (yyval.reference)->node.type_specifier.token = CHAR; + ;} + break; + + case 95: +#line 983 "cod/cod.y" + { + (yyval.reference) = cod_new_type_specifier(); + (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + (yyval.reference)->node.type_specifier.token = SHORT; + ;} + break; + + case 96: +#line 988 "cod/cod.y" + { + (yyval.reference) = cod_new_type_specifier(); + (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + (yyval.reference)->node.type_specifier.token = INT; + ;} + break; + + case 97: +#line 993 "cod/cod.y" + { + (yyval.reference) = cod_new_type_specifier(); + (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + (yyval.reference)->node.type_specifier.token = LONG; + ;} + break; + + case 98: +#line 998 "cod/cod.y" + { + (yyval.reference) = cod_new_type_specifier(); + (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + (yyval.reference)->node.type_specifier.token = FLOAT; + ;} + break; + + case 99: +#line 1003 "cod/cod.y" + { + (yyval.reference) = cod_new_type_specifier(); + (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + (yyval.reference)->node.type_specifier.token = DOUBLE; + ;} + break; + + case 100: +#line 1008 "cod/cod.y" + { + (yyval.reference) = cod_new_type_specifier(); + (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + (yyval.reference)->node.type_specifier.token = VOID; + ;} + break; + + case 101: +#line 1013 "cod/cod.y" + { + (yyval.reference) = cod_new_type_specifier(); + (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + (yyval.reference)->node.type_specifier.token = SIGNED; + ;} + break; + + case 102: +#line 1018 "cod/cod.y" + { + (yyval.reference) = cod_new_type_specifier(); + (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + (yyval.reference)->node.type_specifier.token = UNSIGNED; + ;} + break; + + case 103: +#line 1023 "cod/cod.y" + { + (yyval.reference) = cod_new_type_specifier(); + (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + (yyval.reference)->node.type_specifier.token = STRING; + ;} + break; + + case 104: +#line 1028 "cod/cod.y" + { + (yyval.reference) = cod_new_identifier(); + (yyval.reference)->node.identifier.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + (yyval.reference)->node.identifier.id = (yyvsp[(1) - (1)].info).string; + ;} + break; + + case 105: +#line 1033 "cod/cod.y" + { + (yyval.reference) = (yyvsp[(1) - (1)].reference); + ;} + break; + + case 106: +#line 1036 "cod/cod.y" + { + (yyval.reference) = (yyvsp[(1) - (1)].reference); + ;} + break; + + case 107: +#line 1042 "cod/cod.y" + { + (yyval.reference) = cod_build_parsed_type_node(yycontext, (yyvsp[(2) - (5)].info).string, (yyvsp[(4) - (5)].list)); + ;} + break; + + case 108: +#line 1045 "cod/cod.y" + { + (yyval.reference) = cod_build_parsed_type_node(yycontext, strdup("anon"), (yyvsp[(3) - (4)].list)); + ;} + break; + + case 109: +#line 1048 "cod/cod.y" + { + (yyval.reference) = cod_build_parsed_type_node(yycontext, (yyvsp[(2) - (2)].info).string, NULL); + ;} + break; + + case 111: +#line 1055 "cod/cod.y" + { + yyerror("UNIONs not supported!"); + ;} + break; + + case 113: +#line 1062 "cod/cod.y" + { + sm_list tmp = (yyvsp[(1) - (2)].list); + while (tmp->next != NULL) { + tmp = tmp->next; + } + tmp->next =(yyvsp[(2) - (2)].list); + (yyval.list) = (yyvsp[(1) - (2)].list); + ;} + break; + + case 114: +#line 1074 "cod/cod.y" + { ;} + break; + + case 115: +#line 1075 "cod/cod.y" + { + sm_list type_spec = (yyvsp[(1) - (3)].list); + sm_list decl_list = (yyvsp[(2) - (3)].list); + (yyval.list) = (yyvsp[(2) - (3)].list); +/******** GSE This isn't right. Reusing potentially modified type spec */ + while (decl_list != NULL) { + sm_ref decl = decl_list->node; + if (decl->node_type == cod_declaration) { + if (decl->node.declaration.type_spec == NULL) { + decl->node.declaration.type_spec = type_spec; + } else { + /* + * the pointer type list (with the declarator) + * goes at the end + */ + sm_list tmp = (yyvsp[(1) - (3)].list); + while (tmp->next != NULL) { + tmp = tmp->next; + } + tmp->next = decl->node.declaration.type_spec; + decl->node.declaration.type_spec = type_spec; + } + } else if (decl->node_type == cod_array_type_decl) { + if (decl->node.array_type_decl.type_spec == NULL) { + decl->node.array_type_decl.type_spec = type_spec; + } else { + /* + * the pointer type list (with the declarator) + * goes at the end + */ + sm_list tmp = type_spec; + while (tmp->next != NULL) { + tmp = tmp->next; + } + tmp->next = decl->node.array_type_decl.type_spec; + decl->node.array_type_decl.type_spec = type_spec; + } + } else { + printf("Unknown decl entry\n"); + cod_print(decl); + } + decl_list = decl_list->next; + if (decl_list != NULL) { + type_spec = cod_dup_list(type_spec); + } + } + ;} + break; + + case 116: +#line 1126 "cod/cod.y" + { + (yyval.list) = malloc(sizeof(struct list_struct)); + (yyval.list)->node = (yyvsp[(1) - (1)].reference); + (yyval.list)->next = NULL; + ;} + break; + + case 117: +#line 1131 "cod/cod.y" + { + sm_list tmp = (yyvsp[(1) - (3)].list); + while (tmp->next != NULL) { + tmp = tmp->next; + } + tmp->next = malloc(sizeof(struct list_struct)); + tmp->next->node = (yyvsp[(3) - (3)].reference); + tmp->next->next = NULL; + (yyval.list) = (yyvsp[(1) - (3)].list); + ;} + break; + + case 119: +#line 1146 "cod/cod.y" + { + sm_list tmp = malloc(sizeof(struct list_struct)); + tmp->node = (yyvsp[(1) - (2)].reference); + tmp->next = (yyvsp[(2) - (2)].list); + (yyval.list) = tmp; + ;} + break; + + case 120: +#line 1152 "cod/cod.y" + { + (yyval.list) = malloc(sizeof(struct list_struct)); + (yyval.list)->node = (yyvsp[(1) - (1)].reference); + (yyval.list)->next = NULL; + ;} + break; + + case 121: +#line 1157 "cod/cod.y" + { + sm_list tmp = malloc(sizeof(struct list_struct)); + tmp->node = (yyvsp[(1) - (2)].reference); + tmp->next = (yyvsp[(2) - (2)].list); + (yyval.list) = tmp; + ;} + break; + + case 122: +#line 1163 "cod/cod.y" + { + (yyval.list) = malloc(sizeof(struct list_struct)); + (yyval.list)->node = (yyvsp[(1) - (1)].reference); + (yyval.list)->next = NULL; + ;} + break; + + case 123: +#line 1171 "cod/cod.y" + { + (yyval.reference) = cod_new_enum_type_decl(); + (yyval.reference)->node.enum_type_decl.id = gen_anon(); + (yyval.reference)->node.enum_type_decl.enums = (yyvsp[(3) - (4)].list); + (yyval.reference)->node.enum_type_decl.lx_srcpos = (yyvsp[(1) - (4)].info).lx_srcpos; + // cod_add_defined_type(decl->node.declaration.id, yycontext); + ;} + break; + + case 124: +#line 1178 "cod/cod.y" + { + (yyval.reference) = cod_new_enum_type_decl(); + (yyval.reference)->node.enum_type_decl.id = gen_anon(); + (yyval.reference)->node.enum_type_decl.enums = (yyvsp[(3) - (5)].list); + (yyval.reference)->node.enum_type_decl.lx_srcpos = (yyvsp[(1) - (5)].info).lx_srcpos; + // cod_add_defined_type(decl->node.declaration.id, yycontext); + ;} + break; + + case 125: +#line 1185 "cod/cod.y" + { + (yyval.reference) = cod_new_enum_type_decl(); + (yyval.reference)->node.enum_type_decl.id = (yyvsp[(2) - (5)].info).string; + (yyval.reference)->node.enum_type_decl.enums = (yyvsp[(4) - (5)].list); + (yyval.reference)->node.enum_type_decl.lx_srcpos = (yyvsp[(1) - (5)].info).lx_srcpos; + // cod_add_defined_type(decl->node.declaration.id, yycontext); + ;} + break; + + case 126: +#line 1192 "cod/cod.y" + { + (yyval.reference) = cod_new_enum_type_decl(); + (yyval.reference)->node.enum_type_decl.id = (yyvsp[(2) - (6)].info).string; + (yyval.reference)->node.enum_type_decl.enums = (yyvsp[(4) - (6)].list); + (yyval.reference)->node.enum_type_decl.lx_srcpos = (yyvsp[(1) - (6)].info).lx_srcpos; + // cod_add_defined_type(decl->node.declaration.id, yycontext); + ;} + break; + + case 127: +#line 1199 "cod/cod.y" + { + (yyval.reference) = cod_new_enum_type_decl(); + (yyval.reference)->node.enum_type_decl.id = (yyvsp[(2) - (2)].info).string; + (yyval.reference)->node.enum_type_decl.enums = NULL; + (yyval.reference)->node.enum_type_decl.lx_srcpos = (yyvsp[(1) - (2)].info).lx_srcpos; + // cod_add_defined_type(decl->node.declaration.id, yycontext); + ;} + break; + + case 128: +#line 1209 "cod/cod.y" + { + sm_list tmp = malloc(sizeof(struct list_struct)); + tmp->node = (yyvsp[(1) - (1)].reference); + tmp->next = NULL; + (yyval.list) = tmp; + ;} + break; + + case 129: +#line 1215 "cod/cod.y" + { + sm_list tmp = malloc(sizeof(struct list_struct)); + tmp->node = (yyvsp[(3) - (3)].reference); + tmp->next = (yyvsp[(1) - (3)].list); + (yyval.list) = tmp; + ;} + break; + + case 130: +#line 1225 "cod/cod.y" + { + (yyval.reference) = cod_new_enumerator(); + (yyval.reference)->node.enumerator.id = (yyvsp[(1) - (3)].info).string; + (yyval.reference)->node.enumerator.const_expression = (yyvsp[(3) - (3)].reference); + ;} + break; + + case 131: +#line 1231 "cod/cod.y" + { + (yyval.reference) = cod_new_enumerator(); + (yyval.reference)->node.enumerator.id = (yyvsp[(1) - (1)].info).string; + (yyval.reference)->node.enumerator.const_expression = NULL; + ;} + break; + + case 132: +#line 1239 "cod/cod.y" + { + (yyval.reference) = cod_new_type_specifier(); + (yyval.reference)->node.type_specifier.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + (yyval.reference)->node.type_specifier.token = CONST; + ;} + break; + + case 134: +#line 1249 "cod/cod.y" + { + (yyval.reference) = (yyvsp[(2) - (2)].reference); + if ((yyval.reference)->node_type == cod_declaration) { + (yyval.reference)->node.declaration.type_spec = (yyvsp[(1) - (2)].list); + } else if ((yyval.reference)->node_type == cod_array_type_decl) { + (yyval.reference)->node.array_type_decl.type_spec = (yyvsp[(1) - (2)].list); + } else { + printf("Unknown direct_declarator entry\n"); + cod_print((yyval.reference)); + } + ;} + break; + + case 135: +#line 1263 "cod/cod.y" + { + (yyval.reference) = cod_new_declaration(); + (yyval.reference)->node.declaration.param_num = -1; + (yyval.reference)->node.declaration.id = (yyvsp[(1) - (1)].info).string; + (yyval.reference)->node.declaration.init_value = NULL; + (yyval.reference)->node.declaration.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + (yyval.reference)->node.declaration.is_subroutine = 0; + (yyval.reference)->node.declaration.params = NULL; + ;} + break; + + case 136: +#line 1272 "cod/cod.y" + { + (yyval.reference) = (yyvsp[(2) - (3)].reference); + ;} + break; + + case 137: +#line 1275 "cod/cod.y" + { + (yyval.reference) = cod_new_declaration(); + (yyval.reference)->node.declaration.param_num = -1; + (yyval.reference)->node.declaration.id = (yyvsp[(1) - (4)].info).string; + (yyval.reference)->node.declaration.init_value = NULL; + (yyval.reference)->node.declaration.lx_srcpos = (yyvsp[(1) - (4)].info).lx_srcpos; + (yyval.reference)->node.declaration.is_subroutine = 1; + (yyval.reference)->node.declaration.params = (yyvsp[(3) - (4)].list); + ;} + break; + + case 138: +#line 1284 "cod/cod.y" + { + (yyval.reference) = cod_new_declaration(); + (yyval.reference)->node.declaration.param_num = -1; + (yyval.reference)->node.declaration.id = (yyvsp[(1) - (3)].info).string; + (yyval.reference)->node.declaration.init_value = NULL; + (yyval.reference)->node.declaration.lx_srcpos = (yyvsp[(1) - (3)].info).lx_srcpos; + (yyval.reference)->node.declaration.is_subroutine = 1; + (yyval.reference)->node.declaration.params = NULL; + ;} + break; + + case 139: +#line 1293 "cod/cod.y" + { + (yyval.reference) = cod_new_array_type_decl(); + (yyval.reference)->node.array_type_decl.lx_srcpos = (yyvsp[(2) - (4)].info).lx_srcpos; + (yyval.reference)->node.array_type_decl.size_expr = (yyvsp[(3) - (4)].reference); + (yyval.reference)->node.array_type_decl.element_ref = (yyvsp[(1) - (4)].reference); + (yyval.reference)->node.array_type_decl.sm_dynamic_size = NULL; + ;} + break; + + case 140: +#line 1300 "cod/cod.y" + { + (yyval.reference) = cod_new_array_type_decl(); + (yyval.reference)->node.array_type_decl.lx_srcpos = (yyvsp[(2) - (3)].info).lx_srcpos; + (yyval.reference)->node.array_type_decl.size_expr = NULL; + (yyval.reference)->node.array_type_decl.element_ref = (yyvsp[(1) - (3)].reference); + (yyval.reference)->node.array_type_decl.sm_dynamic_size = NULL; + ;} + break; + + case 141: +#line 1310 "cod/cod.y" + { + sm_ref star = cod_new_type_specifier(); + star->node.type_specifier.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + star->node.type_specifier.token = STAR; + (yyval.list) = malloc(sizeof(struct list_struct)); + (yyval.list)->node = star; + (yyval.list)->next = NULL; + ;} + break; + + case 142: +#line 1318 "cod/cod.y" + { + sm_ref star = cod_new_type_specifier(); + star->node.type_specifier.lx_srcpos = (yyvsp[(1) - (2)].info).lx_srcpos; + star->node.type_specifier.token = STAR; + (yyval.list) = malloc(sizeof(struct list_struct)); + (yyval.list)->node = star; + (yyval.list)->next = (yyvsp[(2) - (2)].list); + ;} + break; + + case 143: +#line 1326 "cod/cod.y" + { + sm_ref star = cod_new_type_specifier(); + star->node.type_specifier.lx_srcpos = (yyvsp[(1) - (2)].info).lx_srcpos; + star->node.type_specifier.token = STAR; + (yyval.list) = malloc(sizeof(struct list_struct)); + (yyval.list)->node = star; + (yyval.list)->next = (yyvsp[(2) - (2)].list); + ;} + break; + + case 144: +#line 1334 "cod/cod.y" + { + sm_list tmp = (yyvsp[(2) - (3)].list); + sm_ref star = cod_new_type_specifier(); + star->node.type_specifier.lx_srcpos = (yyvsp[(1) - (3)].info).lx_srcpos; + star->node.type_specifier.token = STAR; + + while (tmp->next != NULL) { + tmp = tmp->next; + } + tmp->next = (yyvsp[(3) - (3)].list); + (yyval.list) = malloc(sizeof(struct list_struct)); + (yyval.list)->node = star; + (yyval.list)->next = (yyvsp[(2) - (3)].list); + ;} + break; + + case 145: +#line 1348 "cod/cod.y" + { + sm_ref star = cod_new_type_specifier(); + if(!cod_segmented_pointers) { + yyerror("Segmented pointers disabled!"); + } + star->node.type_specifier.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + star->node.type_specifier.token = AT; + (yyval.list) = malloc(sizeof(struct list_struct)); + (yyval.list)->node = star; + (yyval.list)->next = NULL; + ;} + break; + + case 146: +#line 1359 "cod/cod.y" + { + sm_ref star = cod_new_type_specifier(); + if(!cod_segmented_pointers) { + yyerror("Segmented pointers disabled!"); + } + star->node.type_specifier.lx_srcpos = (yyvsp[(1) - (2)].info).lx_srcpos; + star->node.type_specifier.token = AT; + (yyval.list) = malloc(sizeof(struct list_struct)); + (yyval.list)->node = star; + (yyval.list)->next = (yyvsp[(2) - (2)].list); + ;} + break; + + case 147: +#line 1370 "cod/cod.y" + { + sm_ref star = cod_new_type_specifier(); + if(!cod_segmented_pointers) { + yyerror("Segmented pointers disabled!"); + } + star->node.type_specifier.lx_srcpos = (yyvsp[(1) - (2)].info).lx_srcpos; + star->node.type_specifier.token = AT; + (yyval.list) = malloc(sizeof(struct list_struct)); + (yyval.list)->node = star; + (yyval.list)->next = (yyvsp[(2) - (2)].list); + ;} + break; + + case 148: +#line 1381 "cod/cod.y" + { + sm_list tmp = (yyvsp[(2) - (3)].list); + sm_ref star = cod_new_type_specifier(); + if(!cod_segmented_pointers) { + yyerror("Segmented pointers disabled!"); + } + star->node.type_specifier.lx_srcpos = (yyvsp[(1) - (3)].info).lx_srcpos; + star->node.type_specifier.token = AT; + + while (tmp->next != NULL) { + tmp = tmp->next; + } + tmp->next = (yyvsp[(3) - (3)].list); + (yyval.list) = malloc(sizeof(struct list_struct)); + (yyval.list)->node = star; + (yyval.list)->next = (yyvsp[(2) - (3)].list); + ;} + break; + + case 149: +#line 1401 "cod/cod.y" + { + (yyval.list) = malloc(sizeof(struct list_struct)); + (yyval.list)->node = (yyvsp[(1) - (1)].reference); + (yyval.list)->next = NULL; + ;} + break; + + case 150: +#line 1406 "cod/cod.y" + { + sm_list tmp = (yyvsp[(1) - (2)].list); + while (tmp->next != NULL) { + tmp = tmp->next; + } + tmp->next = malloc(sizeof(struct list_struct)); + tmp->next->node = (yyvsp[(2) - (2)].reference); + tmp->next->next = NULL; + (yyval.list) = (yyvsp[(1) - (2)].list); + ;} + break; + + case 152: +#line 1420 "cod/cod.y" + { + sm_list tmp = (yyvsp[(1) - (3)].list); + sm_ref id = cod_new_declaration(); + while (tmp->next != NULL) { + tmp = tmp->next; + } + tmp->next = malloc(sizeof(struct list_struct)); + tmp->next->node = id; + tmp->next->next = NULL; + id->node.declaration.id = strdup("..."); + (yyval.list) = (yyvsp[(1) - (3)].list); + ;} + break; + + case 153: +#line 1435 "cod/cod.y" + { + (yyval.list) = malloc(sizeof(struct list_struct)); + (yyval.list)->node = (yyvsp[(1) - (1)].reference); + (yyval.list)->next = NULL; + ;} + break; + + case 154: +#line 1441 "cod/cod.y" + { + sm_list tmp = (yyvsp[(1) - (3)].list); + while (tmp->next != NULL) { + tmp = tmp->next; + } + tmp->next = malloc(sizeof(struct list_struct)); + tmp->next->node = (yyvsp[(3) - (3)].reference); + tmp->next->next = NULL; + (yyval.list) = (yyvsp[(1) - (3)].list); + ;} + break; + + case 155: +#line 1456 "cod/cod.y" + { + (yyval.reference) = cod_new_declaration(); + (yyval.reference)->node.declaration.param_num = -1; + (yyval.reference)->node.declaration.id = gen_anon(); + (yyval.reference)->node.declaration.init_value = NULL; + (yyval.reference)->node.declaration.is_subroutine = 0; + (yyval.reference)->node.declaration.params = NULL; + (yyval.reference)->node.declaration.type_spec = (yyvsp[(1) - (1)].list); + ;} + break; + + case 156: +#line 1465 "cod/cod.y" + { + (yyval.reference) = (yyvsp[(2) - (2)].reference); + if ((yyval.reference)->node_type == cod_declaration) { + (yyval.reference)->node.declaration.static_var = 0; + if ((yyval.reference)->node.declaration.type_spec == NULL) { + (yyval.reference)->node.declaration.type_spec = (yyvsp[(1) - (2)].list); + } else { + /* + * the pointer type list (with the declarator) + * goes at the end + */ + sm_list tmp = (yyvsp[(1) - (2)].list); + while (tmp->next != NULL) { + tmp = tmp->next; + } + tmp->next = (yyval.reference)->node.declaration.type_spec; + (yyval.reference)->node.declaration.type_spec = (yyvsp[(1) - (2)].list); + } + } else if ((yyval.reference)->node_type == cod_array_type_decl) { + if ((yyval.reference)->node.array_type_decl.type_spec == NULL) { + (yyval.reference)->node.array_type_decl.type_spec = (yyvsp[(1) - (2)].list); + } else { + /* + * the pointer type list (with the declarator) + * goes at the end + */ + sm_list tmp = (yyvsp[(1) - (2)].list); + while (tmp->next != NULL) { + tmp = tmp->next; + } + tmp->next = (yyval.reference)->node.array_type_decl.type_spec; + (yyval.reference)->node.array_type_decl.type_spec = (yyvsp[(1) - (2)].list); + } + } else { + printf("unexpected node in parameter_declaration"); + } + ;} + break; + + case 158: +#line 1505 "cod/cod.y" + { + sm_list tmp = (yyvsp[(1) - (2)].list); + while (tmp->next != NULL) { + tmp = tmp->next; + } + tmp->next = (yyvsp[(2) - (2)].list); + (yyval.list) = (yyvsp[(1) - (2)].list); + ;} + break; + + case 160: +#line 1556 "cod/cod.y" + { + (yyval.reference) = cod_new_initializer_list(); + (yyval.reference)->node.initializer_list.initializers = (yyvsp[(2) - (3)].list); + ;} + break; + + case 161: +#line 1561 "cod/cod.y" + { + (yyval.reference) = cod_new_initializer_list(); + (yyval.reference)->node.initializer_list.initializers = (yyvsp[(2) - (4)].list); + ;} + break; + + case 162: +#line 1565 "cod/cod.y" + { (yyval.reference) = (yyvsp[(1) - (1)].reference);;} + break; + + case 163: +#line 1570 "cod/cod.y" + { + sm_ref initializer = cod_new_initializer(); + initializer->node.initializer.designation = (yyvsp[(1) - (2)].list); + initializer->node.initializer.initializer = (yyvsp[(2) - (2)].reference); + (yyval.list) = malloc(sizeof(struct list_struct)); + (yyval.list)->node = initializer; + (yyval.list)->next = NULL; + ;} + break; + + case 164: +#line 1578 "cod/cod.y" + { + sm_ref initializer = cod_new_initializer(); + initializer->node.initializer.designation = NULL; + initializer->node.initializer.initializer = (yyvsp[(1) - (1)].reference); + (yyval.list) = malloc(sizeof(struct list_struct)); + (yyval.list)->node = initializer; + (yyval.list)->next = NULL; + ;} + break; + + case 165: +#line 1586 "cod/cod.y" + { + sm_list tmp = (yyvsp[(1) - (4)].list); + sm_ref initializer = cod_new_initializer(); + initializer->node.initializer.designation = (yyvsp[(3) - (4)].list); + initializer->node.initializer.initializer = (yyvsp[(4) - (4)].reference); + while (tmp->next != NULL) { + tmp = tmp->next; + } + tmp->next = malloc(sizeof(struct list_struct)); + tmp->next->node = initializer; + tmp->next->next = NULL; + (yyval.list) = (yyvsp[(1) - (4)].list); + ;} + break; + + case 166: +#line 1599 "cod/cod.y" + { + sm_list tmp = (yyvsp[(1) - (3)].list); + sm_ref initializer = cod_new_initializer(); + initializer->node.initializer.designation = NULL; + initializer->node.initializer.initializer = (yyvsp[(3) - (3)].reference); + while (tmp->next != NULL) { + tmp = tmp->next; + } + tmp->next = malloc(sizeof(struct list_struct)); + tmp->next->node = initializer; + tmp->next->next = NULL; + (yyval.list) = (yyvsp[(1) - (3)].list); + ;} + break; + + case 167: +#line 1616 "cod/cod.y" + { (yyval.list) = (yyvsp[(1) - (2)].list);;} + break; + + case 168: +#line 1620 "cod/cod.y" + { + (yyval.list) = malloc(sizeof(struct list_struct)); + (yyval.list)->node = (yyvsp[(1) - (1)].reference); + (yyval.list)->next = NULL; + ;} + break; + + case 169: +#line 1625 "cod/cod.y" + { + sm_list tmp = (yyvsp[(1) - (2)].list); + while (tmp->next != NULL) { + tmp = tmp->next; + } + tmp->next = malloc(sizeof(struct list_struct)); + tmp->next->node = (yyvsp[(2) - (2)].reference); + tmp->next->next = NULL; + (yyval.list) = (yyvsp[(1) - (2)].list); + ;} + break; + + case 170: +#line 1639 "cod/cod.y" + { + (yyval.reference) = cod_new_designator(); + (yyval.reference)->node.designator.expression = (yyvsp[(2) - (3)].reference); + (yyval.reference)->node.designator.id = NULL; + ;} + break; + + case 171: +#line 1645 "cod/cod.y" + { + (yyval.reference) = cod_new_designator(); + (yyval.reference)->node.designator.expression = NULL; + (yyval.reference)->node.designator.id = (yyvsp[(2) - (2)].info).string; + ;} + break; + + case 172: +#line 1653 "cod/cod.y" + { + sm_list tmp = malloc(sizeof(struct list_struct)); + tmp->node = (yyvsp[(1) - (1)].reference); + tmp->next = NULL; + (yyval.list) = tmp; + ;} + break; + + case 173: +#line 1659 "cod/cod.y" + { + (yyval.list) = (yyvsp[(1) - (1)].list); + ;} + break; + + case 174: +#line 1662 "cod/cod.y" + { + (yyval.list) = NULL; + ;} + break; + + case 175: +#line 1665 "cod/cod.y" + { + sm_list tmp = malloc(sizeof(struct list_struct)); + tmp->node = (yyvsp[(2) - (2)].reference); + tmp->next = NULL; + (yyval.list) = cod_append_list((yyvsp[(1) - (2)].list), tmp); + ;} + break; + + case 176: +#line 1671 "cod/cod.y" + { + (yyval.list) = cod_append_list((yyvsp[(1) - (2)].list), (yyvsp[(2) - (2)].list)); + ;} + break; + + case 183: +#line 1688 "cod/cod.y" + { + (yyval.reference) = cod_new_label_statement(); + (yyval.reference)->node.label_statement.name = (yyvsp[(1) - (3)].info).string; + (yyval.reference)->node.label_statement.statement = (yyvsp[(3) - (3)].reference); + ;} + break; + + case 184: +#line 1695 "cod/cod.y" + { + (yyval.reference) = cod_new_compound_statement(); + ;} + break; + + case 185: +#line 1698 "cod/cod.y" + { + int count = (yyvsp[(1) - (3)].info).type_stack_count; + (yyval.reference) = cod_new_compound_statement(); + (yyval.reference)->node.compound_statement.decls = (yyvsp[(2) - (3)].list); + cod_remove_defined_types(yycontext, count); + ;} + break; + + case 186: +#line 1706 "cod/cod.y" + { (yyval.list) = (yyvsp[(1) - (1)].list); ;} + break; + + case 187: +#line 1708 "cod/cod.y" + { + if ((yyvsp[(1) - (2)].list) == NULL) { + (yyval.list) = (yyvsp[(2) - (2)].list); + } else { + sm_list tmp = (yyvsp[(1) - (2)].list); + while (tmp->next != NULL) { + tmp = tmp->next; + } + tmp->next = (yyvsp[(2) - (2)].list); + (yyval.list) = (yyvsp[(1) - (2)].list); + } + ;} + break; + + case 188: +#line 1722 "cod/cod.y" + { + (yyval.reference) = cod_new_return_statement(); + (yyval.reference)->node.return_statement.expression = (yyvsp[(2) - (3)].reference); + (yyval.reference)->node.return_statement.lx_srcpos = (yyvsp[(1) - (3)].info).lx_srcpos; + ;} + break; + + case 189: +#line 1727 "cod/cod.y" + { + (yyval.reference) = cod_new_return_statement(); + (yyval.reference)->node.return_statement.expression = NULL; + (yyval.reference)->node.return_statement.lx_srcpos = (yyvsp[(1) - (2)].info).lx_srcpos; + ;} + break; + + case 190: +#line 1732 "cod/cod.y" + { + (yyval.reference) = cod_new_jump_statement(); + (yyval.reference)->node.jump_statement.continue_flag = 1; + (yyval.reference)->node.jump_statement.goto_target = NULL; + (yyval.reference)->node.jump_statement.lx_srcpos = (yyvsp[(1) - (2)].info).lx_srcpos; + ;} + break; + + case 191: +#line 1738 "cod/cod.y" + { + (yyval.reference) = cod_new_jump_statement(); + (yyval.reference)->node.jump_statement.continue_flag = 0; + (yyval.reference)->node.jump_statement.goto_target = NULL; + (yyval.reference)->node.jump_statement.lx_srcpos = (yyvsp[(1) - (2)].info).lx_srcpos; + ;} + break; + + case 192: +#line 1744 "cod/cod.y" + { + (yyval.reference) = cod_new_jump_statement(); + (yyval.reference)->node.jump_statement.continue_flag = 0; + (yyval.reference)->node.jump_statement.goto_target = (yyvsp[(2) - (3)].info).string; + (yyval.reference)->node.jump_statement.lx_srcpos = (yyvsp[(1) - (3)].info).lx_srcpos; + ;} + break; + + case 193: +#line 1753 "cod/cod.y" + { + (yyval.reference) = NULL; + ;} + break; + + case 194: +#line 1757 "cod/cod.y" + { + (yyval.reference) = cod_new_expression_statement(); + (yyval.reference)->node.expression_statement.expression = (yyvsp[(1) - (2)].reference); + ;} + break; + + case 195: +#line 1768 "cod/cod.y" + { + (yyval.reference) = cod_new_selection_statement(); + (yyval.reference)->node.selection_statement.lx_srcpos = (yyvsp[(1) - (5)].info).lx_srcpos; + (yyval.reference)->node.selection_statement.conditional = (yyvsp[(3) - (5)].reference); + (yyval.reference)->node.selection_statement.then_part = (yyvsp[(5) - (5)].reference); + (yyval.reference)->node.selection_statement.else_part = NULL; + ;} + break; + + case 196: +#line 1777 "cod/cod.y" + { + (yyval.reference) = cod_new_selection_statement(); + (yyval.reference)->node.selection_statement.lx_srcpos = (yyvsp[(1) - (7)].info).lx_srcpos; + (yyval.reference)->node.selection_statement.conditional = (yyvsp[(3) - (7)].reference); + (yyval.reference)->node.selection_statement.then_part = (yyvsp[(5) - (7)].reference); + (yyval.reference)->node.selection_statement.else_part = (yyvsp[(7) - (7)].reference); + ;} + break; + + case 197: +#line 1794 "cod/cod.y" + { + (yyval.reference) = cod_new_iteration_statement(); + (yyval.reference)->node.iteration_statement.lx_srcpos = (yyvsp[(1) - (9)].info).lx_srcpos; + (yyval.reference)->node.iteration_statement.init_expr = (yyvsp[(3) - (9)].reference); + (yyval.reference)->node.iteration_statement.test_expr = (yyvsp[(5) - (9)].reference); + (yyval.reference)->node.iteration_statement.iter_expr = (yyvsp[(7) - (9)].reference); + (yyval.reference)->node.iteration_statement.statement = (yyvsp[(9) - (9)].reference); + ;} + break; + + case 198: +#line 1804 "cod/cod.y" + { + (yyval.reference) = cod_new_iteration_statement(); + (yyval.reference)->node.iteration_statement.lx_srcpos = (yyvsp[(1) - (5)].info).lx_srcpos; + (yyval.reference)->node.iteration_statement.init_expr = NULL; + (yyval.reference)->node.iteration_statement.test_expr = (yyvsp[(3) - (5)].reference); + (yyval.reference)->node.iteration_statement.iter_expr = NULL; + (yyval.reference)->node.iteration_statement.statement = (yyvsp[(5) - (5)].reference); + ;} + break; + + case 199: +#line 1814 "cod/cod.y" + { + (yyval.reference) = cod_new_iteration_statement(); + (yyval.reference)->node.iteration_statement.lx_srcpos = (yyvsp[(1) - (7)].info).lx_srcpos; + (yyval.reference)->node.iteration_statement.init_expr = NULL; + (yyval.reference)->node.iteration_statement.test_expr = NULL; + (yyval.reference)->node.iteration_statement.post_test_expr = (yyvsp[(5) - (7)].reference); + (yyval.reference)->node.iteration_statement.iter_expr = NULL; + (yyval.reference)->node.iteration_statement.statement = (yyvsp[(2) - (7)].reference); + ;} + break; + + case 200: +#line 1827 "cod/cod.y" + { (yyval.reference) = NULL; ;} + break; + + case 202: +#line 1832 "cod/cod.y" + { + (yyval.reference) = cod_new_constant(); + (yyval.reference)->node.constant.token = integer_constant; + (yyval.reference)->node.constant.const_val = (yyvsp[(1) - (1)].info).string; + (yyval.reference)->node.constant.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + ;} + break; + + case 203: +#line 1839 "cod/cod.y" + { + (yyval.reference) = cod_new_constant(); + (yyval.reference)->node.constant.token = floating_constant; + (yyval.reference)->node.constant.const_val = (yyvsp[(1) - (1)].info).string; + (yyval.reference)->node.constant.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + ;} + break; + + case 204: +#line 1846 "cod/cod.y" + { + (yyval.reference) = cod_new_constant(); + (yyval.reference)->node.constant.token = string_constant; + (yyval.reference)->node.constant.const_val = (yyvsp[(1) - (1)].info).string; + (yyval.reference)->node.constant.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + ;} + break; + + case 205: +#line 1853 "cod/cod.y" + { + (yyval.reference) = cod_new_constant(); + (yyval.reference)->node.constant.token = character_constant; + (yyval.reference)->node.constant.const_val = (yyvsp[(1) - (1)].info).string; + (yyval.reference)->node.constant.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + ;} + break; + + case 206: +#line 1860 "cod/cod.y" + { + (yyval.reference) = cod_new_constant(); + (yyval.reference)->node.constant.token = character_constant; + (yyval.reference)->node.constant.const_val = (yyvsp[(1) - (1)].info).string; + (yyval.reference)->node.constant.lx_srcpos = (yyvsp[(1) - (1)].info).lx_srcpos; + ;} + break; + + +/* Line 1267 of yacc.c. */ +#line 3997 "/Users/eisen/prog/ffs/build/cod.tab.c" + default: break; + } + YY_SYMBOL_PRINT ("-> $$ =", yyr1[yyn], &yyval, &yyloc); + + YYPOPSTACK (yylen); + yylen = 0; + YY_STACK_PRINT (yyss, yyssp); + + *++yyvsp = yyval; + + + /* Now `shift' the result of the reduction. Determine what state + that goes to, based on the state we popped back to and the rule + number reduced by. */ + + yyn = yyr1[yyn]; + + yystate = yypgoto[yyn - YYNTOKENS] + *yyssp; + if (0 <= yystate && yystate <= YYLAST && yycheck[yystate] == *yyssp) + yystate = yytable[yystate]; + else + yystate = yydefgoto[yyn - YYNTOKENS]; + + goto yynewstate; + + +/*------------------------------------. +| yyerrlab -- here on detecting error | +`------------------------------------*/ +yyerrlab: + /* If not already recovering from an error, report this error. */ + if (!yyerrstatus) + { + ++yynerrs; +#if ! YYERROR_VERBOSE + yyerror (YY_("syntax error")); +#else + { + YYSIZE_T yysize = yysyntax_error (0, yystate, yychar); + if (yymsg_alloc < yysize && yymsg_alloc < YYSTACK_ALLOC_MAXIMUM) + { + YYSIZE_T yyalloc = 2 * yysize; + if (! (yysize <= yyalloc && yyalloc <= YYSTACK_ALLOC_MAXIMUM)) + yyalloc = YYSTACK_ALLOC_MAXIMUM; + if (yymsg != yymsgbuf) + YYSTACK_FREE (yymsg); + yymsg = (char *) YYSTACK_ALLOC (yyalloc); + if (yymsg) + yymsg_alloc = yyalloc; + else + { + yymsg = yymsgbuf; + yymsg_alloc = sizeof yymsgbuf; + } + } + + if (0 < yysize && yysize <= yymsg_alloc) + { + (void) yysyntax_error (yymsg, yystate, yychar); + yyerror (yymsg); + } + else + { + yyerror (YY_("syntax error")); + if (yysize != 0) + goto yyexhaustedlab; + } + } +#endif + } + + + + if (yyerrstatus == 3) + { + /* If just tried and failed to reuse look-ahead token after an + error, discard it. */ + + if (yychar <= YYEOF) + { + /* Return failure if at end of input. */ + if (yychar == YYEOF) + YYABORT; + } + else + { + yydestruct ("Error: discarding", + yytoken, &yylval); + yychar = YYEMPTY; + } + } + + /* Else will try to reuse look-ahead token after shifting the error + token. */ + goto yyerrlab1; + + +/*---------------------------------------------------. +| yyerrorlab -- error raised explicitly by YYERROR. | +`---------------------------------------------------*/ +yyerrorlab: + + /* Pacify compilers like GCC when the user code never invokes + YYERROR and the label yyerrorlab therefore never appears in user + code. */ + if (/*CONSTCOND*/ 0) + goto yyerrorlab; + + /* Do not reclaim the symbols of the rule which action triggered + this YYERROR. */ + YYPOPSTACK (yylen); + yylen = 0; + YY_STACK_PRINT (yyss, yyssp); + yystate = *yyssp; + goto yyerrlab1; + + +/*-------------------------------------------------------------. +| yyerrlab1 -- common code for both syntax error and YYERROR. | +`-------------------------------------------------------------*/ +yyerrlab1: + yyerrstatus = 3; /* Each real token shifted decrements this. */ + + for (;;) + { + yyn = yypact[yystate]; + if (yyn != YYPACT_NINF) + { + yyn += YYTERROR; + if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYTERROR) + { + yyn = yytable[yyn]; + if (0 < yyn) + break; + } + } + + /* Pop the current state because it cannot handle the error token. */ + if (yyssp == yyss) + YYABORT; + + + yydestruct ("Error: popping", + yystos[yystate], yyvsp); + YYPOPSTACK (1); + yystate = *yyssp; + YY_STACK_PRINT (yyss, yyssp); + } + + if (yyn == YYFINAL) + YYACCEPT; + + *++yyvsp = yylval; + + + /* Shift the error token. */ + YY_SYMBOL_PRINT ("Shifting", yystos[yyn], yyvsp, yylsp); + + yystate = yyn; + goto yynewstate; + + +/*-------------------------------------. +| yyacceptlab -- YYACCEPT comes here. | +`-------------------------------------*/ +yyacceptlab: + yyresult = 0; + goto yyreturn; + +/*-----------------------------------. +| yyabortlab -- YYABORT comes here. | +`-----------------------------------*/ +yyabortlab: + yyresult = 1; + goto yyreturn; + +#ifndef yyoverflow +/*-------------------------------------------------. +| yyexhaustedlab -- memory exhaustion comes here. | +`-------------------------------------------------*/ +yyexhaustedlab: + yyerror (YY_("memory exhausted")); + yyresult = 2; + /* Fall through. */ +#endif + +yyreturn: + if (yychar != YYEOF && yychar != YYEMPTY) + yydestruct ("Cleanup: discarding lookahead", + yytoken, &yylval); + /* Do not reclaim the symbols of the rule which action triggered + this YYABORT or YYACCEPT. */ + YYPOPSTACK (yylen); + YY_STACK_PRINT (yyss, yyssp); + while (yyssp != yyss) + { + yydestruct ("Cleanup: popping", + yystos[*yyssp], yyvsp); + YYPOPSTACK (1); + } +#ifndef yyoverflow + if (yyss != yyssa) + YYSTACK_FREE (yyss); +#endif +#if YYERROR_VERBOSE + if (yymsg != yymsgbuf) + YYSTACK_FREE (yymsg); +#endif + /* Make sure YYID is used. */ + return YYID (yyresult); +} + + +#line 1868 "cod/cod.y" + +#include "lex.yy.c" + +typedef struct scope *scope_ptr; + +struct parse_struct { + sm_list decls; + sm_list standard_decls; + scope_ptr scope; + char **defined_types; + char **enumerated_constants; + err_out_func_t error_func; + void *client_data; + sm_list return_type_list; + int return_cg_type; + sm_ref freeable_declaration; + int has_exec_context; + int dont_coerce_return; + int alloc_globals; +}; + +static int +semanticize_compound_statement(cod_parse_context context, sm_ref compound, + scope_ptr containing_scope, int require_last_return); +static int semanticize_decls_list(cod_parse_context context, sm_list decls, + scope_ptr scope); +static int semanticize_array_type_node(cod_parse_context context, + sm_ref array, scope_ptr scope); +static int semanticize_reference_type_node(cod_parse_context context, + sm_ref decl, scope_ptr scope); +static void add_decl(char *id, sm_ref node, scope_ptr scope); +static sm_ref find_complex_type(sm_ref node, scope_ptr scope); +static const char *cod_code_string; +static int is_string(sm_ref expr); + + +int +cod_semanticize_added_decls(cod_parse_context context) +{ + return semanticize_decls_list(context, context->decls, context->scope); +} + +extern void +cod_swap_decls_to_standard(cod_parse_context context) +{ + context->standard_decls = context->decls; + context->decls = NULL; +} + +void cod_set_error_func(cod_parse_context context, err_out_func_t err_func) +{ + context->error_func = err_func; +} + +void cod_set_dont_coerce_return(cod_parse_context context, int value) +{ + context->dont_coerce_return = value; +} + +static +char * +cod_preprocessor(char *input, cod_parse_context context, int*white) +{ + char *out; + char *ptr; + if (index(input, '#') == NULL) return NULL; + out = strdup(input); + ptr = out; + *white = 0; + while (ptr && (*ptr)) { + if (isspace(*ptr)) ptr++; + if (*ptr == '#') { + char *start = ptr; + char *line_end; + if ((strncmp(ptr, "#include", 8) == 0) && isspace(*(ptr+8))) { + /* got a #include */ + char *include_end; + ptr += 8; + while(isspace(*ptr)) ptr++; + line_end = index(ptr, '\n'); + if (line_end) *line_end = 0; + if ((*ptr == '<') || (*ptr == '"')) { + include_end = (*ptr == '<') ? index(ptr, '>') : index((ptr+1), '"'); + if (!include_end) { + printf("improper #include, \"%s\"\n", ptr); + goto skip; + } + *include_end = 0; + cod_process_include((ptr+1), context); + } else { + printf("improper #include, \"%s\"\n", ptr); + goto skip; + } + /* good #include, replace with spaces */ + if (line_end) *line_end = '\n'; + *include_end = ' '; + while ((start != include_end) && (*start)) { + *(start++) = ' '; + } + } + } + skip: + /* skip to next line */ + ptr = index(ptr, '\n'); + while (ptr && (*(ptr - 1) == '\'')) { + /* continued line */ + ptr = index(ptr, '\n'); + } + } + { + char *tmp = out; + while(isspace(*tmp)) tmp++; + if(*tmp == 0) { + free(out); + *white = 1; + } + } + return out; +} + +int +cod_parse_for_globals(code, context) +char *code; +cod_parse_context context; +{ + int ret; + context->alloc_globals = 1; + ret = cod_parse_for_context(code, context); + context->alloc_globals = 0; + return ret; +} +int +cod_parse_for_context(code, context) +char *code; +cod_parse_context context; +{ + sm_list decls; + int ret; + int all_whitespace = 0; + char *freeable_code = NULL; +#if defined(YYDEBUG) && (YYDEBUG != 0) + extern int yydebug; + yydebug = 1; +#endif + freeable_code = cod_preprocessor(code, context, &all_whitespace); + if (all_whitespace) return 1; + if (freeable_code) { + code = freeable_code; + } + if (code != NULL) { + setup_for_string_parse(code, context->defined_types, context->enumerated_constants); + cod_code_string = code; + } + yyerror_count = 0; + yycontext = context; + yyparse(); + terminate_string_parse(); + + if ((yyparse_value == NULL) || (yyerror_count != 0)) { + if (freeable_code) free(freeable_code); + return 0; + } + + decls = (sm_list) yyparse_value; + if (context->decls) { + sm_list last = context->decls; + while (last->next != NULL) + last = last->next; + last->next = decls; + } else { + context->decls = decls; + } + ret = semanticize_decls_list(context, decls, context->scope); + if (ret == 0) { + cod_rfree_list(decls, NULL); + context->decls = NULL; + } + if (freeable_code) free(freeable_code); + return ret; +} + +static int +semanticize_gotos(cod_parse_context context, sm_ref stmt, sm_list function_context); +static int semanticize_decl(cod_parse_context context, sm_ref decl, + scope_ptr scope); + +static int include_prefix(char *code) +{ + char *tmp = code; + int not_done = 1; + while (not_done) { + while(isspace(*tmp)) tmp++; + if (*tmp == '#') { + /* skip this line */ + while(*tmp != '\n') tmp++; + } else if (*tmp == '{') { + break; + } + } + return tmp - code; +} +cod_code +cod_code_gen(code, context) +char *code; +cod_parse_context context; +{ + sm_ref tmp, tmp2; + cod_code ret_code; + unsigned int offset; + void *func; + int bracket = 0; + + if (code != NULL) { + if ((bracket = include_prefix(code))) { + char *prefix = malloc(bracket+1), *tmp; + strncpy(prefix, code, bracket); + prefix[bracket] = 0; + tmp = prefix; + while(isspace(*tmp)) tmp++; + if (strlen(tmp) > 0) { + cod_parse_for_globals(tmp, context); + } + free(prefix); + code += bracket; + } + setup_for_string_parse(code, context->defined_types, context->enumerated_constants); + cod_code_string = code; + } + + yyerror_count = 0; + yycontext = context; + yyparse(); + terminate_string_parse(); + + if ((yyparse_value == NULL) || (yyerror_count != 0)) { + return 0; + } + tmp = cod_new_compound_statement(); + tmp->node.compound_statement.decls = context->decls; + tmp->node.compound_statement.statements = NULL; + tmp->node.compound_statement.statements = + malloc(sizeof(struct list_struct)); + tmp->node.compound_statement.statements->next = NULL; + tmp->node.compound_statement.statements->node = yyparse_value; + tmp2 = cod_new_compound_statement(); + tmp2->node.compound_statement.decls = context->standard_decls; + tmp2->node.compound_statement.statements = + malloc(sizeof(struct list_struct)); + tmp2->node.compound_statement.statements->next = NULL; + tmp2->node.compound_statement.statements->node = tmp; + if (!semanticize_gotos(context, tmp, tmp2->node.compound_statement.statements) || + !semanticize_compound_statement(context, tmp, context->scope, (context->return_cg_type != DILL_V))) { + tmp->node.compound_statement.decls = NULL; + tmp2->node.compound_statement.decls = NULL; + cod_rfree(tmp2); + return NULL; + } + ret_code = malloc(sizeof(struct _cod_code_struct)); + memset(ret_code, 0, sizeof(struct _cod_code_struct)); + ret_code->code_memory_block = NULL; + ret_code->data = NULL; + ret_code->has_exec_ctx = context->has_exec_context; + ret_code->static_block_address_register = -1; + func = cod_cg_net(tmp, context->return_cg_type, &offset, ret_code); + tmp->node.compound_statement.decls = NULL; + tmp2->node.compound_statement.decls = NULL; + cod_rfree(tmp2); + ret_code->func = (void(*)())(long)func; + return ret_code; +} + +void +cod_dump(cod_code code) +{ + printf("ECL CODE structure %p - \n", code); + printf(" function pointer %p, code memory block %p, data %p, static size %d\n", + code->func, code->code_memory_block, + code->data, code->static_size_required); +#ifdef HAVE_DILL_H + dill_dump((dill_stream) code->drisc_context); +#endif +} + + +int +cod_code_verify(code, context) +char *code; +cod_parse_context context; +{ + sm_ref tmp; + + if (code != NULL) { + setup_for_string_parse(code, context->defined_types, context->enumerated_constants); + cod_code_string = code; + } + + yyerror_count = 0; + yycontext = context; + yyparse(); + terminate_string_parse(); + + if ((yyparse_value == NULL) || (yyerror_count != 0)) { + if (yyparse_value) { + cod_rfree(yyparse_value); + } + return 0; + } + + tmp = cod_new_compound_statement(); + tmp->node.compound_statement.decls = context->decls; + tmp->node.compound_statement.statements = + malloc(sizeof(struct list_struct)); + tmp->node.compound_statement.statements->next = NULL; + tmp->node.compound_statement.statements->node = yyparse_value; + if (semanticize_compound_statement(context, tmp, context->scope, (context->return_cg_type != DILL_V)) == 0) { + tmp->node.compound_statement.decls = NULL; + cod_rfree(tmp); + return 0; + } + tmp->node.compound_statement.decls = NULL; + cod_rfree(tmp); + return 1; +} + +extern void +cod_code_free(code) +cod_code code; +{ + if (code->code_memory_block) free(code->code_memory_block); + if (code->data) free(code->data); +#if defined(HAVE_DILL_H) + if (code->drisc_context) { + dill_free_stream((dill_stream) code->drisc_context); + } + if (code->execution_handle) { + dill_free_handle((dill_exec_handle) code->execution_handle); + } +#endif + free(code); +} + +static char * +copy_line(const char *line_begin) +{ + const char *line_end; + if ((line_end = strchr(line_begin, 10)) == NULL) { + /* no CR */ + return strdup(line_begin); + } else { + char *tmp = malloc(line_end - line_begin + 1); + strncpy(tmp, line_begin, line_end - line_begin); + tmp[line_end - line_begin] = 0; + return tmp; + } +} + +static void +default_error_out(void *client_data, char *string) +{ + fprintf(stderr, "%s", string); +} + +static void +print_context(cod_parse_context context, int line, int character) +{ + const char *tmp = cod_code_string; + const char *line_begin = cod_code_string; + char *line_copy = NULL; + int i, line_len, offset = 0; + + while (line > 1) { + switch(*tmp) { + case 10: + line_begin = tmp + 1; + line--; + break; + case 0: + line = 1; /* end of src */ + break; + } + tmp++; + } + if (character > 40) { + offset = character - 40; + } + line_copy = copy_line(line_begin + offset); + line_len = strlen(line_copy); + if (line_len > 60) { + line_copy[60] = 0; + } + context->error_func(context->client_data, line_copy); + context->error_func(context->client_data, "\n"); + free(line_copy); + for(i=offset + 1; i< character; i++) { + if (line_begin[i-1] == '\t') { + context->error_func(context->client_data, "\t"); + } else { + context->error_func(context->client_data, " "); + } + } + context->error_func(context->client_data, "^\n"); +} + +void yyerror(str) +char *str; +{ + char tmp_str[100]; + sprintf(tmp_str, "## Error %s\n", str); + yycontext->error_func(yycontext->client_data, tmp_str); + yycontext->error_func(yycontext->client_data, "## While parsing near "); + yycontext->error_func(yycontext->client_data, yytext); + sprintf(tmp_str, ", offset = %d, line = %d ####\n",lex_offset,line_count); + yycontext->error_func(yycontext->client_data, tmp_str); + print_context(yycontext, line_count, lex_offset); + yyerror_count++; +} + +#ifdef STDC_HEADERS +static void +cod_src_error(cod_parse_context context, sm_ref expr, char *format, ...) +#else +static void +cod_src_error(context, expr, format, va_alist) +cod_parse_context context; +sm_ref expr; +char *format; +va_dcl +#endif +{ + + va_list ap; + char *tmp = malloc(10240); /* arbitrarily large */ + srcpos lx_srcpos = {0,0}; +#ifdef STDC_HEADERS + va_start(ap, format); +#else + va_start(ap); +#endif + if (expr) lx_srcpos = cod_get_srcpos(expr); + context->error_func(context->client_data, "## Ecode Error: "); + vsprintf(tmp, format, ap); + context->error_func(context->client_data, tmp); + sprintf(tmp, " at line %d, char %d\n", lx_srcpos.line, lx_srcpos.character); + context->error_func(context->client_data, tmp); + free(tmp); + print_context(context, lx_srcpos.line, lx_srcpos.character); +} + +extern void +cod_print_dimen_p(dimen_p d) +{ + int i; + if (!d) { + printf("DIMENS NOT SET YET\n"); + return; + } + for (i=0; i < d->dimen_count; i++) { + if (d->dimens[i].static_size != -1) { + printf("[%d]", d->dimens[i].static_size); + } else { + sm_ref field = d->dimens[i].control_field; + printf("[%s]", field->node.field.name); + } + } + printf("\n"); +} + +extern void +cod_print_operator_t(operator_t o) +{ + switch (o) { + case op_modulus: + printf("MODULUS"); + break; + case op_plus: + printf("PLUS"); + break; + case op_minus: + printf("MINUS"); + break; + case op_leq: + printf("LEQ"); + break; + case op_lt: + printf("LESS THAN"); + break; + case op_geq: + printf("GEQ"); + break; + case op_gt: + printf("GREATER THAN"); + break; + case op_eq: + printf("EQUAL"); + break; + case op_neq: + printf("NOT EQUAL"); + break; + case op_log_or: + printf("LOGICAL OR"); + break; + case op_log_and: + printf("LOGICAL AND"); + break; + case op_log_neg: + printf("LOGICAL NEGATION"); + break; + case op_arith_and: + printf("ARITH AND"); + break; + case op_arith_or: + printf("ARITH OR"); + break; + case op_arith_xor: + printf("ARITH XOR"); + break; + case op_left_shift: + printf("LEFT SHIFT"); + break; + case op_right_shift: + printf("RIGHT SHIFT"); + break; + case op_mult: + printf("MULTIPLY"); + break; + case op_div: + printf("DIVISION"); + break; + case op_deref: + printf("DEREFERENCE"); + break; + case op_inc: + printf("INCREMENT"); + break; + case op_not: + printf("BITWISE NOT"); + break; + case op_dec: + printf("DECREMENT"); + break; + case op_address: + printf("ADDRESS"); + break; + case op_sizeof: + printf("SIZEOF"); + break; + } +} + +extern void +cod_print_srcpos(srcpos pos) +{ + printf("line %d, char %d", pos.line, pos.character); +} + +extern void +cod_print_enc_info(enc_info enc) +{ + if (enc == NULL) { + printf("Not encoded"); + } else { + switch(enc->byte_order) { + case 1: + printf("Bigendian"); + break; + case 2: + printf("Littleendian"); + break; + } + } +} + +extern void +free_enc_info(enc_info enc) +{ + free(enc); +} + +enum namespace { NS_DEFAULT, NS_STRUCT, NS_ENUM }; + +char *namespace_str[] = {"DEFAULT", "STRUCT"}; + +typedef struct st_entry { + char *id; + sm_ref node; + enum namespace ns; + struct st_entry *next; +} *st_entry; + +struct scope { + cod_extern_list externs; + struct st_entry *entry_list; + sm_ref code_container; + struct scope *containing_scope; +}; + + +extern cod_parse_context +cod_copy_context(context) +cod_parse_context context; +{ + int i, count; + int type_count = 0; + cod_parse_context new_context = new_cod_parse_context(); + new_context->has_exec_context = context->has_exec_context; + new_context->decls = cod_copy_list(context->decls); + count = 0; + while (context->scope->externs && context->scope->externs[count].extern_value) count++; + i=0; + while(new_context->scope->externs[i].extern_name) free(new_context->scope->externs[i++].extern_name); + free(new_context->scope->externs); + new_context->scope->externs = malloc(sizeof(context->scope->externs[0]) * + (count+1)); + for (i=0; i < count; i++) { + new_context->scope->externs[i].extern_name = strdup(context->scope->externs[i].extern_name); + new_context->scope->externs[i].extern_value = context->scope->externs[i].extern_value; + } + new_context->scope->externs[count].extern_name = NULL; + new_context->scope->externs[count].extern_value = NULL; + + new_context->error_func = context->error_func; + new_context->client_data = context->client_data; + semanticize_decls_list(new_context, new_context->decls, + new_context->scope); + free(new_context->defined_types); + while(context->defined_types && context->defined_types[type_count]) type_count++; + new_context->defined_types = malloc(sizeof(char*) * (type_count + 2)); + for (i=0; i<= type_count; i++) { + new_context->defined_types[i] = context->defined_types[i]; + } + return new_context; +} + +extern void dump_scope(scope_ptr scope); + +extern cod_parse_context +cod_copy_globals(context) +cod_parse_context context; +{ + int i, count; + int type_count = 0; + cod_parse_context new_context = new_cod_parse_context(); + new_context->has_exec_context = context->has_exec_context; + new_context->decls = cod_copy_list(context->decls); + sm_list new_decls = new_context->decls; + sm_list old_decls = context->decls; + sm_list *last_new_p; + last_new_p = &new_context->decls; + while(new_decls != NULL) { + sm_ref new_decl = new_decls->node; + sm_ref old_decl = old_decls->node; + switch(new_decl->node_type) { + case cod_declaration: + if ((old_decl->node.declaration.param_num != -1) || + (old_decl->node.declaration.cg_address == (void*)-1)){ + /* this is an old parameter or subroutine, we have to kill it */ + *last_new_p = new_decls->next; + new_decls = new_decls->next; + old_decls = old_decls->next; + continue; + } + new_decl->node.declaration.cg_address = old_decl->node.declaration.cg_address; + new_decl->node.declaration.sm_complex_type = NULL; + break; + case cod_array_type_decl: + new_decl->node.array_type_decl.element_ref->node.declaration.cg_address = + old_decl->node.array_type_decl.element_ref->node.declaration.cg_address; + break; + default: + break; + } + last_new_p = &new_decls->next; + new_decls = new_decls->next; + old_decls = old_decls->next; + } + count = 0; + while (context->scope->externs && context->scope->externs[count].extern_value) count++; + i=0; + while(new_context->scope->externs[i].extern_name) free(new_context->scope->externs[i++].extern_name); + free(new_context->scope->externs); + new_context->scope->externs = malloc(sizeof(context->scope->externs[0]) * + (count+1)); + for (i=0; i < count; i++) { + new_context->scope->externs[i].extern_name = strdup(context->scope->externs[i].extern_name); + new_context->scope->externs[i].extern_value = context->scope->externs[i].extern_value; + } + new_context->scope->externs[count].extern_name = NULL; + new_context->scope->externs[count].extern_value = NULL; + + new_context->error_func = context->error_func; + new_context->client_data = context->client_data; + semanticize_decls_list(new_context, new_context->decls, + new_context->scope); + free(new_context->defined_types); + while(context->defined_types && context->defined_types[type_count]) type_count++; + new_context->defined_types = malloc(sizeof(char*) * (type_count + 2)); + for (i=0; i<= type_count; i++) { + new_context->defined_types[i] = context->defined_types[i]; + } + return new_context; +} + +static sm_ref +find_containing_iterator(scope_ptr scope) +{ + if (scope == NULL) return NULL; + if ((scope->code_container != NULL) && + (scope->code_container->node_type == cod_iteration_statement)) { + return scope->code_container; + } + return find_containing_iterator(scope->containing_scope); +} + +static void * +resolve_extern(char *id, scope_ptr scope) +{ + if (scope == NULL) return NULL; + if (scope->externs != NULL) { + cod_extern_list externs = scope->externs; + while(externs->extern_name != NULL) { + if (strcmp(id, externs->extern_name) == 0) { + return externs->extern_value; + } + externs++; + } + } + return resolve_extern(id, scope->containing_scope); +} + +static scope_ptr +push_scope_container(scope_ptr containing_scope, sm_ref container) +{ + scope_ptr new_scope = malloc(sizeof(*new_scope)); + new_scope->externs = NULL; + new_scope->entry_list = NULL; + new_scope->code_container = container; + new_scope->containing_scope = containing_scope; + return new_scope; +} + +static scope_ptr +push_scope(scope_ptr containing_scope) +{ + scope_ptr new_scope = malloc(sizeof(*new_scope)); + new_scope->externs = NULL; + new_scope->entry_list = NULL; + new_scope->code_container = NULL; + new_scope->containing_scope = containing_scope; + return new_scope; +} + +static void +pop_scope(scope_ptr scope) +{ + st_entry list = scope->entry_list; + while (list != NULL) { + st_entry tmp = list->next; + free(list); + list = tmp; + } + free(scope); +} + +extern void +dump_scope(scope_ptr scope) +{ + printf("Containing_scope is %p\n", scope->containing_scope); + printf("Extern list:"); + if (scope->externs != NULL) { + int i = 0; + while (scope->externs[i].extern_name != NULL) { + printf("\t\"%s\" -> 0x%p\n", scope->externs[i].extern_name, + scope->externs[i].extern_value); + i++; + } + } + printf("Symbol list:"); + if (scope->entry_list != NULL) { + st_entry e = scope->entry_list; + while (e != NULL) { + printf("\t\"%s\" -> 0x%p [%s]\n", e->id, e->node, namespace_str[e->ns]); + cod_print(e->node); + e = e->next; + } + } +} +static void +add_decl(char *id, sm_ref node, scope_ptr scope) +{ + st_entry entry = malloc(sizeof(*entry)); + entry->node = node; + entry->id = id; + entry->ns = NS_DEFAULT; + entry->next = scope->entry_list; + scope->entry_list = entry; +} + +static void +add_decl_ns(char *id, sm_ref node, scope_ptr scope, enum namespace ns) +{ + st_entry entry = malloc(sizeof(*entry)); + entry->node = node; + entry->id = id; + entry->ns = ns; + entry->next = scope->entry_list; + scope->entry_list = entry; +} + +extern void +cod_add_decl_to_scope(char *id, sm_ref node, cod_parse_context context) +{ + add_decl(id, node, context->scope); +} + +static sm_ref +resolve_local(char *id, scope_ptr scope) +{ + st_entry list = scope->entry_list; + while(list != NULL) { + if (strcmp(list->id, id) == 0) { + return list->node; + } + list = list->next; + } + return NULL; +} + +static sm_ref +resolve(char *id, scope_ptr scope) +{ + sm_ref tmp; + if (scope == NULL) return NULL; + tmp = resolve_local(id, scope); + if (tmp != NULL) { + return tmp; + } + return resolve(id, scope->containing_scope); +} + +static int +determine_unary_type(cod_parse_context context, sm_ref expr, sm_ref right) +{ + int right_type = cod_sm_get_type(right); + operator_t op = expr->node.operator.op; + switch(right_type) { + case DILL_C: case DILL_UC: case DILL_S: case DILL_US: + right_type = DILL_I; /* integer promotion */ + } + if (op == op_minus) { + switch(right_type) { + case DILL_U: + return DILL_I; + case DILL_UL: + return DILL_L; + } + } + return right_type; +} + +static int +determine_op_type(cod_parse_context context, sm_ref expr, + sm_ref left, sm_ref right) +{ + int unsigned_used = 0; + int left_type = cod_sm_get_type(left); + int right_type = cod_sm_get_type(right); + operator_t op = expr->node.operator.op; + + if (left_type == DILL_P) { + sm_ref ctype; + if (is_string(expr->node.operator.left) && + is_string(expr->node.operator.right) && + (op == op_eq)) { + return DILL_I; + } + + ctype = get_complex_type(context, left); + if(ctype && (ctype->node_type == cod_struct_type_decl)) { + cod_src_error(context, expr, + "Illegal arithmetic. Left side is a structure."); + return DILL_ERR; + } + + switch(right_type) { + case DILL_P: + case DILL_C: + case DILL_UC: + case DILL_S: + case DILL_US: + case DILL_I: + case DILL_U: + case DILL_L: + case DILL_UL: + case DILL_B: + return DILL_P; + + default: + cod_src_error(context, expr, + "Illegal pointer arithmetic. Right side is of incompatible type."); + return DILL_ERR; + } + } + if (right_type == DILL_P) { + sm_ref right_complex = get_complex_type(context, right); + if(right_complex && (right->node_type == cod_struct_type_decl)) { + cod_src_error(context, expr, + "Illegal arithmetic. Right side is a structure."); + return DILL_ERR; + } + + switch(left_type) { + case DILL_P: + case DILL_C: + case DILL_UC: + case DILL_S: + case DILL_US: + case DILL_I: + case DILL_U: + case DILL_L: + case DILL_UL: + return DILL_P; + + default: + cod_src_error(context, expr, + "Illegal pointer arithmetic. Left side is of incompatible type."); + return DILL_ERR; + } + + } + if (left_type == DILL_B) { + cod_src_error(context, expr, + "Illegal arithmetic. Left side is a structured type"); + return DILL_ERR; + } + if (right_type == DILL_B) { + cod_src_error(context, expr, + "Illegal arithmetic. Right side is a structured type"); + return DILL_ERR; + } + if ((left_type == DILL_D) || (right_type == DILL_D)) { + if ((op == op_modulus) || (op == op_log_or) || (op == op_log_and)) { + cod_src_error(context, expr, "Operands must be integral."); + return DILL_ERR; + } else { + return DILL_D; + } + } + if ((left_type == DILL_F) || (right_type == DILL_F)) { + if ((op == op_modulus) || (op == op_log_or) || (op == op_log_and)) { + cod_src_error(context, expr, "Operands must be integral."); + return DILL_ERR; + + } else { + return DILL_F; + } + } + switch(left_type) { + case DILL_C: case DILL_UC: case DILL_S: case DILL_US: + left_type = DILL_I; /* integer promotion */ + } + switch(right_type) { + case DILL_C: case DILL_UC: case DILL_S: case DILL_US: + right_type = DILL_I; /* integer promotion */ + } + switch(left_type) { + case DILL_UC: case DILL_US: case DILL_U: case DILL_UL: + unsigned_used++; + } + switch(right_type) { + case DILL_UC: case DILL_US: case DILL_U: case DILL_UL: + unsigned_used++; + } + if ((op == op_left_shift) || (op == op_right_shift)) return left_type; + if ((left_type == DILL_UL) || (right_type == DILL_UL)) return DILL_UL; + if ((left_type == DILL_L) || (right_type == DILL_L)) { + /* GSE -bug This test should be for *generated* target, not host */ + if (sizeof(long) > sizeof(unsigned int)) { + /* Long can represent all values of unsigned int */ + return DILL_L; + } else { + return unsigned_used? DILL_UL : DILL_L; + } + } + if ((left_type == DILL_U) || (right_type == DILL_U)) return DILL_U; + return unsigned_used? DILL_U: DILL_I; +} + +static sm_ref reduce_type_list(cod_parse_context context, sm_list type_list, + int *cg_type, scope_ptr scope, int *is_typedef, + sm_ref *freeable_type); +static int +assignment_types_match(cod_parse_context context, sm_ref left, sm_ref right, int strict); + +#ifdef NOTDEF +static int +is_n_dimen_array(int dimen, sm_ref expr) +{ + if ((dimen == 0) && (expr == NULL)) return 1; + if ((dimen > 0) && (expr == NULL)) return 0; + if (dimen == 0) { + if (expr->node_type == cod_array_type_decl) { + return 0; + } else { + return 1; + } + } + if (expr->node_type == cod_field_ref) { + return is_n_dimen_array(dimen, expr->node.field_ref.sm_field_ref); + } + if (expr->node_type == cod_element_ref) { + return is_n_dimen_array(dimen + 1, expr); + } + if (expr->node_type != cod_field) return 0; + if (expr->node_type == cod_field) { + return is_n_dimen_array(dimen, expr->node.field.sm_complex_type); + } + /* ought to recurse or handle above */ + assert(0); + return 0; +} +#endif + +static int +is_string(sm_ref expr) +{ + if (expr->node_type == cod_field) { + return (expr->node.field.string_type && (strcmp(expr->node.field.string_type, "string") == 0)); + } else if (expr->node_type == cod_field_ref) { + return is_string(expr->node.field_ref.sm_field_ref); + } else if (expr->node_type == cod_identifier) { + return is_string(expr->node.identifier.sm_declaration); + } else if (expr->node_type == cod_conditional_operator) { + return is_string(expr->node.conditional_operator.e1); /* e2 must be similar */ + } else if (expr->node_type == cod_declaration) { + if (expr->node.declaration.cg_type != DILL_P) return 0; + if (expr->node.declaration.sm_complex_type != NULL) return 0; + /* only strings have pointers without complex types */ + return 1; + } else if (expr->node_type == cod_constant) { + return (expr->node.constant.token == string_constant); + } + return 0; +} + +static int +is_const(sm_ref expr) +{ + switch(expr->node_type) { + case cod_field_ref: + return is_const(expr->node.field_ref.struct_ref); + case cod_element_ref: + return is_const(expr->node.element_ref.array_ref); + case cod_cast: + return is_const(expr->node.cast.expression); + case cod_identifier: + return is_const(expr->node.identifier.sm_declaration); + case cod_declaration: + return expr->node.declaration.const_var; + case cod_constant: + return 1; + case cod_operator: + /* most operator errors handled elsewhere. Here we're only looking for dereference */ + if (expr->node.operator.op == op_deref) { + return is_const(expr->node.operator.right); + } + return 0; + default: + printf("Unhandled case in is_const()\n"); + cod_print(expr); + assert(0); + } + return 0; +} + +extern int +cod_expr_is_string(sm_ref expr) +{ + return is_string(expr); +} + +extern int +is_control_value(sm_ref expr, sm_ref strct) +{ + sm_list fields; + if (expr->node_type == cod_field_ref) { + return is_control_value(expr->node.field_ref.sm_field_ref, + expr->node.field_ref.struct_ref); + } + if (expr->node_type != cod_field) return 0; + assert(strct != NULL); + strct = get_complex_type(0, strct); + if (strct->node_type == cod_reference_type_decl) { + strct = strct->node.reference_type_decl.sm_complex_referenced_type; + } + if (strct->node_type == cod_declaration) { + strct = strct->node.declaration.sm_complex_type; + } + assert(strct->node_type == cod_struct_type_decl); + fields = strct->node.struct_type_decl.fields; + while(fields != NULL) { + sm_ref ctype = fields->node->node.field.sm_complex_type; + if ((ctype != NULL) && (ctype->node_type == cod_reference_type_decl)) + ctype = ctype->node.reference_type_decl.sm_complex_referenced_type; + while (ctype != NULL) { + if (ctype->node_type == cod_array_type_decl) { + if (ctype->node.array_type_decl.sm_dynamic_size == expr) { + return 1; + } + ctype = ctype->node.array_type_decl.sm_complex_element_type; + } else { + ctype = NULL; + } + } + fields = fields->next; + } + return 0; +} + +#ifndef FALSE +#define FALSE 0 +#endif + +static char* +type_list_to_string(cod_parse_context context, sm_list type_list, int *size) +{ + sm_list orig_list = type_list; + int short_appeared = 0; + int long_appeared = 0; + int long_long_appeared = 0; + int int_appeared = 0; + int double_appeared = 0; + int float_appeared = 0; + int char_appeared = 0; + int signed_appeared = 0; + int unsigned_appeared = 0; + int void_appeared = 0; + int string_appeared = 0; + int spec_count = 0; + int prefix_end = 0; + int type_found = 0; + int cg_type; + + cg_type = DILL_ERR; + while ((type_list != NULL) && (prefix_end == 0)) { + sm_ref node = type_list->node; + int typ = -1; + if (node->node_type == cod_type_specifier) { + typ = type_list->node->node.type_specifier.token; + if ((typ == STAR) || (typ == AT)) { + prefix_end = 1; + type_list = type_list->next; + continue; + } + } + if (node->node_type != cod_type_specifier) { + if (node->node_type == cod_identifier) { + return NULL; + } else if (node->node_type == cod_struct_type_decl) { + return NULL; + } else { + printf("Unknown node type in type_list_to_string\n"); + break; + } + } else { + spec_count++; + switch (typ) { + case INT: + int_appeared++; + break; + case LONG: + long_appeared++; + break; + case SHORT: + short_appeared++; + break; + case DOUBLE: + double_appeared++; + break; + case STRING: + string_appeared++; + break; + case VOID: + void_appeared++; + break; + case FLOAT: + float_appeared++; + break; + case CHAR: + char_appeared++; + break; + case SIGNED: + signed_appeared++; + break; + case UNSIGNED: + unsigned_appeared++; + break; + case TYPEDEF: + spec_count--; + break; + case STATIC: + spec_count--; + break; + case EXTERN_TOKEN: + spec_count--; + break; + case CONST: + spec_count--; + break; + default: + printf("Unknown type\n"); + } + type_list = type_list->next; + } + } + if (spec_count == 0) { + if (type_list == NULL) cg_type = DILL_I; /* default to int */ + goto finalize; + } + if (void_appeared && (spec_count > 1)) { + cod_src_error(context, orig_list->node, + "Void type may not appear with other specifiers"); + cg_type = DILL_ERR; + return NULL; + } + if (string_appeared && (spec_count > 1)) { + cod_src_error(context, orig_list->node, + "String type may not appear with other specifiers"); + cg_type = DILL_ERR; + return NULL; + } + if (void_appeared) { + cg_type = DILL_V; + goto finalize; + } + if (string_appeared) { + cg_type = DILL_P; + goto finalize; + } + if (short_appeared && long_appeared) { + cod_src_error(context, orig_list->node, + "Only one of long or short permitted"); + cg_type = DILL_ERR; + return NULL; + } + if (short_appeared && (double_appeared + float_appeared)) { + cod_src_error(context, orig_list->node, + "Short may not be specified with double or float"); + cg_type = DILL_ERR; + return NULL; + } + if (double_appeared + float_appeared) { + if (double_appeared + float_appeared + short_appeared + signed_appeared + unsigned_appeared + char_appeared + int_appeared > 1) { + cod_src_error(context, orig_list->node, "Bad type spec"); + cg_type = DILL_ERR; + return NULL; + } else { + /* not handling LONG plus one of these */ + if (double_appeared) { + cg_type = DILL_D; + goto finalize; + } else { + cg_type = DILL_F; + goto finalize; + } + } + } + + /* neither float or double appeared */ + if (long_appeared == 2) { + long_long_appeared++; + long_appeared = 0; + } + if (short_appeared + char_appeared + long_appeared + long_long_appeared >= 2) { + cod_src_error(context, orig_list->node, + "Only one integer size spec may be specified"); + cg_type = DILL_ERR; + return NULL; + } + if (unsigned_appeared + signed_appeared > 1) { + cod_src_error(context, orig_list->node, "Bad type spec"); + cg_type = DILL_ERR; + return NULL; + } + if (unsigned_appeared) { + if (char_appeared) { + cg_type = DILL_UC; + goto finalize; + } else if (short_appeared) { + cg_type = DILL_US; + goto finalize; + } else if (long_appeared || long_long_appeared) { + cg_type = DILL_UL; + goto finalize; + } else { + cg_type = DILL_U; + goto finalize; + } + } else { + if (char_appeared) { + cg_type = DILL_C; + goto finalize; + } else if (short_appeared) { + cg_type = DILL_S; + goto finalize; + } else if (long_appeared || long_long_appeared) { + cg_type = DILL_L; + goto finalize; + } else { + cg_type = DILL_I; + goto finalize; + } + } + finalize: + if (cg_type != DILL_ERR) { + type_found++; + } + switch(cg_type) { + case DILL_C: + *size = sizeof(char); + return strdup("integer"); + case DILL_UC: + *size = sizeof(char); + return strdup("unsigned integer"); + case DILL_I: + *size = sizeof(int); + return strdup("integer"); + case DILL_L: + *size = sizeof(long); + return strdup("integer"); + case DILL_S: + *size = sizeof(short); + return strdup("integer"); + case DILL_U: + *size = sizeof(int); + return strdup("unsigned integer"); + case DILL_UL: + *size = sizeof(long); + return strdup("unsigned integer"); + case DILL_US: + *size = sizeof(short); + return strdup("unsigned integer"); + case DILL_F: + *size = sizeof(float); + return strdup("float"); + case DILL_D: + *size = sizeof(double); + return strdup("float"); + } + return NULL; +} + +static sm_ref +cod_build_parsed_type_node(cod_parse_context c, char *name, sm_list l) +{ + sm_ref decl = cod_new_struct_type_decl(); + sm_list *end_ptr = &decl->node.struct_type_decl.fields; + + sm_list tmp = l; + sm_list last_type = NULL; + int field_count = 0; + decl->node.struct_type_decl.id = name; + + while(tmp != NULL) { + sm_ref node = tmp->node; + sm_list typ = NULL; + sm_list new_elem; + new_elem = malloc(sizeof(*new_elem)); + new_elem->next = NULL; + new_elem->node = cod_new_field(); + if (node->node_type == cod_declaration) { + typ = cod_dup_list(node->node.declaration.type_spec); + new_elem->node->node.field.name = strdup(node->node.declaration.id); + new_elem->node->node.field.string_type = + type_list_to_string(c, typ, &new_elem->node->node.field.cg_size); + } else if (node->node_type == cod_array_type_decl) { + sm_ref base_decl = node->node.array_type_decl.element_ref; + sm_ref size = node->node.array_type_decl.size_expr; + char *base_string_type = NULL; + char *size_str = NULL, *final_type; + typ = cod_dup_list(node->node.array_type_decl.type_spec); + if (base_decl->node_type != cod_declaration) { + printf("Array base type must be a simple type\n"); + return NULL; + } + new_elem->node->node.field.name = strdup(base_decl->node.declaration.id); + base_string_type = + type_list_to_string(c, typ, &new_elem->node->node.field.cg_size); + if (size->node_type == cod_identifier) { + size_str = size->node.identifier.id; + } else { + int free_val = 0; + sm_ref constant = evaluate_constant_return_expr(c, size, &free_val); + if (constant->node_type == cod_constant) { + if (constant->node.constant.token != integer_constant) { + printf("Array size constant is non-integer\n"); + return NULL; + } else { + size_str = constant->node.constant.const_val; + } + if (free_val) free(constant); + } else { + printf("Unexpected value for array size\n"); + return NULL; + } + } + if (base_string_type) { + final_type = malloc(strlen(base_string_type) + + strlen(size_str) + 3); + sprintf(final_type, "%s[%s]", base_string_type, size_str); + new_elem->node->node.field.string_type = final_type; + free(base_string_type); + } else { + new_elem->node->node.field.string_type = NULL; + } + } + new_elem->node->node.field.cg_offset = -1; + new_elem->node->node.field.cg_type = DILL_ERR; + new_elem->node->node.field.type_spec = typ; + cod_rfree(node); + field_count++; + last_type = tmp; + tmp = tmp->next; + free(last_type); + *end_ptr = new_elem; + end_ptr = &new_elem->next; + } + return decl; +} + +int +is_array(sm_ref expr) +{ + sm_ref typ; + if (expr->node_type == cod_field_ref) { + return is_array(expr->node.field_ref.sm_field_ref); + } + if (expr->node_type == cod_identifier) { + return is_array(expr->node.identifier.sm_declaration); + } + if (expr->node_type == cod_declaration) { + sm_ref ctype = expr->node.declaration.sm_complex_type; + if ((ctype != NULL) && (ctype->node_type == cod_array_type_decl)) { + return 1; + } + } + typ = get_complex_type(NULL, expr); + if (typ == NULL) return 0; + + if (typ->node_type == cod_array_type_decl) { + return 1; + } + + if (typ->node_type == cod_reference_type_decl) { + sm_ref ctype = + typ->node.reference_type_decl.sm_complex_referenced_type; + if (ctype == NULL) return 0; + if (ctype->node_type == cod_array_type_decl) { + return 1; + } + } + return 0; +} + +static sm_ref +get_containing_structure(sm_ref expr) +{ + switch(expr->node_type) { + case cod_element_ref: + return get_containing_structure(expr->node.element_ref.array_ref); + case cod_field_ref: + return expr->node.field_ref.struct_ref; + default: + return NULL; + } +} + + +static void +add_field_list(int *format_count_p, FMStructDescList *format_list_p, sm_ref typ) +{ + sm_list fields = typ->node.struct_type_decl.fields; + FMFieldList field_list = malloc(sizeof(field_list[0]) * 2); + int field_count = 0; + int my_format_num = (*format_count_p)++; + *format_list_p = realloc(*format_list_p, sizeof(*format_list_p[0]) * (*format_count_p + 1)); + while(fields != NULL) { + sm_ref typ = fields->node->node.field.sm_complex_type; + field_list = realloc(field_list, (sizeof(field_list[0]) * (field_count +2))); + field_list[field_count].field_name = strdup(fields->node->node.field.name); + field_list[field_count].field_type = strdup(fields->node->node.field.string_type); + field_list[field_count].field_size = fields->node->node.field.cg_size; + field_list[field_count].field_offset = fields->node->node.field.cg_offset; + while((typ != NULL) && ((typ->node_type == cod_reference_type_decl) || (typ->node_type == cod_declaration) + || (typ->node_type == cod_array_type_decl))) { + if (typ->node_type == cod_reference_type_decl) { + typ = typ->node.reference_type_decl.sm_complex_referenced_type; + } else if (typ->node_type == cod_array_type_decl) { + typ = typ->node.array_type_decl.sm_complex_element_type; + } else if (typ->node_type == cod_declaration) { + typ = typ->node.declaration.sm_complex_type; + } + } + if ((typ != NULL) && (typ->node_type == cod_struct_type_decl)) { + add_field_list(format_count_p, format_list_p, typ); + } + field_count++; + fields = fields->next; + } + field_list[field_count].field_name = field_list[field_count].field_type = NULL; + field_list[field_count].field_size = field_list[field_count].field_offset = 0; + (*format_list_p)[my_format_num].format_name = strdup(typ->node.struct_type_decl.id); + (*format_list_p)[my_format_num].field_list = field_list; + (*format_list_p)[my_format_num].struct_size = typ->node.struct_type_decl.cg_size; + (*format_list_p)[my_format_num].opt_info = NULL; +} + +static FMStructDescList +build_format_list(cod_parse_context context, sm_ref expr) +{ + sm_ref typ = get_complex_type(context, expr); + FMStructDescList formats = malloc(sizeof(formats[0]) * 2); + int format_count = 0; + if (typ == NULL) { + cod_src_error(context, expr->node.field_ref.struct_ref, + "Reference must be structured type", + expr->node.field_ref.lx_field); + return 0; + } + if (typ->node_type == cod_reference_type_decl) { + typ = typ->node.reference_type_decl.sm_complex_referenced_type; + } + if (typ->node_type == cod_declaration) { + typ = typ->node.declaration.sm_complex_type; + } + add_field_list(&format_count, &formats, typ); + formats[format_count].format_name = NULL; + formats[format_count].field_list = NULL; + return formats; +} + +static int is_left_hand_side(sm_ref expr); + +static int semanticize_expr(cod_parse_context context, sm_ref expr, + scope_ptr scope) +{ + switch(expr->node_type) { + case cod_identifier: { + sm_ref tmp = resolve(expr->node.identifier.id, scope); + if (tmp != NULL) { + if (tmp->node_type == cod_constant) { + srcpos old_srcpos = expr->node.identifier.lx_srcpos; + free(expr->node.identifier.id); + /* morph identifier into constant */ + expr->node_type = cod_constant; + expr->node.constant.token = tmp->node.constant.token; + expr->node.constant.const_val = strdup(tmp->node.constant.const_val); + expr->node.constant.freeable_name = NULL; + expr->node.constant.lx_srcpos = old_srcpos; + return semanticize_expr(context, expr, scope); + } else { + expr->node.identifier.sm_declaration = tmp; + return 1; + } + } else { + cod_src_error(context, expr, + "Undefined Symbol \"%s\"", + expr->node.identifier.id); + return 0; + } + } + case cod_comma_expression: + if (!semanticize_expr(context, expr->node.comma_expression.left, scope)) + return 0; + if (!semanticize_expr(context, expr->node.comma_expression.right, scope)) + return 0; + return 1; + case cod_cast: { + int cg_type; + sm_ref typ; + if (expr->node.cast.expression && + !semanticize_expr(context, expr->node.cast.expression, scope)) { + return 0; + } + + typ = reduce_type_list(context, expr->node.cast.type_spec, &cg_type, + scope, NULL, NULL); + if ((cg_type == DILL_ERR) && (typ == NULL)) { + cod_src_error(context, expr, "Illegal cast"); + return 0; + } + expr->node.cast.cg_type = cg_type; + expr->node.cast.sm_complex_type = typ; + return 1; + } + case cod_operator: { + int ret = 1; + if (expr->node.operator.left != NULL) { + if (!semanticize_expr(context, expr->node.operator.left, scope)) { + ret = 0; + } + } + if (expr->node.operator.right != NULL) { + if (!semanticize_expr(context, expr->node.operator.right, scope)) { + ret = 0; + } + } + if (ret == 0) return 0; + if ((expr->node.operator.left != NULL) && + (expr->node.operator.right != NULL)) { + expr->node.operator.operation_type = + determine_op_type(context, expr, + expr->node.operator.left, + expr->node.operator.right); + if (expr->node.operator.operation_type == DILL_ERR) { + return 0; + } + } else if (expr->node.operator.right != NULL) { + expr->node.operator.operation_type = + determine_unary_type(context, expr, expr->node.operator.right); + } else if (expr->node.operator.left != NULL) { + expr->node.operator.operation_type = + determine_unary_type(context, expr, expr->node.operator.left); + } + switch (expr->node.operator.op) { + case op_leq: case op_lt: case op_geq: case op_gt: case op_neq: + case op_eq: case op_log_neg: case op_log_or: case op_log_and: + case op_sizeof: + expr->node.operator.result_type = DILL_I; + break; + case op_address: + expr->node.operator.result_type = DILL_P; + if (expr->node.operator.right->node_type == cod_identifier) { + sm_ref decl = expr->node.operator.right->node.identifier.sm_declaration; + if (decl->node_type == cod_declaration) { + if (decl->node.declaration.param_num != -1) { + if (decl->node.declaration.sm_complex_type == NULL) { + cod_src_error(context, expr, "Cannot take address of a pass-by-value parameter"); + return 0; + } + } + decl->node.declaration.addr_taken = 1; + } + } else { + if (!is_left_hand_side(expr->node.operator.right)) { + cod_src_error(context, expr, "Invalid operand to address operator"); + return 0; + } + } + break; + case op_deref: { + sm_ref typ = get_complex_type(context, expr->node.operator.right); + if (!typ || ((typ->node_type != cod_reference_type_decl) && + (typ->node_type != cod_array_type_decl))) { + cod_src_error(context, expr, "Cannot dereference a non-reference type"); + return 0; + } else if (typ->node_type == cod_reference_type_decl) { + expr->node.operator.result_type = + typ->node.reference_type_decl.cg_referenced_type; + } else if (typ->node_type == cod_array_type_decl) { + expr->node.operator.result_type = + typ->node.array_type_decl.cg_element_type; + } else { + assert(0); + } + break; + } + default: + /* Operator applied to pointer types? Check compatibility... */ + if(expr->node.operator.operation_type == DILL_P) { + + switch(expr->node.operator.op) { + case op_inc: + case op_dec: + break; + + case op_plus: + { + sm_ref left = expr->node.operator.left; + sm_ref right = expr->node.operator.right; + + sm_ref lcplx = NULL; + sm_ref rcplx = NULL; + + if(!left) { + cod_src_error(context, expr, + "Invalid operand to unary plus\n"); + return 0; + } + + /* Extract complex types, if any */ + lcplx = get_complex_type(context, left); + rcplx = get_complex_type(context, right); + + /* Pointers do not add with complex types */ + if(lcplx && rcplx) { + cod_src_error(context, expr, + "Invalid operands to binary plus"); + return 0; + } + + /* + * We're ok if we reach this, since that implies we have subtraction + * between a pointer and an integral type. The suitability of the + * integral type has been checked in determine_op_type() already. + */ + } + break; + + case op_minus: + { + sm_ref left = expr->node.operator.left; + sm_ref right = expr->node.operator.right; + + sm_ref lcplx = NULL; + sm_ref rcplx = NULL; + + if(!left) { + cod_src_error(context, expr, + "Invalid operand to unary minus\n"); + return 0; + } + + /* Extract complex types, if any */ + lcplx = get_complex_type(context, left); + rcplx = get_complex_type(context, right); + + + /* If both are complex types... */ + if(lcplx && rcplx) { + + /* If both are pointers... */ + if(((lcplx->node_type == cod_reference_type_decl) || (lcplx->node_type == cod_array_type_decl)) && + ((rcplx->node_type == cod_reference_type_decl) || (rcplx->node_type == cod_array_type_decl))) { + /* Check if the argument pointers are compatible */ + if(!are_compatible_ptrs(lcplx, rcplx)) { + cod_src_error(context, expr, + "Incompatible pointer arguments to binary minus"); + return 0; + } else { + /* + * Binary minus between two compatible pointers is allowed, + * but it produces an integral type, so we fix that here. + */ + expr->node.operator.result_type=DILL_L; + + /* + * NOTE how we return success directly from here and do not + * break from the switch through the line below setting the + * result_type from the operation_type... In this case this + * would cause problems. We want operation_type to stay DILL_P + * but the final result_type to be a DILL_L. + */ + return 1; + } + } else { + /* + * Pointers and other complex types do not subtract. + * Arithmetic canno be done on non-pointer complex types. + */ + cod_src_error(context, expr, + "Incompatible arguments to binary minus"); + return 0; + } + } + + /* + * We're ok if we reach this, since that implies we have subtraction + * between a pointer and an integral type. The suitability of the + * integral type has been checked in determine_op_type() already. + */ + } + break; + + default: + cod_src_error(context, expr, + "Operator cannot be applied to pointer types!\n"); + return 0; + } + } + /* + * NOTE: If anything here changes, one (potentially) has to + * update the code above which deals with binary minus + * between two compatible pointers, changes the result to + * an integral type, and returns directly without going + * through this code (as all other cases do). + */ + expr->node.operator.result_type=expr->node.operator.operation_type; + } + return ret; + } + case cod_constant: + return 1; + case cod_assignment_expression: { + int ret = 1; + if (!semanticize_expr(context, expr->node.assignment_expression.left, scope)) { + ret = 0; + } else { + expr->node.assignment_expression.cg_type = + cod_sm_get_type(expr->node.assignment_expression.left); + } + if (expr->node.assignment_expression.left && is_const(expr->node.assignment_expression.left)) { + cod_src_error(context, expr->node.assignment_expression.left, "Invalid assignment, left side is const"); + ret = 0; + } + if (!semanticize_expr(context, expr->node.assignment_expression.right, scope)){ + ret = 0; + } else { + int right_type = + cod_sm_get_type(expr->node.assignment_expression.right); + if ((right_type == DILL_P) && + (is_string(expr->node.assignment_expression.right))) { + if (expr->node.assignment_expression.cg_type != DILL_P) { + cod_src_error(context, expr, "assignment mixes string and non-string types"); + ret = 0; + } + } else if ((right_type == DILL_B) || (right_type == DILL_ERR)) { + cod_src_error(context, expr->node.assignment_expression.right, "Invalid assignment, right side must be simple type"); + ret = 0; + } + } + if ((expr->node.assignment_expression.cg_type == DILL_P) || + (expr->node.assignment_expression.cg_type == DILL_ERR)) { + sm_ref ltyp = + get_complex_type(context, + expr->node.assignment_expression.left); +// sm_ref rtyp = +// get_complex_type(context, +// expr->node.assignment_expression.right); + if (ltyp == NULL) { + if (!is_string(expr->node.assignment_expression.left)) { + cod_src_error(context, expr->node.assignment_expression.left, "Invalid assignment, left side must be simple, non-pointer type"); + ret = 0; + } + } else { + if ((ltyp->node_type == cod_struct_type_decl) || (ltyp->node_type == cod_array_type_decl) || (ltyp->node_type == cod_enum_type_decl)) { + /* maybe OK */ + } else if (ltyp->node_type != cod_reference_type_decl) { + cod_src_error(context, expr->node.assignment_expression.left, "Invalid assignment, left side must be simple, non-pointer type"); + ret = 0; + } + } + } + if (ret == 1) { + ret = assignment_types_match(context, + expr->node.assignment_expression.left, + expr->node.assignment_expression.right, + /* strict */ (expr->node.assignment_expression.op == op_eq)); + } + return ret; + } + case cod_field_ref: { + sm_ref typ; + sm_list fields; + if (!semanticize_expr(context, expr->node.field_ref.struct_ref, scope)) { + return 0; + } + typ = get_complex_type(context, expr->node.field_ref.struct_ref); + if (typ == NULL) { + cod_src_error(context, expr->node.field_ref.struct_ref, + "Reference must be structured type", + expr->node.field_ref.lx_field); + return 0; + } + if (typ->node_type == cod_reference_type_decl) { + typ = typ->node.reference_type_decl.sm_complex_referenced_type; + } + if (typ->node_type == cod_declaration) { + typ = typ->node.declaration.sm_complex_type; + } + fields = typ->node.struct_type_decl.fields; + while(fields != NULL) { + if (strcmp(expr->node.field_ref.lx_field, + fields->node->node.field.name) == 0) { + break; + } + fields = fields->next; + } + if (fields == NULL) { + cod_src_error(context, expr, + "Unknown field reference, \"%s\".", + expr->node.field_ref.lx_field); + return 0; + } + expr->node.field_ref.sm_field_ref = fields->node; + return 1; + } + case cod_element_ref: { + if (semanticize_expr(context, expr->node.element_ref.array_ref, scope)) { + int cg_type; + sm_ref arr = get_complex_type(NULL, expr->node.element_ref.array_ref); + if (is_string(expr->node.element_ref.array_ref)) { + expr->node.element_ref.this_index_dimension = 0; + expr->node.element_ref.sm_complex_element_type = NULL; + expr->node.element_ref.cg_element_type = DILL_C; + expr->node.element_ref.sm_containing_structure_ref = + get_containing_structure(expr->node.element_ref.array_ref); + } else if (is_array(expr->node.element_ref.array_ref)) { + if (arr->node_type == cod_reference_type_decl) { + arr = arr->node.reference_type_decl.sm_complex_referenced_type; + } + if (expr->node.element_ref.array_ref->node_type != cod_element_ref) { + /* bottom level of recursion, we're the left-most array index */ + expr->node.element_ref.this_index_dimension = 0; + } else { + sm_ref subindex = expr->node.element_ref.array_ref; + expr->node.element_ref.this_index_dimension = subindex->node.element_ref.this_index_dimension + 1; + } + expr->node.element_ref.sm_complex_element_type = + arr->node.array_type_decl.sm_complex_element_type; + + expr->node.element_ref.cg_element_type = + arr->node.array_type_decl.cg_element_type; + expr->node.element_ref.sm_containing_structure_ref = + get_containing_structure(expr->node.element_ref.array_ref); + } else if (arr && (arr->node_type == cod_reference_type_decl)) { + expr->node.element_ref.sm_complex_element_type = + arr->node.reference_type_decl.sm_complex_referenced_type; + expr->node.element_ref.cg_element_type = + arr->node.reference_type_decl.cg_referenced_type; + expr->node.element_ref.sm_containing_structure_ref = + get_containing_structure(expr->node.element_ref.array_ref); + } else { + cod_src_error(context, expr, "Indexed element must be array, string or reference type."); + return 0; + } + + if (!semanticize_expr(context, expr->node.element_ref.expression, scope)) { + return 0; + } + + cg_type = cod_sm_get_type(expr->node.element_ref.expression); + switch(cg_type) { + case DILL_C: + case DILL_UC: + case DILL_S: + case DILL_US: + case DILL_I: + case DILL_U: + case DILL_L: + case DILL_UL: + return 1; + break; + } + cod_src_error(context, expr, + "Index for element reference must be integer type"); + return 0; + } + return 0; + } + case cod_subroutine_call: { + sm_ref func_ref = expr->node.subroutine_call.sm_func_ref; + char *id; + sm_ref tmp; + sm_list args; + sm_list formals, tmp_formals, tmp_args; + int ret = 1; + if (func_ref->node_type == cod_identifier) { + id = func_ref->node.identifier.id; + } else { + id = func_ref->node.declaration.id; + } + tmp = resolve(id, scope); + args = expr->node.subroutine_call.arguments; + int done; + if (tmp != NULL) { + if ((tmp->node_type != cod_declaration) || + !tmp->node.declaration.is_subroutine) { + cod_src_error(context, expr, + "Identifier is not subroutine \"%s\".", + func_ref->node.identifier.id); + return 0; + } + free(func_ref->node.identifier.id); + free(func_ref); + expr->node.subroutine_call.sm_func_ref = func_ref = tmp; + formals = func_ref->node.declaration.params; + } else { + cod_src_error(context, func_ref, "Undefined Subroutine \"%s\".", + func_ref->node.identifier.id); + + return 0; + } + tmp_formals = formals; + tmp_args = args; + sm_list *last_arg_p = &args; + /* add closure args if required */ + while (tmp_formals != NULL) { + sm_ref formal = tmp_formals->node; + if (formal && (formal->node.declaration.sm_complex_type != NULL)) { + sm_ref ct = formal->node.declaration.sm_complex_type; + if ((ct->node_type == cod_reference_type_decl) && + (strcmp(ct->node.reference_type_decl.name, "cod_closure_context") == 0)) { + sm_list new_arg = malloc(sizeof(struct list_struct)); + char tmp[30]; + new_arg->next = tmp_args; + if (func_ref->node.declaration.closure_id == NULL) { + strcpy(tmp, "0"); + } else { + sprintf(tmp, "%p", func_ref->node.declaration.closure_id); + if (strncmp(tmp, "0x", 2) != 0) { + sprintf(tmp, "0x%p", func_ref->node.declaration.closure_id); + } + } + + new_arg->node = cod_new_constant(); + new_arg->node->node.constant.token = integer_constant; + new_arg->node->node.constant.const_val = strdup(tmp); + *last_arg_p = new_arg; + tmp_args = new_arg; + } else if ((ct->node_type == cod_reference_type_decl) && + (strcmp(ct->node.reference_type_decl.name, "cod_exec_context") == 0)) { + tmp_formals = tmp_formals->next; + continue; + } + + } + tmp_formals = tmp_formals->next; + if (tmp_args) { + last_arg_p = &tmp_args->next; + tmp_args = tmp_args->next; + } + } + /* must do this assigment, in case things changed from the loop above */ + expr->node.subroutine_call.arguments = args; + + done = 0; + while (!done) { + sm_ref arg = NULL; + sm_ref formal = NULL; + if (formals != NULL) { + formal = formals->node; + } + if (args != NULL) { + arg = args->node; + } + if (formal && (formal->node.declaration.sm_complex_type != NULL)) { + sm_ref ct = formal->node.declaration.sm_complex_type; + if ((ct->node_type == cod_reference_type_decl) && + (ct->node.reference_type_decl.name != NULL)) { + if (strcmp(ct->node.reference_type_decl.name, "cod_exec_context") == 0) { + if (context->has_exec_context == 0) { + cod_src_error(context, arg, "Calling subroutine has no cod_exec_context"); + return 0; + } + /* swallow next formal, we'll fill that in ourselves */ + formals = formals->next; + continue; + } + } + } + if ((args == NULL) && (formals != NULL)) { + if (strcmp(formal->node.declaration.id, "...") != 0) { + cod_src_error(context, arg, "Too few arguments to function"); + ret = 0; + } + } + if (args == NULL) { + done++; + continue; + } + if (!semanticize_expr(context, arg, scope) ) { + args = args->next; + continue; + } + if (formal == NULL) { + cod_src_error(context, arg, "Too many arguments to subroutine"); + ret = 0; + return ret; + } + if (strcmp(formal->node.declaration.id, "...") != 0) { + /* we've got a real formal to check against */ + /* do some checking... */ + int mismatch = 0; + switch (cod_sm_get_type(arg)) { + case DILL_D: case DILL_F: + if (formal->node.declaration.cg_type >= DILL_V) { + mismatch++; + } + break; + case DILL_I: case DILL_U: + case DILL_L: case DILL_UL: + if (formal->node.declaration.cg_type == DILL_P) { + sm_ref ct = formal->node.declaration.sm_complex_type; + if (!ct || + (ct->node_type != cod_reference_type_decl) || + ((strcmp(ct->node.reference_type_decl.name, "cod_type_spec") != 0) && + (strcmp(ct->node.reference_type_decl.name, "cod_closure_context") != 0))) { + if ((arg->node_type != cod_constant) || + (arg->node.constant.token != integer_constant)) { + mismatch++; + } else { + int tmp = -1; + sscanf(arg->node.constant.const_val, "%d", &tmp); + /* zero is an acceptable pointer */ + if (tmp != 0) { + mismatch++; + } + } + } + } + break; + case DILL_P: + if (formal->node.declaration.cg_type != DILL_P) { + if (!(formal->node.declaration.sm_complex_type && + (formal->node.declaration.sm_complex_type->node_type == + cod_reference_type_decl))) { + mismatch++; + } + } + break; + } + + if (mismatch) { + cod_src_error(context, arg, + "Type mismatch, parameter \"%s\".", + formal->node.declaration.id); + ret = 0; + } + } + if ((formals != NULL) && + (strcmp(formal->node.declaration.id, "...") != 0)) { + formals = formals->next; + formal = NULL; + if (formals != NULL) formal = formals->node; + } + /* look ahead to next formal and insert an arg if it's cod_type_spec */ + if (formal && + formal->node.declaration.sm_complex_type != NULL) { + sm_ref ct = formal->node.declaration.sm_complex_type; + if ((ct->node_type == cod_reference_type_decl) && + (strcmp(ct->node.reference_type_decl.name, "cod_type_spec") + == 0)) { + /* swallow next formal, we'll fill that in ourselves */ + sm_list tmp_args = malloc(sizeof(struct list_struct)); + FMStructDescList list = build_format_list(context, arg); + char tmp[30]; + sprintf(&tmp[0], "0x%p", list); + tmp_args->node = cod_new_constant(); + tmp_args->node->node.constant.token = integer_constant; + tmp_args->node->node.constant.const_val = strdup(tmp); + tmp_args->next = args->next; + args->next = tmp_args; + } + } + args = args->next; + if ((args == NULL) && (formals != NULL)) { + if (strcmp(formal->node.declaration.id, "...") != 0) { + cod_src_error(context, arg, "Too few arguments to function"); + ret = 0; + } + } + } + return ret; + } + case cod_conditional_operator: { + int ret = 1; + if (expr->node.conditional_operator.condition != NULL) { + if (!semanticize_expr(context, expr->node.conditional_operator.condition, scope)) { + ret = 0; + } + } + if (expr->node.conditional_operator.e1 != NULL) { + if (!semanticize_expr(context, expr->node.conditional_operator.e1, scope)) { + ret = 0; + } + } + if (expr->node.conditional_operator.e2 != NULL) { + if (!semanticize_expr(context, expr->node.conditional_operator.e2, scope)) { + ret = 0; + } + } + expr->node.conditional_operator.result_type = + determine_unary_type(context, expr, expr->node.conditional_operator.e1); + return ret; + } + case cod_initializer_list: { + sm_list items = expr->node.initializer_list.initializers; + int ret = 1; + while (items) { + if (!semanticize_expr(context, items->node, scope)) ret = 0; + items = items->next; + } + return ret; + } + case cod_initializer: { + if (!semanticize_expr(context, expr->node.initializer.initializer, scope)) + return 0; + if (expr->node.initializer.designation) { +// if (!semanticize_expr(context, expr->node.initializer.designation, scope)) return 0; + } + return 1; + } + default: + fprintf(stderr, "Unknown case in semanticize_expression\n"); + cod_print(expr); + } + return 0; +} + +static int +is_left_hand_side(sm_ref expr) +{ + switch(expr->node_type) { + case cod_identifier: + return 1; + case cod_operator: + return 0; + case cod_cast: + return is_left_hand_side(expr->node.cast.expression); + case cod_assignment_expression: + return expr->node.assignment_expression.cg_type; + case cod_declaration: + return 1; + case cod_constant: + return 0; + case cod_field_ref: + return 1; + case cod_element_ref: + return 1; + case cod_subroutine_call: + return 0; + default: + fprintf(stderr, "Unknown case in is_left_hand_side()\n"); + cod_print(expr); + assert(0); + } + return 0; +} + +int +type_of_int_const_string(char *val) +{ +/* +For decimal, it is the first type the value can fit in: int, long, long long + +For hexadecimal, it is the first type the value can fit in: int, unsigned int, long, +unsigned long, long long, unsigned long long +*/ + + long i; + int len = strlen(val); + int hex = 0; + int specified_unsgned = 0, specified_lng = 0; + if (val[0] == '0') { + /* hex or octal */ + hex++; + if (val[1] == 'x') { + /* hex */ + if (sscanf(val+2, "%lx", &i) != 1) + printf("hex sscanf failed, %s\n", val); + } else if (val[1] == 'b') { + /* binary */ + int j = 2; + i = 0; + while (val[j]) { + i <<= 1; + if (val[j] == '1') { + i += 1; + } + j++; + } + } else { + if (sscanf(val, "%lo", &i) != 1) + printf("octal sscanf failed %s\n", val); + } + } else { + if (sscanf(val, "%ld", &i) != 1) + printf("decimal sscanf failed %s\n", val); + } + switch(val[len-1]) { + case 'U': + case 'u': + specified_unsgned++; + break; + case 'l': + case 'L': + specified_lng++; + break; + } + if (len > 2) + switch(val[len-2]) { + case 'U': + case 'u': + specified_unsgned++; + break; + case 'l': + case 'L': + specified_lng++; + break; + } + if (len > 3) + switch(val[len-3]) { + case 'U': + case 'u': + specified_unsgned++; + break; + case 'l': + case 'L': + specified_lng++; + break; + } + if (specified_lng == 0) { + /* unspecified */ + if (hex) { + if (i == (int)i) return DILL_I; + if (i == (unsigned)i) return DILL_U; + if (i == (long)i) return DILL_L; + if (i == (unsigned)i) return DILL_UL; + return DILL_UL; /* don't do long long now */ + } else { + if (i == (int)i) return DILL_I; + if (i == (long)i) return DILL_L; + return DILL_L; /* don't do long long now */ + } + } + /* must have specified long */ + if (specified_unsgned) { + return DILL_UL; + } else { + return DILL_L; + } +} + +extern int +cod_sm_get_type(sm_ref node) +{ + switch(node->node_type) { + case cod_identifier: + if (node->node.identifier.sm_declaration != NULL) { + return cod_sm_get_type(node->node.identifier.sm_declaration); + } + return node->node.identifier.cg_type; + case cod_enumerator: + return DILL_I; + case cod_operator: + return node->node.operator.result_type; + case cod_conditional_operator: + return node->node.conditional_operator.result_type; + case cod_cast: + return node->node.cast.cg_type; + case cod_assignment_expression: + return node->node.assignment_expression.cg_type; + case cod_declaration: + if (is_array(node)) { + return DILL_P; + } else { + return node->node.declaration.cg_type; + } + case cod_constant: + /* need to handle bigger constants */ + if (node->node.constant.token == string_constant) { + return DILL_P; + } else if (node->node.constant.token == floating_constant) { + return DILL_D; + } else if (node->node.constant.token == character_constant) { + return DILL_C; + } else { + return type_of_int_const_string(node->node.constant.const_val); + } + case cod_field_ref: + return cod_sm_get_type(node->node.field_ref.sm_field_ref); + case cod_element_ref: + return node->node.element_ref.cg_element_type; + case cod_field: + if (is_array(node)) { + return DILL_P; + } else { + return node->node.field.cg_type; + } + case cod_initializer_list: + return DILL_ERR; + case cod_subroutine_call: + return cod_sm_get_type(node->node.subroutine_call.sm_func_ref); + case cod_comma_expression: + return cod_sm_get_type(node->node.comma_expression.right); + default: + fprintf(stderr, "Unknown case in cod_sm_get_type()\n"); + cod_print(node); + } + return DILL_ERR; +} + +extern int +are_compatible_ptrs(sm_ref left, sm_ref right) { + sm_ref lTyp = NULL, rTyp = NULL; + int lcgTyp = -1, rcgTyp = -1; + + /* Sanity check */ + if (left->node_type == cod_reference_type_decl) { + lTyp = left->node.reference_type_decl.sm_complex_referenced_type; + lcgTyp = left->node.reference_type_decl.cg_referenced_type; + } else if (left->node_type == cod_array_type_decl) { + lTyp = left->node.array_type_decl.sm_complex_element_type; + lcgTyp = left->node.array_type_decl.cg_element_type; + } else { + return 0; + } + if (right->node_type == cod_reference_type_decl) { + rTyp = right->node.reference_type_decl.sm_complex_referenced_type; + rcgTyp = right->node.reference_type_decl.cg_referenced_type; + } else if (right->node_type == cod_array_type_decl) { + rTyp = right->node.array_type_decl.sm_complex_element_type; + rcgTyp = right->node.array_type_decl.cg_element_type; + } else { + return 0; + } + + if(lTyp && rTyp) { + /* Two complex referenced types */ + if(((lTyp->node_type == cod_reference_type_decl) || (lTyp->node_type == cod_array_type_decl)) && + ((rTyp->node_type == cod_reference_type_decl) || (rTyp->node_type == cod_array_type_decl))) { + /* Recurse if both are pointers */ + return are_compatible_ptrs(lTyp, rTyp); + } + return (lTyp == rTyp)?1:0; + } + + if(!lTyp && !rTyp) { + /* Two integral referenced types */ + return (rcgTyp == lcgTyp)?1:0; + } + + /* Mix of a pointer to a complex type and a pointer to an integral type */ + return 0; +} + +extern sm_ref +get_complex_type(cod_parse_context context, sm_ref node) +{ + if (!node) return NULL; + switch(node->node_type) { + case cod_array_type_decl: + case cod_reference_type_decl: + case cod_struct_type_decl: + case cod_enum_type_decl: + return node; + case cod_subroutine_call: + return get_complex_type(context, + node->node.subroutine_call.sm_func_ref); + case cod_identifier: + return get_complex_type(context, + node->node.identifier.sm_declaration); + case cod_element_ref: + return node->node.element_ref.sm_complex_element_type; + case cod_field: + return node->node.field.sm_complex_type; + case cod_declaration: + return get_complex_type(context, node->node.declaration.sm_complex_type); + case cod_field_ref:{ + sm_ref typ; + sm_list fields; + typ = get_complex_type(context, node->node.field_ref.struct_ref); + if (typ->node_type == cod_reference_type_decl) { + typ = typ->node.reference_type_decl.sm_complex_referenced_type; + } + if (typ->node_type == cod_declaration) { + typ = typ->node.declaration.sm_complex_type; + } + fields = typ->node.struct_type_decl.fields; + while ((fields != NULL) && + (strcmp(node->node.field_ref.lx_field, + fields->node->node.field.name) != 0)) { + fields = fields->next; + } + if (fields == NULL) { + cod_src_error(context, node, "Unknown field reference \"%s\".", + node->node.field_ref.lx_field); + return NULL; + } + return get_complex_type(context, fields->node->node.field.sm_complex_type); + } + case cod_conditional_operator: + return NULL; + case cod_constant: + return NULL; + case cod_operator: + switch (node->node.operator.op) { + case op_deref: { + sm_ref right = get_complex_type(NULL, node->node.operator.right); + if ((right != NULL) && + (right->node_type == cod_reference_type_decl)) { + sm_ref typ = right->node.reference_type_decl.sm_complex_referenced_type; + if (typ && (typ->node_type == cod_declaration)) { + return get_complex_type(context, typ); + } else { + return typ; + } + } + return NULL; + } + case op_plus: case op_minus: case op_inc: case op_dec: { + sm_ref right = NULL; + sm_ref left = NULL; + if (node->node.operator.right) + right = get_complex_type(NULL, node->node.operator.right); + if (node->node.operator.left) + left = get_complex_type(NULL, node->node.operator.left); + if (right && (left == NULL)) return right; + if (left && (right == NULL)) return left; + if ((left == NULL) && (right == NULL)) return NULL; + /* + * GANEV: op_minus can be applied to two pointers, + * i.e. two complex types => both left _and_ right can be + * non-NULL, hence this code. This shouldn't happen in + * other cases... (I think). + */ + if(node->node.operator.op == op_minus && right && left) { + if(left->node_type == cod_reference_type_decl && + right->node_type == cod_reference_type_decl) { + /* Ok, so it's op_minus between two pointers, then check compatibility */ + if(are_compatible_ptrs(left, right)) { + return left; + } else { + cod_src_error(context, node, + "Incompatible pointer args to binary minus"); + return NULL; + } + } + } + cod_src_error(context, node, "Incompatible pointer arguments to operator"); + return NULL; + } + default: + return NULL; + } + + case cod_cast: + return node->node.cast.sm_complex_type; + break; + case cod_initializer_list: + case cod_enumerator: + return NULL; + case cod_assignment_expression: + return get_complex_type(context, node->node.assignment_expression.left); + default: + fprintf(stderr, "Unknown case in get_complex_type()\n"); + cod_print(node); + } + return NULL; +} + +static sm_ref +reduce_type_list(cod_parse_context context, sm_list type_list, int *cg_type, + scope_ptr scope, int*is_typedef, sm_ref *freeable_type) +{ + sm_list orig_list = type_list; + int short_appeared = 0; + int long_appeared = 0; + int long_long_appeared = 0; + int int_appeared = 0; + int double_appeared = 0; + int float_appeared = 0; + int char_appeared = 0; + int signed_appeared = 0; + int unsigned_appeared = 0; + int void_appeared = 0; + int string_appeared = 0; + int spec_count = 0; + int prefix_end = 0; + int type_found = 0; + sm_ref complex_return_type = NULL;; + + *cg_type = DILL_ERR; + while ((type_list != NULL) && (prefix_end == 0)) { + int typ = type_list->node->node.type_specifier.token; + if ((type_list->node->node_type != cod_type_specifier) || + (typ == STAR) || (typ == AT)) { + prefix_end = 1; + } else { + spec_count++; + switch (typ) { + case INT: + int_appeared++; + break; + case LONG: + long_appeared++; + break; + case SHORT: + short_appeared++; + break; + case DOUBLE: + double_appeared++; + break; + case STRING: + string_appeared++; + break; + case VOID: + void_appeared++; + break; + case FLOAT: + float_appeared++; + break; + case CHAR: + char_appeared++; + break; + case SIGNED: + signed_appeared++; + break; + case UNSIGNED: + unsigned_appeared++; + break; + case TYPEDEF: + if (is_typedef) (*is_typedef)++; + spec_count--; + break; + case STATIC: + spec_count--; + break; + case EXTERN_TOKEN: + spec_count--; + break; + case CONST: + spec_count--; + break; + default: + printf("Unknown type\n"); + } + type_list = type_list->next; + } + } + if (spec_count == 0) { + if (type_list == NULL) *cg_type = DILL_I; /* default to int */ + goto finalize; + } + if (void_appeared && (spec_count > 1)) { + cod_src_error(context, orig_list->node, + "Void type may not appear with other specifiers"); + *cg_type = DILL_ERR; + return NULL; + } + if (string_appeared && (spec_count > 1)) { + cod_src_error(context, orig_list->node, + "String type may not appear with other specifiers"); + *cg_type = DILL_ERR; + return NULL; + } + if (void_appeared) { + *cg_type = DILL_V; + goto finalize; + } + if (string_appeared) { + *cg_type = DILL_P; + goto finalize; + } + if (short_appeared && long_appeared ) { + cod_src_error(context, orig_list->node, + "Only one of long or short permitted"); + *cg_type = DILL_ERR; + return NULL; + } + if (short_appeared && (double_appeared + float_appeared)) { + cod_src_error(context, orig_list->node, + "Short may not be specified with double or float"); + *cg_type = DILL_ERR; + return NULL; + } + if (double_appeared + float_appeared) { + if (double_appeared + float_appeared + short_appeared + signed_appeared + unsigned_appeared + char_appeared + int_appeared > 1) { + cod_src_error(context, orig_list->node, "Bad type spec"); + *cg_type = DILL_ERR; + return NULL; + } else { + /* not handling LONG plus one of these */ + if (double_appeared) { + *cg_type = DILL_D; + goto finalize; + } else { + *cg_type = DILL_F; + goto finalize; + } + } + } + + /* neither float or double appeared */ + if (long_appeared == 2) { + long_long_appeared++; + long_appeared = 0; + } + if (short_appeared + char_appeared + long_appeared + long_long_appeared >= 2) { + cod_src_error(context, orig_list->node, + "Only one integer size spec may be specified"); + *cg_type = DILL_ERR; + return NULL; + } + if (unsigned_appeared + signed_appeared > 1) { + cod_src_error(context, orig_list->node, "Bad type spec"); + *cg_type = DILL_ERR; + return NULL; + } + if (unsigned_appeared) { + if (char_appeared) { + *cg_type = DILL_UC; + goto finalize; + } else if (short_appeared) { + *cg_type = DILL_US; + goto finalize; + } else if (long_appeared || long_long_appeared) { + *cg_type = DILL_UL; + goto finalize; + } else { + *cg_type = DILL_U; + goto finalize; + } + } else { + if (char_appeared) { + *cg_type = DILL_C; + goto finalize; + } else if (short_appeared) { + *cg_type = DILL_S; + goto finalize; + } else if (long_appeared || long_long_appeared) { + *cg_type = DILL_L; + goto finalize; + } else if (int_appeared) { + *cg_type = DILL_I; + goto finalize; + } + } + finalize: + if (type_list == NULL) { + /* no error and no more to process */ + return NULL; + } + if (*cg_type != DILL_ERR) { + type_found++; + } + while (type_list != NULL) { + sm_ref node = type_list->node; + switch (node->node_type) { + case cod_identifier: + { + if (type_found != 0) { + cod_src_error(context, node, + "Type identifier cannot follow prior identifiers"); + *cg_type = DILL_ERR; + return NULL; + } + complex_return_type = find_complex_type(node, scope); + if ((complex_return_type != NULL)&& + (complex_return_type->node_type == cod_declaration)) { + if (complex_return_type->node.declaration.sm_complex_type) { + complex_return_type = complex_return_type->node.declaration.sm_complex_type; + } else { + *cg_type = complex_return_type->node.declaration.cg_type; + } + } + if ((complex_return_type != NULL)&& + (complex_return_type->node_type == cod_reference_type_decl)) { + *cg_type = DILL_P; + } + if ((complex_return_type != NULL)&& + (complex_return_type->node_type == cod_struct_type_decl)) { + *cg_type = DILL_B; + } + if ((complex_return_type != NULL)&& + (complex_return_type->node_type == cod_enum_type_decl)) { + *cg_type = DILL_I; + } + if ((complex_return_type == NULL) && node->node.identifier.id && + ((strcmp(node->node.identifier.id, "cod_type_spec") == 0) || + (strcmp(node->node.identifier.id, "cod_closure_context") == 0) || + (strcmp(node->node.identifier.id, "cod_exec_context") == 0))) { + /* special ECL type information for prior arg */ + sm_ref typ = cod_new_reference_type_decl(); + typ->node.reference_type_decl.name = strdup(node->node.identifier.id); + if (strcmp(node->node.identifier.id, "cod_type_spec") == 0) { + *cg_type = DILL_P; + } else if (strcmp(node->node.identifier.id, "cod_closure_context") == 0) { + *cg_type = DILL_P; + } else { + context->has_exec_context = 1; + *cg_type = DILL_P; + } + typ->node.reference_type_decl.cg_referenced_type = *cg_type; + typ->node.reference_type_decl.sm_complex_referenced_type = + complex_return_type; + typ->node.reference_type_decl.kernel_ref = 0; + complex_return_type = typ; + if (*freeable_type) { + cod_rfree(*freeable_type); + *freeable_type = NULL; + } + *freeable_type = typ; + } + assert((complex_return_type != NULL) || (*cg_type != DILL_ERR)); + type_found++; + } + break; + case cod_type_specifier: + switch (node->node.type_specifier.token) { + case STAR: + { + if (node->node.type_specifier.created_type_decl == NULL) { + /* GSE create anon-type */ + sm_ref typ = cod_new_reference_type_decl(); + typ->node.reference_type_decl.name = gen_anon(); + typ->node.reference_type_decl.cg_referenced_type = *cg_type; + *cg_type = DILL_P; + typ->node.reference_type_decl.sm_complex_referenced_type = + complex_return_type; + typ->node.reference_type_decl.kernel_ref = 0; + complex_return_type = typ; + node->node.type_specifier.created_type_decl = typ; + } else { + complex_return_type = node->node.type_specifier.created_type_decl; + *cg_type = DILL_P; + } + } + break; + case AT: + { + /* GSE create anon-type */ + sm_ref typ = cod_new_reference_type_decl(); + typ->node.reference_type_decl.name = gen_anon(); + typ->node.reference_type_decl.cg_referenced_type = *cg_type; + *cg_type = DILL_P; + typ->node.reference_type_decl.sm_complex_referenced_type = + complex_return_type; + typ->node.reference_type_decl.kernel_ref = 1; + complex_return_type = typ; + } + break; + default: + if (type_found != 0) { + cod_src_error(context, node, + "Only '*', '@', and CONST can follow valid type"); + *cg_type = DILL_ERR; + return NULL; + } + } + break; + case cod_struct_type_decl: { + if (node->node.struct_type_decl.fields != NULL) { + semanticize_decl(context, node, scope); + complex_return_type = node; + } else { + complex_return_type = resolve(node->node.struct_type_decl.id, scope); + if (complex_return_type == NULL) { + cod_src_error(context, node, + "Struct declaration not found"); + return NULL; + } + } + *cg_type = DILL_B; + break; + } + case cod_enum_type_decl: { + if (node->node.enum_type_decl.enums != NULL) { + semanticize_decl(context, node, scope); + complex_return_type = node; + } else { + complex_return_type = resolve(node->node.enum_type_decl.id, scope); + if (complex_return_type == NULL) { + cod_src_error(context, node, + "Enum declaration not found"); + return NULL; + } + } + *cg_type = DILL_I; + break; + } + default: + printf("Unexpected node in reduce_type_list\n"); + return NULL; + } + type_list = type_list->next; + } + return complex_return_type; +} + +static int +assignment_types_match(cod_parse_context context, sm_ref left, sm_ref right, int strict) +{ + sm_ref left_smt, right_smt; + int left_cgt, right_cgt; + left_smt = get_complex_type(context, left); + right_smt = get_complex_type(context, right); + left_cgt = cod_sm_get_type(left); + right_cgt = cod_sm_get_type(right); + if ((left_smt == NULL) && (right_smt == NULL)) { + /* just check cgts */ + /* don't mix DILL_P, DILL_B and anything else */ + switch (left_cgt) { + case DILL_P: + switch (right_cgt) { + case DILL_P: + case DILL_L: + case DILL_UL: + return 1; + break; + default: + cod_src_error(context, left, "Trying to assign a pointer variable with a non-pointer value."); + return 0; + } + default: + switch (right_cgt) { + case DILL_P: + cod_src_error(context, left, "Trying to assign pointer to an incompatible variable."); + return 0; + default: + return 1; + break; + } + } + } + if ((left_smt != NULL) && + ((left_smt->node_type != cod_reference_type_decl) && + (left_smt->node_type != cod_array_type_decl) && + (left_smt->node_type != cod_struct_type_decl) && + (left_smt->node_type != cod_enum_type_decl))) { + if ((left_cgt == DILL_P) || (left_cgt == DILL_B)) { + cod_src_error(context, left, "Only pointer, array, struct or enum complex types allowed as LHS in assignment"); + return 0; + } + } + if ((right_smt != NULL) && + ((right_smt->node_type != cod_reference_type_decl) && + (right_smt->node_type != cod_array_type_decl) && + (right_smt->node_type != cod_struct_type_decl) && + (right_smt->node_type != cod_enum_type_decl))) { + if ((right_cgt == DILL_P) || (right_cgt == DILL_B)) { + cod_src_error(context, right, "Only pointer, array, struct or enum complex types allowed as RHS in assignment"); + return 0; + } + } + if (left_smt && (left_smt->node_type == cod_reference_type_decl) && + (right_smt == NULL)) { + + switch(right_cgt) { + case DILL_P: + case DILL_L: + case DILL_UL: + return 1; + case DILL_I: + case DILL_U: + if (!strict) return 1; + if ((right->node_type == cod_constant) && + (right->node.constant.token == integer_constant)) { + int i = -1; + sscanf(right->node.constant.const_val, "%d", &i); + if (i== 0) return 1; + } + /* falling through */ + default: + cod_src_error(context, right, "Right hand side must be pointer type"); + return 0; + } + } + if (right_smt && (left_smt == NULL)) { + switch(left_cgt) { + case DILL_C: + case DILL_UC: + case DILL_S: + case DILL_US: + case DILL_I: + case DILL_U: + case DILL_L: + case DILL_UL: + case DILL_P: + /* GANEV: should we have a warning here? */ + return 1; + + default: + cod_src_error(context, right, "Pointer converted without explicit cast"); + return 0; + } + } + return 1; +} + +static int semanticize_struct_type_node(cod_parse_context context, sm_ref decl, + scope_ptr scope); + +static int semanticize_enum_type_node(cod_parse_context context, sm_ref decl, + scope_ptr scope); + +static int +is_constant_expr(sm_ref expr) +{ + switch(expr->node_type) { + case cod_constant: { + return 1; + break; + } + case cod_identifier: + if (!expr->node.identifier.sm_declaration) return 0; + return is_constant_expr(expr->node.identifier.sm_declaration); + case cod_declaration: + if (!expr->node.declaration.const_var) return 0; + return is_constant_expr(expr->node.declaration.init_value); + case cod_operator: { + if (expr->node.operator.left != NULL) { + if (!is_constant_expr(expr->node.operator.left)) return 0; + } + if (expr->node.operator.op == op_sizeof) { + return 1; + } + if (expr->node.operator.right != NULL) { + if (!is_constant_expr(expr->node.operator.right)) return 0; + } + switch(expr->node.operator.op) { + case op_modulus: + case op_not: + case op_plus: + case op_minus: + case op_leq: + case op_lt: + case op_geq: + case op_gt: + case op_eq: + case op_neq: + case op_log_or: + case op_arith_or: + case op_arith_xor: + case op_log_and: + case op_arith_and: + case op_mult: + case op_div: + case op_log_neg: + case op_left_shift: + case op_right_shift: + return 1; + break; + case op_deref: + case op_address: + case op_inc: + case op_dec: + case op_sizeof: + return 0; + } + return 1; + } + case cod_cast: + return is_constant_expr(expr->node.cast.expression); + case cod_assignment_expression: + case cod_field_ref: + case cod_element_ref: + case cod_subroutine_call: + return 0; + default: + assert(0); + } + return 0; +} + +static int +possibly_set_sizes_to_match(cod_parse_context context, sm_ref decl, sm_ref init_value) +{ + sm_ref array_type = get_complex_type(context, decl); + if (array_type->node.array_type_decl.size_expr) return 1; + if ((init_value->node_type == cod_constant) && + (init_value->node.constant.token == string_constant)) { + /* init value is a string, set the array size to strlen + 1 */ + sm_ref size_expr = cod_new_constant(); + char *str = malloc(40); /* plenty */ + size_expr->node.constant.token = integer_constant; + sprintf(str, "%ld\n", (long) strlen(init_value->node.constant.const_val) + 1); + size_expr->node.constant.const_val = str; + array_type->node.array_type_decl.size_expr = size_expr; + return 1; + } + + if (is_array(decl)) { + sm_list items; + long size = 0; + assert(init_value->node_type == cod_initializer_list); + items = init_value->node.initializer_list.initializers; + /* init value is a list of initializers, count */ + while (items) { + size++; + items = items->next; + } + sm_ref size_expr = cod_new_constant(); + char *str = malloc(40); /* plenty */ + size_expr->node.constant.token = integer_constant; + sprintf(str, "%ld\n", size); + size_expr->node.constant.const_val = str; + array_type->node.array_type_decl.size_expr = size_expr; + return 1; + } + printf("Decl is : \n"); cod_print(decl); + printf("init_value is : \n"); cod_print(init_value); + return 1; +} +static int semanticize_decl(cod_parse_context context, sm_ref decl, + scope_ptr scope) +{ + switch(decl->node_type) { + case cod_declaration: { + sm_ref ctype; + int is_block_type = 0; + + if (resolve_local(decl->node.declaration.id, scope) != NULL) { + if (resolve_local(decl->node.declaration.id, scope) != decl) { + cod_src_error(context, decl, "Duplicate Symbol \"%s\"", + decl->node.declaration.id); + return 0; + } else { + /* been here, done that */ + return 1; + } + } else { + add_decl(decl->node.declaration.id, decl, scope); + } + if (scope->containing_scope == NULL) { + /* must be external variable */ + void *extern_value = + resolve_extern(decl->node.declaration.id, scope); + if ((extern_value == NULL) && (context->alloc_globals)) { + ; + } else if ((extern_value == NULL) && (decl->node.declaration.cg_address == NULL) && + (decl->node.declaration.const_var == 0)) { + cod_src_error(context, decl, + "External symbol lacking address \"%s\"", + decl->node.declaration.id); + return 0; + } + if (extern_value) { + decl->node.declaration.cg_address = extern_value; + } + decl->node.declaration.is_extern = 1; + } + if (decl->node.declaration.type_spec != NULL) { + sm_list l = decl->node.declaration.type_spec; + if ((l->node->node_type == cod_type_specifier) && + ((l->node->node.type_specifier.token == STATIC) || + (l->node->node.type_specifier.token == CONST))) { + if ((l->node->node.type_specifier.token == STATIC) && + !decl->node.declaration.is_subroutine) { + decl->node.declaration.static_var = 1; + } else { + decl->node.declaration.const_var = 1; + } + decl->node.declaration.type_spec = l->next; + free(l->node); + free(l); + } + } + if (decl->node.declaration.static_var) { + if (decl->node.declaration.init_value != NULL) { + sm_ref const_val = decl->node.declaration.init_value; + if (const_val->node_type == cod_initializer_list) { + sm_list items = const_val->node.initializer_list.initializers; + /* init value is a list of initializers, count */ + while (items) { + sm_ref sub_init = items->node->node.initializer.initializer; + if (!is_constant_expr(sub_init)) { + cod_src_error(context, sub_init, + "Static initializer not constant. Variable \"%s\"", + decl->node.declaration.id); + return 0; + } + items = items->next; + } + + } else if (!is_constant_expr(const_val)) { + cod_src_error(context, const_val, + "Static initializer not constant. Variable \"%s\"", + decl->node.declaration.id); + return 0; + } + } + } + if ((decl->node.declaration.sm_complex_type != NULL) && + (decl->node.declaration.param_num != -1)) { + /* complex type + param, must be pass by reference */ + sm_ref type = decl->node.declaration.sm_complex_type; + decl->node.declaration.cg_type = DILL_P; + if (type->node_type == cod_array_type_decl) { + int ret = semanticize_array_type_node(context, type, + scope); + if (ret == 0) return ret; + } + } + /* some array decls have sm_complex_type set already */ + if (decl->node.declaration.sm_complex_type == NULL) { + sm_ref typ = NULL; + int cg_type = DILL_I; + if (decl->node.declaration.type_spec != NULL) { + int type_def = 0; + typ = reduce_type_list(context, decl->node.declaration.type_spec, + &cg_type, scope, &type_def, &decl->node.declaration.freeable_complex_type); + if (type_def) { + decl->node.declaration.is_typedef = 1; + } + } else { + sm_ref arr = decl->node.declaration.sm_complex_type; + if ((arr != NULL) && + (arr->node_type == cod_array_type_decl)) { + typ = reduce_type_list(context, + arr->node.array_type_decl.type_spec, + &cg_type, scope, NULL, &decl->node.declaration.freeable_complex_type); + } else if ((arr != NULL) && (arr->node_type == cod_enum_type_decl)) { + cg_type = DILL_I; + } + } + if ((typ == NULL) && (cg_type == DILL_ERR)) return 0; + decl->node.declaration.cg_type = cg_type; + decl->node.declaration.sm_complex_type = typ; + } + ctype = decl->node.declaration.sm_complex_type; + if ((ctype != NULL) && ((ctype->node_type == cod_array_type_decl) || (ctype->node_type == cod_struct_type_decl))) { + is_block_type = 1; + } + if (decl->node.declaration.init_value != NULL) { + int ret; + ret = semanticize_expr(context, decl->node.declaration.init_value, + scope); + if (ret == 0) return ret; + if (is_array(decl)) { + ret = possibly_set_sizes_to_match(context, decl, decl->node.declaration.init_value); + } + ret = assignment_types_match(context, decl, + decl->node.declaration.init_value, 1); + return ret; + } + if (decl->node.declaration.is_subroutine) { + int ret; + int param_count = 0; + sm_list params = decl->node.declaration.params; + scope_ptr sub_scope = push_scope(scope); + ret = semanticize_decls_list(context, + decl->node.declaration.params, + sub_scope); + decl->node.declaration.varidiac_subroutine_param_count = -1; + while(params) { + sm_ref formal = params->node; + while(formal->node_type == cod_array_type_decl) { + formal = formal->node.array_type_decl.element_ref; + } + if (strcmp(formal->node.declaration.id, "...") == 0) { + decl->node.declaration.varidiac_subroutine_param_count = param_count; + } + params = params->next; + param_count++; + } + pop_scope(sub_scope); + return ret; + } + return 1; + break; + } + case cod_struct_type_decl: + return semanticize_struct_type_node(context, decl, scope); + break; + case cod_array_type_decl: + if (decl->node.array_type_decl.type_spec != NULL) { + sm_list l = decl->node.array_type_decl.type_spec; + if ((l->node->node_type == cod_type_specifier) && + (l->node->node.type_specifier.token == STATIC)) { + decl->node.array_type_decl.type_spec = l->next; + decl->node.array_type_decl.element_ref->node.declaration.static_var = 1; + free(l->node); + free(l); + } + } + return semanticize_array_type_node(context, decl, scope); + break; + case cod_reference_type_decl: + return semanticize_reference_type_node(context, decl, scope); + break; + case cod_constant: + return 1; + break; + case cod_enum_type_decl: + return semanticize_enum_type_node(context, decl, scope); + break; + default: + printf("Unhandled case in semanticize decls_list\n"); + cod_print(decl); + } + return 0; +} + +static int semanticize_statement(cod_parse_context context, sm_ref stmt, + scope_ptr scope); +static int +check_last_statement_return_list(cod_parse_context context, sm_list stmts); + +static int +check_last_statement_return(cod_parse_context context, sm_ref stmt) +{ + switch (stmt->node_type) { + case cod_selection_statement: + if (!check_last_statement_return(context, stmt->node.selection_statement.then_part)) return 0; + if (stmt->node.selection_statement.else_part && + !check_last_statement_return(context, stmt->node.selection_statement.else_part)) return 0; + return 1; + case cod_compound_statement: { + sm_list list = stmt->node.compound_statement.statements; + if (!list) list = stmt->node.compound_statement.decls; + if (list) return check_last_statement_return_list(context, list); + return 1; + } + case cod_return_statement: + return 1; + case cod_expression_statement: + return check_last_statement_return(context, stmt->node.expression_statement.expression); + case cod_label_statement: + return check_last_statement_return(context, stmt->node.label_statement.statement); + case cod_subroutine_call: { + sm_ref func_ref = stmt->node.subroutine_call.sm_func_ref; + char *id; + if (func_ref->node_type == cod_identifier) { + id = func_ref->node.identifier.id; + } else { + id = func_ref->node.declaration.id; + } + if (strcmp(id, "exit") == 0) return 1; + if (strcmp(id, "abort") == 0) return 1; + return 0; + } + default: + return 0; + } +} + +static int +check_last_statement_return_list(cod_parse_context context, sm_list stmts) +{ + sm_ref stmt; + while (stmts != NULL) { + stmt = stmts->node; + stmts = stmts->next; + } + if (!stmt) return 0; + return check_last_statement_return(context, stmt); +} + +static int +semanticize_decls_stmts_list(cod_parse_context context, sm_list decls_stmts, scope_ptr scope) +{ + int ret = 1; + while (decls_stmts != NULL) { + sm_ref item = decls_stmts->node; + switch(item->node_type) { + case cod_declaration: + case cod_struct_type_decl: + case cod_array_type_decl: + case cod_reference_type_decl: + case cod_enum_type_decl: + case cod_constant: + if (!semanticize_decl(context, item, scope)) { + ret = 0; + } + break; + default: { + int t = semanticize_statement(context, item, scope); + if (!t) { + ret = 0; + } + } + } + decls_stmts = decls_stmts->next; + } + return ret; +} + +static int +semanticize_decls_list(cod_parse_context context, sm_list decls, + scope_ptr scope) +{ + int ret = 1; + while (decls != NULL) { + if (!semanticize_decl(context, decls->node, scope)) { + ret = 0; + } + decls = decls->next; + } + return ret; +} + +static int +semanticize_selection_statement(cod_parse_context context, sm_ref selection, + scope_ptr scope) +{ + int ret = 1; + if (!semanticize_expr(context, + selection->node.selection_statement.conditional, + scope)) { + ret = 0; + } + if (!semanticize_statement(context, + selection->node.selection_statement.then_part, + scope)) { + ret = 0; + } + if (selection->node.selection_statement.else_part) { + if (!semanticize_statement(context, + selection->node.selection_statement.else_part, + scope)) { + ret = 0; + } + } + return ret; +} + +static int +semanticize_iteration_statement(cod_parse_context context, sm_ref iteration, + scope_ptr scope) +{ + int ret = 1; + if (iteration->node.iteration_statement.init_expr != NULL) { + if (!semanticize_expr(context, + iteration->node.iteration_statement.init_expr, + scope)) { + ret = 0; + } + } + + if (iteration->node.iteration_statement.test_expr != NULL) { + if (!semanticize_expr(context, + iteration->node.iteration_statement.test_expr, + scope)) { + ret = 0; + } + } + + if (iteration->node.iteration_statement.iter_expr != NULL) { + if (!semanticize_expr(context, + iteration->node.iteration_statement.iter_expr, + scope)) { + ret = 0; + } + } + + if (iteration->node.iteration_statement.statement != NULL) { + scope_ptr sub_scope = push_scope_container(scope, iteration); + if (!semanticize_statement(context, + iteration->node.iteration_statement.statement, + sub_scope)) { + ret = 0; + } + pop_scope(sub_scope); + } + if (iteration->node.iteration_statement.post_test_expr != NULL) { + if (!semanticize_expr(context, + iteration->node.iteration_statement.post_test_expr, + scope)) { + ret = 0; + } + } + + return ret; +} + +static int +semanticize_statement(cod_parse_context context, sm_ref stmt, + scope_ptr scope) +{ + if (!stmt) return 1; + switch (stmt->node_type) { + case cod_selection_statement: + return semanticize_selection_statement(context, stmt, scope); + case cod_iteration_statement: + return semanticize_iteration_statement(context, stmt, scope); + case cod_expression_statement: { + return semanticize_expr(context, + stmt->node.expression_statement.expression, + scope); + } + case cod_compound_statement: + return semanticize_compound_statement(context, stmt, scope, 0); + case cod_return_statement:{ + int expr_type; + stmt->node.return_statement.cg_func_type = context->return_cg_type; + if (stmt->node.return_statement.cg_func_type == DILL_V) { + if (stmt->node.return_statement.expression != NULL) { + cod_src_error(context, stmt, + "Return value supplied in subroutine declared to return VOID"); + return 0; + } + } else { + if (stmt->node.return_statement.expression == NULL) { + cod_src_error(context, stmt, + "Return value missing in non-VOID subroutine"); + return 0; + } + } + if (stmt->node.return_statement.expression == NULL) return 1; + if (!semanticize_expr(context, stmt->node.return_statement.expression, + scope)) return 0; + expr_type = cod_sm_get_type(stmt->node.return_statement.expression); + if (context->dont_coerce_return) { + int type_failure = 0; + switch (stmt->node.return_statement.cg_func_type) { + case DILL_C: case DILL_UC: case DILL_S: case DILL_US: case DILL_I: case DILL_U: case DILL_L: case DILL_UL: + if (expr_type > DILL_UL) type_failure++; + break; + case DILL_F: case DILL_D: + if ((expr_type != DILL_F) && (expr_type != DILL_D)) type_failure++; + break; + } + if (type_failure) { + cod_src_error(context, stmt, + "Return value doesn't match procedure type declaration and now allowed to use coercion"); + return 0; + } + } + return 1; + } + case cod_label_statement:{ +// add_decl(stmt->node.label_statement.name, stmt, scope); + return semanticize_statement(context, stmt->node.label_statement.statement, scope); + } + case cod_jump_statement:{ + if (stmt->node.jump_statement.goto_target != NULL) { + if (!stmt->node.jump_statement.sm_target_stmt) { + cod_src_error(context, stmt, + "Label \"%s\" not found. Goto has no target.", stmt->node.jump_statement.goto_target); + return 0; + } + } else { + /* this is a continue or a break */ + sm_ref tmp = find_containing_iterator(scope); + if (!tmp) { + cod_src_error(context, stmt, + "Continue or Break statement not contained inside an iterator."); + return 0; + } + stmt->node.jump_statement.sm_target_stmt = tmp; + } + return 1; + break; + } + default: + printf("unhandled case in semanticize statement\n"); + return 1; + } + return 1; +} + +typedef struct goto_semantic_state { + int backward_jump; + int passed_init_decl; + int already_found; +} *goto_state; + +static int semanticize_goto_l(cod_parse_context context, sm_ref this_goto, sm_list stmts, goto_state gs); + +static int semanticize_goto(cod_parse_context context, sm_ref this_goto, sm_ref stmt, goto_state gs) +{ + int ret = 1; + if (!stmt) return ret; + switch (stmt->node_type) { + case cod_declaration: + if (!gs->backward_jump && stmt->node.declaration.init_value) { + gs->passed_init_decl = 1; + } + break; + case cod_struct_type_decl: + case cod_array_type_decl: + case cod_reference_type_decl: + case cod_enum_type_decl: + case cod_constant: + /* no action for decls */ + break; + case cod_selection_statement: + ret &= semanticize_goto(context, this_goto, stmt->node.selection_statement.then_part, gs); + if (stmt->node.selection_statement.else_part) + ret &= semanticize_goto(context, this_goto, stmt->node.selection_statement.else_part, gs); + break; + case cod_iteration_statement: + ret &= semanticize_goto(context, this_goto, stmt->node.iteration_statement.statement, gs); + break; + case cod_compound_statement: + ret &= semanticize_goto_l(context, this_goto, stmt->node.compound_statement.decls, gs); + ret &= semanticize_goto_l(context, this_goto, stmt->node.compound_statement.statements, gs); + break; + case cod_return_statement: + break; + case cod_label_statement: + if (strcmp(this_goto->node.jump_statement.goto_target, stmt->node.label_statement.name) == 0) { + /* found target */ + if (!gs->backward_jump && gs->passed_init_decl) { + cod_src_error(context, stmt, "Goto jumps over initialized declaration, illegal forward jump."); + ret = 0; + } else if (gs->already_found) { + cod_src_error(context, stmt, "Duplicate label \"%s\".", stmt->node.label_statement.name); + ret = 0; + } else { + this_goto->node.jump_statement.sm_target_stmt = stmt; + gs->already_found = 1; + } + } + ret &= semanticize_goto(context, this_goto, stmt->node.label_statement.statement, gs); + break; + case cod_jump_statement: + if (stmt == this_goto) { + gs->backward_jump = 0; + } + break; + case cod_expression_statement: + break; + default: + printf("unhandled case in semanticize goto\n"); + return 0; + } + return ret; +} + +static int semanticize_goto_l(cod_parse_context context, sm_ref this_goto, sm_list stmts, goto_state gs) +{ + int saved_passed_init_decl = gs->passed_init_decl; + int ret = 1; + while(stmts) { + ret &= semanticize_goto(context, this_goto, stmts->node, gs); + stmts = stmts->next; + } + gs->passed_init_decl = saved_passed_init_decl; + return ret; +} + +static int +semanticize_gotos_list(cod_parse_context context, sm_list stmts, sm_list function_context) +{ + int ret = 1; + while(stmts) { + ret &= semanticize_gotos(context, stmts->node, function_context); + stmts = stmts->next; + } + return ret; +} + +static int +semanticize_gotos(cod_parse_context context, sm_ref stmt, sm_list function_context) +{ + /* + * recursive descent looking for goto's, followed by a recursive descent in + * the entire scope looking for their target + */ + int ret = 1; + if (!stmt) return 1; + switch (stmt->node_type) { + case cod_declaration: + case cod_struct_type_decl: + case cod_array_type_decl: + case cod_reference_type_decl: + case cod_enum_type_decl: + case cod_constant: + /* no action for most decls */ + break; + case cod_selection_statement: + ret &= semanticize_gotos(context, stmt->node.selection_statement.then_part, function_context); + if (stmt->node.selection_statement.else_part) + ret &= semanticize_gotos(context, stmt->node.selection_statement.else_part, function_context); + break; + case cod_iteration_statement: + ret &= semanticize_gotos(context, stmt->node.iteration_statement.statement, function_context); + break; + case cod_compound_statement: + ret &= semanticize_gotos_list(context, stmt->node.compound_statement.decls, function_context); + ret &= semanticize_gotos_list(context, stmt->node.compound_statement.statements, function_context); + break; + case cod_return_statement: + break; + case cod_label_statement: + ret &= semanticize_gotos(context, stmt->node.label_statement.statement, function_context); + break; + case cod_jump_statement: + if (stmt->node.jump_statement.goto_target != NULL) { + /* this is a goto */ + struct goto_semantic_state gs; + gs.backward_jump = 1; + gs.passed_init_decl = 0; + gs.already_found = 0; + ret &= semanticize_goto_l(context, stmt, function_context, &gs); + } + break; + case cod_expression_statement: + break; + default: + printf("unhandled case in semanticize gotos\n"); + return 0; + } + return ret; +} + +extern int +semanticize_compound_statement(cod_parse_context context, sm_ref compound, + scope_ptr containing_scope, int require_last_return) +{ + int ret = 1; + scope_ptr current_scope = push_scope(containing_scope); + + ret &= semanticize_decls_stmts_list(context, + compound->node.compound_statement.decls, + current_scope); + ret &= semanticize_decls_stmts_list(context, + compound->node.compound_statement.statements, + current_scope); + if (ret && require_last_return) { + int tmp; + sm_list list = compound->node.compound_statement.statements; + if (!list) list = compound->node.compound_statement.decls; + tmp = check_last_statement_return_list(context, compound->node.compound_statement.statements); + if (!tmp) { + cod_src_error(context, NULL, + "Control reaches end of non-void function."); + } + ret &= tmp; + } + pop_scope(current_scope); + return ret; +} + +extern sm_ref +cod_build_type_node(const char *name, FMFieldList field_list) +{ + sm_ref decl = cod_new_struct_type_decl(); + sm_list *end_ptr = &decl->node.struct_type_decl.fields; + + decl->node.struct_type_decl.id = strdup(name); + while ((field_list != NULL) && (field_list->field_name != NULL)) { + sm_list new_elem; + new_elem = malloc(sizeof(*new_elem)); + new_elem->next = NULL; + new_elem->node = cod_new_field(); + new_elem->node->node.field.name = strdup(field_list->field_name); + new_elem->node->node.field.string_type = strdup(field_list->field_type); + new_elem->node->node.field.cg_size = field_list->field_size; + new_elem->node->node.field.cg_offset = field_list->field_offset; + new_elem->node->node.field.cg_type = DILL_ERR; + *end_ptr = new_elem; + end_ptr = &new_elem->next; + field_list++; + } + return decl; +} + + +extern void +cod_remove_defined_types(cod_parse_context context, int count) +{ + char **types = context->defined_types; + while(types && types[count]) types[count++] = NULL; +} + +void +cod_add_defined_type(id, context) +char *id; +cod_parse_context context; +{ + int count = 0; + while(context->defined_types && context->defined_types[count]) count++; + if (count == 0) { + context->defined_types = malloc(sizeof(char*) * 2); + } else { + context->defined_types = realloc(context->defined_types, + (count+2)*sizeof(char*)); + } + context->defined_types[count] = id; + context->defined_types[count+1] = NULL; + reset_types_table(context->defined_types, context->enumerated_constants); + +} + +void +cod_add_enum_const(id, context) +char *id; +cod_parse_context context; +{ + int count = 0; + while(context->enumerated_constants && context->enumerated_constants[count]) count++; + if (count == 0) { + context->enumerated_constants = malloc(sizeof(char*) * 2); + } else { + context->enumerated_constants = realloc(context->enumerated_constants, + (count+2)*sizeof(char*)); + } + context->enumerated_constants[count] = id; + context->enumerated_constants[count+1] = NULL; + reset_types_table(context->defined_types, context->enumerated_constants); +} + +extern void +cod_add_simple_struct_type(const char *name, FMFieldList field_list, + cod_parse_context context) +{ + sm_ref node = cod_build_type_node(name, field_list); + cod_add_decl_to_parse_context(name, node, context); + cod_add_decl_to_scope((char*)name, node, context); +} + +extern void +cod_add_struct_type(FMStructDescList format_list, + cod_parse_context context) +{ + int count=0; + while(format_list && format_list[count].format_name) { + count++; + } + count = count-1; + for ( ; count >= 0; count--) { + cod_add_simple_struct_type(format_list[count].format_name, + format_list[count].field_list, + context); + } +} + +static int +str_to_data_type(str, size) +char *str; +int size; +{ + char *tmp = malloc(strlen(str) + 1); + char *free_str = tmp; + strcpy(tmp, str); + str = tmp; /* make a copy of str parameter */ + + while (isspace((int)*str) || (*str == '*') || (*str == '(')) { /* skip preceeding space */ + str++; + } + tmp = str + strlen(str) - 1; + while (isspace((int)*tmp) || (*tmp == ')')) { /* test trailing space */ + *tmp = 0; + tmp--; + } + tmp = str; + while (*tmp) { /* map to lower case */ + *tmp = tolower(*tmp); + tmp++; + } + if ((strcmp(str, "integer") == 0) || (strcmp(str, "enumeration") == 0)) { + free(free_str); + if (size == sizeof(long)) { + return DILL_L; + } else if (size == sizeof(int)) { + return DILL_I; + } else if (size == sizeof(short)) { + return DILL_S; + } else if (size == sizeof(char)) { + return DILL_C; + } else { + return DILL_L; + } + } else if (strcmp(str, "unsigned integer") == 0) { + free(free_str); + if (size == sizeof(long)) { + return DILL_UL; + } else if (size == sizeof(int)) { + return DILL_U; + } else if (size == sizeof(short)) { + return DILL_US; + } else if (size == sizeof(char)) { + return DILL_UC; + } else { + return DILL_UL; + } + } else if ((strcmp(str, "float") == 0) || (strcmp(str, "double") == 0)) { + free(free_str); + if (size == sizeof(double)) { + return DILL_D; + } else if (size == sizeof(float)) { + return DILL_F; + } else { + fprintf(stderr, "unsupported float size %d\n", size); + return DILL_D; + } + } else if (strcmp(str, "char") == 0) { + free(free_str); + assert(size == 1); + return DILL_C; + } else if (strcmp(str, "string") == 0) { + free(free_str); + return DILL_P; + } else { + free(free_str); + return DILL_ERR; + } +} + +static int +array_str_to_data_type(str, size) +char *str; +int size; +{ + int ret_type; + char field_type[1024]; + char *left_paren; + if ((left_paren = strchr(str, '[')) == NULL) { + ret_type = str_to_data_type(str, size); + } else { + char *tmp = str; + int i = 0; + for( ; tmp < left_paren; tmp++) { + field_type[i++] = *tmp; + } + field_type[i] = 0; + ret_type = str_to_data_type(field_type, size); + } + return ret_type; +} + +static sm_ref +build_subtype_nodes(context, decl, f, desc, err, scope, must_free_p) +cod_parse_context context; +sm_ref decl; +field* f; +FMTypeDesc *desc; +int *err; +scope_ptr scope; +int *must_free_p; +{ + sm_ref ret = NULL; + sm_ref subtype = NULL; + int must_free_flag = 0; + if (desc->next != NULL) { + subtype = build_subtype_nodes(context, decl, f, desc->next, err, scope, &must_free_flag); + if (*err != 0) { + printf("Subtype node failure\n"); + return NULL; + } + } + switch (desc->type) { + case FMType_array: { + sm_list fields = decl->node.struct_type_decl.fields; + sm_ref cf; + int i; + ret = cod_new_array_type_decl(); + *must_free_p = 1; + ret->node.array_type_decl.cg_static_size = desc->static_size; + if (desc->static_size == 0) { + ret->node.array_type_decl.cg_static_size = -1; + } + ret->node.array_type_decl.cg_element_type = DILL_B; + ret->node.array_type_decl.sm_complex_element_type = subtype; + if (must_free_flag) { + if (ret->node.array_type_decl.freeable_complex_element_type) { + cod_rfree(ret->node.array_type_decl.freeable_complex_element_type); + } + ret->node.array_type_decl.freeable_complex_element_type = subtype; + } + if (subtype == NULL) { + ret->node.array_type_decl.cg_element_type = + array_str_to_data_type(f->string_type, f->cg_size); + ret->node.array_type_decl.cg_element_size = f->cg_size; + ret->node.array_type_decl.dimensions = malloc(sizeof(struct dimen_p)); + ret->node.array_type_decl.dimensions->dimen_count = 1; + } else { + if (subtype->node_type == cod_array_type_decl) { + int sub_size = subtype->node.array_type_decl.cg_static_size; + int sub_dimensions = subtype->node.array_type_decl.dimensions->dimen_count; + if (sub_size == -1) { + /* element of *this* array has varying elements */ + ret->node.array_type_decl.cg_element_size = -1; + } else { + ret->node.array_type_decl.cg_element_size = + sub_size * subtype->node.array_type_decl.cg_element_size;; + } + + ret->node.array_type_decl.dimensions = malloc(sizeof(struct dimen_p) + sub_dimensions * sizeof(dimen_s)); + ret->node.array_type_decl.dimensions->dimen_count = sub_dimensions+1;; + memcpy(&ret->node.array_type_decl.dimensions->dimens[1], &subtype->node.array_type_decl.dimensions->dimens[0], sub_dimensions * sizeof(dimen_s)); + } else { + ret->node.array_type_decl.cg_element_size = f->cg_size; + ret->node.array_type_decl.dimensions = malloc(sizeof(struct dimen_p)); + ret->node.array_type_decl.dimensions->dimen_count = 1; + if (subtype->node_type == cod_reference_type_decl) { + ret->node.array_type_decl.cg_element_type = DILL_P; + } + } + } + + if (ret->node.array_type_decl.cg_static_size != -1) { + ret->node.array_type_decl.sm_dynamic_size = NULL; + ret->node.array_type_decl.dimensions->dimens[0].static_size = ret->node.array_type_decl.cg_static_size; + ret->node.array_type_decl.dimensions->dimens[0].control_field = NULL; + } else { + for (i=0; i < desc->control_field_index; i++) { + fields = fields->next; + } + cf = fields->node; + switch (str_to_data_type(cf->node.field.string_type, + (int)sizeof(int))) { + case DILL_C: case DILL_UC: case DILL_S: case DILL_US: + case DILL_I: case DILL_U: case DILL_L: case DILL_UL: + break; + default: + cod_src_error(context, NULL, + "Variable length control field \"%s\"not of integer type.", cf->node.field.string_type); + *err = 1; + return NULL; + break; + } + ret->node.array_type_decl.sm_dynamic_size = cf; + ret->node.array_type_decl.dimensions->dimens[0].static_size = -1; + ret->node.array_type_decl.dimensions->dimens[0].control_field = cf; + } + break; + } + case FMType_pointer: + ret = cod_new_reference_type_decl(); + *must_free_p = 1; + ret->node.reference_type_decl.name = gen_anon(); + ret->node.reference_type_decl.cg_referenced_type = DILL_ERR; + ret->node.reference_type_decl.sm_complex_referenced_type = subtype; + if (must_free_flag) { + if (ret->node.array_type_decl.freeable_complex_element_type) { + cod_rfree(ret->node.array_type_decl.freeable_complex_element_type); + } + ret->node.reference_type_decl.freeable_complex_referenced_type = subtype; + } + ret->node.reference_type_decl.cg_referenced_size = -1; + break; + case FMType_subformat: { + char *tmp_str = FMbase_type(f->string_type); + ret = resolve(tmp_str, scope); + free(tmp_str); + if (ret == NULL) { + printf("Didn't find base type %s\n", tmp_str); + *err = 1; + } + break; + } + case FMType_simple: + case FMType_string: + ret = NULL; + break; + } + return ret; + +} + +static void +build_type_nodes(context, decl, f, fields, cg_size, cg_type, desc, err, scope) +cod_parse_context context; +sm_ref decl; +field* f; +sm_list fields; +int cg_size; +int cg_type; +FMTypeDesc* desc; +int *err; +scope_ptr scope; +{ + int must_free_flag = 0; + sm_ref complex_type = build_subtype_nodes(context, decl, f, desc, err, scope, &must_free_flag); + f->sm_complex_type = complex_type; + if (must_free_flag) { + if (f->freeable_complex_type) { + cod_rfree(f->freeable_complex_type); + } + f->freeable_complex_type = complex_type; + } +} + +static int +semanticize_array_element_node(context, array, super_type, base_type_spec, + scope) +cod_parse_context context; +sm_ref array; +sm_ref super_type; +sm_list base_type_spec; +scope_ptr scope; +{ + if (array->node.array_type_decl.size_expr != NULL) { + if (!is_constant_expr(array->node.array_type_decl.size_expr)) { + cod_src_error(context, array, + "Array size expression must be constant."); + return 0; + } + if (semanticize_expr(context, + array->node.array_type_decl.size_expr, scope) == 0) { + return 0; + } + } else { + sm_ref element_ref = array->node.array_type_decl.element_ref; + /* allow NULL array type sizes */ + if (element_ref->node_type != cod_declaration) { + cod_src_error(context, element_ref, + "Null sizes only allowed in parameter contexts"); + return 0; + } + + } + dimen_p d = super_type->node.array_type_decl.dimensions; + d->dimen_count++; + d = realloc(d, sizeof(*d) + (sizeof(d->dimens[0]) * d->dimen_count)); + d->dimens[d->dimen_count].control_field = NULL; + super_type->node.array_type_decl.dimensions = d; + + if (array->node.array_type_decl.element_ref->node_type + == cod_declaration) { + /* we're the last in line */ + sm_ref typ = NULL; + int cg_type = DILL_ERR; + int ret; + sm_ref decl = array->node.array_type_decl.element_ref; + decl->node.declaration.sm_complex_type = super_type; + decl->node.declaration.cg_type = DILL_B; + ret = semanticize_decl(context, decl, scope); + if (ret == 0) return 0; + + if (decl->node.declaration.type_spec != NULL) { + typ = reduce_type_list(context, decl->node.declaration.type_spec, + &cg_type, scope, NULL, &decl->node.declaration.freeable_complex_type); + } else { + sm_ref arr = decl->node.declaration.sm_complex_type; + if ((arr != NULL) && + (arr->node_type == cod_array_type_decl)) { + typ = reduce_type_list(context, + arr->node.array_type_decl.type_spec, + &cg_type, scope, NULL, &decl->node.declaration.freeable_complex_type); + } + } + if ((typ == NULL) && (cg_type == DILL_ERR)) return 0; + array->node.array_type_decl.cg_element_type = cg_type; + array->node.array_type_decl.sm_complex_element_type = typ; + super_type->node.array_type_decl.cg_element_type = cg_type; + } else { + assert(array->node.array_type_decl.element_ref->node_type == cod_array_type_decl); + array->node.array_type_decl.sm_complex_element_type = array->node.array_type_decl.element_ref; + return semanticize_array_element_node(context, + array->node.array_type_decl.element_ref, + array, + base_type_spec, scope); + } + return 1; +} + +static int +semanticize_array_type_node(context, array, scope) +cod_parse_context context; +sm_ref array; +scope_ptr scope; +{ + if (!array->node.array_type_decl.dimensions) { + array->node.array_type_decl.dimensions = malloc(sizeof(dimen_s)); + memset(array->node.array_type_decl.dimensions, 0, sizeof(dimen_s)); + } + array->node.array_type_decl.dimensions->dimen_count = 0; + return semanticize_array_element_node(context, array, array, + array->node.array_type_decl.type_spec, + scope); +} + +#define Max(i,j) ((inext; + free(desc); + desc = tmp; + } + return; +} + +static int +semanticize_struct_type_node(cod_parse_context context, sm_ref decl, + scope_ptr scope) +{ + FMFieldList fl = malloc(sizeof(fl[0])); + int field_num = 0; + int ret = 1; + int struct_size = 0; + sm_list fields = decl->node.struct_type_decl.fields; + add_decl_ns(decl->node.struct_type_decl.id, decl, scope, NS_STRUCT); + while(fields != NULL) { + field *f = &fields->node->node.field; + fl[field_num].field_name = f->name; + fl[field_num].field_type = f->string_type; + fl = realloc(fl, sizeof(fl[0]) * (field_num + 2)); + field_num++; + fields = fields->next; + } + fl[field_num].field_name = NULL; + fl[field_num].field_type = NULL; + field_num = 0; + fields = decl->node.struct_type_decl.fields; + while(fields != NULL) { + field *f = &fields->node->node.field; + int err = 0; + int field_size = f->cg_size; + + if (f->string_type != NULL) { + /* FFS-compatible field type */ + FMTypeDesc* desc = gen_FMTypeDesc(fl, field_num, f->string_type); + if (desc == NULL) { + cod_src_error(context, decl, + "Field \"%s\" has unknown type \"%s\".", + f->name, f->string_type); + ret = 0; + } + build_type_nodes(context, decl, f, fields, f->cg_size, f->cg_type, + desc, &err, scope); + + free_FMTypeDesc(desc); + f->cg_type = str_to_data_type(f->string_type, f->cg_size); + field_size = f->cg_size; + if (f->sm_complex_type) { + if (f->sm_complex_type->node_type == cod_reference_type_decl) { + field_size = sizeof(char*); + } else if (f->sm_complex_type->node_type == cod_array_type_decl) { + sm_ref arr = f->sm_complex_type; + while (arr && (arr->node_type == cod_array_type_decl)) { + if (arr->node.array_type_decl.cg_static_size != -1) { + field_size *= arr->node.array_type_decl.cg_static_size; + } + arr = arr->node.array_type_decl.sm_complex_element_type; + } + } + } + } else { + /* not able to get a FFS-compatible form */ + int type_def = 0; + int cg_type; + sm_ref typ; + typ = reduce_type_list(context, f->type_spec, &cg_type, scope, + &type_def, &f->freeable_complex_type); + f->sm_complex_type = typ; + f->cg_type = cg_type; + field_size = -1; + } + if (err == 1) ret = 0; + struct_size = Max(struct_size, + (f->cg_offset + field_size)); + fields = fields->next; + field_num++; + } + free(fl); + decl->node.struct_type_decl.cg_size = struct_size; + return ret; +} + +static int +semanticize_enum_type_node(cod_parse_context context, sm_ref decl, + scope_ptr scope) +{ + sm_list enums = decl->node.enum_type_decl.enums; + while(enums != NULL) { + if (enums->node->node.enumerator.const_expression) { + if (!is_constant_expr(enums->node->node.enumerator.const_expression)) { + cod_src_error(context, enums->node, + "Enumerator value expression must be constant."); + return 0; + } + } + add_decl(enums->node->node.enumerator.id, enums->node, context->scope); + enums = enums->next; + } + add_decl_ns(decl->node.enum_type_decl.id, decl, scope, NS_ENUM); + return 1; +} + +static int +semanticize_reference_type_node(cod_parse_context context, sm_ref decl, + scope_ptr scope) +{ + int ret = 1; + if (decl->node.reference_type_decl.name != NULL) { + add_decl(decl->node.reference_type_decl.name, decl, scope); + } + return ret; +} + +extern sm_ref +cod_build_param_node(const char *id, sm_ref typ, int param_num) +{ + sm_ref node = cod_new_declaration(); + sm_ref ident; + node->node.declaration.param_num = param_num; + node->node.declaration.id = strdup(id); + node->node.declaration.sm_complex_type = typ; + if (typ != NULL) { + ident = cod_new_identifier(); + node->node.declaration.type_spec = malloc(sizeof(struct list_struct)); + node->node.declaration.type_spec->next = NULL; + node->node.declaration.type_spec->node = ident; + ident->node.identifier.id = strdup(typ->node.struct_type_decl.id); + } + return node; +} + +extern +void +get_FMformat_characteristics(FMFormat format, FMfloat_format *ff, FMinteger_format *intf, int *column_major, int *pointer_size); + +static +sm_ref cod_build_type_node_FMformat(FMFormat format, cod_parse_context context) +{ + sm_ref decl = cod_new_struct_type_decl(); + sm_list *end_ptr = &decl->node.struct_type_decl.fields; + FMfloat_format data_float; + FMinteger_format data_int; + int column_major; + int pointer_size; + FMFieldList field_list = format->field_list; + get_FMformat_characteristics(format, &data_float, &data_int, &column_major, &pointer_size); + + decl->node.struct_type_decl.id = strdup(name_of_FMformat(format)); + decl->node.struct_type_decl.encode_info = malloc(sizeof(struct enc_struct)); + decl->node.struct_type_decl.encode_info->byte_order = data_int; + decl->node.struct_type_decl.encode_info->float_order = data_float; + decl->node.struct_type_decl.encode_info->pointer_size = pointer_size; + while ((field_list != NULL) && (field_list->field_name != NULL)) { + sm_list new_elem; + char *colon = strchr(field_list->field_type, ':'); + char *bracket = strchr(field_list->field_type, '['); + + if (colon != NULL) { + *colon = 0; + if (bracket != NULL) strcpy(colon, bracket); + } + new_elem = malloc(sizeof(*new_elem)); + new_elem->next = NULL; + new_elem->node = cod_new_field(); + new_elem->node->node.field.name = strdup(field_list->field_name); + new_elem->node->node.field.string_type = strdup(field_list->field_type); + new_elem->node->node.field.cg_size = field_list->field_size; + new_elem->node->node.field.cg_offset = field_list->field_offset; + new_elem->node->node.field.cg_type = DILL_ERR; + *end_ptr = new_elem; + end_ptr = &new_elem->next; + field_list++; + } + return decl; +} + +extern void +cod_add_encoded_param(const char *id, char *data, int param_num, + FMContext c, cod_parse_context context) +{ + int i = 0; + FMFormat format = FMformat_from_ID(c, data); + FMFormat *formats; + sm_ref top_type = NULL, param_node; + sm_ref node; + if (format == NULL) { + printf("No FMFormat ID found in buffer supplied to cod_add_encoded_param()\n"); + printf("No parameter added\n"); + return; + } + formats = format->subformats; + while (formats[i] != NULL) { + node = cod_build_type_node_FMformat(formats[i], context); + cod_add_decl_to_parse_context(name_of_FMformat(formats[i]), node, context); + cod_add_decl_to_scope(name_of_FMformat(formats[i]), node, context); + top_type = node; + i++; + } + + node = cod_build_type_node_FMformat(format, context); + cod_add_decl_to_parse_context(name_of_FMformat(format), node, context); + cod_add_decl_to_scope(name_of_FMformat(format), node, context); + top_type = node; + param_node = cod_build_param_node(id, NULL, param_num); + param_node->node.declaration.sm_complex_type = top_type; + cod_add_decl_to_parse_context(id, param_node, context); +} + +extern void +cod_add_param(const char *id, const char *typ, int param_num, + cod_parse_context context) +{ + sm_list type_list; + sm_ref node; + setup_for_string_parse(typ, context->defined_types, context->enumerated_constants); + cod_code_string = typ; + parsing_type = 1; + yyerror_count = 0; + yycontext = context; + yyparse(); + parsing_type = 0; + terminate_string_parse(); + + if ((yyparse_value == NULL) || (yyerror_count != 0)) { + return; + } + type_list = (sm_list) yyparse_value; + + node = cod_build_param_node(id, NULL, param_num); + node->node.declaration.type_spec = type_list; + cod_add_decl_to_parse_context(id, node, context); +} + +extern void +cod_subroutine_declaration(const char *decl, cod_parse_context context) +{ + sm_list type_list, params; + sm_ref complex_type, declaration, freeable_complex_type = NULL; + int cg_type, param_num; + + setup_for_string_parse(decl, context->defined_types, context->enumerated_constants); + cod_code_string = decl; + parsing_param_spec = 1; + yyerror_count = 0; + yycontext = context; + yyparse(); + parsing_param_spec = 0; + terminate_string_parse(); + + if ((yyparse_value == NULL) || (yyerror_count != 0)) { + return; + } + declaration = yyparse_value; + context->freeable_declaration = declaration; + type_list = declaration->node.declaration.type_spec; + + /* handle return type */ + complex_type = reduce_type_list(context, type_list, &cg_type, context->scope, NULL, &freeable_complex_type); + if (freeable_complex_type) cod_rfree(freeable_complex_type); + /* context->return_type_list = type_list; - Free'd as part of parse, not here*/ + if (complex_type != NULL) { + cg_type = DILL_P; + } + context->return_cg_type = cg_type; + + /* handle params */ + params = declaration->node.declaration.params; + param_num = 0; + while (params != NULL) { + sm_ref param = NULL; + switch (params->node->node_type) { + case cod_declaration: + param = params->node; + break; + case cod_array_type_decl: + param = params->node->node.array_type_decl.element_ref; + param->node.declaration.sm_complex_type = params->node; + break; + default: + printf("unhandled case in cod_subroutine_declaration\n"); + } + param->node.declaration.param_num = param_num; + cod_add_decl_to_parse_context(param->node.declaration.id, + cod_copy(params->node), context); + param_num++; + params = params->next; + } +} + +extern void +cod_set_return_type(char *typ, cod_parse_context context) +{ + sm_list type_list; + int cg_type; + sm_ref complex_type, freeable_complex_type = NULL; + setup_for_string_parse(typ, context->defined_types, context->enumerated_constants); + cod_code_string = typ; + parsing_type = 1; + yyerror_count = 0; + yycontext = context; + yyparse(); + parsing_type = 0; + terminate_string_parse(); + + if ((yyparse_value == NULL) || (yyerror_count != 0)) { + return; + } + type_list = (sm_list) yyparse_value; + + complex_type = reduce_type_list(context, type_list, &cg_type, context->scope, NULL, &freeable_complex_type); + context->return_type_list = type_list; + if (complex_type != NULL) { + cg_type = DILL_P; + if (freeable_complex_type) cod_rfree(freeable_complex_type); + } + context->return_cg_type = cg_type; +} + +static sm_ref +find_complex_type(sm_ref node, scope_ptr scope) +{ + assert(node->node_type == cod_identifier); + return resolve(node->node.identifier.id, scope); +} + +extern cod_parse_context +new_cod_parse_context() +{ + cod_parse_context context = malloc(sizeof(struct parse_struct)); + context->decls = NULL; + context->standard_decls = NULL; + context->scope = push_scope(NULL); + context->defined_types = NULL; + context->enumerated_constants = NULL; + context->error_func = default_error_out; + context->client_data = NULL; + context->return_type_list = NULL; + context->return_cg_type = DILL_I; + context->freeable_declaration = NULL; + context->has_exec_context = 0; + context->dont_coerce_return = 0; + context->alloc_globals = 0; + cod_add_standard_elements(context); + return context; +} + +extern void +cod_free_parse_context(cod_parse_context parse_context) +{ + if (parse_context->scope->externs) { + int i = 0; + while(parse_context->scope->externs[i].extern_name) { + free(parse_context->scope->externs[i].extern_name); + i++; + } + free(parse_context->scope->externs); + } + pop_scope(parse_context->scope); + if (parse_context->defined_types) { + free(parse_context->defined_types); + } + if (parse_context->decls) { + cod_rfree_list(parse_context->decls, NULL); + } + if (parse_context->return_type_list) { + cod_rfree_list(parse_context->return_type_list, NULL); + } + if (parse_context->standard_decls) { + cod_rfree_list(parse_context->standard_decls, NULL); + } + if (parse_context->freeable_declaration) { + cod_rfree(parse_context->freeable_declaration); + } + free(parse_context); +} + +extern void +cod_assoc_externs(cod_parse_context context, cod_extern_list externs) +{ + int new_count = 0; + + while(externs[new_count].extern_value) new_count++; + + if (context->scope->externs == NULL) { + int i; + context->scope->externs = malloc((new_count+1) * sizeof(externs[0])); + for (i=0; i < new_count; i++) { + context->scope->externs[i].extern_name = strdup(externs[i].extern_name); + context->scope->externs[i].extern_value = externs[i].extern_value; + } + context->scope->externs[new_count].extern_name = NULL; + context->scope->externs[new_count].extern_value = NULL; + } else { + int old_count = 0; + int i; + while(context->scope->externs[old_count++].extern_value); + context->scope->externs = realloc(context->scope->externs, (new_count + old_count) * sizeof(externs[0])); + + for (i=0; i < new_count; i++) { + int j; + for (j=0; j < old_count-1; j++) { + if (strcmp(externs[i].extern_name, context->scope->externs[j].extern_name) == 0) { + context->scope->externs[j].extern_value = externs[i].extern_value; + } + } + context->scope->externs[old_count + i - 1].extern_name = strdup(externs[i].extern_name); + context->scope->externs[old_count + i - 1].extern_value = externs[i].extern_value; + } + context->scope->externs[new_count + old_count -1].extern_name = NULL; + context->scope->externs[new_count + old_count -1].extern_value = NULL; + } +} + +extern void +cod_add_decl_to_parse_context(const char *name, sm_ref item, cod_parse_context context) +{ + sm_list *last_ptr = &context->decls; + sm_list list = context->decls; + while (list != NULL) { + last_ptr = &list->next; + list = list->next; + } + *last_ptr = malloc(sizeof(*list)); + (*last_ptr)->next = NULL; + (*last_ptr)->node = item; + if (item->node_type == cod_struct_type_decl) { + cod_add_defined_type((char *)name, context); + } +} + +extern void +cod_add_int_constant_to_parse_context(const char *const_name, int value, cod_parse_context context) +{ + sm_ref constant; + char str_value[64]; + char *name = strdup(const_name); + sprintf(str_value, "%d", value); + constant = cod_new_constant(); + constant->node.constant.token = integer_constant; + constant->node.constant.const_val = strdup(str_value); + constant->node.constant.freeable_name = name; + cod_add_decl_to_scope((char*) name, constant, context); + cod_add_decl_to_parse_context(name, constant, context); +} + +extern void +cod_set_closure(char *name, void* closure_context, cod_parse_context context) +{ + sm_ref decl; + decl = resolve(name, context->scope); + assert(decl->node_type == cod_declaration); + assert(decl->node.declaration.is_subroutine); + decl->node.declaration.closure_id = closure_context; +} + +static void +space_to_underscore(char *str){ + while(*str != '\0'){ + if(isspace(*str)) + *str = '_'; + str++; + } +} + +static void +purify_name(FMStructDescList list){ + int i,j; + for(i=0; list[i].format_name; i++){ + FMFieldList fl = list[i].field_list; + space_to_underscore((char*)list[i].format_name); + for(j=0; fl[j].field_name; j++){ + space_to_underscore((char*)fl[j].field_name); + space_to_underscore((char*)fl[j].field_type); + } + } +} + +static void +uniqueify_names(FMStructDescList list, char *prefix) +{ + int i = 0; + int prefix_len = strlen(prefix); + while (list[i].format_name != NULL) { + int j = 0; + FMFieldList fl = list[i].field_list; + char *new_name = + malloc(strlen(list[i].format_name) + prefix_len + 1); + strcpy(new_name, prefix); + strcpy(new_name + prefix_len, list[i].format_name); + free(list[i].format_name); + list[i].format_name = new_name; + while (fl[j].field_name != 0) { + int field_type_len = strlen(fl[j].field_type); + char *bracket = strchr(fl[j].field_type, '['); + int k; + if (bracket != NULL) { + field_type_len = (long) bracket - (long) fl[j].field_type; + } + for (k = 0; k < i; k++) { + char *new_type; + if (strncmp + (fl[j].field_type, list[k].format_name + prefix_len, + field_type_len) != 0) { + /* don't match in N chars */ + continue; + } + if ((list[k].format_name + prefix_len)[field_type_len] != + 0) { + /* list name is longer */ + continue; + } + new_type = + malloc(strlen(fl[j].field_type) + prefix_len + 1); + strcpy(new_type, prefix); + strcpy(new_type + prefix_len, fl[j].field_type); + free((void *) fl[j].field_type); + fl[j].field_type = new_type; + break; + } + j++; + } + i++; + } + purify_name(list); +} + +/* Returns the ecode function which will do message format conversion */ +extern cod_code +gen_rollback_code(FMStructDescList format1, FMStructDescList format2, char *xform_code) +{ + /* setup ECL generation */ + /* + * NOTE: We have to make the type names (structure names) + * in format1 and format2 to be unique. + * Because of the nature of the problem, they are likely to be + * identical and this may cause namespace collisions in ECL. + */ + cod_code code; + cod_parse_context parse_context = new_cod_parse_context(); + + int i = 0; + uniqueify_names(format1, "f0_"); + while (format1[i].format_name != NULL) { + cod_add_simple_struct_type(format1[i].format_name, + format1[i].field_list, parse_context); + i++; + } + cod_add_param("new", format1[i - 1].format_name, 0, parse_context); + + i = 0; + uniqueify_names(format2, "f1_"); + while (format2[i].format_name != NULL) { + cod_add_simple_struct_type(format2[i].format_name, + format2[i].field_list, parse_context); + i++; + } + cod_add_param("old", format2[i - 1].format_name, 1, parse_context); + + code = cod_code_gen(xform_code, parse_context); + cod_free_parse_context(parse_context); + + /* + * the "code" structure is the only output from this block, + * all else is free'd. + */ + return code; +} + +static double +get_constant_float_value(cod_parse_context context, sm_ref expr) +{ + double result; + switch(expr->node.constant.token) { + case integer_constant: + case floating_constant: + sscanf(expr->node.constant.const_val, "%lg", &result); + return result; + case string_constant: + return 0.0; + case character_constant: + return (double)(unsigned char)expr->node.constant.const_val[0]; + default: + assert(FALSE); + } +} + +static long +get_constant_long_value(cod_parse_context context, sm_ref expr) +{ + double dresult; + long result; + switch(expr->node.constant.token) { + case integer_constant: + sscanf(expr->node.constant.const_val, "%ld", &result); + return result; + case floating_constant: + sscanf(expr->node.constant.const_val, "%lg", &dresult); + return (long)dresult; + case string_constant: + return -1; + case character_constant: + return (long)(unsigned char)expr->node.constant.const_val[0]; + default: + assert(FALSE); + } +} + +extern sm_ref +evaluate_constant_return_expr(cod_parse_context context, sm_ref expr, int *free_result) +{ + switch(expr->node_type) { + case cod_constant: + *free_result = 0; + return expr; + case cod_identifier: + return evaluate_constant_return_expr(context, expr->node.identifier.sm_declaration, free_result); + case cod_declaration: + if (!expr->node.declaration.const_var) return NULL; + return evaluate_constant_return_expr(context, expr->node.identifier.sm_declaration, free_result); + case cod_operator: { + sm_ref left = NULL, right = NULL, ret; + int free_left = 0, free_right = 0; + int left_token, right_token; + if (expr->node.operator.left != NULL) { + if (!(left = evaluate_constant_return_expr(context, expr->node.operator.left, &free_left))) return NULL; + left_token = left->node.constant.token; + } + if (expr->node.operator.op == op_sizeof) { + int cg_type; + sm_ref cast = expr->node.operator.right; + sm_ref typ; + long size; + assert(cast->node_type == cod_cast); + typ = reduce_type_list(context, cast->node.cast.type_spec, &cg_type, context? context->scope:NULL, NULL, NULL); + static dill_stream s = NULL; + char str_val[40]; + extern int cg_get_size(dill_stream s, sm_ref node); + + if (s == NULL) { + s = dill_create_stream(); + } + if (typ == NULL) { + size = dill_type_size(s, cg_type); + } else { + size = cg_get_size(s, cast); + } + ret = cod_new_constant(); + ret->node.constant.token = integer_constant; + sprintf(str_val, "%ld", size); + ret->node.constant.const_val = strdup(str_val); + *free_result = 1; + return ret; + } + if (expr->node.operator.right != NULL) { + if (!(right = evaluate_constant_return_expr(context, expr->node.operator.right, &free_right))) return NULL; + right_token = right->node.constant.token; + if (!expr->node.operator.left) { + left_token = right_token; + } + } + if ((left_token == string_constant) || + (right_token == string_constant)) { + /* blech. No operators on strings. */ + return NULL; + } + if ((left_token == floating_constant) || + (right_token == floating_constant)) { + double left_val, right_val, fvalue; + int ivalue, is_ivalue = 0; + char str_val[40]; + if (left) + left_val = get_constant_float_value(context, left); + if (right) + right_val = get_constant_float_value(context, right); + switch(expr->node.operator.op) { + case op_plus: + fvalue = left_val + right_val; + break; + case op_minus: + fvalue = left_val - right_val; + break; + case op_leq: + ivalue = left_val <= right_val; + is_ivalue=1; + break; + case op_lt: + ivalue = left_val < right_val; + is_ivalue=1; + break; + case op_geq: + ivalue = left_val >= right_val; + is_ivalue=1; + break; + case op_gt: + ivalue = left_val > right_val; + is_ivalue=1; + break; + case op_eq: + ivalue = left_val = right_val; + is_ivalue=1; + break; + case op_neq: + ivalue = left_val != right_val; + is_ivalue=1; + break; + case op_mult: + fvalue = left_val * right_val; + break; + case op_div: + if (right_val == 0) { + return NULL; + } + fvalue = left_val / right_val; + break; + case op_log_neg: + case op_not: + case op_left_shift: + case op_right_shift: + case op_modulus: + case op_log_or: + case op_arith_or: + case op_arith_xor: + case op_log_and: + case op_arith_and: + case op_deref: + case op_address: + case op_inc: + case op_dec: + case op_sizeof: + assert(FALSE); + } + ret = cod_new_constant(); + if (is_ivalue) { + ret->node.constant.token = integer_constant; + sprintf(str_val, "%d", ivalue); + } else { + ret->node.constant.token = floating_constant; + sprintf(str_val, "%.*e\n", OP_DBL_Digs - 1, fvalue); + } + ret->node.constant.const_val = strdup(str_val); + *free_result = 1; + } else { + /* we get an integer result */ + long left_val = 0, right_val = 0, value; + char str_val[40]; + if (expr->node.operator.left) + left_val = get_constant_long_value(context, left); + if (expr->node.operator.right) + right_val = get_constant_long_value(context, right); + switch(expr->node.operator.op) { + case op_modulus: + if (right_val == 0) { + return NULL; + } + value = left_val % right_val; + break; + case op_plus: + value = left_val + right_val; + break; + case op_minus: + value = left_val - right_val; + break; + case op_leq: + value = left_val <= right_val; + break; + case op_lt: + value = left_val < right_val; + break; + case op_geq: + value = left_val >= right_val; + break; + case op_gt: + value = left_val > right_val; + break; + case op_eq: + value = left_val = right_val; + break; + case op_neq: + value = left_val != right_val; + break; + case op_log_or: + value = left_val || right_val; + break; + case op_arith_or: + value = left_val | right_val; + break; + case op_arith_xor: + value = left_val ^ right_val; + break; + case op_log_and: + value = left_val && right_val; + break; + case op_arith_and: + value = left_val & right_val; + break; + case op_mult: + value = left_val * right_val; + break; + case op_div: + value = left_val / right_val; + break; + case op_log_neg: + value = !right_val; + break; + case op_not: + value = ~right_val; + break; + case op_left_shift: + value = left_val << right_val; + break; + case op_right_shift: + value = left_val >> right_val; + break; + case op_deref: + case op_address: + case op_inc: + case op_dec: + case op_sizeof: + assert(FALSE); + } + ret = cod_new_constant(); + ret->node.constant.token = integer_constant; + sprintf(str_val, "%ld", value); + ret->node.constant.const_val = strdup(str_val); + *free_result = 1; + } + if (free_left) { + /* do stuff*/ + } + if (free_right) { + /* do stuff*/ + } + return ret; + } + case cod_cast: + return evaluate_constant_return_expr(context, expr->node.cast.expression, free_result); + case cod_assignment_expression: + case cod_field_ref: + case cod_element_ref: + case cod_subroutine_call: + assert(FALSE); + default: + assert(FALSE); + } +} + + diff --git a/thirdparty/ffs/ffs/cod/pregen_source/Windows/lex.yy.c b/thirdparty/ffs/ffs/cod/pregen_source/Windows/lex.yy.c new file mode 100644 index 0000000000..e754ca5f93 --- /dev/null +++ b/thirdparty/ffs/ffs/cod/pregen_source/Windows/lex.yy.c @@ -0,0 +1,2829 @@ +#line 2 "/Users/eisen/prog/ffs/build/lex.yy.c" + +#line 4 "/Users/eisen/prog/ffs/build/lex.yy.c" + +#define YY_INT_ALIGNED short int + +/* A lexical scanner generated by flex */ + +#define FLEX_SCANNER +#define YY_FLEX_MAJOR_VERSION 2 +#define YY_FLEX_MINOR_VERSION 5 +#define YY_FLEX_SUBMINOR_VERSION 35 +#if YY_FLEX_SUBMINOR_VERSION > 0 +#define FLEX_BETA +#endif + +/* First, we deal with platform-specific or compiler-specific issues. */ + +/* begin standard C headers. */ +#include +#include +#include +#include + +/* end standard C headers. */ + +/* flex integer type definitions */ + +#ifndef FLEXINT_H +#define FLEXINT_H + +/* C99 systems have . Non-C99 systems may or may not. */ + +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + +/* C99 says to define __STDC_LIMIT_MACROS before including stdint.h, + * if you want the limit (max/min) macros for int types. + */ +#ifndef __STDC_LIMIT_MACROS +#define __STDC_LIMIT_MACROS 1 +#endif + +#include +typedef int8_t flex_int8_t; +typedef uint8_t flex_uint8_t; +typedef int16_t flex_int16_t; +typedef uint16_t flex_uint16_t; +typedef int32_t flex_int32_t; +typedef uint32_t flex_uint32_t; +typedef uint64_t flex_uint64_t; +#else +typedef signed char flex_int8_t; +typedef short int flex_int16_t; +typedef int flex_int32_t; +typedef unsigned char flex_uint8_t; +typedef unsigned short int flex_uint16_t; +typedef unsigned int flex_uint32_t; +#endif /* ! C99 */ + +/* Limits of integral types. */ +#ifndef INT8_MIN +#define INT8_MIN (-128) +#endif +#ifndef INT16_MIN +#define INT16_MIN (-32767-1) +#endif +#ifndef INT32_MIN +#define INT32_MIN (-2147483647-1) +#endif +#ifndef INT8_MAX +#define INT8_MAX (127) +#endif +#ifndef INT16_MAX +#define INT16_MAX (32767) +#endif +#ifndef INT32_MAX +#define INT32_MAX (2147483647) +#endif +#ifndef UINT8_MAX +#define UINT8_MAX (255U) +#endif +#ifndef UINT16_MAX +#define UINT16_MAX (65535U) +#endif +#ifndef UINT32_MAX +#define UINT32_MAX (4294967295U) +#endif + +#endif /* ! FLEXINT_H */ + +#ifdef __cplusplus + +/* The "const" storage-class-modifier is valid. */ +#define YY_USE_CONST + +#else /* ! __cplusplus */ + +/* C99 requires __STDC__ to be defined as 1. */ +#if defined (__STDC__) + +#define YY_USE_CONST + +#endif /* defined (__STDC__) */ +#endif /* ! __cplusplus */ + +#ifdef YY_USE_CONST +#define yyconst const +#else +#define yyconst +#endif + +/* Returned upon end-of-file. */ +#define YY_NULL 0 + +/* Promotes a possibly negative, possibly signed char to an unsigned + * integer for use as an array index. If the signed char is negative, + * we want to instead treat it as an 8-bit unsigned char, hence the + * double cast. + */ +#define YY_SC_TO_UI(c) ((unsigned int) (unsigned char) c) + +/* Enter a start condition. This macro really ought to take a parameter, + * but we do it the disgusting crufty way forced on us by the ()-less + * definition of BEGIN. + */ +#define BEGIN (yy_start) = 1 + 2 * + +/* Translate the current start state into a value that can be later handed + * to BEGIN to return to the state. The YYSTATE alias is for lex + * compatibility. + */ +#define YY_START (((yy_start) - 1) / 2) +#define YYSTATE YY_START + +/* Action number for EOF rule of a given start state. */ +#define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1) + +/* Special action meaning "start processing a new file". */ +#define YY_NEW_FILE yyrestart(yyin ) + +#define YY_END_OF_BUFFER_CHAR 0 + +/* Size of default input buffer. */ +#ifndef YY_BUF_SIZE +#define YY_BUF_SIZE 16384 +#endif + +/* The state buf must be large enough to hold one state per character in the main buffer. + */ +#define YY_STATE_BUF_SIZE ((YY_BUF_SIZE + 2) * sizeof(yy_state_type)) + +#ifndef YY_TYPEDEF_YY_BUFFER_STATE +#define YY_TYPEDEF_YY_BUFFER_STATE +typedef struct yy_buffer_state *YY_BUFFER_STATE; +#endif + +#ifndef YY_TYPEDEF_YY_SIZE_T +#define YY_TYPEDEF_YY_SIZE_T +typedef size_t yy_size_t; +#endif + +extern yy_size_t yyleng; + +extern FILE *yyin, *yyout; + +#define EOB_ACT_CONTINUE_SCAN 0 +#define EOB_ACT_END_OF_FILE 1 +#define EOB_ACT_LAST_MATCH 2 + + #define YY_LESS_LINENO(n) + +/* Return all but the first "n" matched characters back to the input stream. */ +#define yyless(n) \ + do \ + { \ + /* Undo effects of setting up yytext. */ \ + int yyless_macro_arg = (n); \ + YY_LESS_LINENO(yyless_macro_arg);\ + *yy_cp = (yy_hold_char); \ + YY_RESTORE_YY_MORE_OFFSET \ + (yy_c_buf_p) = yy_cp = yy_bp + yyless_macro_arg - YY_MORE_ADJ; \ + YY_DO_BEFORE_ACTION; /* set up yytext again */ \ + } \ + while ( 0 ) + +#define unput(c) yyunput( c, (yytext_ptr) ) + +#ifndef YY_STRUCT_YY_BUFFER_STATE +#define YY_STRUCT_YY_BUFFER_STATE +struct yy_buffer_state + { + FILE *yy_input_file; + + char *yy_ch_buf; /* input buffer */ + char *yy_buf_pos; /* current position in input buffer */ + + /* Size of input buffer in bytes, not including room for EOB + * characters. + */ + yy_size_t yy_buf_size; + + /* Number of characters read into yy_ch_buf, not including EOB + * characters. + */ + yy_size_t yy_n_chars; + + /* Whether we "own" the buffer - i.e., we know we created it, + * and can realloc() it to grow it, and should free() it to + * delete it. + */ + int yy_is_our_buffer; + + /* Whether this is an "interactive" input source; if so, and + * if we're using stdio for input, then we want to use getc() + * instead of fread(), to make sure we stop fetching input after + * each newline. + */ + int yy_is_interactive; + + /* Whether we're considered to be at the beginning of a line. + * If so, '^' rules will be active on the next match, otherwise + * not. + */ + int yy_at_bol; + + int yy_bs_lineno; /**< The line count. */ + int yy_bs_column; /**< The column count. */ + + /* Whether to try to fill the input buffer when we reach the + * end of it. + */ + int yy_fill_buffer; + + int yy_buffer_status; + +#define YY_BUFFER_NEW 0 +#define YY_BUFFER_NORMAL 1 + /* When an EOF's been seen but there's still some text to process + * then we mark the buffer as YY_EOF_PENDING, to indicate that we + * shouldn't try reading from the input source any more. We might + * still have a bunch of tokens to match, though, because of + * possible backing-up. + * + * When we actually see the EOF, we change the status to "new" + * (via yyrestart()), so that the user can continue scanning by + * just pointing yyin at a new input file. + */ +#define YY_BUFFER_EOF_PENDING 2 + + }; +#endif /* !YY_STRUCT_YY_BUFFER_STATE */ + +/* Stack of input buffers. */ +static size_t yy_buffer_stack_top = 0; /**< index of top of stack. */ +static size_t yy_buffer_stack_max = 0; /**< capacity of stack. */ +static YY_BUFFER_STATE * yy_buffer_stack = 0; /**< Stack as an array. */ + +/* We provide macros for accessing buffer states in case in the + * future we want to put the buffer states in a more general + * "scanner state". + * + * Returns the top of the stack, or NULL. + */ +#define YY_CURRENT_BUFFER ( (yy_buffer_stack) \ + ? (yy_buffer_stack)[(yy_buffer_stack_top)] \ + : NULL) + +/* Same as previous macro, but useful when we know that the buffer stack is not + * NULL or when we need an lvalue. For internal use only. + */ +#define YY_CURRENT_BUFFER_LVALUE (yy_buffer_stack)[(yy_buffer_stack_top)] + +/* yy_hold_char holds the character lost when yytext is formed. */ +static char yy_hold_char; +static yy_size_t yy_n_chars; /* number of characters read into yy_ch_buf */ +yy_size_t yyleng; + +/* Points to current character in buffer. */ +static char *yy_c_buf_p = (char *) 0; +static int yy_init = 0; /* whether we need to initialize */ +static int yy_start = 0; /* start state number */ + +/* Flag which is used to allow yywrap()'s to do buffer switches + * instead of setting up a fresh yyin. A bit of a hack ... + */ +static int yy_did_buffer_switch_on_eof; + +void yyrestart (FILE *input_file ); +void yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ); +YY_BUFFER_STATE yy_create_buffer (FILE *file,int size ); +void yy_delete_buffer (YY_BUFFER_STATE b ); +void yy_flush_buffer (YY_BUFFER_STATE b ); +void yypush_buffer_state (YY_BUFFER_STATE new_buffer ); +void yypop_buffer_state (void ); + +static void yyensure_buffer_stack (void ); +static void yy_load_buffer_state (void ); +static void yy_init_buffer (YY_BUFFER_STATE b,FILE *file ); + +#define YY_FLUSH_BUFFER yy_flush_buffer(YY_CURRENT_BUFFER ) + +YY_BUFFER_STATE yy_scan_buffer (char *base,yy_size_t size ); +YY_BUFFER_STATE yy_scan_string (yyconst char *yy_str ); +YY_BUFFER_STATE yy_scan_bytes (yyconst char *bytes,yy_size_t len ); + +void *yyalloc (yy_size_t ); +void *yyrealloc (void *,yy_size_t ); +void yyfree (void * ); + +#define yy_new_buffer yy_create_buffer + +#define yy_set_interactive(is_interactive) \ + { \ + if ( ! YY_CURRENT_BUFFER ){ \ + yyensure_buffer_stack (); \ + YY_CURRENT_BUFFER_LVALUE = \ + yy_create_buffer(yyin,YY_BUF_SIZE ); \ + } \ + YY_CURRENT_BUFFER_LVALUE->yy_is_interactive = is_interactive; \ + } + +#define yy_set_bol(at_bol) \ + { \ + if ( ! YY_CURRENT_BUFFER ){\ + yyensure_buffer_stack (); \ + YY_CURRENT_BUFFER_LVALUE = \ + yy_create_buffer(yyin,YY_BUF_SIZE ); \ + } \ + YY_CURRENT_BUFFER_LVALUE->yy_at_bol = at_bol; \ + } + +#define YY_AT_BOL() (YY_CURRENT_BUFFER_LVALUE->yy_at_bol) + +/* Begin user sect3 */ + +typedef unsigned char YY_CHAR; + +FILE *yyin = (FILE *) 0, *yyout = (FILE *) 0; + +typedef int yy_state_type; + +extern int yylineno; + +int yylineno = 1; + +extern char *yytext; +#define yytext_ptr yytext + +static yy_state_type yy_get_previous_state (void ); +static yy_state_type yy_try_NUL_trans (yy_state_type current_state ); +static int yy_get_next_buffer (void ); +static void yy_fatal_error (yyconst char msg[] ); + +/* Done after the current pattern has been matched and before the + * corresponding action - sets up yytext. + */ +#define YY_DO_BEFORE_ACTION \ + (yytext_ptr) = yy_bp; \ + yyleng = (yy_size_t) (yy_cp - yy_bp); \ + (yy_hold_char) = *yy_cp; \ + *yy_cp = '\0'; \ + (yy_c_buf_p) = yy_cp; + +#define YY_NUM_RULES 107 +#define YY_END_OF_BUFFER 108 +/* This struct is not used in this scanner, + but its presence is necessary. */ +struct yy_trans_info + { + flex_int32_t yy_verify; + flex_int32_t yy_nxt; + }; +static yyconst flex_int16_t yy_accept[284] = + { 0, + 0, 0, 0, 0, 90, 90, 108, 107, 105, 106, + 72, 76, 12, 38, 107, 2, 3, 9, 13, 8, + 14, 7, 11, 96, 96, 73, 42, 17, 24, 19, + 74, 10, 75, 75, 4, 5, 39, 75, 75, 75, + 75, 75, 75, 75, 75, 75, 75, 75, 75, 75, + 75, 70, 37, 71, 15, 88, 78, 77, 107, 90, + 92, 91, 23, 27, 36, 32, 0, 0, 25, 40, + 28, 41, 29, 1, 0, 103, 89, 94, 26, 104, + 96, 0, 96, 96, 0, 0, 96, 0, 20, 16, + 22, 18, 21, 75, 0, 33, 75, 75, 75, 47, + + 75, 75, 75, 75, 75, 75, 43, 75, 75, 75, + 75, 75, 75, 75, 75, 75, 75, 34, 35, 88, + 87, 80, 81, 85, 86, 82, 84, 83, 87, 90, + 91, 91, 93, 95, 0, 0, 6, 0, 103, 94, + 103, 0, 104, 0, 102, 96, 96, 96, 96, 0, + 0, 98, 97, 30, 31, 75, 75, 75, 75, 75, + 75, 75, 75, 45, 75, 51, 75, 75, 75, 75, + 75, 75, 75, 75, 75, 75, 75, 75, 80, 81, + 79, 0, 0, 0, 103, 0, 103, 0, 104, 102, + 96, 0, 0, 0, 98, 98, 98, 97, 97, 97, + + 75, 53, 75, 75, 75, 44, 67, 75, 75, 63, + 52, 75, 75, 75, 75, 75, 75, 75, 75, 75, + 75, 57, 75, 80, 0, 0, 0, 103, 0, 0, + 0, 99, 98, 98, 98, 98, 97, 97, 97, 97, + 62, 64, 75, 75, 75, 55, 75, 50, 75, 75, + 75, 75, 75, 75, 68, 75, 46, 0, 100, 0, + 101, 99, 98, 97, 75, 56, 59, 69, 49, 65, + 58, 54, 66, 75, 75, 100, 101, 75, 60, 75, + 61, 48, 0 + } ; + +static yyconst flex_int32_t yy_ec[256] = + { 0, + 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 2, 4, 5, 1, 1, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 19, 19, 19, 19, 19, 20, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 28, 28, 28, 29, 30, + 31, 31, 31, 31, 31, 32, 31, 31, 31, 33, + 31, 31, 31, 31, 34, 31, 31, 35, 31, 31, + 36, 37, 38, 39, 31, 1, 40, 41, 42, 43, + + 44, 45, 46, 47, 48, 31, 49, 50, 51, 52, + 53, 54, 31, 55, 56, 57, 58, 59, 60, 61, + 62, 63, 64, 65, 66, 67, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1 + } ; + +static yyconst flex_int32_t yy_meta[68] = + { 0, + 1, 1, 2, 1, 3, 1, 1, 4, 1, 1, + 5, 1, 1, 1, 6, 1, 7, 7, 7, 8, + 1, 1, 1, 1, 1, 9, 1, 8, 8, 8, + 10, 10, 10, 10, 10, 1, 3, 1, 1, 7, + 7, 8, 8, 8, 7, 10, 10, 10, 10, 10, + 10, 11, 10, 10, 11, 10, 11, 10, 11, 10, + 11, 10, 10, 1, 1, 1, 1 + } ; + +static yyconst flex_int16_t yy_base[299] = + { 0, + 0, 0, 65, 66, 69, 70, 614, 615, 615, 615, + 589, 615, 588, 67, 574, 615, 615, 586, 63, 615, + 64, 75, 85, 95, 44, 615, 615, 53, 585, 58, + 615, 615, 0, 600, 615, 615, 583, 551, 51, 552, + 47, 66, 551, 55, 550, 558, 74, 539, 76, 547, + 552, 615, 82, 615, 615, 0, 615, 615, 123, 0, + 615, 107, 615, 615, 615, 615, 109, 115, 615, 615, + 615, 615, 615, 615, 583, 141, 615, 0, 615, 170, + 0, 189, 116, 88, 582, 108, 147, 560, 548, 615, + 615, 615, 546, 0, 531, 615, 523, 515, 495, 486, + + 467, 464, 461, 455, 451, 424, 0, 423, 427, 421, + 423, 89, 111, 421, 101, 426, 425, 615, 615, 0, + 615, 175, 193, 615, 615, 615, 615, 615, 0, 0, + 151, 161, 615, 615, 208, 0, 615, 216, 615, 0, + 220, 239, 615, 204, 243, 121, 615, 440, 421, 0, + 150, 237, 251, 615, 615, 430, 414, 161, 427, 423, + 414, 419, 420, 0, 406, 0, 411, 398, 400, 402, + 403, 388, 96, 395, 384, 370, 373, 364, 224, 259, + 0, 273, 303, 279, 307, 341, 615, 295, 345, 615, + 615, 165, 213, 354, 248, 197, 284, 270, 236, 320, + + 364, 0, 355, 359, 352, 0, 0, 346, 343, 0, + 0, 334, 331, 343, 323, 321, 315, 314, 308, 298, + 303, 0, 294, 362, 266, 0, 366, 374, 391, 403, + 379, 408, 214, 615, 304, 285, 283, 615, 298, 279, + 0, 0, 267, 272, 256, 0, 255, 0, 257, 249, + 247, 186, 174, 152, 0, 130, 0, 412, 416, 424, + 432, 615, 615, 615, 111, 0, 0, 0, 0, 0, + 0, 0, 0, 118, 93, 615, 615, 67, 0, 43, + 0, 0, 615, 482, 493, 504, 509, 520, 531, 542, + 553, 562, 573, 579, 582, 584, 586, 588 + + } ; + +static yyconst flex_int16_t yy_def[299] = + { 0, + 283, 1, 284, 284, 285, 285, 283, 283, 283, 283, + 283, 283, 283, 283, 286, 283, 283, 283, 283, 283, + 283, 283, 283, 283, 24, 283, 283, 283, 283, 283, + 283, 283, 287, 287, 283, 283, 283, 287, 287, 287, + 287, 287, 287, 287, 287, 287, 287, 287, 287, 287, + 287, 283, 283, 283, 283, 288, 283, 283, 289, 290, + 283, 291, 283, 283, 283, 283, 286, 292, 283, 283, + 283, 283, 283, 283, 283, 283, 283, 293, 283, 283, + 25, 283, 283, 283, 294, 283, 283, 295, 283, 283, + 283, 283, 283, 287, 286, 283, 287, 287, 287, 287, + + 287, 287, 287, 287, 287, 287, 287, 287, 287, 287, + 287, 287, 287, 287, 287, 287, 287, 283, 283, 288, + 283, 283, 283, 283, 283, 283, 283, 283, 296, 290, + 291, 291, 283, 283, 286, 297, 283, 283, 283, 293, + 283, 283, 283, 283, 283, 283, 283, 283, 283, 298, + 294, 283, 295, 283, 283, 287, 287, 287, 287, 287, + 287, 287, 287, 287, 287, 287, 287, 287, 287, 287, + 287, 287, 287, 287, 287, 287, 287, 287, 283, 283, + 296, 286, 286, 283, 283, 283, 283, 283, 283, 283, + 283, 298, 298, 283, 283, 283, 283, 283, 283, 283, + + 287, 287, 287, 287, 287, 287, 287, 287, 287, 287, + 287, 287, 287, 287, 287, 287, 287, 287, 287, 287, + 287, 287, 287, 283, 286, 183, 283, 283, 283, 283, + 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, + 287, 287, 287, 287, 287, 287, 287, 287, 287, 287, + 287, 287, 287, 287, 287, 287, 287, 283, 283, 283, + 283, 283, 283, 283, 287, 287, 287, 287, 287, 287, + 287, 287, 287, 287, 287, 283, 283, 287, 287, 287, + 287, 287, 0, 283, 283, 283, 283, 283, 283, 283, + 283, 283, 283, 283, 283, 283, 283, 283 + + } ; + +static yyconst flex_int16_t yy_nxt[683] = + { 0, + 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 18, 19, 20, 21, 22, 23, 24, 25, 25, 25, + 26, 27, 28, 29, 30, 31, 32, 33, 33, 33, + 33, 34, 33, 34, 33, 35, 8, 36, 37, 33, + 38, 39, 40, 41, 42, 43, 33, 44, 33, 45, + 33, 33, 33, 33, 46, 47, 48, 49, 50, 51, + 33, 33, 33, 52, 53, 54, 55, 57, 57, 58, + 58, 61, 61, 65, 70, 89, 90, 72, 283, 62, + 62, 92, 93, 95, 283, 282, 71, 73, 74, 75, + 66, 76, 76, 76, 76, 77, 101, 98, 102, 107, + + 78, 59, 59, 99, 283, 118, 108, 103, 79, 80, + 281, 81, 81, 81, 81, 104, 134, 132, 105, 148, + 111, 112, 133, 82, 152, 152, 83, 115, 84, 85, + 113, 135, 135, 135, 170, 86, 280, 149, 82, 122, + 122, 122, 123, 217, 87, 68, 119, 146, 175, 147, + 172, 171, 84, 218, 147, 88, 176, 76, 76, 76, + 76, 283, 279, 124, 193, 173, 283, 125, 278, 138, + 139, 132, 139, 147, 126, 136, 133, 127, 147, 128, + 147, 275, 194, 129, 138, 139, 141, 141, 141, 141, + 139, 179, 179, 179, 180, 274, 146, 229, 142, 143, + + 144, 143, 144, 194, 147, 145, 145, 145, 145, 180, + 180, 180, 180, 142, 143, 134, 203, 204, 229, 143, + 145, 145, 145, 145, 182, 182, 182, 184, 235, 184, + 273, 272, 185, 185, 185, 185, 141, 141, 141, 141, + 224, 224, 224, 180, 68, 230, 236, 234, 186, 187, + 188, 187, 188, 152, 152, 189, 189, 189, 189, 145, + 145, 145, 145, 186, 187, 193, 230, 239, 195, 187, + 196, 234, 190, 134, 190, 180, 180, 180, 180, 233, + 134, 234, 198, 194, 199, 240, 197, 190, 271, 225, + 225, 225, 190, 270, 196, 185, 185, 185, 185, 269, + + 200, 237, 68, 238, 194, 234, 268, 267, 199, 68, + 134, 189, 189, 189, 189, 266, 238, 234, 265, 226, + 226, 226, 226, 185, 185, 185, 185, 238, 264, 264, + 226, 226, 226, 233, 263, 263, 139, 257, 139, 68, + 238, 234, 226, 226, 226, 226, 226, 226, 256, 255, + 254, 139, 227, 238, 227, 253, 139, 228, 228, 228, + 228, 189, 189, 189, 189, 231, 252, 231, 251, 237, + 232, 232, 232, 232, 143, 250, 143, 238, 180, 180, + 180, 180, 228, 228, 228, 228, 249, 248, 247, 143, + 228, 228, 228, 228, 143, 232, 232, 232, 232, 246, + + 245, 244, 258, 187, 258, 187, 243, 259, 259, 259, + 259, 242, 241, 223, 260, 222, 260, 221, 187, 261, + 261, 261, 261, 187, 232, 232, 232, 232, 259, 259, + 259, 259, 259, 259, 259, 259, 220, 262, 219, 262, + 261, 261, 261, 261, 216, 276, 215, 276, 261, 261, + 261, 261, 262, 214, 213, 212, 211, 262, 210, 209, + 276, 277, 208, 277, 207, 276, 206, 205, 202, 201, + 191, 191, 178, 177, 174, 169, 277, 168, 167, 166, + 165, 277, 56, 56, 56, 56, 56, 56, 56, 56, + 56, 56, 56, 60, 60, 60, 60, 60, 60, 60, + + 60, 60, 60, 60, 67, 164, 67, 163, 67, 67, + 67, 67, 67, 67, 67, 94, 94, 162, 94, 94, + 120, 161, 160, 120, 120, 120, 120, 120, 120, 120, + 120, 121, 121, 121, 121, 121, 121, 121, 121, 121, + 121, 121, 130, 159, 130, 130, 158, 130, 130, 130, + 130, 130, 130, 131, 157, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 67, 67, 156, 68, 67, 155, + 67, 154, 67, 140, 150, 140, 140, 140, 140, 140, + 140, 140, 140, 140, 151, 151, 151, 153, 153, 153, + 181, 181, 183, 183, 192, 192, 150, 137, 117, 116, + + 114, 110, 109, 106, 100, 97, 96, 95, 91, 69, + 68, 64, 63, 283, 7, 283, 283, 283, 283, 283, + 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, + 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, + 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, + 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, + 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, + 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, + 283, 283 + } ; + +static yyconst flex_int16_t yy_chk[683] = + { 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 3, 4, 3, + 4, 5, 6, 14, 19, 28, 28, 21, 25, 5, + 6, 30, 30, 49, 25, 280, 19, 21, 21, 22, + 14, 22, 22, 22, 22, 23, 41, 39, 41, 44, + + 23, 3, 4, 39, 25, 53, 44, 41, 23, 24, + 278, 24, 24, 24, 24, 42, 67, 62, 42, 84, + 47, 47, 62, 24, 86, 86, 24, 49, 24, 24, + 47, 68, 68, 68, 112, 24, 275, 84, 24, 59, + 59, 59, 59, 173, 24, 67, 53, 83, 115, 83, + 113, 112, 24, 173, 146, 24, 115, 76, 76, 76, + 76, 131, 274, 59, 151, 113, 131, 59, 265, 76, + 76, 132, 76, 83, 59, 68, 132, 59, 146, 59, + 87, 256, 151, 59, 76, 76, 80, 80, 80, 80, + 76, 122, 122, 122, 122, 254, 87, 192, 80, 80, + + 82, 80, 82, 151, 87, 82, 82, 82, 82, 123, + 123, 123, 123, 80, 80, 135, 158, 158, 192, 80, + 144, 144, 144, 144, 135, 135, 135, 138, 196, 138, + 253, 252, 138, 138, 138, 138, 141, 141, 141, 141, + 179, 179, 179, 179, 135, 193, 196, 233, 141, 141, + 142, 141, 142, 152, 152, 142, 142, 142, 142, 145, + 145, 145, 145, 141, 141, 153, 193, 199, 152, 141, + 152, 233, 145, 225, 145, 180, 180, 180, 180, 195, + 182, 195, 153, 153, 153, 199, 152, 145, 251, 182, + 182, 182, 145, 250, 152, 184, 184, 184, 184, 249, + + 153, 198, 225, 198, 153, 195, 247, 245, 153, 182, + 183, 188, 188, 188, 188, 244, 237, 197, 243, 183, + 183, 183, 183, 185, 185, 185, 185, 198, 240, 239, + 183, 183, 183, 197, 236, 235, 185, 223, 185, 183, + 237, 197, 183, 183, 183, 183, 183, 183, 221, 220, + 219, 185, 186, 200, 186, 218, 185, 186, 186, 186, + 186, 189, 189, 189, 189, 194, 217, 194, 216, 200, + 194, 194, 194, 194, 189, 215, 189, 200, 224, 224, + 224, 224, 227, 227, 227, 227, 214, 213, 212, 189, + 228, 228, 228, 228, 189, 231, 231, 231, 231, 209, + + 208, 205, 229, 228, 229, 228, 204, 229, 229, 229, + 229, 203, 201, 178, 230, 177, 230, 176, 228, 230, + 230, 230, 230, 228, 232, 232, 232, 232, 258, 258, + 258, 258, 259, 259, 259, 259, 175, 232, 174, 232, + 260, 260, 260, 260, 172, 259, 171, 259, 261, 261, + 261, 261, 232, 170, 169, 168, 167, 232, 165, 163, + 259, 261, 162, 261, 161, 259, 160, 159, 157, 156, + 149, 148, 117, 116, 114, 111, 261, 110, 109, 108, + 106, 261, 284, 284, 284, 284, 284, 284, 284, 284, + 284, 284, 284, 285, 285, 285, 285, 285, 285, 285, + + 285, 285, 285, 285, 286, 105, 286, 104, 286, 286, + 286, 286, 286, 286, 286, 287, 287, 103, 287, 287, + 288, 102, 101, 288, 288, 288, 288, 288, 288, 288, + 288, 289, 289, 289, 289, 289, 289, 289, 289, 289, + 289, 289, 290, 100, 290, 290, 99, 290, 290, 290, + 290, 290, 290, 291, 98, 291, 291, 291, 291, 291, + 291, 291, 291, 291, 292, 292, 97, 95, 292, 93, + 292, 89, 292, 293, 88, 293, 293, 293, 293, 293, + 293, 293, 293, 293, 294, 294, 294, 295, 295, 295, + 296, 296, 297, 297, 298, 298, 85, 75, 51, 50, + + 48, 46, 45, 43, 40, 38, 37, 34, 29, 18, + 15, 13, 11, 7, 283, 283, 283, 283, 283, 283, + 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, + 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, + 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, + 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, + 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, + 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, + 283, 283 + } ; + +static yy_state_type yy_last_accepting_state; +static char *yy_last_accepting_cpos; + +extern int yy_flex_debug; +int yy_flex_debug = 0; + +/* The intent behind this definition is that it'll catch + * any uses of REJECT which flex missed. + */ +#define REJECT reject_used_but_not_detected +#define yymore() yymore_used_but_not_detected +#define YY_MORE_ADJ 0 +#define YY_RESTORE_YY_MORE_OFFSET +char *yytext; +#line 1 "cod/cod.l" +#line 18 "cod/cod.l" +static int lex_offset = 1; +static int line_count = 1; +static char *create_string_from_yytext(); +#define RETURN(val) {yylval.info.lx_srcpos.line = line_count; yylval.info.lx_srcpos.character = lex_offset; lex_offset +=yyleng; return(val);} +static int type_count; +static char**types; +static char**enums; +#ifdef input +/* lex, not flex */ +static char *saved_input_str = NULL; +static void +terminate_string_parse() +{ + if (saved_input_str) { + free(saved_input_str); + saved_input_str = NULL; + } +} + +static char *input_str = NULL; +static void +setup_for_string_parse(string, defined_types, enum_constants) +const char *string; +char **defined_types; +char **enum_constants; +{ + int len = strlen(string); + + types = defined_types; + enums = enum_constants; + if (saved_input_str != NULL) free(input_str); + input_str = malloc(len + 2); + saved_input_str = input_str; + strcpy(input_str, string); + input_str[len] = 0; + input_str[len + 1] = 0; + lex_offset = 1; + line_count = 1; +} +#undef input +#define input() (((yytchar=*input_str)==0)?0:(input_str++,yytchar==10?(yylineno++,yytchar):yytchar)) +#undef unput +#define unput(c) (input_str--,c==10?yylineno--:yylineno) +#else +/* flex */ + +#ifdef YY_INPUT +#undef YY_INPUT +extern int my_yy_input(); +#define YY_INPUT(buf,x,y) x=my_yy_input(buf,x,y) +#endif + +/* end of lex ifdef */ +#endif +static +int +is_defined_type(id) +char *id; +{ + int i = 0; + while(types && types[i]) { + if (strcmp(id, types[i]) == 0) return 1; + i++; + } + return 0; +} +static +int +is_enumeration_constant(id) +char *id; +{ + int i = 0; + while(enums && enums[i]) { + if (strcmp(id, enums[i]) == 0) return 1; + i++; + } + return 0; +} +static void check_strbuf(); +static int buffer_len; +static char *string_buffer; +static char *string_buf_ptr; + +#line 783 "/Users/eisen/prog/ffs/build/lex.yy.c" + +#define INITIAL 0 +#define string_cond 1 +#define comment 2 + +#ifndef YY_NO_UNISTD_H +/* Special case for "unistd.h", since it is non-ANSI. We include it way + * down here because we want the user's section 1 to have been scanned first. + * The user has a chance to override it with an option. + */ +#include +#endif + +#ifndef YY_EXTRA_TYPE +#define YY_EXTRA_TYPE void * +#endif + +static int yy_init_globals (void ); + +/* Accessor methods to globals. + These are made visible to non-reentrant scanners for convenience. */ + +int yylex_destroy (void ); + +int yyget_debug (void ); + +void yyset_debug (int debug_flag ); + +YY_EXTRA_TYPE yyget_extra (void ); + +void yyset_extra (YY_EXTRA_TYPE user_defined ); + +FILE *yyget_in (void ); + +void yyset_in (FILE * in_str ); + +FILE *yyget_out (void ); + +void yyset_out (FILE * out_str ); + +yy_size_t yyget_leng (void ); + +char *yyget_text (void ); + +int yyget_lineno (void ); + +void yyset_lineno (int line_number ); + +/* Macros after this point can all be overridden by user definitions in + * section 1. + */ + +#ifndef YY_SKIP_YYWRAP +#ifdef __cplusplus +extern "C" int yywrap (void ); +#else +extern int yywrap (void ); +#endif +#endif + + static void yyunput (int c,char *buf_ptr ); + +#ifndef yytext_ptr +static void yy_flex_strncpy (char *,yyconst char *,int ); +#endif + +#ifdef YY_NEED_STRLEN +static int yy_flex_strlen (yyconst char * ); +#endif + +#ifndef YY_NO_INPUT + +#ifdef __cplusplus +static int yyinput (void ); +#else +static int input (void ); +#endif + +#endif + +/* Amount of stuff to slurp up with each read. */ +#ifndef YY_READ_BUF_SIZE +#define YY_READ_BUF_SIZE 8192 +#endif + +/* Copy whatever the last rule matched to the standard output. */ +#ifndef ECHO +/* This used to be an fputs(), but since the string might contain NUL's, + * we now use fwrite(). + */ +#define ECHO fwrite( yytext, yyleng, 1, yyout ) +#endif + +/* Gets input and stuffs it into "buf". number of characters read, or YY_NULL, + * is returned in "result". + */ +#ifndef YY_INPUT +#define YY_INPUT(buf,result,max_size) \ + if ( YY_CURRENT_BUFFER_LVALUE->yy_is_interactive ) \ + { \ + int c = '*'; \ + yy_size_t n; \ + for ( n = 0; n < max_size && \ + (c = getc( yyin )) != EOF && c != '\n'; ++n ) \ + buf[n] = (char) c; \ + if ( c == '\n' ) \ + buf[n++] = (char) c; \ + if ( c == EOF && ferror( yyin ) ) \ + YY_FATAL_ERROR( "input in flex scanner failed" ); \ + result = n; \ + } \ + else \ + { \ + errno=0; \ + while ( (result = fread(buf, 1, max_size, yyin))==0 && ferror(yyin)) \ + { \ + if( errno != EINTR) \ + { \ + YY_FATAL_ERROR( "input in flex scanner failed" ); \ + break; \ + } \ + errno=0; \ + clearerr(yyin); \ + } \ + }\ +\ + +#endif + +/* No semi-colon after return; correct usage is to write "yyterminate();" - + * we don't want an extra ';' after the "return" because that will cause + * some compilers to complain about unreachable statements. + */ +#ifndef yyterminate +#define yyterminate() return YY_NULL +#endif + +/* Number of entries by which start-condition stack grows. */ +#ifndef YY_START_STACK_INCR +#define YY_START_STACK_INCR 25 +#endif + +/* Report a fatal error. */ +#ifndef YY_FATAL_ERROR +#define YY_FATAL_ERROR(msg) yy_fatal_error( msg ) +#endif + +/* end tables serialization structures and prototypes */ + +/* Default declaration of generated scanner - a define so the user can + * easily add parameters. + */ +#ifndef YY_DECL +#define YY_DECL_IS_OURS 1 + +extern int yylex (void); + +#define YY_DECL int yylex (void) +#endif /* !YY_DECL */ + +/* Code executed at the beginning of each rule, after yytext and yyleng + * have been set up. + */ +#ifndef YY_USER_ACTION +#define YY_USER_ACTION +#endif + +/* Code executed at the end of each rule. */ +#ifndef YY_BREAK +#define YY_BREAK break; +#endif + +#define YY_RULE_SETUP \ + YY_USER_ACTION + +/** The main scanner function which does all the work. + */ +YY_DECL +{ + register yy_state_type yy_current_state; + register char *yy_cp, *yy_bp; + register int yy_act; + +#line 103 "cod/cod.l" + + + +#line 971 "/Users/eisen/prog/ffs/build/lex.yy.c" + + if ( !(yy_init) ) + { + (yy_init) = 1; + +#ifdef YY_USER_INIT + YY_USER_INIT; +#endif + + if ( ! (yy_start) ) + (yy_start) = 1; /* first start state */ + + if ( ! yyin ) + yyin = stdin; + + if ( ! yyout ) + yyout = stdout; + + if ( ! YY_CURRENT_BUFFER ) { + yyensure_buffer_stack (); + YY_CURRENT_BUFFER_LVALUE = + yy_create_buffer(yyin,YY_BUF_SIZE ); + } + + yy_load_buffer_state( ); + } + + while ( 1 ) /* loops until end-of-file is reached */ + { + yy_cp = (yy_c_buf_p); + + /* Support of yytext. */ + *yy_cp = (yy_hold_char); + + /* yy_bp points to the position in yy_ch_buf of the start of + * the current run. + */ + yy_bp = yy_cp; + + yy_current_state = (yy_start); +yy_match: + do + { + register YY_CHAR yy_c = yy_ec[YY_SC_TO_UI(*yy_cp)]; + if ( yy_accept[yy_current_state] ) + { + (yy_last_accepting_state) = yy_current_state; + (yy_last_accepting_cpos) = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = (int) yy_def[yy_current_state]; + if ( yy_current_state >= 284 ) + yy_c = yy_meta[(unsigned int) yy_c]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; + ++yy_cp; + } + while ( yy_base[yy_current_state] != 615 ); + +yy_find_action: + yy_act = yy_accept[yy_current_state]; + if ( yy_act == 0 ) + { /* have to back up */ + yy_cp = (yy_last_accepting_cpos); + yy_current_state = (yy_last_accepting_state); + yy_act = yy_accept[yy_current_state]; + } + + YY_DO_BEFORE_ACTION; + +do_action: /* This label is used only to access EOF actions. */ + + switch ( yy_act ) + { /* beginning of action switch */ + case 0: /* must back up */ + /* undo the effects of YY_DO_BEFORE_ACTION */ + *yy_cp = (yy_hold_char); + yy_cp = (yy_last_accepting_cpos); + yy_current_state = (yy_last_accepting_state); + goto yy_find_action; + +case 1: +YY_RULE_SETUP +#line 106 "cod/cod.l" +{RETURN(ARROW);} + YY_BREAK +case 2: +YY_RULE_SETUP +#line 107 "cod/cod.l" +{RETURN(LPAREN);} + YY_BREAK +case 3: +YY_RULE_SETUP +#line 108 "cod/cod.l" +{RETURN(RPAREN);} + YY_BREAK +case 4: +YY_RULE_SETUP +#line 109 "cod/cod.l" +{RETURN(LBRACKET);} + YY_BREAK +case 5: +YY_RULE_SETUP +#line 110 "cod/cod.l" +{RETURN(RBRACKET);} + YY_BREAK +case 6: +YY_RULE_SETUP +#line 111 "cod/cod.l" +{RETURN(DOTDOTDOT);} + YY_BREAK +case 7: +YY_RULE_SETUP +#line 112 "cod/cod.l" +{RETURN(DOT);} + YY_BREAK +case 8: +YY_RULE_SETUP +#line 113 "cod/cod.l" +{RETURN(COMMA);} + YY_BREAK +case 9: +YY_RULE_SETUP +#line 114 "cod/cod.l" +{RETURN(STAR);} + YY_BREAK +case 10: +YY_RULE_SETUP +#line 115 "cod/cod.l" +{RETURN(AT);} + YY_BREAK +case 11: +YY_RULE_SETUP +#line 116 "cod/cod.l" +{RETURN(SLASH);} + YY_BREAK +case 12: +YY_RULE_SETUP +#line 117 "cod/cod.l" +{RETURN(MODULUS);} + YY_BREAK +case 13: +YY_RULE_SETUP +#line 118 "cod/cod.l" +{RETURN(PLUS);} + YY_BREAK +case 14: +YY_RULE_SETUP +#line 119 "cod/cod.l" +{RETURN(MINUS);} + YY_BREAK +case 15: +YY_RULE_SETUP +#line 120 "cod/cod.l" +{RETURN(TILDE);} + YY_BREAK +case 16: +YY_RULE_SETUP +#line 121 "cod/cod.l" +{RETURN(LEQ);} + YY_BREAK +case 17: +YY_RULE_SETUP +#line 122 "cod/cod.l" +{RETURN(LT);} + YY_BREAK +case 18: +YY_RULE_SETUP +#line 123 "cod/cod.l" +{RETURN(GEQ);} + YY_BREAK +case 19: +YY_RULE_SETUP +#line 124 "cod/cod.l" +{RETURN(GT);} + YY_BREAK +case 20: +YY_RULE_SETUP +#line 125 "cod/cod.l" +{RETURN(LEFT_SHIFT);} + YY_BREAK +case 21: +YY_RULE_SETUP +#line 126 "cod/cod.l" +{RETURN(RIGHT_SHIFT);} + YY_BREAK +case 22: +YY_RULE_SETUP +#line 127 "cod/cod.l" +{RETURN(EQ);} + YY_BREAK +case 23: +YY_RULE_SETUP +#line 128 "cod/cod.l" +{RETURN(NEQ);} + YY_BREAK +case 24: +YY_RULE_SETUP +#line 129 "cod/cod.l" +{RETURN(ASSIGN);} + YY_BREAK +case 25: +YY_RULE_SETUP +#line 130 "cod/cod.l" +{RETURN(MUL_ASSIGN);} + YY_BREAK +case 26: +YY_RULE_SETUP +#line 131 "cod/cod.l" +{RETURN(DIV_ASSIGN);} + YY_BREAK +case 27: +YY_RULE_SETUP +#line 132 "cod/cod.l" +{RETURN(MOD_ASSIGN);} + YY_BREAK +case 28: +YY_RULE_SETUP +#line 133 "cod/cod.l" +{RETURN(ADD_ASSIGN);} + YY_BREAK +case 29: +YY_RULE_SETUP +#line 134 "cod/cod.l" +{RETURN(SUB_ASSIGN);} + YY_BREAK +case 30: +YY_RULE_SETUP +#line 135 "cod/cod.l" +{RETURN(LEFT_ASSIGN);} + YY_BREAK +case 31: +YY_RULE_SETUP +#line 136 "cod/cod.l" +{RETURN(RIGHT_ASSIGN);} + YY_BREAK +case 32: +YY_RULE_SETUP +#line 137 "cod/cod.l" +{RETURN(AND_ASSIGN);} + YY_BREAK +case 33: +YY_RULE_SETUP +#line 138 "cod/cod.l" +{RETURN(XOR_ASSIGN);} + YY_BREAK +case 34: +YY_RULE_SETUP +#line 139 "cod/cod.l" +{RETURN(OR_ASSIGN);} + YY_BREAK +case 35: +YY_RULE_SETUP +#line 140 "cod/cod.l" +{RETURN(LOG_OR);} + YY_BREAK +case 36: +YY_RULE_SETUP +#line 141 "cod/cod.l" +{RETURN(LOG_AND);} + YY_BREAK +case 37: +YY_RULE_SETUP +#line 142 "cod/cod.l" +{RETURN(ARITH_OR);} + YY_BREAK +case 38: +YY_RULE_SETUP +#line 143 "cod/cod.l" +{RETURN(ARITH_AND);} + YY_BREAK +case 39: +YY_RULE_SETUP +#line 144 "cod/cod.l" +{RETURN(ARITH_XOR);} + YY_BREAK +case 40: +YY_RULE_SETUP +#line 145 "cod/cod.l" +{RETURN(INC_OP);} + YY_BREAK +case 41: +YY_RULE_SETUP +#line 146 "cod/cod.l" +{RETURN(DEC_OP);} + YY_BREAK +case 42: +YY_RULE_SETUP +#line 147 "cod/cod.l" +{RETURN(SEMI);} + YY_BREAK +case 43: +YY_RULE_SETUP +#line 148 "cod/cod.l" +{RETURN(IF);} + YY_BREAK +case 44: +YY_RULE_SETUP +#line 149 "cod/cod.l" +{RETURN(ELSE);} + YY_BREAK +case 45: +YY_RULE_SETUP +#line 150 "cod/cod.l" +{RETURN(FOR);} + YY_BREAK +case 46: +YY_RULE_SETUP +#line 151 "cod/cod.l" +{RETURN(WHILE);} + YY_BREAK +case 47: +YY_RULE_SETUP +#line 152 "cod/cod.l" +{RETURN(DO);} + YY_BREAK +case 48: +YY_RULE_SETUP +#line 153 "cod/cod.l" +{RETURN(UNSIGNED);} + YY_BREAK +case 49: +YY_RULE_SETUP +#line 154 "cod/cod.l" +{RETURN(SIGNED);} + YY_BREAK +case 50: +YY_RULE_SETUP +#line 155 "cod/cod.l" +{RETURN(SHORT);} + YY_BREAK +case 51: +YY_RULE_SETUP +#line 156 "cod/cod.l" +{RETURN(INT);} + YY_BREAK +case 52: +YY_RULE_SETUP +#line 157 "cod/cod.l" +{RETURN(LONG);} + YY_BREAK +case 53: +YY_RULE_SETUP +#line 158 "cod/cod.l" +{RETURN(CHAR);} + YY_BREAK +case 54: +YY_RULE_SETUP +#line 159 "cod/cod.l" +{RETURN(STRING);} + YY_BREAK +case 55: +YY_RULE_SETUP +#line 160 "cod/cod.l" +{RETURN(FLOAT);} + YY_BREAK +case 56: +YY_RULE_SETUP +#line 161 "cod/cod.l" +{RETURN(DOUBLE);} + YY_BREAK +case 57: +YY_RULE_SETUP +#line 162 "cod/cod.l" +{RETURN(VOID);} + YY_BREAK +case 58: +YY_RULE_SETUP +#line 163 "cod/cod.l" +{RETURN(STATIC);} + YY_BREAK +case 59: +YY_RULE_SETUP +#line 164 "cod/cod.l" +{RETURN(EXTERN_TOKEN);} + YY_BREAK +case 60: +YY_RULE_SETUP +#line 165 "cod/cod.l" +{RETURN(TYPEDEF);} + YY_BREAK +case 61: +YY_RULE_SETUP +#line 166 "cod/cod.l" +{RETURN(CONTINUE);} + YY_BREAK +case 62: +YY_RULE_SETUP +#line 167 "cod/cod.l" +{RETURN(BREAK);} + YY_BREAK +case 63: +YY_RULE_SETUP +#line 168 "cod/cod.l" +{RETURN(GOTO);} + YY_BREAK +case 64: +YY_RULE_SETUP +#line 169 "cod/cod.l" +{RETURN(CONST);} + YY_BREAK +case 65: +YY_RULE_SETUP +#line 170 "cod/cod.l" +{RETURN(SIZEOF);} + YY_BREAK +case 66: +YY_RULE_SETUP +#line 171 "cod/cod.l" +{RETURN(STRUCT);} + YY_BREAK +case 67: +YY_RULE_SETUP +#line 172 "cod/cod.l" +{RETURN(ENUM);} + YY_BREAK +case 68: +YY_RULE_SETUP +#line 173 "cod/cod.l" +{RETURN(UNION);} + YY_BREAK +case 69: +YY_RULE_SETUP +#line 174 "cod/cod.l" +{RETURN(RETURN_TOKEN);} + YY_BREAK +case 70: +YY_RULE_SETUP +#line 175 "cod/cod.l" +{ + int count = 0; + while(types && types[count]) count++; + yylval.info.type_stack_count = count; + RETURN(LCURLY); + } + YY_BREAK +case 71: +YY_RULE_SETUP +#line 181 "cod/cod.l" +{RETURN(RCURLY);} + YY_BREAK +case 72: +YY_RULE_SETUP +#line 182 "cod/cod.l" +{RETURN(BANG);} + YY_BREAK +case 73: +YY_RULE_SETUP +#line 183 "cod/cod.l" +{RETURN(COLON);} + YY_BREAK +case 74: +YY_RULE_SETUP +#line 184 "cod/cod.l" +{RETURN(QUESTION);} + YY_BREAK +case 75: +YY_RULE_SETUP +#line 185 "cod/cod.l" +{ + yylval.info.string = create_string_from_yytext(); + if (is_defined_type(yylval.info.string)) { + RETURN(type_id); + } else if (is_enumeration_constant(yylval.info.string)) { + RETURN(enumeration_constant); + } else { + RETURN(identifier_ref); + } + } + YY_BREAK +case 76: +YY_RULE_SETUP +#line 196 "cod/cod.l" +{ + buffer_len = 20; + string_buffer = malloc(20 + 1); + string_buf_ptr = string_buffer; BEGIN(string_cond); +} + YY_BREAK +case 77: +YY_RULE_SETUP +#line 202 "cod/cod.l" +{ /* saw closing quote - all done */ + BEGIN(INITIAL); + *string_buf_ptr = '\0'; + /* return string constant token type and + * value to parser + */ + yylval.info.string = string_buffer; + RETURN(string_constant); +} + YY_BREAK +case 78: +/* rule 78 can match eol */ +YY_RULE_SETUP +#line 212 "cod/cod.l" +{ + yyerror("Unterminated string constant"); +} + YY_BREAK +case 79: +YY_RULE_SETUP +#line 216 "cod/cod.l" +{ + /* hex escape sequence */ + int result; + + (void) sscanf( yytext + 2, "%x", &result ); + + if ( result > 0xff ) { + yyerror("bad hex escape character"); + } + + check_strbuf(); + *string_buf_ptr++ = result; +} + YY_BREAK +case 80: +YY_RULE_SETUP +#line 230 "cod/cod.l" +{ + /* octal escape sequence */ + int result; + + (void) sscanf( yytext + 1, "%o", &result ); + + if ( result > 0xff ) { + yyerror("bad octal escape character"); + } + + check_strbuf(); + *string_buf_ptr++ = result; +} + YY_BREAK +case 81: +YY_RULE_SETUP +#line 244 "cod/cod.l" +{ + yyerror("bad character escape"); +} + YY_BREAK +case 82: +YY_RULE_SETUP +#line 248 "cod/cod.l" +{check_strbuf();*string_buf_ptr++ = '\n';} + YY_BREAK +case 83: +YY_RULE_SETUP +#line 249 "cod/cod.l" +{check_strbuf();*string_buf_ptr++ = '\t';} + YY_BREAK +case 84: +YY_RULE_SETUP +#line 250 "cod/cod.l" +{check_strbuf();*string_buf_ptr++ = '\r';} + YY_BREAK +case 85: +YY_RULE_SETUP +#line 251 "cod/cod.l" +{check_strbuf();*string_buf_ptr++ = '\b';} + YY_BREAK +case 86: +YY_RULE_SETUP +#line 252 "cod/cod.l" +{check_strbuf();*string_buf_ptr++ = '\f';} + YY_BREAK +case 87: +/* rule 87 can match eol */ +YY_RULE_SETUP +#line 254 "cod/cod.l" +{check_strbuf();*string_buf_ptr++ = yytext[1];} + YY_BREAK +case 88: +YY_RULE_SETUP +#line 256 "cod/cod.l" +{ + char *yptr = yytext; + + while ( *yptr ) { + check_strbuf(); + *string_buf_ptr++ = *yptr++; + } +} + YY_BREAK +case 89: +YY_RULE_SETUP +#line 267 "cod/cod.l" +BEGIN(comment); + YY_BREAK +case 90: +YY_RULE_SETUP +#line 269 "cod/cod.l" +{lex_offset += yyleng;} /* eat anything that's not a '*' */ + YY_BREAK +case 91: +YY_RULE_SETUP +#line 270 "cod/cod.l" +{lex_offset += yyleng;} /* eat up '*'s not followed by '/'s */ + YY_BREAK +case 92: +/* rule 92 can match eol */ +YY_RULE_SETUP +#line 271 "cod/cod.l" +{++line_count;lex_offset = 1;} + YY_BREAK +case 93: +YY_RULE_SETUP +#line 272 "cod/cod.l" +{lex_offset += yyleng;BEGIN(INITIAL);} + YY_BREAK +case 94: +YY_RULE_SETUP +#line 273 "cod/cod.l" +{ /* consume //-comment */ } + YY_BREAK +case 95: +YY_RULE_SETUP +#line 275 "cod/cod.l" +{ + yylval.info.string = create_string_from_yytext(); + RETURN(character_constant); + } + YY_BREAK +case 96: +YY_RULE_SETUP +#line 279 "cod/cod.l" +{ + yylval.info.string = create_string_from_yytext(); + RETURN(integer_constant); + } + YY_BREAK +case 97: +YY_RULE_SETUP +#line 283 "cod/cod.l" +{ + yylval.info.string = create_string_from_yytext(); + RETURN(integer_constant); + } + YY_BREAK +case 98: +YY_RULE_SETUP +#line 288 "cod/cod.l" +{ + yylval.info.string = create_string_from_yytext(); + RETURN(integer_constant); + } + YY_BREAK +case 99: +YY_RULE_SETUP +#line 293 "cod/cod.l" +{ + yylval.info.string = create_string_from_yytext(); + RETURN(floating_constant); + } + YY_BREAK +case 100: +YY_RULE_SETUP +#line 297 "cod/cod.l" +{ + yylval.info.string = create_string_from_yytext(); + RETURN(floating_constant); + } + YY_BREAK +case 101: +YY_RULE_SETUP +#line 301 "cod/cod.l" +{ + yylval.info.string = create_string_from_yytext(); + RETURN(floating_constant); + } + YY_BREAK +case 102: +YY_RULE_SETUP +#line 305 "cod/cod.l" +{ + yylval.info.string = create_string_from_yytext(); + RETURN(floating_constant); + } + YY_BREAK +case 103: +YY_RULE_SETUP +#line 310 "cod/cod.l" +{ + yylval.info.string = create_string_from_yytext(); + RETURN(floating_constant); + } + YY_BREAK +case 104: +YY_RULE_SETUP +#line 315 "cod/cod.l" +{ + yylval.info.string = create_string_from_yytext(); + RETURN(floating_constant); + } + YY_BREAK +case 105: +YY_RULE_SETUP +#line 320 "cod/cod.l" +{lex_offset += yyleng;} + YY_BREAK +case 106: +/* rule 106 can match eol */ +YY_RULE_SETUP +#line 321 "cod/cod.l" +{lex_offset = 1; line_count++;} + YY_BREAK +case 107: +YY_RULE_SETUP +#line 322 "cod/cod.l" +ECHO; + YY_BREAK +#line 1684 "/Users/eisen/prog/ffs/build/lex.yy.c" +case YY_STATE_EOF(INITIAL): +case YY_STATE_EOF(string_cond): +case YY_STATE_EOF(comment): + yyterminate(); + + case YY_END_OF_BUFFER: + { + /* Amount of text matched not including the EOB char. */ + int yy_amount_of_matched_text = (int) (yy_cp - (yytext_ptr)) - 1; + + /* Undo the effects of YY_DO_BEFORE_ACTION. */ + *yy_cp = (yy_hold_char); + YY_RESTORE_YY_MORE_OFFSET + + if ( YY_CURRENT_BUFFER_LVALUE->yy_buffer_status == YY_BUFFER_NEW ) + { + /* We're scanning a new file or input source. It's + * possible that this happened because the user + * just pointed yyin at a new source and called + * yylex(). If so, then we have to assure + * consistency between YY_CURRENT_BUFFER and our + * globals. Here is the right place to do so, because + * this is the first action (other than possibly a + * back-up) that will match for the new input source. + */ + (yy_n_chars) = YY_CURRENT_BUFFER_LVALUE->yy_n_chars; + YY_CURRENT_BUFFER_LVALUE->yy_input_file = yyin; + YY_CURRENT_BUFFER_LVALUE->yy_buffer_status = YY_BUFFER_NORMAL; + } + + /* Note that here we test for yy_c_buf_p "<=" to the position + * of the first EOB in the buffer, since yy_c_buf_p will + * already have been incremented past the NUL character + * (since all states make transitions on EOB to the + * end-of-buffer state). Contrast this with the test + * in input(). + */ + if ( (yy_c_buf_p) <= &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)] ) + { /* This was really a NUL. */ + yy_state_type yy_next_state; + + (yy_c_buf_p) = (yytext_ptr) + yy_amount_of_matched_text; + + yy_current_state = yy_get_previous_state( ); + + /* Okay, we're now positioned to make the NUL + * transition. We couldn't have + * yy_get_previous_state() go ahead and do it + * for us because it doesn't know how to deal + * with the possibility of jamming (and we don't + * want to build jamming into it because then it + * will run more slowly). + */ + + yy_next_state = yy_try_NUL_trans( yy_current_state ); + + yy_bp = (yytext_ptr) + YY_MORE_ADJ; + + if ( yy_next_state ) + { + /* Consume the NUL. */ + yy_cp = ++(yy_c_buf_p); + yy_current_state = yy_next_state; + goto yy_match; + } + + else + { + yy_cp = (yy_c_buf_p); + goto yy_find_action; + } + } + + else switch ( yy_get_next_buffer( ) ) + { + case EOB_ACT_END_OF_FILE: + { + (yy_did_buffer_switch_on_eof) = 0; + + if ( yywrap( ) ) + { + /* Note: because we've taken care in + * yy_get_next_buffer() to have set up + * yytext, we can now set up + * yy_c_buf_p so that if some total + * hoser (like flex itself) wants to + * call the scanner after we return the + * YY_NULL, it'll still work - another + * YY_NULL will get returned. + */ + (yy_c_buf_p) = (yytext_ptr) + YY_MORE_ADJ; + + yy_act = YY_STATE_EOF(YY_START); + goto do_action; + } + + else + { + if ( ! (yy_did_buffer_switch_on_eof) ) + YY_NEW_FILE; + } + break; + } + + case EOB_ACT_CONTINUE_SCAN: + (yy_c_buf_p) = + (yytext_ptr) + yy_amount_of_matched_text; + + yy_current_state = yy_get_previous_state( ); + + yy_cp = (yy_c_buf_p); + yy_bp = (yytext_ptr) + YY_MORE_ADJ; + goto yy_match; + + case EOB_ACT_LAST_MATCH: + (yy_c_buf_p) = + &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)]; + + yy_current_state = yy_get_previous_state( ); + + yy_cp = (yy_c_buf_p); + yy_bp = (yytext_ptr) + YY_MORE_ADJ; + goto yy_find_action; + } + break; + } + + default: + YY_FATAL_ERROR( + "fatal flex scanner internal error--no action found" ); + } /* end of action switch */ + } /* end of scanning one token */ +} /* end of yylex */ + +/* yy_get_next_buffer - try to read in a new buffer + * + * Returns a code representing an action: + * EOB_ACT_LAST_MATCH - + * EOB_ACT_CONTINUE_SCAN - continue scanning from current position + * EOB_ACT_END_OF_FILE - end of file + */ +static int yy_get_next_buffer (void) +{ + register char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf; + register char *source = (yytext_ptr); + register int number_to_move, i; + int ret_val; + + if ( (yy_c_buf_p) > &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars) + 1] ) + YY_FATAL_ERROR( + "fatal flex scanner internal error--end of buffer missed" ); + + if ( YY_CURRENT_BUFFER_LVALUE->yy_fill_buffer == 0 ) + { /* Don't try to fill the buffer, so this is an EOF. */ + if ( (yy_c_buf_p) - (yytext_ptr) - YY_MORE_ADJ == 1 ) + { + /* We matched a single character, the EOB, so + * treat this as a final EOF. + */ + return EOB_ACT_END_OF_FILE; + } + + else + { + /* We matched some text prior to the EOB, first + * process it. + */ + return EOB_ACT_LAST_MATCH; + } + } + + /* Try to read more data. */ + + /* First move last chars to start of buffer. */ + number_to_move = (int) ((yy_c_buf_p) - (yytext_ptr)) - 1; + + for ( i = 0; i < number_to_move; ++i ) + *(dest++) = *(source++); + + if ( YY_CURRENT_BUFFER_LVALUE->yy_buffer_status == YY_BUFFER_EOF_PENDING ) + /* don't do the read, it's not guaranteed to return an EOF, + * just force an EOF + */ + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars) = 0; + + else + { + yy_size_t num_to_read = + YY_CURRENT_BUFFER_LVALUE->yy_buf_size - number_to_move - 1; + + while ( num_to_read <= 0 ) + { /* Not enough room in the buffer - grow it. */ + + /* just a shorter name for the current buffer */ + YY_BUFFER_STATE b = YY_CURRENT_BUFFER; + + int yy_c_buf_p_offset = + (int) ((yy_c_buf_p) - b->yy_ch_buf); + + if ( b->yy_is_our_buffer ) + { + yy_size_t new_size = b->yy_buf_size * 2; + + if ( new_size <= 0 ) + b->yy_buf_size += b->yy_buf_size / 8; + else + b->yy_buf_size *= 2; + + b->yy_ch_buf = (char *) + /* Include room in for 2 EOB chars. */ + yyrealloc((void *) b->yy_ch_buf,b->yy_buf_size + 2 ); + } + else + /* Can't grow it, we don't own it. */ + b->yy_ch_buf = 0; + + if ( ! b->yy_ch_buf ) + YY_FATAL_ERROR( + "fatal error - scanner input buffer overflow" ); + + (yy_c_buf_p) = &b->yy_ch_buf[yy_c_buf_p_offset]; + + num_to_read = YY_CURRENT_BUFFER_LVALUE->yy_buf_size - + number_to_move - 1; + + } + + if ( num_to_read > YY_READ_BUF_SIZE ) + num_to_read = YY_READ_BUF_SIZE; + + /* Read in more data. */ + YY_INPUT( (&YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[number_to_move]), + (yy_n_chars), num_to_read ); + + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars); + } + + if ( (yy_n_chars) == 0 ) + { + if ( number_to_move == YY_MORE_ADJ ) + { + ret_val = EOB_ACT_END_OF_FILE; + yyrestart(yyin ); + } + + else + { + ret_val = EOB_ACT_LAST_MATCH; + YY_CURRENT_BUFFER_LVALUE->yy_buffer_status = + YY_BUFFER_EOF_PENDING; + } + } + + else + ret_val = EOB_ACT_CONTINUE_SCAN; + + if ((yy_size_t) ((yy_n_chars) + number_to_move) > YY_CURRENT_BUFFER_LVALUE->yy_buf_size) { + /* Extend the array by 50%, plus the number we really need. */ + yy_size_t new_size = (yy_n_chars) + number_to_move + ((yy_n_chars) >> 1); + YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = (char *) yyrealloc((void *) YY_CURRENT_BUFFER_LVALUE->yy_ch_buf,new_size ); + if ( ! YY_CURRENT_BUFFER_LVALUE->yy_ch_buf ) + YY_FATAL_ERROR( "out of dynamic memory in yy_get_next_buffer()" ); + } + + (yy_n_chars) += number_to_move; + YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)] = YY_END_OF_BUFFER_CHAR; + YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars) + 1] = YY_END_OF_BUFFER_CHAR; + + (yytext_ptr) = &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[0]; + + return ret_val; +} + +/* yy_get_previous_state - get the state just before the EOB char was reached */ + + static yy_state_type yy_get_previous_state (void) +{ + register yy_state_type yy_current_state; + register char *yy_cp; + + yy_current_state = (yy_start); + + for ( yy_cp = (yytext_ptr) + YY_MORE_ADJ; yy_cp < (yy_c_buf_p); ++yy_cp ) + { + register YY_CHAR yy_c = (*yy_cp ? yy_ec[YY_SC_TO_UI(*yy_cp)] : 1); + if ( yy_accept[yy_current_state] ) + { + (yy_last_accepting_state) = yy_current_state; + (yy_last_accepting_cpos) = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = (int) yy_def[yy_current_state]; + if ( yy_current_state >= 284 ) + yy_c = yy_meta[(unsigned int) yy_c]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; + } + + return yy_current_state; +} + +/* yy_try_NUL_trans - try to make a transition on the NUL character + * + * synopsis + * next_state = yy_try_NUL_trans( current_state ); + */ + static yy_state_type yy_try_NUL_trans (yy_state_type yy_current_state ) +{ + register int yy_is_jam; + register char *yy_cp = (yy_c_buf_p); + + register YY_CHAR yy_c = 1; + if ( yy_accept[yy_current_state] ) + { + (yy_last_accepting_state) = yy_current_state; + (yy_last_accepting_cpos) = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = (int) yy_def[yy_current_state]; + if ( yy_current_state >= 284 ) + yy_c = yy_meta[(unsigned int) yy_c]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; + yy_is_jam = (yy_current_state == 283); + + return yy_is_jam ? 0 : yy_current_state; +} + + static void yyunput (int c, register char * yy_bp ) +{ + register char *yy_cp; + + yy_cp = (yy_c_buf_p); + + /* undo effects of setting up yytext */ + *yy_cp = (yy_hold_char); + + if ( yy_cp < YY_CURRENT_BUFFER_LVALUE->yy_ch_buf + 2 ) + { /* need to shift things up to make room */ + /* +2 for EOB chars. */ + register yy_size_t number_to_move = (yy_n_chars) + 2; + register char *dest = &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[ + YY_CURRENT_BUFFER_LVALUE->yy_buf_size + 2]; + register char *source = + &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[number_to_move]; + + while ( source > YY_CURRENT_BUFFER_LVALUE->yy_ch_buf ) + *--dest = *--source; + + yy_cp += (int) (dest - source); + yy_bp += (int) (dest - source); + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = + (yy_n_chars) = YY_CURRENT_BUFFER_LVALUE->yy_buf_size; + + if ( yy_cp < YY_CURRENT_BUFFER_LVALUE->yy_ch_buf + 2 ) + YY_FATAL_ERROR( "flex scanner push-back overflow" ); + } + + *--yy_cp = (char) c; + + (yytext_ptr) = yy_bp; + (yy_hold_char) = *yy_cp; + (yy_c_buf_p) = yy_cp; +} + +#ifndef YY_NO_INPUT +#ifdef __cplusplus + static int yyinput (void) +#else + static int input (void) +#endif + +{ + int c; + + *(yy_c_buf_p) = (yy_hold_char); + + if ( *(yy_c_buf_p) == YY_END_OF_BUFFER_CHAR ) + { + /* yy_c_buf_p now points to the character we want to return. + * If this occurs *before* the EOB characters, then it's a + * valid NUL; if not, then we've hit the end of the buffer. + */ + if ( (yy_c_buf_p) < &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[(yy_n_chars)] ) + /* This was really a NUL. */ + *(yy_c_buf_p) = '\0'; + + else + { /* need more input */ + yy_size_t offset = (yy_c_buf_p) - (yytext_ptr); + ++(yy_c_buf_p); + + switch ( yy_get_next_buffer( ) ) + { + case EOB_ACT_LAST_MATCH: + /* This happens because yy_g_n_b() + * sees that we've accumulated a + * token and flags that we need to + * try matching the token before + * proceeding. But for input(), + * there's no matching to consider. + * So convert the EOB_ACT_LAST_MATCH + * to EOB_ACT_END_OF_FILE. + */ + + /* Reset buffer status. */ + yyrestart(yyin ); + + /*FALLTHROUGH*/ + + case EOB_ACT_END_OF_FILE: + { + if ( yywrap( ) ) + return 0; + + if ( ! (yy_did_buffer_switch_on_eof) ) + YY_NEW_FILE; +#ifdef __cplusplus + return yyinput(); +#else + return input(); +#endif + } + + case EOB_ACT_CONTINUE_SCAN: + (yy_c_buf_p) = (yytext_ptr) + offset; + break; + } + } + } + + c = *(unsigned char *) (yy_c_buf_p); /* cast for 8-bit char's */ + *(yy_c_buf_p) = '\0'; /* preserve yytext */ + (yy_hold_char) = *++(yy_c_buf_p); + + return c; +} +#endif /* ifndef YY_NO_INPUT */ + +/** Immediately switch to a different input stream. + * @param input_file A readable stream. + * + * @note This function does not reset the start condition to @c INITIAL . + */ + void yyrestart (FILE * input_file ) +{ + + if ( ! YY_CURRENT_BUFFER ){ + yyensure_buffer_stack (); + YY_CURRENT_BUFFER_LVALUE = + yy_create_buffer(yyin,YY_BUF_SIZE ); + } + + yy_init_buffer(YY_CURRENT_BUFFER,input_file ); + yy_load_buffer_state( ); +} + +/** Switch to a different input buffer. + * @param new_buffer The new input buffer. + * + */ + void yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ) +{ + + /* TODO. We should be able to replace this entire function body + * with + * yypop_buffer_state(); + * yypush_buffer_state(new_buffer); + */ + yyensure_buffer_stack (); + if ( YY_CURRENT_BUFFER == new_buffer ) + return; + + if ( YY_CURRENT_BUFFER ) + { + /* Flush out information for old buffer. */ + *(yy_c_buf_p) = (yy_hold_char); + YY_CURRENT_BUFFER_LVALUE->yy_buf_pos = (yy_c_buf_p); + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars); + } + + YY_CURRENT_BUFFER_LVALUE = new_buffer; + yy_load_buffer_state( ); + + /* We don't actually know whether we did this switch during + * EOF (yywrap()) processing, but the only time this flag + * is looked at is after yywrap() is called, so it's safe + * to go ahead and always set it. + */ + (yy_did_buffer_switch_on_eof) = 1; +} + +static void yy_load_buffer_state (void) +{ + (yy_n_chars) = YY_CURRENT_BUFFER_LVALUE->yy_n_chars; + (yytext_ptr) = (yy_c_buf_p) = YY_CURRENT_BUFFER_LVALUE->yy_buf_pos; + yyin = YY_CURRENT_BUFFER_LVALUE->yy_input_file; + (yy_hold_char) = *(yy_c_buf_p); +} + +/** Allocate and initialize an input buffer state. + * @param file A readable stream. + * @param size The character buffer size in bytes. When in doubt, use @c YY_BUF_SIZE. + * + * @return the allocated buffer state. + */ + YY_BUFFER_STATE yy_create_buffer (FILE * file, int size ) +{ + YY_BUFFER_STATE b; + + b = (YY_BUFFER_STATE) yyalloc(sizeof( struct yy_buffer_state ) ); + if ( ! b ) + YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + + b->yy_buf_size = size; + + /* yy_ch_buf has to be 2 characters longer than the size given because + * we need to put in 2 end-of-buffer characters. + */ + b->yy_ch_buf = (char *) yyalloc(b->yy_buf_size + 2 ); + if ( ! b->yy_ch_buf ) + YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + + b->yy_is_our_buffer = 1; + + yy_init_buffer(b,file ); + + return b; +} + +/** Destroy the buffer. + * @param b a buffer created with yy_create_buffer() + * + */ + void yy_delete_buffer (YY_BUFFER_STATE b ) +{ + + if ( ! b ) + return; + + if ( b == YY_CURRENT_BUFFER ) /* Not sure if we should pop here. */ + YY_CURRENT_BUFFER_LVALUE = (YY_BUFFER_STATE) 0; + + if ( b->yy_is_our_buffer ) + yyfree((void *) b->yy_ch_buf ); + + yyfree((void *) b ); +} + +#ifndef __cplusplus +extern int isatty (int ); +#endif /* __cplusplus */ + +/* Initializes or reinitializes a buffer. + * This function is sometimes called more than once on the same buffer, + * such as during a yyrestart() or at EOF. + */ + static void yy_init_buffer (YY_BUFFER_STATE b, FILE * file ) + +{ + int oerrno = errno; + + yy_flush_buffer(b ); + + b->yy_input_file = file; + b->yy_fill_buffer = 1; + + /* If b is the current buffer, then yy_init_buffer was _probably_ + * called from yyrestart() or through yy_get_next_buffer. + * In that case, we don't want to reset the lineno or column. + */ + if (b != YY_CURRENT_BUFFER){ + b->yy_bs_lineno = 1; + b->yy_bs_column = 0; + } + + b->yy_is_interactive = file ? (isatty( fileno(file) ) > 0) : 0; + + errno = oerrno; +} + +/** Discard all buffered characters. On the next scan, YY_INPUT will be called. + * @param b the buffer state to be flushed, usually @c YY_CURRENT_BUFFER. + * + */ + void yy_flush_buffer (YY_BUFFER_STATE b ) +{ + if ( ! b ) + return; + + b->yy_n_chars = 0; + + /* We always need two end-of-buffer characters. The first causes + * a transition to the end-of-buffer state. The second causes + * a jam in that state. + */ + b->yy_ch_buf[0] = YY_END_OF_BUFFER_CHAR; + b->yy_ch_buf[1] = YY_END_OF_BUFFER_CHAR; + + b->yy_buf_pos = &b->yy_ch_buf[0]; + + b->yy_at_bol = 1; + b->yy_buffer_status = YY_BUFFER_NEW; + + if ( b == YY_CURRENT_BUFFER ) + yy_load_buffer_state( ); +} + +/** Pushes the new state onto the stack. The new state becomes + * the current state. This function will allocate the stack + * if necessary. + * @param new_buffer The new state. + * + */ +void yypush_buffer_state (YY_BUFFER_STATE new_buffer ) +{ + if (new_buffer == NULL) + return; + + yyensure_buffer_stack(); + + /* This block is copied from yy_switch_to_buffer. */ + if ( YY_CURRENT_BUFFER ) + { + /* Flush out information for old buffer. */ + *(yy_c_buf_p) = (yy_hold_char); + YY_CURRENT_BUFFER_LVALUE->yy_buf_pos = (yy_c_buf_p); + YY_CURRENT_BUFFER_LVALUE->yy_n_chars = (yy_n_chars); + } + + /* Only push if top exists. Otherwise, replace top. */ + if (YY_CURRENT_BUFFER) + (yy_buffer_stack_top)++; + YY_CURRENT_BUFFER_LVALUE = new_buffer; + + /* copied from yy_switch_to_buffer. */ + yy_load_buffer_state( ); + (yy_did_buffer_switch_on_eof) = 1; +} + +/** Removes and deletes the top of the stack, if present. + * The next element becomes the new top. + * + */ +void yypop_buffer_state (void) +{ + if (!YY_CURRENT_BUFFER) + return; + + yy_delete_buffer(YY_CURRENT_BUFFER ); + YY_CURRENT_BUFFER_LVALUE = NULL; + if ((yy_buffer_stack_top) > 0) + --(yy_buffer_stack_top); + + if (YY_CURRENT_BUFFER) { + yy_load_buffer_state( ); + (yy_did_buffer_switch_on_eof) = 1; + } +} + +/* Allocates the stack if it does not exist. + * Guarantees space for at least one push. + */ +static void yyensure_buffer_stack (void) +{ + yy_size_t num_to_alloc; + + if (!(yy_buffer_stack)) { + + /* First allocation is just for 2 elements, since we don't know if this + * scanner will even need a stack. We use 2 instead of 1 to avoid an + * immediate realloc on the next call. + */ + num_to_alloc = 1; + (yy_buffer_stack) = (struct yy_buffer_state**)yyalloc + (num_to_alloc * sizeof(struct yy_buffer_state*) + ); + if ( ! (yy_buffer_stack) ) + YY_FATAL_ERROR( "out of dynamic memory in yyensure_buffer_stack()" ); + + memset((yy_buffer_stack), 0, num_to_alloc * sizeof(struct yy_buffer_state*)); + + (yy_buffer_stack_max) = num_to_alloc; + (yy_buffer_stack_top) = 0; + return; + } + + if ((yy_buffer_stack_top) >= ((yy_buffer_stack_max)) - 1){ + + /* Increase the buffer to prepare for a possible push. */ + int grow_size = 8 /* arbitrary grow size */; + + num_to_alloc = (yy_buffer_stack_max) + grow_size; + (yy_buffer_stack) = (struct yy_buffer_state**)yyrealloc + ((yy_buffer_stack), + num_to_alloc * sizeof(struct yy_buffer_state*) + ); + if ( ! (yy_buffer_stack) ) + YY_FATAL_ERROR( "out of dynamic memory in yyensure_buffer_stack()" ); + + /* zero only the new slots.*/ + memset((yy_buffer_stack) + (yy_buffer_stack_max), 0, grow_size * sizeof(struct yy_buffer_state*)); + (yy_buffer_stack_max) = num_to_alloc; + } +} + +/** Setup the input buffer state to scan directly from a user-specified character buffer. + * @param base the character buffer + * @param size the size in bytes of the character buffer + * + * @return the newly allocated buffer state object. + */ +YY_BUFFER_STATE yy_scan_buffer (char * base, yy_size_t size ) +{ + YY_BUFFER_STATE b; + + if ( size < 2 || + base[size-2] != YY_END_OF_BUFFER_CHAR || + base[size-1] != YY_END_OF_BUFFER_CHAR ) + /* They forgot to leave room for the EOB's. */ + return 0; + + b = (YY_BUFFER_STATE) yyalloc(sizeof( struct yy_buffer_state ) ); + if ( ! b ) + YY_FATAL_ERROR( "out of dynamic memory in yy_scan_buffer()" ); + + b->yy_buf_size = size - 2; /* "- 2" to take care of EOB's */ + b->yy_buf_pos = b->yy_ch_buf = base; + b->yy_is_our_buffer = 0; + b->yy_input_file = 0; + b->yy_n_chars = b->yy_buf_size; + b->yy_is_interactive = 0; + b->yy_at_bol = 1; + b->yy_fill_buffer = 0; + b->yy_buffer_status = YY_BUFFER_NEW; + + yy_switch_to_buffer(b ); + + return b; +} + +/** Setup the input buffer state to scan a string. The next call to yylex() will + * scan from a @e copy of @a str. + * @param yystr a NUL-terminated string to scan + * + * @return the newly allocated buffer state object. + * @note If you want to scan bytes that may contain NUL values, then use + * yy_scan_bytes() instead. + */ +YY_BUFFER_STATE yy_scan_string (yyconst char * yystr ) +{ + + return yy_scan_bytes(yystr,strlen(yystr) ); +} + +/** Setup the input buffer state to scan the given bytes. The next call to yylex() will + * scan from a @e copy of @a bytes. + * @param bytes the byte buffer to scan + * @param len the number of bytes in the buffer pointed to by @a bytes. + * + * @return the newly allocated buffer state object. + */ +YY_BUFFER_STATE yy_scan_bytes (yyconst char * yybytes, yy_size_t _yybytes_len ) +{ + YY_BUFFER_STATE b; + char *buf; + yy_size_t n, i; + + /* Get memory for full buffer, including space for trailing EOB's. */ + n = _yybytes_len + 2; + buf = (char *) yyalloc(n ); + if ( ! buf ) + YY_FATAL_ERROR( "out of dynamic memory in yy_scan_bytes()" ); + + for ( i = 0; i < _yybytes_len; ++i ) + buf[i] = yybytes[i]; + + buf[_yybytes_len] = buf[_yybytes_len+1] = YY_END_OF_BUFFER_CHAR; + + b = yy_scan_buffer(buf,n ); + if ( ! b ) + YY_FATAL_ERROR( "bad buffer in yy_scan_bytes()" ); + + /* It's okay to grow etc. this buffer, and we should throw it + * away when we're done. + */ + b->yy_is_our_buffer = 1; + + return b; +} + +#ifndef YY_EXIT_FAILURE +#define YY_EXIT_FAILURE 2 +#endif + +static void yy_fatal_error (yyconst char* msg ) +{ + (void) fprintf( stderr, "%s\n", msg ); + exit( YY_EXIT_FAILURE ); +} + +/* Redefine yyless() so it works in section 3 code. */ + +#undef yyless +#define yyless(n) \ + do \ + { \ + /* Undo effects of setting up yytext. */ \ + int yyless_macro_arg = (n); \ + YY_LESS_LINENO(yyless_macro_arg);\ + yytext[yyleng] = (yy_hold_char); \ + (yy_c_buf_p) = yytext + yyless_macro_arg; \ + (yy_hold_char) = *(yy_c_buf_p); \ + *(yy_c_buf_p) = '\0'; \ + yyleng = yyless_macro_arg; \ + } \ + while ( 0 ) + +/* Accessor methods (get/set functions) to struct members. */ + +/** Get the current line number. + * + */ +int yyget_lineno (void) +{ + + return yylineno; +} + +/** Get the input stream. + * + */ +FILE *yyget_in (void) +{ + return yyin; +} + +/** Get the output stream. + * + */ +FILE *yyget_out (void) +{ + return yyout; +} + +/** Get the length of the current token. + * + */ +yy_size_t yyget_leng (void) +{ + return yyleng; +} + +/** Get the current token. + * + */ + +char *yyget_text (void) +{ + return yytext; +} + +/** Set the current line number. + * @param line_number + * + */ +void yyset_lineno (int line_number ) +{ + + yylineno = line_number; +} + +/** Set the input stream. This does not discard the current + * input buffer. + * @param in_str A readable stream. + * + * @see yy_switch_to_buffer + */ +void yyset_in (FILE * in_str ) +{ + yyin = in_str ; +} + +void yyset_out (FILE * out_str ) +{ + yyout = out_str ; +} + +int yyget_debug (void) +{ + return yy_flex_debug; +} + +void yyset_debug (int bdebug ) +{ + yy_flex_debug = bdebug ; +} + +static int yy_init_globals (void) +{ + /* Initialization is the same as for the non-reentrant scanner. + * This function is called from yylex_destroy(), so don't allocate here. + */ + + (yy_buffer_stack) = 0; + (yy_buffer_stack_top) = 0; + (yy_buffer_stack_max) = 0; + (yy_c_buf_p) = (char *) 0; + (yy_init) = 0; + (yy_start) = 0; + +/* Defined in main.c */ +#ifdef YY_STDINIT + yyin = stdin; + yyout = stdout; +#else + yyin = (FILE *) 0; + yyout = (FILE *) 0; +#endif + + /* For future reference: Set errno on error, since we are called by + * yylex_init() + */ + return 0; +} + +/* yylex_destroy is for both reentrant and non-reentrant scanners. */ +int yylex_destroy (void) +{ + + /* Pop the buffer stack, destroying each element. */ + while(YY_CURRENT_BUFFER){ + yy_delete_buffer(YY_CURRENT_BUFFER ); + YY_CURRENT_BUFFER_LVALUE = NULL; + yypop_buffer_state(); + } + + /* Destroy the stack itself. */ + yyfree((yy_buffer_stack) ); + (yy_buffer_stack) = NULL; + + /* Reset the globals. This is important in a non-reentrant scanner so the next time + * yylex() is called, initialization will occur. */ + yy_init_globals( ); + + return 0; +} + +/* + * Internal utility routines. + */ + +#ifndef yytext_ptr +static void yy_flex_strncpy (char* s1, yyconst char * s2, int n ) +{ + register int i; + for ( i = 0; i < n; ++i ) + s1[i] = s2[i]; +} +#endif + +#ifdef YY_NEED_STRLEN +static int yy_flex_strlen (yyconst char * s ) +{ + register int n; + for ( n = 0; s[n]; ++n ) + ; + + return n; +} +#endif + +void *yyalloc (yy_size_t size ) +{ + return (void *) malloc( size ); +} + +void *yyrealloc (void * ptr, yy_size_t size ) +{ + /* The cast to (char *) in the following accommodates both + * implementations that use char* generic pointers, and those + * that use void* generic pointers. It works with the latter + * because both ANSI C and C++ allow castless assignment from + * any pointer type to void*, and deal with argument conversions + * as though doing an assignment. + */ + return (void *) realloc( (char *) ptr, size ); +} + +void yyfree (void * ptr ) +{ + free( (char *) ptr ); /* see yyrealloc() for (char *) cast */ +} + +#define YYTABLES_NAME "yytables" + +#line 322 "cod/cod.l" + + +static char *create_string_from_yytext() +{ + char *st = (char *) malloc((yyleng+1)*sizeof(char)); + strcpy (st, yytext); + return(st); +} + +static void check_strbuf() +{ + int cur_len = string_buf_ptr - string_buffer; + if ((cur_len + 1) == buffer_len) { + buffer_len += 20; + string_buffer = realloc(string_buffer, buffer_len + 1); + string_buf_ptr = string_buffer + cur_len; + } +} + +#define yy_size_t int +#ifndef yyconst +#define yyconst +#endif + +#if defined(__STDC__) || defined(__cplusplus) || defined(c_plusplus) +#define YY_USE_PROTOS +#endif + +#ifndef YY_PROTO +#ifdef YY_USE_PROTOS +#define YY_PROTO(proto) proto +#else +#define YY_PROTO(proto) () +#endif +#endif + +extern int +yywrap YY_PROTO(( void )) +{ + return 1; +} + + +#ifndef input +/* flex, not lex */ +void yy_delete_buffer YY_PROTO((YY_BUFFER_STATE b)); + +#ifdef WINNT +/* old Windows code for MKS Toolkit version of flex */ + +static void +terminate_string_parse() +{ + yyrestart(NULL); +} + +#ifdef YY_USE_PROTOS +static void *yy_flex_alloc( yy_size_t size ) +#else +static void *yy_flex_alloc( size ) +yy_size_t size; +#endif + { + return (void *) malloc( size ); + } + +static char* current_input_string; + +int my_yy_input(buf,result,max_size) { + + if (current_input_string == NULL) + { + + result = 0; + } + else + if (max_size < strlen(current_input_string)) + { + memcpy((void*)buf, current_input_string, max_size); + current_input_string += max_size; + result = max_size; + } else { + int n = strlen(current_input_string); + memcpy((void*)buf, current_input_string, n+1); + current_input_string = NULL; + result = n; + } + +/* printf("my_yy_input buf[%s],result[%d]\n",buf,result);*/ + return result; +} + +static void +setup_for_string_parse(string, defined_types, enum_constants) +const char *string; +char **defined_types; +char **enum_constants; +{ + type_count = defined_type_count; + types = defined_types; + enums = enum_constants; + + current_input_string = string; + lex_offset = 1; + line_count = 1; +} +#else + +static YY_BUFFER_STATE bb = NULL; + +static void +reset_types_table(defined_types, enumerated_constants) +char **defined_types; +char **enumerated_constants; +{ + types = defined_types; + enums = enumerated_constants; +} + +static void +setup_for_string_parse(string, defined_types, enum_constants) +const char *string; +char **defined_types; +char **enum_constants; +{ + types = defined_types; + enums = enum_constants; + + if ((bb = yy_scan_string(string)) == NULL) { + fprintf(stderr, "yyscan_buffer_failed\n"); + } + lex_offset = 1; + line_count = 1; +} + +static void +terminate_string_parse() +{ + if (bb) { + yy_delete_buffer(bb); + bb = NULL; + } +} + +#endif +#endif + diff --git a/thirdparty/ffs/ffs/cod/struct.pl b/thirdparty/ffs/ffs/cod/struct.pl index 54f26c4641..ec6ace2276 100755 --- a/thirdparty/ffs/ffs/cod/struct.pl +++ b/thirdparty/ffs/ffs/cod/struct.pl @@ -8,8 +8,10 @@ sub read_structs { $name = $1; $obj_ref = {}; $obj_ref->{types} = {}; + push @display_order, $name; } elsif (/\s*(\S+.*\W)(\w*);/) { $obj_ref->{types}->{$2} = $1; + push @{$obj_ref->{order}}, $2; $obj_ref->{types}->{$2} =~ s/ //g; $structs{$name} = $obj_ref; } @@ -17,9 +19,9 @@ sub read_structs { } sub dump_structs { - foreach $name (reverse keys %structs) { + foreach my $name (@display_order) { print "structure $name ->\n"; - foreach $field (reverse keys %{$structs{$name}->{types}}) { + foreach my $field ( @{$structs{$name}->{order}} ) { print " $structs{$name}->{types}->{$field} $field;\n"; } print "\n"; @@ -33,20 +35,20 @@ sub gen_typedef { print $outfile " struct list_struct *next;\n"; print $outfile "};\n\n"; print $outfile "typedef enum {\n"; - foreach $name (reverse keys %structs) { + foreach my $name (@display_order) { print $outfile " cod_$name,\n"; } print $outfile " cod_last_node_type\n"; print $outfile "} cod_node_type;\n\n"; - foreach $name (reverse keys %structs) { + foreach my $name (@display_order) { print $outfile "typedef struct {\n"; - foreach $field (reverse keys %{$structs{$name}->{types}}) { + foreach my $field ( @{$structs{$name}->{order}} ) { print $outfile " $structs{$name}->{types}->{$field} $field;\n"; } print $outfile "} $name;\n\n"; } print $outfile "typedef union {\n"; - foreach $name (reverse keys %structs) { + foreach my $name (@display_order) { print $outfile " $name $name;\n"; } print $outfile "} sm_union;\n\n"; @@ -59,7 +61,7 @@ sub gen_typedef { sub gen_create { my($houtfile, $coutfile) = @_; - foreach $name (reverse keys %structs) { + foreach my $name (@display_order) { print $houtfile "extern sm_ref cod_new_$name();\n"; print $coutfile "extern sm_ref\ncod_new_$name()\n{\n"; print $coutfile " sm_ref tmp = malloc(sizeof(*tmp));\n"; @@ -91,9 +93,9 @@ sub gen_apply { print $coutfile " node->visited++;\n"; print $coutfile " if(pre_func) (pre_func)(node, data);\n"; print $coutfile " switch(node->node_type) {\n"; - foreach $name (reverse keys %structs) { + foreach my $name (@display_order) { print $coutfile " case cod_$name: {\n"; - foreach $field (reverse keys %{$structs{$name}->{types}}) { + foreach my $field ( @{$structs{$name}->{order}} ) { if (($structs{$name}->{types}->{$field} eq "sm_ref") && (substr($field,0,2) ne "sm")) { print $coutfile " cod_apply(node->node.$name.$field, pre_func, post_func, list_func, data);\n"; @@ -122,10 +124,10 @@ sub gen_dump { print $coutfile "}\n\n"; print $coutfile "extern void cod_print(sm_ref node)\n{\n"; print $coutfile " switch(node->node_type) {\n"; - foreach $name (reverse keys %structs) { + foreach my $name (@display_order) { print $coutfile " case cod_$name: {\n"; print $coutfile " printf(\"0x%p -- $name ->\\n\", node);\n"; - foreach $field (reverse keys %{$structs{$name}->{types}}) { + foreach my $field ( @{$structs{$name}->{order}} ) { print $coutfile " printf(\"\t$field : "; if ($structs{$name}->{types}->{$field} eq "sm_ref") { print $coutfile "%p\\n\", node->node.$name.$field);\n"; @@ -163,9 +165,9 @@ sub gen_free { print $coutfile "}\n\n"; print $coutfile "extern void cod_free(sm_ref node)\n{\n"; print $coutfile " switch(node->node_type) {\n"; - foreach $name (reverse keys %structs) { + foreach my $name (@display_order) { print $coutfile " case cod_$name: {\n"; - foreach $field (reverse keys %{$structs{$name}->{types}}) { + foreach my $field ( @{$structs{$name}->{order}} ) { if ($structs{$name}->{types}->{$field} eq "char*") { print $coutfile " free(node->node.$name.$field);\n"; } @@ -190,9 +192,9 @@ sub gen_free { print $coutfile " new_free->node = node;\n"; print $coutfile " free_list = new_free;\n"; print $coutfile " switch(node->node_type) {\n"; - foreach $name (reverse keys %structs) { + foreach my $name (@display_order) { print $coutfile " case cod_$name: {\n"; - foreach $field (reverse keys %{$structs{$name}->{types}}) { + foreach my $field ( @{$structs{$name}->{order}} ) { if ($structs{$name}->{types}->{$field} eq "sm_list") { print $coutfile " node->node.$name.$field = NULL;\n"; } @@ -256,11 +258,11 @@ sub gen_copy { print $coutfile " sm_ref new_node = NULL;\n"; print $coutfile " if (node == NULL) return NULL;\n\n"; print $coutfile " switch(node->node_type) {\n"; - foreach $name (reverse keys %structs) { + foreach my $name (@display_order) { print $coutfile " case cod_$name: {\n"; print $coutfile " new_node = cod_new_$name();\n"; print $coutfile " new_node->node.$name = node->node.$name;\n"; - foreach $field (reverse keys %{$structs{$name}->{types}}) { + foreach my $field ( @{$structs{$name}->{order}} ) { if ($structs{$name}->{types}->{$field} eq "char*") { print $coutfile " new_node->node.$name.$field = node->node.$name.$field? strdup(node->node.$name.$field):NULL;\n"; } elsif ($structs{$name}->{types}->{$field} eq "sm_list") { @@ -284,8 +286,8 @@ sub gen_srcpos { print $houtfile "extern srcpos cod_get_srcpos(sm_ref expr);\n"; print $coutfile "extern srcpos cod_get_srcpos(expr)\nsm_ref expr;\n{\n"; print $coutfile " switch(expr->node_type) {\n"; - foreach $name (reverse keys %structs) { - foreach $field (reverse keys %{$structs{$name}->{types}}) { + foreach my $name (@display_order) { + foreach my $field ( @{$structs{$name}->{order}} ) { if ($structs{$name}->{types}->{$field} eq "srcpos") { print $coutfile " case cod_$name: return expr->node.$name.$field;\n"; } diff --git a/thirdparty/ffs/ffs/cod/tests/control.c b/thirdparty/ffs/ffs/cod/tests/control.c index 24620cfa08..6a876c25be 100644 --- a/thirdparty/ffs/ffs/cod/tests/control.c +++ b/thirdparty/ffs/ffs/cod/tests/control.c @@ -34,25 +34,9 @@ main(int argc, char**argv) { int test_num = 0; int run_only = -1; - char *read_file = NULL; - char *write_file = NULL; while (argc > 1) { if (strcmp(argv[1], "-v") == 0) { verbose++; - } else if (strcmp(argv[1], "-w") == 0) { - if (argc <= 1) { - printf("Need argument to \"-w\"\n"); - } else { - write_file = strdup(argv[2]); - } - argc--; argv++; - } else if (strcmp(argv[1], "-r") == 0) { - if (argc <= 1) { - printf("Need argument to \"-r\"\n"); - } else { - read_file = strdup(argv[2]); - } - argc--; argv++; } else if (strcmp(argv[1], "-o") == 0) { sscanf(argv[2], "%d", &run_only); argc--; argv++; diff --git a/thirdparty/ffs/ffs/cod/tests/structs.c b/thirdparty/ffs/ffs/cod/tests/structs.c index 6be2f3a291..d9d9788ef6 100644 --- a/thirdparty/ffs/ffs/cod/tests/structs.c +++ b/thirdparty/ffs/ffs/cod/tests/structs.c @@ -37,33 +37,10 @@ main(int argc, char**argv) { int test_num = 0; int run_only = -1; - char *read_file = NULL; - char *write_file = NULL; - - struct fool { - int x; - } foo; - foo.x = 1; - struct fool y; - y.x = 2; while (argc > 1) { if (strcmp(argv[1], "-v") == 0) { verbose++; - } else if (strcmp(argv[1], "-w") == 0) { - if (argc <= 1) { - printf("Need argument to \"-w\"\n"); - } else { - write_file = strdup(argv[2]); - } - argc--; argv++; - } else if (strcmp(argv[1], "-r") == 0) { - if (argc <= 1) { - printf("Need argument to \"-r\"\n"); - } else { - read_file = strdup(argv[2]); - } - argc--; argv++; } else if (strcmp(argv[1], "-o") == 0) { sscanf(argv[2], "%d", &run_only); argc--; argv++; diff --git a/thirdparty/ffs/ffs/cod/tests/t8.c b/thirdparty/ffs/ffs/cod/tests/t8.c index 2263da580b..fb64474339 100644 --- a/thirdparty/ffs/ffs/cod/tests/t8.c +++ b/thirdparty/ffs/ffs/cod/tests/t8.c @@ -444,7 +444,7 @@ main(int argc, char** argv) } func = (long(*)()) (long) gen_code->func; result = func(); - assert(result == -2*sizeof(double)); + assert(result == -2*(long)sizeof(double)); cod_code_free(gen_code); cod_free_parse_context(context); } @@ -475,7 +475,7 @@ main(int argc, char** argv) } func = (long(*)()) (long) gen_code->func; result = func(); - assert(result == -2*sizeof(double *)); + assert(result == -2*(long)sizeof(double *)); cod_code_free(gen_code); cod_free_parse_context(context); } diff --git a/thirdparty/ffs/ffs/ffs/ffs.c b/thirdparty/ffs/ffs/ffs/ffs.c index 22e462eddd..d0c23c959d 100755 --- a/thirdparty/ffs/ffs/ffs/ffs.c +++ b/thirdparty/ffs/ffs/ffs/ffs.c @@ -634,7 +634,6 @@ handle_subfield(FFSBuffer buf, FMFormat f, estate s, int data_offset, int parent } if (field_is_flat(f, t->next)) return 1; return handle_subfield(buf, f, s, tmp_data_loc, parent_offset, t->next); - break; } case FMType_string: { diff --git a/thirdparty/ffs/ffs/ffs/ffs_conv.c b/thirdparty/ffs/ffs/ffs/ffs_conv.c index fb99a00f92..2e85fbbb4c 100755 --- a/thirdparty/ffs/ffs/ffs/ffs_conv.c +++ b/thirdparty/ffs/ffs/ffs/ffs_conv.c @@ -214,7 +214,7 @@ void **default_val; char *s, *s1; char *base_type = base_data_type(io_field->field_type); FMdata_type data_type = FMstr_to_data_type(base_type); - strncpy(out_field_name, io_field->field_name, 64); + strncpy(out_field_name, io_field->field_name, 128); s = strchr(out_field_name, '('); *default_val = NULL; free(base_type); @@ -249,6 +249,37 @@ create_default_conversion(FMField iofield, void *default_value, conv_ptr->conversions[conv_index].rc_swap = no_row_column_swap; } +/* + * differs from find_field in fm in that it includes cur_field in search shortcut + */ +static long +find_field_for_conv(char *field_name, FMFieldList fields, int cur_field, void *search_help) +{ + int i; + if (cur_field > 10) { + /* search close first */ + for (i = cur_field; i > cur_field - 10; i--) { + if (strcmp(field_name, fields[i].field_name) == 0) { + return i; + } + } + for (i = cur_field + 1; i < cur_field + 10; i++) { + if (fields[i].field_name == NULL) break; + if (strcmp(field_name, fields[i].field_name) == 0) { + return i; + } + } + } + i = 0; + while (fields[i].field_name != NULL) { + if (strcmp(field_name, fields[i].field_name) == 0) { + return i; + } + i++; + } + return -1; +} + static IOConversionPtr create_conversion(src_ioformat, target_field_list, target_struct_size, @@ -345,7 +376,8 @@ FMStructDescList target_list; FMdata_type in_data_type, target_data_type; long in_elements, target_elements; void *default_val = NULL; - char tmp_field_name[64]; + char *tmp_field_name = NULL; + char *search_name; int multi_dimen_array = 0; /* @@ -354,33 +386,40 @@ FMStructDescList target_list; */ input_index = 0; - field_name_strip_get_default(&nfl_sort[i], tmp_field_name, &default_val); - while (strcmp(tmp_field_name, - input_field_list[input_index].field_name) != 0) { - input_index++; - if (input_index >= src_ioformat->body->field_count) { - if(default_val){ - if ((conv == buffer_and_convert) || - (conv == copy_dynamic_portion)) { - input_index = -1; /* Basically invalidating input_index - Indication for using default_val */ - break; - } else { - if (default_val) { - free(default_val); - default_val = NULL; - } - conv = buffer_and_convert; - goto restart; + if (strchr(nfl_sort[i].field_name, '(') == NULL) { + /* no default value */ + search_name = (char *) nfl_sort[i].field_name; + } else { + tmp_field_name = malloc(strlen(nfl_sort[i].field_name)); /* certainly big enough */ + field_name_strip_get_default(&nfl_sort[i], tmp_field_name, &default_val); + search_name = tmp_field_name; + } + input_index = find_field_for_conv(search_name, input_field_list, i, NULL); + if (input_index == -1) { + if(default_val){ + if ((conv == buffer_and_convert) || + (conv == copy_dynamic_portion)) { + input_index = -1; /* Basically invalidating input_index + Indication for using default_val */ + break; + } else { + if (default_val) { + free(default_val); + default_val = NULL; } + conv = buffer_and_convert; + if (tmp_field_name) free(tmp_field_name); + goto restart; } - fprintf(stderr, - "Requested field %s missing from input format\n", - nfl_sort[i].field_name); - FFSfree_conversion(conv_ptr); - return NULL; } + fprintf(stderr, + "Requested field %s missing from input format\n", + nfl_sort[i].field_name); + FFSfree_conversion(conv_ptr); + if (tmp_field_name) free(tmp_field_name); + return NULL; } + if (tmp_field_name) free(tmp_field_name); if(input_index == -1){ create_default_conversion(nfl_sort[i], default_val, &conv_ptr, conv_index); @@ -2735,7 +2774,7 @@ int data_already_copied; } if (conv_status->register_args) { - dill_reg new_src, new_dest, ret; + dill_reg new_src, new_dest; if (!ffs_getreg(c, &new_src, DILL_P, DILL_TEMP) || !ffs_getreg(c, &new_dest, DILL_P, DILL_TEMP)) gen_fatal("temp vals in subcall\n"); @@ -2743,7 +2782,7 @@ int data_already_copied; new_src, new_dest)); dill_addpi(c, new_src, src_addr, src_offset); dill_addpi(c, new_dest, dest_addr, dest_offset); - ret = dill_scallp(c, (void*)conv->subconversion->conv_func, name, "%p%p%p", new_src, + (void) dill_scallp(c, (void*)conv->subconversion->conv_func, name, "%p%p%p", new_src, new_dest, rt_conv_status); REG_DEBUG(("Putting %d and %d for new src & dest\n", new_src, new_dest)); @@ -2923,6 +2962,7 @@ int data_already_copied; rt_conv_status, conv, next, data_already_copied); + (void)loop_var_type; /* avoid warning */ #if defined(NOT) & defined(RAW) /* generate end of loop */ if (!register_args) { diff --git a/thirdparty/ffs/ffs/ffs/ffs_file.c b/thirdparty/ffs/ffs/ffs/ffs_file.c index 06f18f5c80..c723f79455 100644 --- a/thirdparty/ffs/ffs/ffs/ffs_file.c +++ b/thirdparty/ffs/ffs/ffs/ffs_file.c @@ -670,6 +670,7 @@ parse_index_block(char *index_base) int done = 0; item = malloc(sizeof(FFSIndexItemStruct)); block_size = htonl(*((int*)(index_base+4))) & 0xffffff; + (void) block_size; /* avoid warning */ item->next_index_offset = htonl(*((int*)(index_base+4))); item->start_data_count = htonl(*((int*)(index_base+8))); item->last_data_count = htonl(*((int*)(index_base+12))); @@ -1249,9 +1250,6 @@ FFSseek(FFSFile file, int data_item) off_t fpos; int index_item; FFSIndexItem prev_index_tail = NULL; - int data_item_bak = data_item; - - data_item_bak = file->data_block_no; if (data_item < 0) /* Or should it be set to 0 */ @@ -1363,9 +1361,9 @@ static void convert_last_index_block(FFSFile ffsfile) { FFSIndexItem read_index = ffsfile->index_tail; - FFSIndexType write_index; + init_write_index_block(ffsfile); - write_index = ffsfile->cur_index; + unsigned char *index_data; if (read_index == NULL) return; @@ -1609,7 +1607,6 @@ FFSnext_data_length(FFSFile file) extern int FFSread(FFSFile file, void *dest) { - FFSTypeHandle f; int header_size; int read_size; char *tmp_buf; @@ -1624,7 +1621,6 @@ FFSread(FFSFile file, void *dest) if (!FFSconsume_next_item(file)) return 0; } - f = file->next_data_handle; header_size = FFSheader_size(file->next_actual_handle); read_size = file->next_data_len - header_size; tmp_buf = file->tmp_buffer->tmp_buffer; @@ -1706,7 +1702,6 @@ FFSread_raw(FFSFile file, void *dest, int buffer_size, FFSTypeHandle *fp) FFSTypeHandle f; int header_size; int read_size; - char *tmp_buf; if (file->status != OpenForRead) return 0; @@ -1722,7 +1717,6 @@ FFSread_raw(FFSFile file, void *dest, int buffer_size, FFSTypeHandle *fp) *fp = f; header_size = FFSheader_size(f); read_size = file->next_data_len - header_size; - tmp_buf = file->tmp_buffer->tmp_buffer; if (file->read_func(file->file_id, dest, read_size, NULL, NULL) != read_size) { file->next_record_type = (file->errno_val) ? FFSerror : FFSend; diff --git a/thirdparty/ffs/ffs/fm/fm_dump.c b/thirdparty/ffs/ffs/fm/fm_dump.c index 5c10de0081..42733e634d 100644 --- a/thirdparty/ffs/ffs/fm/fm_dump.c +++ b/thirdparty/ffs/ffs/fm/fm_dump.c @@ -570,7 +570,6 @@ dump_subfield(void*base, FMFormat f, dstate s, int data_offset, void* parent_bas add_to_addr_list(s, ptr_value, new_offset); } return dump_subfield(ptr_value, f, s, 0, parent_base, t->next); - break; } case FMType_string: { @@ -594,7 +593,6 @@ dump_subfield(void*base, FMFormat f, dstate s, int data_offset, void* parent_bas { int elements = 1, i; int element_size; - int var_array = 0; FMTypeDesc *next = t; while (next->type == FMType_array) { if (next->static_size == 0) { @@ -605,7 +603,6 @@ dump_subfield(void*base, FMFormat f, dstate s, int data_offset, void* parent_bas src_spec.offset = f->field_list[field].field_offset; int tmp = quick_get_ulong(&src_spec, parent_base); elements = elements * tmp; - var_array = 1; } else { elements = elements * next->static_size; } @@ -628,7 +625,6 @@ dump_subfield(void*base, FMFormat f, dstate s, int data_offset, void* parent_bas ret = dump_subfields(base, subformat, s, data_offset); stop_field(s, fmfield, &f->var_list[field_index].type_desc); return ret; - break; } case FMType_simple: { FMFieldList fmfield = &f->field_list[t->field_index]; diff --git a/thirdparty/ffs/ffs/fm/fm_formats.c b/thirdparty/ffs/ffs/fm/fm_formats.c index cb2ab8910f..1dce586470 100755 --- a/thirdparty/ffs/ffs/fm/fm_formats.c +++ b/thirdparty/ffs/ffs/fm/fm_formats.c @@ -57,7 +57,7 @@ static char *stringify_field_type(const char *type, static int is_self_server(FMContext fmc); static void expand_FMContext(FMContext fmc); static int IOget_array_size_dimen(const char *str, FMFieldList fields, - int dimen, int *control_field); + int dimen, int *control_field, int cur_field); static int field_type_eq(const char *str1, const char *str2); /* @@ -700,31 +700,25 @@ FMContext fmc; } static int -is_var_array_field(FMFieldList field_list, int field) +is_all_static_array_dimens(const char *str) { - int done = 0; - int ret = 0; - int dimen_count = 0; - int control_val; - while (!done) { - int static_size = IOget_array_size_dimen(field_list[field].field_type, - field_list, dimen_count, - &control_val); - dimen_count++; - if (static_size == 0) { - done++; - continue; - } - if ((static_size == -1) && (control_val == -1)) { - /* failed validation, errors already delivered */ - return -1; - } - if (control_val != -1) { - /* dynamic array */ - ret = 1; - } + char *left_paren, *end; + + if ((left_paren = strchr(str, '[')) == NULL) { + return 1; + } + (void) strtol(left_paren + 1, &end, 0); + if (*end != ']') { + return 0; + } else { + return is_all_static_array_dimens(end+1); } - return ret; +} + +static int +is_var_array_field(FMFieldList field_list, int field) +{ + return !is_all_static_array_dimens(field_list[field].field_type); } static FMTypeDesc * @@ -831,7 +825,7 @@ gen_FMTypeDesc(FMFieldList fl, int field, const char *typ) int control_val; FMTypeDesc *tmp; int static_size = IOget_array_size_dimen(typ, fl, dimen_count, - &control_val); + &control_val, field); tmp = new_FMTypeDesc(); tmp->type = FMType_array; tmp->field_index = field; @@ -929,7 +923,6 @@ int field; if ((size < sizeof(int)) || (size < sizeof(long))) return FMOffset(si, i); return FMOffset(sl, l); - break; case unknown_type: case string_type: assert(0); case float_type: @@ -1165,7 +1158,7 @@ gen_var_dimens(FMFormat fmformat, int field) int control_val; int static_size = IOget_array_size_dimen(field_list[field].field_type, field_list, dimen_count, - &control_val); + &control_val, field); if (static_size == 0) { done++; continue; @@ -1564,7 +1557,6 @@ validate_and_copy_field_list(FMFieldList field_list, FMFormat fmformat) int field; FMFieldList new_field_list; int field_count = count_FMfield(field_list); - int simple_string = 0; new_field_list = (FMFieldList) malloc((size_t) sizeof(FMField) * (field_count + 1)); for (field = 0; field < field_count; field++) { @@ -2766,12 +2758,36 @@ const char *str; return unknown_type; } +static long +find_field(char *field_name, FMFieldList fields, int cur_field, void *search_help) +{ + int i; + if (cur_field > 10) { + /* search close first */ + for (i = cur_field -1; i > cur_field - 10; i--) { + if (strcmp(field_name, fields[i].field_name) == 0) { + return i; + } + } + for (i = cur_field + 1; i < cur_field + 10; i++) { + if (fields[i].field_name == NULL) break; + if (strcmp(field_name, fields[i].field_name) == 0) { + return i; + } + } + } + i = 0; + while (fields[i].field_name != NULL) { + if (strcmp(field_name, fields[i].field_name) == 0) { + return i; + } + i++; + } + return -1; +} + extern int -IOget_array_size_dimen(str, fields, dimen, control_field) -const char *str; -FMFieldList fields; -int dimen; -int *control_field; +IOget_array_size_dimen(const char *str, FMFieldList fields, int dimen, int *control_field, int cur_field) { char *left_paren, *end; long static_size; @@ -2790,27 +2806,25 @@ int *control_field; /* dynamic element */ char field_name[1024]; int count = 0; - int i = 0; while (((left_paren+1)[count] != ']') && ((left_paren+1)[count] != 0)) { field_name[count] = (left_paren+1)[count]; count++; } field_name[count] = 0; - while (fields[i].field_name != NULL) { - if (strcmp(field_name, fields[i].field_name) == 0) { - if ((FMstr_to_data_type(fields[i].field_type) == - integer_type) || - (FMstr_to_data_type(fields[i].field_type) == - unsigned_type)) { - *control_field = i; - return -1; - } else { - fprintf(stderr, "Variable length control field \"%s\" not of integer type.\n", field_name); - return 0; - } + void * search_help = NULL; + long search_field = find_field(field_name, fields, cur_field, search_help); + if (search_field != -1) { + if ((FMstr_to_data_type(fields[search_field].field_type) == + integer_type) || + (FMstr_to_data_type(fields[search_field].field_type) == + unsigned_type)) { + *control_field = search_field; + return -1; + } else { + fprintf(stderr, "Variable length control field \"%s\" not of integer type.\n", field_name); + return 0; } - i++; } fprintf(stderr, "Array dimension \"%s\" in type spec\"%s\" not recognized.\n", field_name, str); @@ -3510,6 +3524,8 @@ struct _subformat_wire_format *rep; if (tmp != 0) { offset = tmp; format->opt_info = malloc(sizeof(FMOptInfo)); + INT2 len = rep->f.f0.subformat_rep_length; + if (byte_reversal) byte_swap((char*)&len, 2); do { memcpy(&tmp_info, offset + (char*) rep, sizeof(tmp_info)); if (tmp_info.info_type != 0) { @@ -3531,7 +3547,7 @@ struct _subformat_wire_format *rep; info_count++; offset += sizeof(tmp_info); } - } while (tmp_info.info_type != 0); + } while ((tmp_info.info_type != 0) && (offset < len)); format->opt_info[info_count].info_type = 0; format->opt_info[info_count].info_len = 0; format->opt_info[info_count].info_block = 0; @@ -3739,6 +3755,7 @@ fill_derived_format_values(FMContext fmc, FMFormat format) } field_size = field_list[field].field_size * elements; } + (void) field_size; } generate_var_list(format, format->subformats); for (field = 0; field < format->field_count; field++) { @@ -4090,7 +4107,6 @@ void *format_ID; memcpy(&tmp, &id2->rep_len, 2); tmp = ntohs(tmp); return tmp << 2; - break; } case 0: case 1: diff --git a/thirdparty/ffs/ffs/fm/progs/server.c b/thirdparty/ffs/ffs/fm/progs/server.c index 325a963a5a..503944ec0a 100755 --- a/thirdparty/ffs/ffs/fm/progs/server.c +++ b/thirdparty/ffs/ffs/fm/progs/server.c @@ -288,8 +288,14 @@ general_format_server(int port, int do_restart, int verbose, int do_proxy) while (1) { format_server_poll_and_handle(fs); } +#ifdef __NVCOMPILER +#pragma diag_suppress 111 +#endif LOG(fs, "Doing Mutex unLock at Line %d ", __LINE__); pthread_mutex_unlock(&fs->lock); +#ifdef __NVCOMPILER +#pragma diag_default 111 +#endif } return; } @@ -340,7 +346,13 @@ format_server_poll_and_handle(format_server fs) fd_set test_set; timeout.tv_usec = 0; timeout.tv_sec = 0; +#ifdef __NVCOMPILER +#pragma diag_suppress 550 +#endif FD_ZERO(&test_set); +#ifdef __NVCOMPILER +#pragma diag_default 550 +#endif FD_SET(i, &test_set); errno = 0; select(FD_SETSIZE, &test_set, (fd_set *) NULL, @@ -776,8 +788,13 @@ get_format_from_master(format_server fs, IOFormatRep ioformat) ioformat->server_format_rep = rep; return ioformat; } - +#ifdef __NVCOMPILER +#pragma diag_suppress 111 +#endif return NULL; +#ifdef __NVCOMPILER +#pragma diag_default 111 +#endif } @@ -1085,28 +1102,28 @@ FSClient fsc; case MAGIC_NUMBER + 1: version = 1; break; - case REVERSE_MAGIC_NUMBER + 0x1000000: + case (FILE_INT) REVERSE_MAGIC_NUMBER + 0x1000000: version = 1; byte_reversal = 1; break; case MAGIC_NUMBER + 2: version = 2; break; - case REVERSE_MAGIC_NUMBER + 0x2000000: + case (FILE_INT) REVERSE_MAGIC_NUMBER + 0x2000000: version = 2; byte_reversal = 1; break; case MAGIC_NUMBER + 3: version = 3; break; - case REVERSE_MAGIC_NUMBER + 0x3000000: + case (FILE_INT) REVERSE_MAGIC_NUMBER + 0x3000000: version = 3; byte_reversal = 1; break; case MAGIC_NUMBER + 4: version = 4; break; - case REVERSE_MAGIC_NUMBER + 0x4000000: + case (FILE_INT) REVERSE_MAGIC_NUMBER + 0x4000000: version = 4; byte_reversal = 1; break; @@ -1773,7 +1790,13 @@ format_server_create() fs->timestamp = (time_t *) malloc(sizeof(time_t) * max_fd); memset((char *) fs->timestamp, 0, sizeof(FSClient) * max_fd); +#ifdef __NVCOMPILER +#pragma diag_suppress 550 +#endif FD_ZERO(&fs->fdset); +#ifdef __NVCOMPILER +#pragma diag_default 550 +#endif fs->data_buffer = (char *) malloc(1); fs->buffer_size = 1; fs->proxy_context_to_master = NULL; @@ -1914,7 +1937,13 @@ format_server_accept_conn_sock(format_server fs, void *conn_sock) if ((long) conn_sock == -1) { fd_set fds; struct timeval timeout; +#ifdef __NVCOMPILER +#pragma diag_suppress 550 +#endif FD_ZERO(&fds); +#ifdef __NVCOMPILER +#pragma diag_default 550 +#endif LOG(fs, "minus 1 variation"); if ((long) fs->conn_sock_inet >= 0) { FD_SET((unsigned long) fs->conn_sock_inet, &fds); diff --git a/thirdparty/ffs/ffs/fm/self_ip_addr.c b/thirdparty/ffs/ffs/fm/self_ip_addr.c index 996108f3f9..ef6a774f20 100755 --- a/thirdparty/ffs/ffs/fm/self_ip_addr.c +++ b/thirdparty/ffs/ffs/fm/self_ip_addr.c @@ -50,7 +50,7 @@ get_self_ip_addr() char *IP_string = getenv("CERCS_IP"); if (IP_string != NULL) { in_addr_t ip = inet_addr(IP_string); - if (ip != -1) return ntohl(ip); + if (ip != (in_addr_t) -1) return ntohl(ip); } gethostname(buf, sizeof(buf)); host = gethostbyname(buf); diff --git a/thirdparty/ffs/ffs/fm/server_acts.c b/thirdparty/ffs/ffs/fm/server_acts.c index a9cc2203f0..3a428d08da 100755 --- a/thirdparty/ffs/ffs/fm/server_acts.c +++ b/thirdparty/ffs/ffs/fm/server_acts.c @@ -189,7 +189,13 @@ action_t action; struct timeval timeout; int ret; +#ifdef __NVCOMPILER +#pragma diag_suppress 550 +#endif FD_ZERO(&rd_set); +#ifdef __NVCOMPILER +#pragma diag_default 550 +#endif timeout.tv_sec = 0; timeout.tv_usec = 0; FD_SET( (int)(long)iofile->server_fd, &rd_set); diff --git a/thirdparty/ffs/ffs/fm/tests/CMakeLists.txt b/thirdparty/ffs/ffs/fm/tests/CMakeLists.txt index 7aa44bd050..dd261f63fc 100644 --- a/thirdparty/ffs/ffs/fm/tests/CMakeLists.txt +++ b/thirdparty/ffs/ffs/fm/tests/CMakeLists.txt @@ -1,5 +1,5 @@ set (TESTS align_test compat_test) -set (PROGS format_test self_format_test) +set (PROGS format_test self_format_test scale_test) foreach (TEST ${TESTS} ) ADD_EXECUTABLE(${TEST} ${TEST}.c test_funcs.c) diff --git a/thirdparty/ffs/ffs/fm/tests/format_test.c b/thirdparty/ffs/ffs/fm/tests/format_test.c index 1dab85200b..2ebb110af4 100755 --- a/thirdparty/ffs/ffs/fm/tests/format_test.c +++ b/thirdparty/ffs/ffs/fm/tests/format_test.c @@ -50,7 +50,7 @@ char **argv; FMContext context = create_FMcontext(); FMcontext_allow_self_formats(context); printf("Format server identifier is %x\n", FMcontext_get_format_server_identifier(context)); - char id[] = {02, 00, 00, 37, 103, 189, 231, 165, 33, 254, 42, 32}; + unsigned char id[] = {02, 00, 00, 37, 103, 189, 231, 165, 33, 254, 42, 32}; printf("Doing get test\n"); first_rec_ioformat = FMformat_from_ID(context, (char *) &id[0]); printf("format is %lx\n", (long)first_rec_ioformat); diff --git a/thirdparty/ffs/ffs/fm/tests/scale_test.c b/thirdparty/ffs/ffs/fm/tests/scale_test.c new file mode 100644 index 0000000000..dfe98965c9 --- /dev/null +++ b/thirdparty/ffs/ffs/fm/tests/scale_test.c @@ -0,0 +1,121 @@ + +#include +#include + +#include "config.h" +#include +#include +#include +#include "fm.h" +#include "ffs.h" + +#ifdef HAVE_WINDOWS_H +#include +#define sleep(x) Sleep(1000*x) +#else +extern int sleep(); +#endif + +#include "test_funcs.h" + +char *gen_name(int i) +{ + char tmp_name[128]; + sprintf(tmp_name, "SST_Variable_FieldName that's really really long because I can't imagine why %d", i); + return strdup(tmp_name); +} + +struct base_elem { + int64_t elem_count1; + int64_t elem_count2; + int64_t *array1; + int64_t *array2; +}; + +int +main(argc, argv) +int argc; +char **argv; +{ + + FMStructDescRec str_list[5]; + struct timespec start, stop; + + FMContext context; + int field_count = 20000; + field_count = ((field_count >> 2 ) << 2); // ensure field count is divisible by 4; + FMFieldList list = malloc(sizeof(struct _FMField) * (field_count + 1)); + int cur_count = 0; + while (cur_count < field_count) { + /* do 4 at a time */ + char tmp[128]; + char *n1 = gen_name(cur_count); + char *n2 = gen_name(cur_count + 1); + char *n3 = gen_name(cur_count + 2); + char *n4 = gen_name(cur_count + 3); + list[cur_count].field_name = n1; + list[cur_count].field_type = strdup("integer"); + list[cur_count].field_size = 8; + list[cur_count].field_offset = cur_count * 8; + list[cur_count+1].field_name = n2; + list[cur_count+1].field_type = strdup("integer"); + list[cur_count+1].field_size = 8; + list[cur_count+1].field_offset = (cur_count+1) * 8; + list[cur_count+2].field_name = n3; + sprintf(tmp, "integer[%s]", n1); + list[cur_count+2].field_type = strdup(tmp); + list[cur_count+2].field_size = 8; + list[cur_count+2].field_offset = (cur_count+2) * 8; + list[cur_count+3].field_name = n4; + sprintf(tmp, "integer[%s]", n2); + list[cur_count+3].field_type = strdup(tmp); + list[cur_count+3].field_size = 8; + list[cur_count+3].field_offset = (cur_count+3) * 8; + cur_count +=4; + } + list[cur_count].field_name = list[cur_count].field_type = NULL; + + clock_gettime(CLOCK_MONOTONIC, &start); + + str_list[0].format_name = "first format"; + str_list[0].field_list = list; + str_list[0].struct_size = sizeof(first_rec); + str_list[0].opt_info = NULL; + str_list[1].format_name = NULL; + FFSContext fc = create_FFSContext(); + context = FMContext_from_FFS(fc); + + FMFormat format = register_data_format(context, str_list); + + clock_gettime(CLOCK_MONOTONIC, &stop); + double duration = (stop.tv_sec + 1.0e-9*stop.tv_nsec) - (start.tv_sec + 1.0e-9*start.tv_nsec); + printf("Registration took %g seconds\n", duration); + + char * buf= malloc(sizeof(struct base_elem) * (field_count / 4)); + int i; + for (i = 0; i < field_count ; i+= 4) { + struct base_elem *tmp = (struct base_elem *) (buf + i * sizeof(int64_t)); + tmp->elem_count1 = 3; + tmp->elem_count2 = 3; + tmp->array1 = malloc(3*sizeof(tmp->array1[0])); + tmp->array2 = malloc(3*sizeof(tmp->array2[0])); + } + + clock_gettime(CLOCK_MONOTONIC, &start); + FFSBuffer b = create_FFSBuffer(); + + int buf_size; + char *encode = FFSencode(b, format, buf, &buf_size); + FFSTypeHandle th = FFSTypeHandle_from_encode(fc, encode); + clock_gettime(CLOCK_MONOTONIC, &stop); + duration = (stop.tv_sec + 1.0e-9*stop.tv_nsec) - (start.tv_sec + 1.0e-9*start.tv_nsec); + printf("encode took %g seconds\n", duration); + clock_gettime(CLOCK_MONOTONIC, &start); + establish_conversion(fc, th, str_list); + clock_gettime(CLOCK_MONOTONIC, &stop); + duration = (stop.tv_sec + 1.0e-9*stop.tv_nsec) - (start.tv_sec + 1.0e-9*start.tv_nsec); + printf("establish took %g seconds\n", duration); + +} + + diff --git a/thirdparty/ffs/ffs/fm/unix_io.c b/thirdparty/ffs/ffs/fm/unix_io.c index 1192c0736f..ffbb854251 100755 --- a/thirdparty/ffs/ffs/fm/unix_io.c +++ b/thirdparty/ffs/ffs/fm/unix_io.c @@ -261,7 +261,13 @@ void *conn; int ret_val; time.tv_sec = time.tv_usec = 0; +#ifdef __NVCOMPILER +#pragma diag_suppress 550 +#endif FD_ZERO(&read_fds); +#ifdef __NVCOMPILER +#pragma diag_default 550 +#endif FD_SET(fd, &read_fds); ret_val = select(FD_SETSIZE, &read_fds, NULL, NULL, &time); return (ret_val > 0); diff --git a/thirdparty/nlohmann_json/nlohmann_json/CMakeLists.txt b/thirdparty/nlohmann_json/nlohmann_json/CMakeLists.txt index b3ed049b42..1014e73978 100644 --- a/thirdparty/nlohmann_json/nlohmann_json/CMakeLists.txt +++ b/thirdparty/nlohmann_json/nlohmann_json/CMakeLists.txt @@ -1,7 +1,14 @@ add_library(nlohmann_json INTERFACE) target_sources(nlohmann_json INTERFACE - $ + $ ) + +target_sources(nlohmann_json INTERFACE + $ +) + +target_compile_options(nlohmann_json INTERFACE $<$,$>:-Wno-all -Wno-error>) + target_include_directories(nlohmann_json INTERFACE $ ) diff --git a/thirdparty/nlohmann_json/nlohmann_json/src/single_include/nlohmann_json.hpp b/thirdparty/nlohmann_json/nlohmann_json/src/single_include/nlohmann_json.hpp new file mode 100644 index 0000000000..b136629b74 --- /dev/null +++ b/thirdparty/nlohmann_json/nlohmann_json/src/single_include/nlohmann_json.hpp @@ -0,0 +1,36 @@ +#ifndef __NLOHMANN_JSON_HPP__ +#define __NLOHMANN_JSON_HPP__ + +#if defined(_MSC_VER) && !defined(__clang__) +#pragma warning(push) + +#elif defined(__INTEL_COMPILER) && !defined(__clang__) +#pragma warning push +#pragma warning disable 1011 +_Pragma("warning(disable:1011)") + +#elif defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" + +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wall" +#endif + +#include + +#if defined(_MSC_VER) && !defined(__clang__) +#pragma warning(pop) + +#elif defined(__INTEL_COMPILER) && !defined(__clang__) +#pragma warning pop + +#elif defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop + +#elif defined(__clang__) +#pragma clang diagnostic pop +#endif + +#endif diff --git a/thirdparty/nlohmann_json/nlohmann_json/src/single_include/nlohmann/json.hpp b/thirdparty/nlohmann_json/nlohmann_json/src/single_include/upstream/nlohmann/json.hpp similarity index 100% rename from thirdparty/nlohmann_json/nlohmann_json/src/single_include/nlohmann/json.hpp rename to thirdparty/nlohmann_json/nlohmann_json/src/single_include/upstream/nlohmann/json.hpp diff --git a/thirdparty/nlohmann_json/update.sh b/thirdparty/nlohmann_json/update.sh index dfd8eaf94d..dac32324da 100755 --- a/thirdparty/nlohmann_json/update.sh +++ b/thirdparty/nlohmann_json/update.sh @@ -6,7 +6,7 @@ shopt -s dotglob readonly name="nlohmann_json/single_include" readonly ownership="JSON For Modern C++ Upstream " -readonly subtree="thirdparty/nlohmann_json/nlohmann_json/src/single_include" +readonly subtree="thirdparty/nlohmann_json/nlohmann_json/src/single_include/upstream" readonly repo="https://github.com/nlohmann/json.git" readonly tag="master" readonly shortlog="true"