From 14b2434be66f9cafee714642b935bd53ddd31b6f Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 25 Jun 2020 15:36:16 -0400 Subject: [PATCH 1/7] core: Remove unused IO member m_Compound --- source/adios2/core/IO.cpp | 3 --- source/adios2/core/IO.h | 2 -- 2 files changed, 5 deletions(-) diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp index 7044e7e16b..67a8b8d482 100644 --- a/source/adios2/core/IO.cpp +++ b/source/adios2/core/IO.cpp @@ -326,8 +326,6 @@ bool IO::RemoveVariable(const std::string &name) noexcept if (type == DataType::Compound) { - auto variableMap = m_Compound; - variableMap.erase(index); } #define declare_type(T) \ else if (type == helper::GetDataType()) \ @@ -355,7 +353,6 @@ void IO::RemoveAllVariables() noexcept #define declare_type(T) GetVariableMap().clear(); ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type - m_Compound.clear(); } bool IO::RemoveAttribute(const std::string &name) noexcept diff --git a/source/adios2/core/IO.h b/source/adios2/core/IO.h index 1cdda01bf3..972b53f07a 100644 --- a/source/adios2/core/IO.h +++ b/source/adios2/core/IO.h @@ -518,8 +518,6 @@ class IO ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_2ARGS(declare_map) #undef declare_map - std::map m_Compound; - std::map> m_Engines; /** From 32774d92dfbe2424f03b9d6922049c407c777219 Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 24 Jun 2020 15:22:03 -0400 Subject: [PATCH 2/7] core: Move IO destructor to implementation file --- source/adios2/core/IO.cpp | 2 ++ source/adios2/core/IO.h | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp index 67a8b8d482..9b350bd6db 100644 --- a/source/adios2/core/IO.cpp +++ b/source/adios2/core/IO.cpp @@ -179,6 +179,8 @@ IO::IO(ADIOS &adios, const std::string name, const bool inConfigFile, { } +IO::~IO() = default; + void IO::SetEngine(const std::string engineType) noexcept { auto lf_InsertParam = [&](const std::string &key, diff --git a/source/adios2/core/IO.h b/source/adios2/core/IO.h index 972b53f07a..1c0c363e86 100644 --- a/source/adios2/core/IO.h +++ b/source/adios2/core/IO.h @@ -123,7 +123,7 @@ class IO IO(ADIOS &adios, const std::string name, const bool inConfigFile, const std::string hostLanguage); - ~IO() = default; + ~IO(); /** * @brief Sets the engine type for this IO class object From f609d8a84e760d0005221de38d2c6e9f9f2abba7 Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 24 Jun 2020 15:21:39 -0400 Subject: [PATCH 3/7] core: Make IO non-copyable --- source/adios2/core/ADIOS.cpp | 4 +++- source/adios2/core/IO.h | 3 +++ source/adios2/helper/adiosXML.cpp | 7 ++++--- source/adios2/helper/adiosYAML.cpp | 3 ++- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/source/adios2/core/ADIOS.cpp b/source/adios2/core/ADIOS.cpp index 5d773feabd..f0341919bb 100644 --- a/source/adios2/core/ADIOS.cpp +++ b/source/adios2/core/ADIOS.cpp @@ -123,7 +123,9 @@ IO &ADIOS::DeclareIO(const std::string name) } } - auto ioPair = m_IOs.emplace(name, IO(*this, name, false, m_HostLanguage)); + auto ioPair = m_IOs.emplace( + std::piecewise_construct, std::forward_as_tuple(name), + std::forward_as_tuple(*this, name, false, m_HostLanguage)); IO &io = ioPair.first->second; io.SetDeclared(); return io; diff --git a/source/adios2/core/IO.h b/source/adios2/core/IO.h index 1c0c363e86..eab40aa3c5 100644 --- a/source/adios2/core/IO.h +++ b/source/adios2/core/IO.h @@ -125,6 +125,9 @@ class IO ~IO(); + IO(IO const &) = delete; + IO &operator=(IO const &) = delete; + /** * @brief Sets the engine type for this IO class object * @param engine predefined engine type, default is bpfile diff --git a/source/adios2/helper/adiosXML.cpp b/source/adios2/helper/adiosXML.cpp index e969aa6e5b..dbbb3056c9 100644 --- a/source/adios2/helper/adiosXML.cpp +++ b/source/adios2/helper/adiosXML.cpp @@ -147,9 +147,10 @@ void ParseConfigXML( helper::XMLAttribute("name", io, hint); // Build the IO object - auto itCurrentIO = - ios.emplace(ioName->value(), core::IO(adios, ioName->value(), true, - adios.m_HostLanguage)); + auto itCurrentIO = ios.emplace( + std::piecewise_construct, std::forward_as_tuple(ioName->value()), + std::forward_as_tuple(adios, ioName->value(), true, + adios.m_HostLanguage)); core::IO ¤tIO = itCurrentIO.first->second; // must be unique per io diff --git a/source/adios2/helper/adiosYAML.cpp b/source/adios2/helper/adiosYAML.cpp index e12c0a30b1..8cf19179f9 100644 --- a/source/adios2/helper/adiosYAML.cpp +++ b/source/adios2/helper/adiosYAML.cpp @@ -123,7 +123,8 @@ void ParseConfigYAML( auto lf_IOYAML = [&](const std::string &ioName, const YAML::Node &ioMap) { // Build the IO object auto itCurrentIO = ios.emplace( - ioName, core::IO(adios, ioName, true, adios.m_HostLanguage)); + std::piecewise_construct, std::forward_as_tuple(ioName), + std::forward_as_tuple(adios, ioName, true, adios.m_HostLanguage)); core::IO ¤tIO = itCurrentIO.first->second; // Engine parameters From dd6fcaef404e2192bfe8b6d3f8ab123d9a947d09 Mon Sep 17 00:00:00 2001 From: Brad King Date: Tue, 23 Jun 2020 14:34:56 -0400 Subject: [PATCH 4/7] core: Refactor Attribute::GetInfo implementation Most of the implementation is common to all `Attribute`. Move it to `AttributeBase` and use virtual dispatch to get the `T`-specific part. --- source/adios2/core/Attribute.cpp | 6 ------ source/adios2/core/Attribute.h | 4 +--- source/adios2/core/Attribute.tcc | 14 +++++--------- source/adios2/core/AttributeBase.cpp | 9 +++++++++ source/adios2/core/AttributeBase.h | 5 +++++ 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/source/adios2/core/Attribute.cpp b/source/adios2/core/Attribute.cpp index ff599618ee..ee68e8f3ac 100644 --- a/source/adios2/core/Attribute.cpp +++ b/source/adios2/core/Attribute.cpp @@ -64,12 +64,6 @@ struct RequiresZeroPadding : std::true_type if (RequiresZeroPadding::value) \ std::memset(&m_DataSingleValue, 0, sizeof(m_DataSingleValue)); \ m_DataSingleValue = value; \ - } \ - \ - template <> \ - Params Attribute::GetInfo() const noexcept \ - { \ - return DoGetInfo(); \ } ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_1ARG(declare_type) diff --git a/source/adios2/core/Attribute.h b/source/adios2/core/Attribute.h index 178ae60b50..83fb5fcf1b 100644 --- a/source/adios2/core/Attribute.h +++ b/source/adios2/core/Attribute.h @@ -51,10 +51,8 @@ class Attribute : public AttributeBase ~Attribute() = default; - Params GetInfo() const noexcept; - private: - Params DoGetInfo() const noexcept; + std::string DoGetInfoValue() const noexcept override; }; } // end namespace core diff --git a/source/adios2/core/Attribute.tcc b/source/adios2/core/Attribute.tcc index e789b7d12a..a047f64f96 100644 --- a/source/adios2/core/Attribute.tcc +++ b/source/adios2/core/Attribute.tcc @@ -13,7 +13,6 @@ #include "Attribute.h" -#include "adios2/helper/adiosFunctions.h" #include "adios2/helper/adiosType.h" namespace adios2 @@ -22,21 +21,18 @@ namespace core { template -Params Attribute::DoGetInfo() const noexcept +std::string Attribute::DoGetInfoValue() const noexcept { - Params info; - info["Type"] = ToString(m_Type); - info["Elements"] = std::to_string(m_Elements); - + std::string value; if (m_IsSingleValue) { - info["Value"] = helper::ValueToString(m_DataSingleValue); + value = helper::ValueToString(m_DataSingleValue); } else { - info["Value"] = "{ " + helper::VectorToCSV(m_DataArray) + " }"; + value = "{ " + helper::VectorToCSV(m_DataArray) + " }"; } - return info; + return value; } } // end namespace core diff --git a/source/adios2/core/AttributeBase.cpp b/source/adios2/core/AttributeBase.cpp index cdbccb61cc..f77ccdb4c5 100644 --- a/source/adios2/core/AttributeBase.cpp +++ b/source/adios2/core/AttributeBase.cpp @@ -26,5 +26,14 @@ AttributeBase::AttributeBase(const std::string &name, const DataType type, { } +Params AttributeBase::GetInfo() const noexcept +{ + Params info; + info["Type"] = ToString(m_Type); + info["Elements"] = std::to_string(m_Elements); + info["Value"] = this->DoGetInfoValue(); + return info; +} + } // end namespace core } // end namespace adios2 diff --git a/source/adios2/core/AttributeBase.h b/source/adios2/core/AttributeBase.h index 19ea783942..94d2916f5c 100644 --- a/source/adios2/core/AttributeBase.h +++ b/source/adios2/core/AttributeBase.h @@ -49,6 +49,11 @@ class AttributeBase const size_t elements); virtual ~AttributeBase() = default; + + Params GetInfo() const noexcept; + +private: + virtual std::string DoGetInfoValue() const noexcept = 0; }; } // end namespace core From 4325a5e35bea70a024e0628f1ff796bba9ac6975 Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 24 Jun 2020 15:22:26 -0400 Subject: [PATCH 5/7] core: Make IO attribute storage non-templated Replace the per-type attribute map storing `Attribute` with a single map storing `unique_ptr` and downcast as needed. --- source/adios2/core/IO.cpp | 40 +++++++++++------------- source/adios2/core/IO.h | 10 +++--- source/adios2/core/IO.tcc | 48 +++++++++++++---------------- source/adios2/core/VariableBase.cpp | 15 ++++----- 4 files changed, 49 insertions(+), 64 deletions(-) diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp index 9b350bd6db..1dad72ebe9 100644 --- a/source/adios2/core/IO.cpp +++ b/source/adios2/core/IO.cpp @@ -373,15 +373,11 @@ bool IO::RemoveAttribute(const std::string &name) noexcept { // nothing to do } -#define declare_type(T) \ - else if (type == helper::GetDataType()) \ - { \ - auto &attributeMap = GetAttributeMap(); \ - attributeMap.erase(index); \ - isRemoved = true; \ - } - ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_1ARG(declare_type) -#undef declare_type + else + { + m_AttributeMap.erase(index); + isRemoved = true; + } } if (isRemoved) @@ -396,10 +392,7 @@ void IO::RemoveAllAttributes() noexcept { TAU_SCOPED_TIMER("IO::RemoveAllAttributes"); m_Attributes.clear(); - -#define declare_type(T) GetAttributeMap().clear(); - ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_1ARG(declare_type) -#undef declare_type + m_AttributeMap.clear(); } std::map @@ -467,15 +460,12 @@ IO::GetAvailableAttributes(const std::string &variableName, if (type == DataType::Compound) { } -#define declare_template_instantiation(T) \ - else if (type == helper::GetDataType()) \ - { \ - Attribute &attribute = \ - GetAttributeMap().at(attributePair.second.second); \ - attributesInfo[name] = attribute.GetInfo(); \ - } - ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_1ARG(declare_template_instantiation) -#undef declare_template_instantiation + else + { + AttributeBase &attribute = + *m_AttributeMap.at(attributePair.second.second); + attributesInfo[name] = attribute.GetInfo(); + } } return attributesInfo; } @@ -775,6 +765,12 @@ void IO::SetPrefixedNames(const bool isStep) noexcept m_IsPrefixedNames = true; } +std::map> & +IO::GetAttributeMap() noexcept +{ + return m_AttributeMap; +} + // PRIVATE int IO::GetMapIndex(const std::string &name, const DataMap &dataMap) const noexcept diff --git a/source/adios2/core/IO.h b/source/adios2/core/IO.h index eab40aa3c5..1d6ab9f8a7 100644 --- a/source/adios2/core/IO.h +++ b/source/adios2/core/IO.h @@ -458,9 +458,9 @@ class IO template std::map> &GetVariableMap() noexcept; - /** Gets the internal reference to an attribute map for type T */ - template - std::map> &GetAttributeMap() noexcept; + /** Gets the internal reference to the attribute map. */ + std::map> & + GetAttributeMap() noexcept; using MakeEngineFunc = std::function( IO &, const std::string &, const Mode, helper::Comm)>; @@ -517,9 +517,7 @@ class IO ADIOS2_FOREACH_STDTYPE_2ARGS(declare_map) #undef declare_map -#define declare_map(T, NAME) std::map> m_##NAME##A; - ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_2ARGS(declare_map) -#undef declare_map + std::map> m_AttributeMap; std::map> m_Engines; diff --git a/source/adios2/core/IO.tcc b/source/adios2/core/IO.tcc index 1251ab2f76..8eef678133 100644 --- a/source/adios2/core/IO.tcc +++ b/source/adios2/core/IO.tcc @@ -15,6 +15,7 @@ /// \cond EXCLUDE_FROM_DOXYGEN #include +#include #include //std::invalid_argument /// \endcond @@ -116,15 +117,15 @@ Attribute &IO::DefineAttribute(const std::string &name, const T &value, const std::string globalName = helper::GlobalName(name, variableName, separator); - auto &attributeMap = GetAttributeMap(); auto itExistingAttribute = m_Attributes.find(globalName); if (!IsEnd(itExistingAttribute, m_Attributes)) { if (helper::ValueToString(value) == - attributeMap.at(itExistingAttribute->second.second) - .GetInfo()["Value"]) + m_AttributeMap.at(itExistingAttribute->second.second) + ->GetInfo()["Value"]) { - return attributeMap.at(itExistingAttribute->second.second); + return static_cast &>( + *m_AttributeMap.at(itExistingAttribute->second.second)); } else { @@ -135,14 +136,15 @@ Attribute &IO::DefineAttribute(const std::string &name, const T &value, } } const unsigned int newIndex = - attributeMap.empty() ? 0 : attributeMap.rbegin()->first + 1; + m_AttributeMap.empty() ? 0 : m_AttributeMap.rbegin()->first + 1; - auto itAttributePair = - attributeMap.emplace(newIndex, Attribute(globalName, value)); + auto itAttributePair = m_AttributeMap.emplace( + newIndex, + std::unique_ptr(new Attribute(globalName, value))); m_Attributes.emplace(globalName, std::make_pair(helper::GetDataType(), newIndex)); - return itAttributePair.first->second; + return static_cast &>(*itAttributePair.first->second); } template @@ -164,7 +166,6 @@ Attribute &IO::DefineAttribute(const std::string &name, const T *array, const std::string globalName = helper::GlobalName(name, variableName, separator); - auto &attributeMap = GetAttributeMap(); auto itExistingAttribute = m_Attributes.find(globalName); if (!IsEnd(itExistingAttribute, m_Attributes)) { @@ -173,10 +174,11 @@ Attribute &IO::DefineAttribute(const std::string &name, const T *array, helper::VectorToCSV(std::vector(array, array + elements)) + " }"); - if (attributeMap.at(itExistingAttribute->second.second) - .GetInfo()["Value"] == arrayValues) + if (m_AttributeMap.at(itExistingAttribute->second.second) + ->GetInfo()["Value"] == arrayValues) { - return attributeMap.at(itExistingAttribute->second.second); + return static_cast &>( + *m_AttributeMap.at(itExistingAttribute->second.second)); } else { @@ -187,14 +189,15 @@ Attribute &IO::DefineAttribute(const std::string &name, const T *array, } } const unsigned int newIndex = - attributeMap.empty() ? 0 : attributeMap.rbegin()->first + 1; + m_AttributeMap.empty() ? 0 : m_AttributeMap.rbegin()->first + 1; - auto itAttributePair = attributeMap.emplace( - newIndex, Attribute(globalName, array, elements)); + auto itAttributePair = m_AttributeMap.emplace( + newIndex, std::unique_ptr( + new Attribute(globalName, array, elements))); m_Attributes.emplace(globalName, std::make_pair(helper::GetDataType(), newIndex)); - return itAttributePair.first->second; + return static_cast &>(*itAttributePair.first->second); } template @@ -217,7 +220,8 @@ Attribute *IO::InquireAttribute(const std::string &name, return nullptr; } - return &GetAttributeMap().at(itAttribute->second.second); + return static_cast *>( + m_AttributeMap.at(itAttribute->second.second).get()); } // PRIVATE @@ -232,16 +236,6 @@ Attribute *IO::InquireAttribute(const std::string &name, ADIOS2_FOREACH_STDTYPE_2ARGS(make_GetVariableMap) #undef make_GetVariableMap -// GetAttributeMap -#define make_GetAttributeMap(T, NAME) \ - template <> \ - std::map> &IO::GetAttributeMap() noexcept \ - { \ - return m_##NAME##A; \ - } -ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_2ARGS(make_GetAttributeMap) -#undef make_GetAttributeMap - template Params IO::GetVariableInfo(const std::string &variableName, const std::set &keys) diff --git a/source/adios2/core/VariableBase.cpp b/source/adios2/core/VariableBase.cpp index 5ec9fdb177..e95c03a8fc 100644 --- a/source/adios2/core/VariableBase.cpp +++ b/source/adios2/core/VariableBase.cpp @@ -313,15 +313,12 @@ VariableBase::GetAttributesInfo(core::IO &io, const std::string separator, if (type == DataType::Compound) { } -#define declare_template_instantiation(T) \ - else if (type == helper::GetDataType()) \ - { \ - Attribute &attribute = \ - io.GetAttributeMap().at(itAttribute->second.second); \ - attributesInfo[key] = attribute.GetInfo(); \ - } - ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_1ARG(declare_template_instantiation) -#undef declare_template_instantiation + else + { + AttributeBase &attribute = + *io.GetAttributeMap().at(itAttribute->second.second); + attributesInfo[key] = attribute.GetInfo(); + } }; // BODY OF FUNCTION STARTS HERE From d3b2aa0e2bb4e2c17567c9d1588d62c7b4c16eb6 Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 25 Jun 2020 15:26:53 -0400 Subject: [PATCH 6/7] core: Make IO variable storage non-templated Replace the per-type variable map storing `Variable` with a single map storing `unique_ptr` and downcast as needed. --- source/adios2/core/IO.cpp | 98 ++++++++++++++++----------------------- source/adios2/core/IO.h | 9 +--- source/adios2/core/IO.tcc | 24 ++++------ 3 files changed, 48 insertions(+), 83 deletions(-) diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp index 1dad72ebe9..c5e58ebdaf 100644 --- a/source/adios2/core/IO.cpp +++ b/source/adios2/core/IO.cpp @@ -329,15 +329,11 @@ bool IO::RemoveVariable(const std::string &name) noexcept if (type == DataType::Compound) { } -#define declare_type(T) \ - else if (type == helper::GetDataType()) \ - { \ - auto &variableMap = GetVariableMap(); \ - variableMap.erase(index); \ - isRemoved = true; \ - } - ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) -#undef declare_type + else + { + m_VariableMap.erase(index); + isRemoved = true; + } } if (isRemoved) @@ -352,9 +348,7 @@ void IO::RemoveAllVariables() noexcept { TAU_SCOPED_TIMER("IO::RemoveAllVariables"); m_Variables.clear(); -#define declare_type(T) GetVariableMap().clear(); - ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) -#undef declare_type + m_VariableMap.clear(); } bool IO::RemoveAttribute(const std::string &name) noexcept @@ -437,16 +431,13 @@ IO::GetAvailableAttributes(const std::string &variableName, if (type == DataType::Compound) { } -#define declare_template_instantiation(T) \ - else if (type == helper::GetDataType()) \ - { \ - Variable &variable = \ - GetVariableMap().at(itVariable->second.second); \ - attributesInfo = \ - variable.GetAttributesInfo(*this, separator, fullNameKeys); \ - } - ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation) -#undef declare_template_instantiation + else + { + VariableBase &variable = + *m_VariableMap.at(itVariable->second.second); + attributesInfo = + variable.GetAttributesInfo(*this, separator, fullNameKeys); + } return attributesInfo; } @@ -492,19 +483,15 @@ DataType IO::InquireVariableType(const DataMap::const_iterator itVariable) const if (type == DataType::Compound) { } -#define declare_template_instantiation(T) \ - else if (type == helper::GetDataType()) \ - { \ - const Variable &variable = \ - const_cast(this)->GetVariableMap().at( \ - itVariable->second.second); \ - if (!variable.IsValidStep(m_EngineStep + 1)) \ - { \ - return DataType::None; \ - } \ - } - ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation) -#undef declare_template_instantiation + else + { + VariableBase &variable = + *m_VariableMap.at(itVariable->second.second); + if (!variable.IsValidStep(m_EngineStep + 1)) + { + return DataType::None; + } + } } return type; @@ -711,18 +698,14 @@ void IO::ResetVariablesStepSelection(const bool zeroStart, if (type == DataType::Compound) { } -// using relative start -#define declare_type(T) \ - else if (type == helper::GetDataType()) \ - { \ - Variable &variable = \ - GetVariableMap().at(itVariable->second.second); \ - variable.CheckRandomAccessConflict(hint); \ - variable.ResetStepsSelection(zeroStart); \ - variable.m_RandomAccess = false; \ - } - ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) -#undef declare_type + else + { + VariableBase &variable = + *m_VariableMap.at(itVariable->second.second); + variable.CheckRandomAccessConflict(hint); + variable.ResetStepsSelection(zeroStart); + variable.m_RandomAccess = false; + } } } @@ -748,18 +731,15 @@ void IO::SetPrefixedNames(const bool isStep) noexcept if (type == DataType::Compound) { } -#define declare_type(T) \ - else if (type == helper::GetDataType()) \ - { \ - Variable &variable = \ - GetVariableMap().at(itVariable->second.second); \ - variable.m_PrefixedVariables = \ - helper::PrefixMatches(variable.m_Name, variables); \ - variable.m_PrefixedAttributes = \ - helper::PrefixMatches(variable.m_Name, attributes); \ - } - ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) -#undef declare_type + else + { + VariableBase &variable = + *m_VariableMap.at(itVariable->second.second); + variable.m_PrefixedVariables = + helper::PrefixMatches(variable.m_Name, variables); + variable.m_PrefixedAttributes = + helper::PrefixMatches(variable.m_Name, attributes); + } } m_IsPrefixedNames = true; diff --git a/source/adios2/core/IO.h b/source/adios2/core/IO.h index 1d6ab9f8a7..74597ae841 100644 --- a/source/adios2/core/IO.h +++ b/source/adios2/core/IO.h @@ -454,10 +454,6 @@ class IO void SetPrefixedNames(const bool isStep) noexcept; - /** Gets the internal reference to a variable map for type T */ - template - std::map> &GetVariableMap() noexcept; - /** Gets the internal reference to the attribute map. */ std::map> & GetAttributeMap() noexcept; @@ -512,10 +508,7 @@ class IO /** Independent (default) or Collective */ adios2::IOMode m_IOMode = adios2::IOMode::Independent; -/** Variable containers based on fixed-size type */ -#define declare_map(T, NAME) std::map> m_##NAME; - ADIOS2_FOREACH_STDTYPE_2ARGS(declare_map) -#undef declare_map + std::map> m_VariableMap; std::map> m_AttributeMap; diff --git a/source/adios2/core/IO.tcc b/source/adios2/core/IO.tcc index 8eef678133..e2103059f8 100644 --- a/source/adios2/core/IO.tcc +++ b/source/adios2/core/IO.tcc @@ -46,16 +46,17 @@ Variable &IO::DefineVariable(const std::string &name, const Dims &shape, } } - auto &variableMap = GetVariableMap(); const unsigned int newIndex = - variableMap.empty() ? 0 : variableMap.rbegin()->first + 1; + m_AttributeMap.empty() ? 0 : m_AttributeMap.rbegin()->first + 1; - auto itVariablePair = variableMap.emplace( - newIndex, Variable(name, shape, start, count, constantDims)); + auto itVariablePair = m_VariableMap.emplace( + newIndex, std::unique_ptr(new Variable( + name, shape, start, count, constantDims))); m_Variables.emplace(name, std::make_pair(helper::GetDataType(), newIndex)); - Variable &variable = itVariablePair.first->second; + Variable &variable = + static_cast &>(*itVariablePair.first->second); // check IO placeholder for variable operations auto itOperations = m_VarOpsPlaceholder.find(name); @@ -88,7 +89,8 @@ Variable *IO::InquireVariable(const std::string &name) noexcept return nullptr; } - Variable *variable = &GetVariableMap().at(itVariable->second.second); + Variable *variable = static_cast *>( + m_VariableMap.at(itVariable->second.second).get()); if (m_ReadStreaming) { if (!variable->IsValidStep(m_EngineStep + 1)) @@ -226,16 +228,6 @@ Attribute *IO::InquireAttribute(const std::string &name, // PRIVATE -// GetVariableMap -#define make_GetVariableMap(T, NAME) \ - template <> \ - std::map> &IO::GetVariableMap() noexcept \ - { \ - return m_##NAME; \ - } -ADIOS2_FOREACH_STDTYPE_2ARGS(make_GetVariableMap) -#undef make_GetVariableMap - template Params IO::GetVariableInfo(const std::string &variableName, const std::set &keys) From 9f71ec6de698a2f07fbcc24fe0227e4e93c89739 Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 25 Jun 2020 16:03:20 -0400 Subject: [PATCH 7/7] core: Flatten IO variable and attribute storage maps Replace the two-layer map and intermediate index with one direct map. --- bindings/C/adios2/c/adios2_c_io.cpp | 14 +-- source/adios2/core/IO.cpp | 101 ++++-------------- source/adios2/core/IO.h | 63 ++--------- source/adios2/core/IO.tcc | 59 ++++------ source/adios2/core/VariableBase.cpp | 11 +- .../engine/insitumpi/InSituMPIReader.cpp | 4 +- source/adios2/engine/ssc/SscHelper.cpp | 4 +- source/adios2/engine/ssc/SscReader.cpp | 7 +- source/adios2/engine/sst/SstWriter.cpp | 9 +- .../adios2/toolkit/format/bp/BPSerializer.cpp | 13 ++- .../format/dataman/DataManSerializer.cpp | 12 +-- .../toolkit/interop/hdf5/HDF5Common.cpp | 4 +- source/utils/adios_reorganize/Reorganize.cpp | 17 ++- source/utils/adios_reorganize/Reorganize.h | 8 +- source/utils/bpls/bpls.cpp | 8 +- source/utils/bpls/bpls.h | 6 +- 16 files changed, 101 insertions(+), 239 deletions(-) diff --git a/bindings/C/adios2/c/adios2_c_io.cpp b/bindings/C/adios2/c/adios2_c_io.cpp index 3ccf5e207e..f6c2f6a370 100644 --- a/bindings/C/adios2/c/adios2_c_io.cpp +++ b/bindings/C/adios2/c/adios2_c_io.cpp @@ -250,7 +250,7 @@ adios2_variable *adios2_inquire_variable(adios2_io *io, const char *name) io, "for adios2_io, in call to adios2_inquire_variable"); adios2::core::IO &ioCpp = *reinterpret_cast(io); - const auto &dataMap = ioCpp.GetVariablesDataMap(); + const auto &dataMap = ioCpp.GetVariables(); auto itVariable = dataMap.find(name); if (itVariable == dataMap.end()) // not found @@ -292,7 +292,7 @@ adios2_error adios2_inquire_all_variables(adios2_variable ***variables, io, "for adios2_io, in call to adios2_inquire_all_variables"); adios2::core::IO &ioCpp = *reinterpret_cast(io); - const auto &dataMap = ioCpp.GetVariablesDataMap(); + const auto &dataMap = ioCpp.GetVariables(); *size = dataMap.size(); adios2_variable **list = @@ -308,7 +308,7 @@ adios2_error adios2_inquire_all_variables(adios2_variable ***variables, for (auto &name : names) { auto it = dataMap.find(name); - const adios2::DataType type(it->second.first); + const adios2::DataType type(it->second->m_Type); adios2::core::VariableBase *variable = nullptr; if (type == adios2::DataType::Compound) @@ -498,7 +498,7 @@ adios2_attribute *adios2_inquire_attribute(adios2_io *io, const char *name) io, "for adios2_io, in call to adios2_inquire_attribute"); adios2::core::IO &ioCpp = *reinterpret_cast(io); - const auto &dataMap = ioCpp.GetAttributesDataMap(); + const auto &dataMap = ioCpp.GetAttributes(); auto itAttribute = dataMap.find(name); if (itAttribute == dataMap.end()) // not found @@ -506,7 +506,7 @@ adios2_attribute *adios2_inquire_attribute(adios2_io *io, const char *name) return attribute; } - const adios2::DataType type(itAttribute->second.first); + const adios2::DataType type(itAttribute->second->m_Type); adios2::core::AttributeBase *attributeCpp = nullptr; if (type == adios2::DataType::Compound) @@ -551,7 +551,7 @@ adios2_error adios2_inquire_all_attributes(adios2_attribute ***attributes, io, "for adios2_io, in call to adios2_inquire_all_attributes"); adios2::core::IO &ioCpp = *reinterpret_cast(io); - const auto &dataMap = ioCpp.GetAttributesDataMap(); + const auto &dataMap = ioCpp.GetAttributes(); *size = dataMap.size(); adios2_attribute **list = @@ -567,7 +567,7 @@ adios2_error adios2_inquire_all_attributes(adios2_attribute ***attributes, for (auto &name : names) { auto it = dataMap.find(name); - const adios2::DataType type(it->second.first); + const adios2::DataType type(it->second->m_Type); adios2::core::AttributeBase *attribute = nullptr; if (type == adios2::DataType::Compound) diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp index c5e58ebdaf..d844a6b2e2 100644 --- a/source/adios2/core/IO.cpp +++ b/source/adios2/core/IO.cpp @@ -301,12 +301,9 @@ void IO::SetTransportParameter(const size_t transportIndex, m_TransportsParameters[transportIndex][key] = value; } -const DataMap &IO::GetVariablesDataMap() const noexcept { return m_Variables; } +const VarMap &IO::GetVariables() const noexcept { return m_Variables; } -const DataMap &IO::GetAttributesDataMap() const noexcept -{ - return m_Attributes; -} +const AttrMap &IO::GetAttributes() const noexcept { return m_Attributes; } bool IO::InConfigFile() const noexcept { return m_InConfigFile; } @@ -322,25 +319,9 @@ bool IO::RemoveVariable(const std::string &name) noexcept // variable exists if (itVariable != m_Variables.end()) { - // first remove the Variable object - const DataType type(itVariable->second.first); - const unsigned int index(itVariable->second.second); - - if (type == DataType::Compound) - { - } - else - { - m_VariableMap.erase(index); - isRemoved = true; - } - } - - if (isRemoved) - { - m_Variables.erase(name); + m_Variables.erase(itVariable); + isRemoved = true; } - return isRemoved; } @@ -348,7 +329,6 @@ void IO::RemoveAllVariables() noexcept { TAU_SCOPED_TIMER("IO::RemoveAllVariables"); m_Variables.clear(); - m_VariableMap.clear(); } bool IO::RemoveAttribute(const std::string &name) noexcept @@ -360,8 +340,7 @@ bool IO::RemoveAttribute(const std::string &name) noexcept if (itAttribute != m_Attributes.end()) { // first remove the Attribute object - const DataType type(itAttribute->second.first); - const unsigned int index(itAttribute->second.second); + const DataType type(itAttribute->second->m_Type); if (type == DataType::None) { @@ -369,16 +348,11 @@ bool IO::RemoveAttribute(const std::string &name) noexcept } else { - m_AttributeMap.erase(index); + m_Attributes.erase(itAttribute); isRemoved = true; } } - if (isRemoved) - { - m_Attributes.erase(name); - } - return isRemoved; } @@ -386,7 +360,6 @@ void IO::RemoveAllAttributes() noexcept { TAU_SCOPED_TIMER("IO::RemoveAllAttributes"); m_Attributes.clear(); - m_AttributeMap.clear(); } std::map @@ -433,12 +406,9 @@ IO::GetAvailableAttributes(const std::string &variableName, } else { - VariableBase &variable = - *m_VariableMap.at(itVariable->second.second); - attributesInfo = - variable.GetAttributesInfo(*this, separator, fullNameKeys); + attributesInfo = itVariable->second->GetAttributesInfo( + *this, separator, fullNameKeys); } - return attributesInfo; } @@ -446,16 +416,13 @@ IO::GetAvailableAttributes(const std::string &variableName, for (const auto &attributePair : m_Attributes) { const std::string &name = attributePair.first; - const DataType type = attributePair.second.first; - if (type == DataType::Compound) + if (attributePair.second->m_Type == DataType::Compound) { } else { - AttributeBase &attribute = - *m_AttributeMap.at(attributePair.second.second); - attributesInfo[name] = attribute.GetInfo(); + attributesInfo[name] = attributePair.second->GetInfo(); } } return attributesInfo; @@ -468,7 +435,7 @@ DataType IO::InquireVariableType(const std::string &name) const noexcept return InquireVariableType(itVariable); } -DataType IO::InquireVariableType(const DataMap::const_iterator itVariable) const +DataType IO::InquireVariableType(const VarMap::const_iterator itVariable) const noexcept { if (itVariable == m_Variables.end()) @@ -476,7 +443,7 @@ DataType IO::InquireVariableType(const DataMap::const_iterator itVariable) const return DataType::None; } - const DataType type = itVariable->second.first; + const DataType type = itVariable->second->m_Type; if (m_ReadStreaming) { @@ -485,9 +452,7 @@ DataType IO::InquireVariableType(const DataMap::const_iterator itVariable) const } else { - VariableBase &variable = - *m_VariableMap.at(itVariable->second.second); - if (!variable.IsValidStep(m_EngineStep + 1)) + if (!itVariable->second->IsValidStep(m_EngineStep + 1)) { return DataType::None; } @@ -511,7 +476,7 @@ DataType IO::InquireAttributeType(const std::string &name, return DataType::None; } - return itAttribute->second.first; + return itAttribute->second->m_Type; } size_t IO::AddOperation(Operator &op, const Params ¶meters) noexcept @@ -700,8 +665,7 @@ void IO::ResetVariablesStepSelection(const bool zeroStart, } else { - VariableBase &variable = - *m_VariableMap.at(itVariable->second.second); + VariableBase &variable = *itVariable->second; variable.CheckRandomAccessConflict(hint); variable.ResetStepsSelection(zeroStart); variable.m_RandomAccess = false; @@ -720,8 +684,8 @@ void IO::SetPrefixedNames(const bool isStep) noexcept const std::string &name = itVariable->first; // if for each step (BP4), check if variable type is not empty (means // variable exist in that step) - const DataType type = - isStep ? InquireVariableType(itVariable) : itVariable->second.first; + const DataType type = isStep ? InquireVariableType(itVariable) + : itVariable->second->m_Type; if (type == DataType::None) { @@ -733,8 +697,7 @@ void IO::SetPrefixedNames(const bool isStep) noexcept } else { - VariableBase &variable = - *m_VariableMap.at(itVariable->second.second); + VariableBase &variable = *itVariable->second; variable.m_PrefixedVariables = helper::PrefixMatches(variable.m_Name, variables); variable.m_PrefixedAttributes = @@ -745,28 +708,11 @@ void IO::SetPrefixedNames(const bool isStep) noexcept m_IsPrefixedNames = true; } -std::map> & -IO::GetAttributeMap() noexcept -{ - return m_AttributeMap; -} - // PRIVATE -int IO::GetMapIndex(const std::string &name, const DataMap &dataMap) const - noexcept -{ - auto itName = dataMap.find(name); - if (itName == dataMap.end()) - { - return -1; - } - return itName->second.second; -} - void IO::CheckAttributeCommon(const std::string &name) const { auto itAttribute = m_Attributes.find(name); - if (!IsEnd(itAttribute, m_Attributes)) + if (itAttribute != m_Attributes.end()) { throw std::invalid_argument("ERROR: attribute " + name + " exists in IO object " + m_Name + @@ -774,15 +720,6 @@ void IO::CheckAttributeCommon(const std::string &name) const } } -bool IO::IsEnd(DataMap::const_iterator itDataMap, const DataMap &dataMap) const -{ - if (itDataMap == dataMap.end()) - { - return true; - } - return false; -} - void IO::CheckTransportType(const std::string type) const { if (type.empty() || type.find("=") != type.npos) diff --git a/source/adios2/core/IO.h b/source/adios2/core/IO.h index 74597ae841..81fd8c4183 100644 --- a/source/adios2/core/IO.h +++ b/source/adios2/core/IO.h @@ -34,9 +34,8 @@ namespace adios2 namespace core { -/** used for Variables and Attributes, name, type, type-index */ -using DataMap = - std::unordered_map>; +using VarMap = std::unordered_map>; +using AttrMap = std::unordered_map>; // forward declaration needed as IO is passed to Engine derived // classes @@ -56,27 +55,6 @@ class IO /** from ADIOS class passed to Engine created with Open */ const std::string m_HostLanguage = "C++"; - /** - * Map holding variable identifiers - * 
-     * key: unique variable name,
-     * value: pair.first = type as string GetDataType from adiosTemplates.h
-     *        pair.second = index in fixed size map (e.g. m_Int8, m_Double)
-     *
- */ - DataMap m_Variables; - - /** - * Map holding attribute identifiers - * 
-     * key: unique attribute name,
-     * value: pair.first = type as string GetDataType from
-     *                     helper/adiosTemplates.h
-     *        pair.second = index in fixed size map (e.g. m_Int8, m_Double)
-     *
- */ - DataMap m_Attributes; - /** From SetParameter, parameters for a particular engine from m_Type */ Params m_Parameters; @@ -304,7 +282,7 @@ class IO * @param itVariable * @return type primitive type */ - DataType InquireVariableType(const DataMap::const_iterator itVariable) const + DataType InquireVariableType(const VarMap::const_iterator itVariable) const noexcept; /** @@ -312,22 +290,20 @@ class IO * @return * 
      * key: unique variable name,
-     * value: pair.first = string type
-     *        pair.second = order in the type bucket
+     * value: pointer to VariableBase
      *
*/ - const DataMap &GetVariablesDataMap() const noexcept; + const VarMap &GetVariables() const noexcept; /** * Retrieves hash holding internal Attributes identifiers * @return * 
      * key: unique attribute name,
-     * value: pair.first = string type
-     *        pair.second = order in the type bucket
+     * value: pointer to AttributeBase
      *
*/ - const DataMap &GetAttributesDataMap() const noexcept; + const AttrMap &GetAttributes() const noexcept; /** * Gets an existing attribute of primitive type by name @@ -454,10 +430,6 @@ class IO void SetPrefixedNames(const bool isStep) noexcept; - /** Gets the internal reference to the attribute map. */ - std::map> & - GetAttributeMap() noexcept; - using MakeEngineFunc = std::function( IO &, const std::string &, const Mode, helper::Comm)>; struct EngineFactoryEntry @@ -508,33 +480,16 @@ class IO /** Independent (default) or Collective */ adios2::IOMode m_IOMode = adios2::IOMode::Independent; - std::map> m_VariableMap; + VarMap m_Variables; - std::map> m_AttributeMap; + AttrMap m_Attributes; std::map> m_Engines; - /** - * Gets map index for Variables or Attributes - * @param name - * @param dataMap m_Variables or m_Attributes - * @return index in type map, -1 if not found - */ - int GetMapIndex(const std::string &name, const DataMap &dataMap) const - noexcept; - /** Checks if attribute exists, called from DefineAttribute different * signatures */ void CheckAttributeCommon(const std::string &name) const; - /** - * Checks if iterator points to end. Used for Variables and Attributes. - * @param itDataMap iterator to be tested - * @param dataMap map - * @return true: itDataMap == dataMap.end(), false otherwise - */ - bool IsEnd(DataMap::const_iterator itDataMap, const DataMap &dataMap) const; - void CheckTransportType(const std::string type) const; template diff --git a/source/adios2/core/IO.tcc b/source/adios2/core/IO.tcc index e2103059f8..fc5f3440a9 100644 --- a/source/adios2/core/IO.tcc +++ b/source/adios2/core/IO.tcc @@ -38,7 +38,7 @@ Variable &IO::DefineVariable(const std::string &name, const Dims &shape, { auto itVariable = m_Variables.find(name); - if (!IsEnd(itVariable, m_Variables)) + if (itVariable != m_Variables.end()) { throw std::invalid_argument("ERROR: variable " + name + " exists in IO object " + m_Name + @@ -46,14 +46,9 @@ Variable &IO::DefineVariable(const std::string &name, const Dims &shape, } } - const unsigned int newIndex = - m_AttributeMap.empty() ? 0 : m_AttributeMap.rbegin()->first + 1; - - auto itVariablePair = m_VariableMap.emplace( - newIndex, std::unique_ptr(new Variable( - name, shape, start, count, constantDims))); - m_Variables.emplace(name, - std::make_pair(helper::GetDataType(), newIndex)); + auto itVariablePair = m_Variables.emplace( + name, std::unique_ptr( + new Variable(name, shape, start, count, constantDims))); Variable &variable = static_cast &>(*itVariablePair.first->second); @@ -84,13 +79,13 @@ Variable *IO::InquireVariable(const std::string &name) noexcept return nullptr; } - if (itVariable->second.first != helper::GetDataType()) + if (itVariable->second->m_Type != helper::GetDataType()) { return nullptr; } - Variable *variable = static_cast *>( - m_VariableMap.at(itVariable->second.second).get()); + Variable *variable = + static_cast *>(itVariable->second.get()); if (m_ReadStreaming) { if (!variable->IsValidStep(m_EngineStep + 1)) @@ -120,14 +115,12 @@ Attribute &IO::DefineAttribute(const std::string &name, const T &value, helper::GlobalName(name, variableName, separator); auto itExistingAttribute = m_Attributes.find(globalName); - if (!IsEnd(itExistingAttribute, m_Attributes)) + if (itExistingAttribute != m_Attributes.end()) { if (helper::ValueToString(value) == - m_AttributeMap.at(itExistingAttribute->second.second) - ->GetInfo()["Value"]) + itExistingAttribute->second->GetInfo()["Value"]) { - return static_cast &>( - *m_AttributeMap.at(itExistingAttribute->second.second)); + return static_cast &>(*itExistingAttribute->second); } else { @@ -137,14 +130,10 @@ Attribute &IO::DefineAttribute(const std::string &name, const T &value, "DefineAttribute\n"); } } - const unsigned int newIndex = - m_AttributeMap.empty() ? 0 : m_AttributeMap.rbegin()->first + 1; - auto itAttributePair = m_AttributeMap.emplace( - newIndex, + auto itAttributePair = m_Attributes.emplace( + globalName, std::unique_ptr(new Attribute(globalName, value))); - m_Attributes.emplace(globalName, - std::make_pair(helper::GetDataType(), newIndex)); return static_cast &>(*itAttributePair.first->second); } @@ -169,18 +158,16 @@ Attribute &IO::DefineAttribute(const std::string &name, const T *array, helper::GlobalName(name, variableName, separator); auto itExistingAttribute = m_Attributes.find(globalName); - if (!IsEnd(itExistingAttribute, m_Attributes)) + if (itExistingAttribute != m_Attributes.end()) { const std::string arrayValues( "{ " + helper::VectorToCSV(std::vector(array, array + elements)) + " }"); - if (m_AttributeMap.at(itExistingAttribute->second.second) - ->GetInfo()["Value"] == arrayValues) + if (itExistingAttribute->second->GetInfo()["Value"] == arrayValues) { - return static_cast &>( - *m_AttributeMap.at(itExistingAttribute->second.second)); + return static_cast &>(*itExistingAttribute->second); } else { @@ -190,15 +177,10 @@ Attribute &IO::DefineAttribute(const std::string &name, const T *array, "DefineAttribute\n"); } } - const unsigned int newIndex = - m_AttributeMap.empty() ? 0 : m_AttributeMap.rbegin()->first + 1; - - auto itAttributePair = m_AttributeMap.emplace( - newIndex, std::unique_ptr( - new Attribute(globalName, array, elements))); - m_Attributes.emplace(globalName, - std::make_pair(helper::GetDataType(), newIndex)); + auto itAttributePair = m_Attributes.emplace( + globalName, std::unique_ptr( + new Attribute(globalName, array, elements))); return static_cast &>(*itAttributePair.first->second); } @@ -217,13 +199,12 @@ Attribute *IO::InquireAttribute(const std::string &name, return nullptr; } - if (itAttribute->second.first != helper::GetDataType()) + if (itAttribute->second->m_Type != helper::GetDataType()) { return nullptr; } - return static_cast *>( - m_AttributeMap.at(itAttribute->second.second).get()); + return static_cast *>(itAttribute->second.get()); } // PRIVATE diff --git a/source/adios2/core/VariableBase.cpp b/source/adios2/core/VariableBase.cpp index e95c03a8fc..2e602a670d 100644 --- a/source/adios2/core/VariableBase.cpp +++ b/source/adios2/core/VariableBase.cpp @@ -304,20 +304,17 @@ VariableBase::GetAttributesInfo(core::IO &io, const std::string separator, return; } - auto itAttribute = io.m_Attributes.find(attributeName); - const DataType type = itAttribute->second.first; + auto itAttribute = io.GetAttributes().find(attributeName); const std::string key = fullNameKeys ? attributeName : attributeName.substr(prefix.size()); - if (type == DataType::Compound) + if (itAttribute->second->m_Type == DataType::Compound) { } else { - AttributeBase &attribute = - *io.GetAttributeMap().at(itAttribute->second.second); - attributesInfo[key] = attribute.GetInfo(); + attributesInfo[key] = itAttribute->second->GetInfo(); } }; @@ -336,7 +333,7 @@ VariableBase::GetAttributesInfo(core::IO &io, const std::string separator, } else { // get prefixed attributes on-the-fly (expensive) - for (const auto &attributePair : io.m_Attributes) + for (const auto &attributePair : io.GetAttributes()) { const std::string &attributeName = attributePair.first; lf_GetAttributeInfo(prefix, attributeName, io, attributesInfo, diff --git a/source/adios2/engine/insitumpi/InSituMPIReader.cpp b/source/adios2/engine/insitumpi/InSituMPIReader.cpp index 3dd6487b62..0b604c5456 100644 --- a/source/adios2/engine/insitumpi/InSituMPIReader.cpp +++ b/source/adios2/engine/insitumpi/InSituMPIReader.cpp @@ -265,8 +265,8 @@ StepStatus InSituMPIReader::BeginStep(const StepMode mode, if (m_Verbosity == 5) { std::cout << "InSituMPI Reader " << m_ReaderRank << " found " - << m_IO.GetVariablesDataMap().size() << " variables and " - << m_IO.GetAttributesDataMap().size() + << m_IO.GetVariables().size() << " variables and " + << m_IO.GetAttributes().size() << " attributes in metadata. Is source row major = " << m_BP3Deserializer.m_IsRowMajor << std::endl; } diff --git a/source/adios2/engine/ssc/SscHelper.cpp b/source/adios2/engine/ssc/SscHelper.cpp index b0cec4d94e..47f1e7e6e5 100644 --- a/source/adios2/engine/ssc/SscHelper.cpp +++ b/source/adios2/engine/ssc/SscHelper.cpp @@ -144,12 +144,12 @@ void BlockVecToJson(const BlockVec &input, nlohmann::json &output) void AttributeMapToJson(IO &input, nlohmann::json &output) { - const auto &attributeMap = input.GetAttributesDataMap(); + const auto &attributeMap = input.GetAttributes(); auto &attributesJson = output["Attributes"]; for (const auto &attributePair : attributeMap) { const std::string name(attributePair.first); - const DataType type(attributePair.second.first); + const DataType type(attributePair.second->m_Type); if (type == DataType::None) { } diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp index 8e79045bf5..6ec60472e6 100644 --- a/source/adios2/engine/ssc/SscReader.cpp +++ b/source/adios2/engine/ssc/SscReader.cpp @@ -386,10 +386,9 @@ void SscReader::SyncWritePattern() #define declare_type(T) \ else if (type == helper::GetDataType()) \ { \ - const auto &attributesDataMap = m_IO.GetAttributesDataMap(); \ - auto it = \ - attributesDataMap.find(attributeJson["Name"].get()); \ - if (it == attributesDataMap.end()) \ + const auto &attributes = m_IO.GetAttributes(); \ + auto it = attributes.find(attributeJson["Name"].get()); \ + if (it == attributes.end()) \ { \ if (attributeJson["IsSingleValue"].get()) \ { \ diff --git a/source/adios2/engine/sst/SstWriter.cpp b/source/adios2/engine/sst/SstWriter.cpp index 5f7d2fcf5b..6a541f63b5 100644 --- a/source/adios2/engine/sst/SstWriter.cpp +++ b/source/adios2/engine/sst/SstWriter.cpp @@ -157,19 +157,18 @@ StepStatus SstWriter::BeginStep(StepMode mode, const float timeout_sec) void SstWriter::FFSMarshalAttributes() { TAU_SCOPED_TIMER_FUNC(); - const auto &attributesDataMap = m_IO.GetAttributesDataMap(); + const auto &attributes = m_IO.GetAttributes(); - const uint32_t attributesCount = - static_cast(attributesDataMap.size()); + const uint32_t attributesCount = static_cast(attributes.size()); // if there are no new attributes, nothing to do if (attributesCount == m_FFSMarshaledAttributesCount) return; - for (const auto &attributePair : attributesDataMap) + for (const auto &attributePair : attributes) { const std::string name(attributePair.first); - const DataType type(attributePair.second.first); + const DataType type(attributePair.second->m_Type); if (type == DataType::None) { diff --git a/source/adios2/toolkit/format/bp/BPSerializer.cpp b/source/adios2/toolkit/format/bp/BPSerializer.cpp index d91b32c780..e5fdead283 100644 --- a/source/adios2/toolkit/format/bp/BPSerializer.cpp +++ b/source/adios2/toolkit/format/bp/BPSerializer.cpp @@ -739,11 +739,11 @@ size_t BPSerializer::GetAttributesSizeInData(core::IO &io) const noexcept { size_t attributesSizeInData = 0; - auto &attributes = io.GetAttributesDataMap(); + auto &attributes = io.GetAttributes(); for (const auto &attribute : attributes) { - const DataType type = attribute.second.first; + const DataType type = attribute.second->m_Type; // each attribute is only written to output once // so filter out the ones already written @@ -772,7 +772,7 @@ size_t BPSerializer::GetAttributesSizeInData(core::IO &io) const noexcept void BPSerializer::PutAttributes(core::IO &io) { - const auto &attributesDataMap = io.GetAttributesDataMap(); + const auto &attributes = io.GetAttributes(); auto &buffer = m_Data.m_Buffer; auto &position = m_Data.m_Position; @@ -781,8 +781,7 @@ void BPSerializer::PutAttributes(core::IO &io) const size_t attributesCountPosition = position; // count is known ahead of time - const uint32_t attributesCount = - static_cast(attributesDataMap.size()); + const uint32_t attributesCount = static_cast(attributes.size()); helper::CopyToBuffer(buffer, position, &attributesCount); // will go back @@ -793,10 +792,10 @@ void BPSerializer::PutAttributes(core::IO &io) uint32_t memberID = 0; - for (const auto &attributePair : attributesDataMap) + for (const auto &attributePair : attributes) { const std::string name(attributePair.first); - const DataType type(attributePair.second.first); + const DataType type(attributePair.second->m_Type); // each attribute is only written to output once // so filter out the ones already written diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp index fc8ae72f61..f14ab50faa 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp @@ -332,12 +332,12 @@ bool DataManSerializer::IsCompressionAvailable(const std::string &method, void DataManSerializer::PutAttributes(core::IO &io) { TAU_SCOPED_TIMER_FUNC(); - const auto &attributesDataMap = io.GetAttributesDataMap(); + const auto &attributes = io.GetAttributes(); bool attributePut = false; - for (const auto &attributePair : attributesDataMap) + for (const auto &attributePair : attributes) { const std::string name(attributePair.first); - const DataType type(attributePair.second.first); + const DataType type(attributePair.second->m_Type); if (type == DataType::None) { } @@ -383,9 +383,9 @@ void DataManSerializer::GetAttributes(core::IO &io) #define declare_type(T) \ else if (type == helper::GetDataType()) \ { \ - const auto &attributesDataMap = io.GetAttributesDataMap(); \ - auto it = attributesDataMap.find(staticVar["N"].get()); \ - if (it == attributesDataMap.end()) \ + const auto &attributes = io.GetAttributes(); \ + auto it = attributes.find(staticVar["N"].get()); \ + if (it == attributes.end()) \ { \ if (staticVar["V"].get()) \ { \ diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp index e47e9c3101..8a13a35edf 100644 --- a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp +++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp @@ -1189,11 +1189,11 @@ void HDF5Common::LocateAttrParent(const std::string &attrName, void HDF5Common::CreateVarsFromIO(core::IO &io) { CheckWriteGroup(); - const core::DataMap &variables = io.GetVariablesDataMap(); + const core::VarMap &variables = io.GetVariables(); for (const auto &vpair : variables) { const std::string &varName = vpair.first; - const DataType varType = vpair.second.first; + const DataType varType = vpair.second->m_Type; #define declare_template_instantiation(T) \ if (varType == helper::GetDataType()) \ { \ diff --git a/source/utils/adios_reorganize/Reorganize.cpp b/source/utils/adios_reorganize/Reorganize.cpp index bb1def8dfe..d09372a2b1 100644 --- a/source/utils/adios_reorganize/Reorganize.cpp +++ b/source/utils/adios_reorganize/Reorganize.cpp @@ -194,8 +194,8 @@ void Reorganize::Run() } curr_step = static_cast(rStream.CurrentStep()); - const core::DataMap &variables = io.GetVariablesDataMap(); - const core::DataMap &attributes = io.GetAttributesDataMap(); + const core::VarMap &variables = io.GetVariables(); + const core::AttrMap &attributes = io.GetAttributes(); print0("____________________\n\nFile info:"); print0(" current step: ", curr_step); @@ -480,8 +480,8 @@ Reorganize::Decompose(int numproc, int rank, VarInfo &vi, } int Reorganize::ProcessMetadata(core::Engine &rStream, core::IO &io, - const core::DataMap &variables, - const core::DataMap &attributes, int step) + const core::VarMap &variables, + const core::AttrMap &attributes, int step) { int retval = 0; @@ -494,7 +494,7 @@ int Reorganize::ProcessMetadata(core::Engine &rStream, core::IO &io, for (const auto &variablePair : variables) { const std::string &name(variablePair.first); - const DataType type(variablePair.second.first); + const DataType type(variablePair.second->m_Type); core::VariableBase *variable = nullptr; print0("Get info on variable ", varidx, ": ", name); size_t nBlocks = 1; @@ -608,8 +608,7 @@ int Reorganize::ProcessMetadata(core::Engine &rStream, core::IO &io, } int Reorganize::ReadWrite(core::Engine &rStream, core::Engine &wStream, - core::IO &io, const core::DataMap &variables, - int step) + core::IO &io, const core::VarMap &variables, int step) { int retval = 0; @@ -639,7 +638,7 @@ int Reorganize::ReadWrite(core::Engine &rStream, core::Engine &wStream, // read variable subset std::cout << "rank " << m_Rank << ": Read variable " << name << std::endl; - const DataType type = variables.at(name).first; + const DataType type = variables.at(name)->m_Type; if (type == DataType::Compound) { // not supported @@ -683,7 +682,7 @@ int Reorganize::ReadWrite(core::Engine &rStream, core::Engine &wStream, // Write variable subset std::cout << "rank " << m_Rank << ": Write variable " << name << std::endl; - const DataType type = variables.at(name).first; + const DataType type = variables.at(name)->m_Type; if (type == DataType::Compound) { // not supported diff --git a/source/utils/adios_reorganize/Reorganize.h b/source/utils/adios_reorganize/Reorganize.h index ee6a7d7844..9a41827bb2 100644 --- a/source/utils/adios_reorganize/Reorganize.h +++ b/source/utils/adios_reorganize/Reorganize.h @@ -11,7 +11,7 @@ #ifndef UTILS_REORGANIZE_REORGANIZE_H_ #define UTILS_REORGANIZE_REORGANIZE_H_ -#include "adios2/core/IO.h" // DataMap +#include "adios2/core/IO.h" #include "adios2/helper/adiosComm.h" #include "utils/Utils.h" @@ -61,10 +61,10 @@ class Reorganize : public Utils const int *np // number of processes in each dimension ); int ProcessMetadata(core::Engine &rStream, core::IO &io, - const core::DataMap &variables, - const core::DataMap &attributes, int step); + const core::VarMap &variables, + const core::AttrMap &attributes, int step); int ReadWrite(core::Engine &rStream, core::Engine &wStream, core::IO &io, - const core::DataMap &variables, int step); + const core::VarMap &variables, int step); Params parseParams(const std::string ¶m_str); // Input arguments diff --git a/source/utils/bpls/bpls.cpp b/source/utils/bpls/bpls.cpp index 7327705dbe..d4077f5982 100644 --- a/source/utils/bpls/bpls.cpp +++ b/source/utils/bpls/bpls.cpp @@ -898,8 +898,8 @@ int nEntriesMatched = 0; int doList_vars(core::Engine *fp, core::IO *io) { - const core::DataMap &variables = io->GetVariablesDataMap(); - const core::DataMap &attributes = io->GetAttributesDataMap(); + const core::VarMap &variables = io->GetVariables(); + const core::AttrMap &attributes = io->GetAttributes(); // make a sorted list of all variables and attributes EntryMap entries; @@ -907,7 +907,7 @@ int doList_vars(core::Engine *fp, core::IO *io) { for (const auto &vpair : variables) { - Entry e(true, vpair.second.first, vpair.second.second); + Entry e(true, vpair.second->m_Type); entries.emplace(vpair.first, e); } } @@ -915,7 +915,7 @@ int doList_vars(core::Engine *fp, core::IO *io) { for (const auto &apair : attributes) { - Entry e(false, apair.second.first, apair.second.second); + Entry e(false, apair.second->m_Type); entries.emplace(apair.first, e); } } diff --git a/source/utils/bpls/bpls.h b/source/utils/bpls/bpls.h index f6b2564c38..66bd7aa8ac 100644 --- a/source/utils/bpls/bpls.h +++ b/source/utils/bpls/bpls.h @@ -39,11 +39,7 @@ struct Entry { bool isVar; DataType typeName; - unsigned int typeIndex; - Entry(bool b, DataType name, unsigned idx) - : isVar(b), typeName(name), typeIndex(idx) - { - } + Entry(bool b, DataType type) : isVar(b), typeName(type) {} }; // how to print one data item of an array