diff --git a/source/adios2/core/VariableDerived.cpp b/source/adios2/core/VariableDerived.cpp index e0c7a11c00..ea5f0aae63 100644 --- a/source/adios2/core/VariableDerived.cpp +++ b/source/adios2/core/VariableDerived.cpp @@ -31,7 +31,7 @@ void VariableDerived::UpdateExprDim(std::map> VariableDerived::ApplyExpression(std::map> &NameToMVI, - bool DoCompute) + bool DoCompute, int nproc) { size_t numBlocks = 0; // check that all variables have the same number of blocks @@ -71,7 +71,7 @@ VariableDerived::ApplyExpression(std::map outputData = - m_Expr.ApplyExpression(m_Type, numBlocks, inputData); + m_Expr.ApplyExpression(m_Type, numBlocks, inputData, nproc); std::vector> blockData; for (size_t i = 0; i < numBlocks; i++) diff --git a/source/adios2/core/VariableDerived.h b/source/adios2/core/VariableDerived.h index fbb7b6c59f..1897043c5a 100644 --- a/source/adios2/core/VariableDerived.h +++ b/source/adios2/core/VariableDerived.h @@ -33,7 +33,8 @@ class VariableDerived : public VariableBase void UpdateExprDim(std::map> NameToDims); std::vector> - ApplyExpression(std::map> &mvi, bool DoCompute = true); + ApplyExpression(std::map> &mvi, bool DoCompute = true, + int nproc = 1); }; } // end namespace core diff --git a/source/adios2/engine/bp5/BP5Writer.cpp b/source/adios2/engine/bp5/BP5Writer.cpp index ef26b2b4c0..fcb56e8ebe 100644 --- a/source/adios2/engine/bp5/BP5Writer.cpp +++ b/source/adios2/engine/bp5/BP5Writer.cpp @@ -548,7 +548,7 @@ void BP5Writer::ComputeDerivedVariables() std::vector> DerivedBlockData; // for expressionString, just generate the blocksinfo bool DoCompute = derivedVar->GetDerivedType() != DerivedVarType::ExpressionString; - DerivedBlockData = derivedVar->ApplyExpression(nameToVarInfo, DoCompute); + DerivedBlockData = derivedVar->ApplyExpression(nameToVarInfo, DoCompute, m_Comm.Size()); // Send the derived variable to ADIOS2 internal logic for (auto derivedBlock : DerivedBlockData) @@ -884,7 +884,8 @@ uint64_t BP5Writer::CountStepsInMetadataIndex(format::BufferSTL &bufferSTL) switch (recordID) { - case IndexRecord::WriterMapRecord: { + case IndexRecord::WriterMapRecord: + { m_AppendWriterCount = (uint32_t)helper::ReadValue(buffer, position, IsLittleEndian); m_AppendAggregatorCount = @@ -899,7 +900,8 @@ uint64_t BP5Writer::CountStepsInMetadataIndex(format::BufferSTL &bufferSTL) position += m_AppendWriterCount * sizeof(uint64_t); break; } - case IndexRecord::StepRecord: { + case IndexRecord::StepRecord: + { position += 2 * sizeof(uint64_t); // MetadataPos, MetadataSize const uint64_t FlushCount = helper::ReadValue(buffer, position, IsLittleEndian); @@ -968,7 +970,8 @@ uint64_t BP5Writer::CountStepsInMetadataIndex(format::BufferSTL &bufferSTL) switch (recordID) { - case IndexRecord::WriterMapRecord: { + case IndexRecord::WriterMapRecord: + { m_AppendWriterCount = (uint32_t)helper::ReadValue(buffer, position, IsLittleEndian); m_AppendAggregatorCount = @@ -986,7 +989,8 @@ uint64_t BP5Writer::CountStepsInMetadataIndex(format::BufferSTL &bufferSTL) } break; } - case IndexRecord::StepRecord: { + case IndexRecord::StepRecord: + { m_AppendMetadataIndexPos = position - sizeof(unsigned char) - sizeof(uint64_t); // pos of RecordID const uint64_t MetadataPos = diff --git a/source/adios2/toolkit/derived/Expression.cpp b/source/adios2/toolkit/derived/Expression.cpp index 95783663fa..70b7c97dd0 100644 --- a/source/adios2/toolkit/derived/Expression.cpp +++ b/source/adios2/toolkit/derived/Expression.cpp @@ -38,26 +38,58 @@ const std::map op_property = { {ExpressionOperator::OP_ATAN, {"ATAN", false}}, {ExpressionOperator::OP_MAGN, {"MAGNITUDE", false}}, {ExpressionOperator::OP_CROSS, {"CROSS", false}}, - {ExpressionOperator::OP_CURL, {"CURL", false}}}; + {ExpressionOperator::OP_CURL, {"CURL", false}}, + {ExpressionOperator::OP_MIN, {"MIN", false}}, + {ExpressionOperator::OP_MAX, {"MAX", false}}, + {ExpressionOperator::OP_SUM, {"SUM", false}}, + {ExpressionOperator::OP_MEAN, {"MEAN", false}}, + {ExpressionOperator::OP_VARIANCE, {"VARIANCE", false}}, + {ExpressionOperator::OP_STDEV, {"STDEV", false}}}; const std::map string_to_op = { {"ALIAS", ExpressionOperator::OP_ALIAS}, /* Parser-use only */ {"PATH", ExpressionOperator::OP_PATH}, /* Parser-use only */ {"NUM", ExpressionOperator::OP_NUM}, /* Parser-use only */ - {"INDEX", ExpressionOperator::OP_INDEX}, {"+", ExpressionOperator::OP_ADD}, - {"add", ExpressionOperator::OP_ADD}, {"ADD", ExpressionOperator::OP_ADD}, - {"-", ExpressionOperator::OP_SUBTRACT}, {"SUBTRACT", ExpressionOperator::OP_SUBTRACT}, - {"/", ExpressionOperator::OP_DIV}, {"divide", ExpressionOperator::OP_DIV}, - {"DIVIDE", ExpressionOperator::OP_DIV}, {"*", ExpressionOperator::OP_MULT}, - {"multiply", ExpressionOperator::OP_MULT}, {"MULTIPLY", ExpressionOperator::OP_MULT}, - {"SQRT", ExpressionOperator::OP_SQRT}, {"sqrt", ExpressionOperator::OP_SQRT}, - {"pow", ExpressionOperator::OP_POW}, {"POW", ExpressionOperator::OP_POW}, - {"sin", ExpressionOperator::OP_SIN}, {"cos", ExpressionOperator::OP_COS}, - {"tan", ExpressionOperator::OP_TAN}, {"asin", ExpressionOperator::OP_ASIN}, - {"acos", ExpressionOperator::OP_ACOS}, {"atan", ExpressionOperator::OP_ATAN}, - {"^", ExpressionOperator::OP_POW}, {"magnitude", ExpressionOperator::OP_MAGN}, - {"MAGNITUDE", ExpressionOperator::OP_MAGN}, {"cross", ExpressionOperator::OP_CROSS}, - {"curl", ExpressionOperator::OP_CURL}, {"CURL", ExpressionOperator::OP_CURL}}; + {"INDEX", ExpressionOperator::OP_INDEX}, + {"+", ExpressionOperator::OP_ADD}, + {"add", ExpressionOperator::OP_ADD}, + {"ADD", ExpressionOperator::OP_ADD}, + {"-", ExpressionOperator::OP_SUBTRACT}, + {"SUBTRACT", ExpressionOperator::OP_SUBTRACT}, + {"/", ExpressionOperator::OP_DIV}, + {"divide", ExpressionOperator::OP_DIV}, + {"DIVIDE", ExpressionOperator::OP_DIV}, + {"*", ExpressionOperator::OP_MULT}, + {"multiply", ExpressionOperator::OP_MULT}, + {"MULTIPLY", ExpressionOperator::OP_MULT}, + {"SQRT", ExpressionOperator::OP_SQRT}, + {"sqrt", ExpressionOperator::OP_SQRT}, + {"pow", ExpressionOperator::OP_POW}, + {"POW", ExpressionOperator::OP_POW}, + {"sin", ExpressionOperator::OP_SIN}, + {"cos", ExpressionOperator::OP_COS}, + {"tan", ExpressionOperator::OP_TAN}, + {"asin", ExpressionOperator::OP_ASIN}, + {"acos", ExpressionOperator::OP_ACOS}, + {"atan", ExpressionOperator::OP_ATAN}, + {"^", ExpressionOperator::OP_POW}, + {"magnitude", ExpressionOperator::OP_MAGN}, + {"MAGNITUDE", ExpressionOperator::OP_MAGN}, + {"cross", ExpressionOperator::OP_CROSS}, + {"curl", ExpressionOperator::OP_CURL}, + {"CURL", ExpressionOperator::OP_CURL}, + {"min", ExpressionOperator::OP_MIN}, + {"MIN", ExpressionOperator::OP_MIN}, + {"max", ExpressionOperator::OP_MAX}, + {"MAX", ExpressionOperator::OP_MAX}, + {"sum", ExpressionOperator::OP_SUM}, + {"SUM", ExpressionOperator::OP_SUM}, + {"mean", ExpressionOperator::OP_MEAN}, + {"MEAN", ExpressionOperator::OP_MEAN}, + {"variance", ExpressionOperator::OP_VARIANCE}, + {"VARIANCE", ExpressionOperator::OP_VARIANCE}, + {"stdev", ExpressionOperator::OP_STDEV}, + {"STDEV", ExpressionOperator::OP_STDEV}}; inline std::string get_op_name(ExpressionOperator op) { return op_property.at(op).name; } @@ -134,7 +166,7 @@ namespace derived { struct OperatorFunctions { - std::function, DataType)> ComputeFct; + std::function, DataType, int)> ComputeFct; std::function(std::vector>)> DimsFct; std::function TypeFct; }; @@ -155,7 +187,13 @@ std::map OpFunctions = { {adios2::detail::ExpressionOperator::OP_MAGN, {MagnitudeFunc, SameDimsWithAgrFunc, SameTypeFunc}}, {adios2::detail::ExpressionOperator::OP_CROSS, {Cross3DFunc, Cross3DDimsFunc, SameTypeFunc}}, - {adios2::detail::ExpressionOperator::OP_CURL, {Curl3DFunc, CurlDimsFunc, SameTypeFunc}}}; + {adios2::detail::ExpressionOperator::OP_CURL, {Curl3DFunc, CurlDimsFunc, SameTypeFunc}}, + {adios2::detail::ExpressionOperator::OP_MIN, {MinFunc, ScalarDimsFunc, SameTypeFunc}}, + {adios2::detail::ExpressionOperator::OP_MAX, {MaxFunc, ScalarDimsFunc, SameTypeFunc}}, + {adios2::detail::ExpressionOperator::OP_SUM, {SumFunc, ScalarDimsFunc, SameTypeFunc}}, + {adios2::detail::ExpressionOperator::OP_MEAN, {MeanFunc, ScalarDimsFunc, SameTypeFunc}}, + {adios2::detail::ExpressionOperator::OP_VARIANCE, {VarianceFunc, ScalarDimsFunc, SameTypeFunc}}, + {adios2::detail::ExpressionOperator::OP_STDEV, {StDevFunc, ScalarDimsFunc, FloatTypeFunc}}}; Expression::Expression(std::string string_exp) : m_Shape({0}), m_Start({0}), m_Count({0}), ExprString(string_exp) @@ -195,9 +233,9 @@ DataType Expression::GetType(std::map NameToType) std::vector Expression::ApplyExpression(DataType type, size_t numBlocks, - std::map> nameToData) + std::map> nameToData, int nproc) { - return m_Expr.ApplyExpression(type, numBlocks, nameToData); + return m_Expr.ApplyExpression(type, numBlocks, nameToData, nproc); } void ExpressionTree::set_base(double c) { detail.constant = c; } @@ -341,7 +379,8 @@ DataType ExpressionTree::GetType(std::map NameToType) std::vector ExpressionTree::ApplyExpression(DataType type, size_t numBlocks, - std::map> nameToData) + std::map> nameToData, + int nproc) { // create operands for the computation function // exprData[0] = list of void* data for block 0 for each variable @@ -361,7 +400,8 @@ ExpressionTree::ApplyExpression(DataType type, size_t numBlocks, else { deallocate.push_back(true); - auto subexpData = std::get<0>(subexp).ApplyExpression(type, numBlocks, nameToData); + auto subexpData = + std::get<0>(subexp).ApplyExpression(type, numBlocks, nameToData, nproc); for (size_t blk = 0; blk < numBlocks; blk++) { exprData[blk].push_back(subexpData[blk]); @@ -373,7 +413,7 @@ ExpressionTree::ApplyExpression(DataType type, size_t numBlocks, auto op_fct = OpFunctions.at(detail.operation); for (size_t blk = 0; blk < numBlocks; blk++) { - outputData[blk] = op_fct.ComputeFct(exprData[blk], type); + outputData[blk] = op_fct.ComputeFct(exprData[blk], type, nproc); } // deallocate intermediate data after computing the operation for (size_t blk = 0; blk < numBlocks; blk++) diff --git a/source/adios2/toolkit/derived/Expression.h b/source/adios2/toolkit/derived/Expression.h index 01bbb871e3..c999a8a5ab 100644 --- a/source/adios2/toolkit/derived/Expression.h +++ b/source/adios2/toolkit/derived/Expression.h @@ -30,7 +30,14 @@ enum ExpressionOperator OP_ATAN, OP_MAGN, OP_CROSS, - OP_CURL + OP_CURL, + OP_MIN, + OP_MAX, + OP_SUM, + OP_MEAN, + OP_MEDIAN, + OP_VARIANCE, + OP_STDEV }; } @@ -79,7 +86,7 @@ class ExpressionTree DataType GetType(std::map NameToType); std::vector ApplyExpression(DataType type, size_t numBlocks, - std::map> nameToData); + std::map> nameToData, int nproc); void print(); std::string toStringExpr(); }; @@ -109,7 +116,7 @@ class Expression std::vector VariableNameList(); std::vector ApplyExpression(DataType type, size_t numBlocks, - std::map> nameToData); + std::map> nameToData, int nproc); }; } diff --git a/source/adios2/toolkit/derived/Function.cpp b/source/adios2/toolkit/derived/Function.cpp index 99f5726295..b3c9afe87b 100644 --- a/source/adios2/toolkit/derived/Function.cpp +++ b/source/adios2/toolkit/derived/Function.cpp @@ -39,6 +39,18 @@ T *ApplyOneToOne(Iterator inputBegin, Iterator inputEnd, size_t dataSize, return outValues; } +template +T ApplyAllToOne(T *inputData, size_t dataSize, std::function compFct, T initVal = (T)0) +{ + T outVal = initVal; + for (size_t i = 0; i < dataSize; i++) + { + T data = *(reinterpret_cast(inputData + i)); + outVal = compFct(outVal, data); + } + return outVal; +} + template T *AggregateOnLastDim(T *data, size_t dataSize, size_t nVariables, std::function compFct) { @@ -153,7 +165,7 @@ DerivedData AddAggregatedFunc(DerivedData inputData, DataType type) return DerivedData(); } -DerivedData AddFunc(std::vector inputData, DataType type) +DerivedData AddFunc(std::vector inputData, DataType type, int nproc) { PERFSTUBS_SCOPED_TIMER("derived::Function::AddFunc"); // if there is only one element return the aggregate result @@ -176,7 +188,7 @@ DerivedData AddFunc(std::vector inputData, DataType type) } // Perform a subtraction from the first variable of all other variables in the std::vector -DerivedData SubtractFunc(std::vector inputData, DataType type) +DerivedData SubtractFunc(std::vector inputData, DataType type, int nproc) { PERFSTUBS_SCOPED_TIMER("derived::Function::SubtractFunc"); size_t dataSize = std::accumulate(std::begin(inputData[0].Count), std::end(inputData[0].Count), @@ -201,7 +213,7 @@ DerivedData SubtractFunc(std::vector inputData, DataType type) } // Perform a reduce multiply over all variables in the std::vector -DerivedData MultFunc(std::vector inputData, DataType type) +DerivedData MultFunc(std::vector inputData, DataType type, int nproc) { PERFSTUBS_SCOPED_TIMER("derived::Function::MultFunc"); size_t dataSize = std::accumulate(std::begin(inputData[0].Count), std::end(inputData[0].Count), @@ -210,8 +222,8 @@ DerivedData MultFunc(std::vector inputData, DataType type) #define declare_type_mult(T) \ if (type == helper::GetDataType()) \ { \ - T *multValues = detail::ApplyOneToOne( \ - inputData.begin(), inputData.end(), dataSize, [](T a, T b) { return a * b; }, 1); \ + T *multValues = detail::ApplyOneToOne(inputData.begin(), inputData.end(), dataSize, \ + [](T a, T b) { return a * b; }, 1); \ return DerivedData({(void *)multValues, inputData[0].Start, inputData[0].Count}); \ } ADIOS2_FOREACH_ATTRIBUTE_PRIMITIVE_STDTYPE_1ARG(declare_type_mult) @@ -221,7 +233,7 @@ DerivedData MultFunc(std::vector inputData, DataType type) } // Perform a division from the first variable of all other variables in the std::vector -DerivedData DivFunc(std::vector inputData, DataType type) +DerivedData DivFunc(std::vector inputData, DataType type, int nproc) { PERFSTUBS_SCOPED_TIMER("derived::Function::DivFunc"); size_t dataSize = std::accumulate(std::begin(inputData[0].Count), std::end(inputData[0].Count), @@ -232,8 +244,8 @@ DerivedData DivFunc(std::vector inputData, DataType type) #define declare_type_div(T) \ if (type == helper::GetDataType()) \ { \ - T *divValues = detail::ApplyOneToOne( \ - inputData.begin() + 1, inputData.end(), dataSize, [](T a, T b) { return a * b; }, 1); \ + T *divValues = detail::ApplyOneToOne(inputData.begin() + 1, inputData.end(), dataSize, \ + [](T a, T b) { return a * b; }, 1); \ for (size_t i = 0; i < dataSize; i++) \ divValues[i] = *(reinterpret_cast(inputData[0].Data) + i) / divValues[i]; \ return DerivedData({(void *)divValues, inputData[0].Start, inputData[0].Count}); \ @@ -245,7 +257,7 @@ DerivedData DivFunc(std::vector inputData, DataType type) } // Apply Sqrt over all elements in the variable -DerivedData SqrtFunc(std::vector inputData, DataType type) +DerivedData SqrtFunc(std::vector inputData, DataType type, int nproc) { PERFSTUBS_SCOPED_TIMER("derived::Function::SqrtFunc"); if (inputData.size() != 1) @@ -282,7 +294,7 @@ DerivedData SqrtFunc(std::vector inputData, DataType type) } // Apply Pow over all elements in the variable -DerivedData PowFunc(std::vector inputData, DataType type) +DerivedData PowFunc(std::vector inputData, DataType type, int nproc) { PERFSTUBS_SCOPED_TIMER("derived::Function::PowFunc"); if (inputData.size() != 1) @@ -317,7 +329,7 @@ DerivedData PowFunc(std::vector inputData, DataType type) return DerivedData(); } -DerivedData SinFunc(std::vector inputData, DataType type) +DerivedData SinFunc(std::vector inputData, DataType type, int nproc) { PERFSTUBS_SCOPED_TIMER("derived::Function::SinFunc"); if (inputData.size() != 1) @@ -355,7 +367,7 @@ DerivedData SinFunc(std::vector inputData, DataType type) return DerivedData(); } -DerivedData CosFunc(std::vector inputData, DataType type) +DerivedData CosFunc(std::vector inputData, DataType type, int nproc) { PERFSTUBS_SCOPED_TIMER("derived::Function::CosFunc"); if (inputData.size() != 1) @@ -393,7 +405,7 @@ DerivedData CosFunc(std::vector inputData, DataType type) return DerivedData(); } -DerivedData TanFunc(std::vector inputData, DataType type) +DerivedData TanFunc(std::vector inputData, DataType type, int nproc) { PERFSTUBS_SCOPED_TIMER("derived::Function::TanFunc"); if (inputData.size() != 1) @@ -431,7 +443,7 @@ DerivedData TanFunc(std::vector inputData, DataType type) return DerivedData(); } -DerivedData AsinFunc(std::vector inputData, DataType type) +DerivedData AsinFunc(std::vector inputData, DataType type, int nproc) { PERFSTUBS_SCOPED_TIMER("derived::Function::AsinFunc"); if (inputData.size() != 1) @@ -469,7 +481,7 @@ DerivedData AsinFunc(std::vector inputData, DataType type) return DerivedData(); } -DerivedData AcosFunc(std::vector inputData, DataType type) +DerivedData AcosFunc(std::vector inputData, DataType type, int nproc) { PERFSTUBS_SCOPED_TIMER("derived::Function::AcosFunc"); if (inputData.size() != 1) @@ -507,7 +519,7 @@ DerivedData AcosFunc(std::vector inputData, DataType type) return DerivedData(); } -DerivedData AtanFunc(std::vector inputData, DataType type) +DerivedData AtanFunc(std::vector inputData, DataType type, int nproc) { PERFSTUBS_SCOPED_TIMER("derived::Function::AtanFunc"); if (inputData.size() != 1) @@ -575,7 +587,7 @@ DerivedData MagAggregatedFunc(DerivedData inputData, DataType type) return DerivedData(); } -DerivedData MagnitudeFunc(std::vector inputData, DataType type) +DerivedData MagnitudeFunc(std::vector inputData, DataType type, int nproc) { PERFSTUBS_SCOPED_TIMER("derived::Function::MagnitudeFunc"); // if there is only one element return the aggregate result @@ -600,7 +612,7 @@ DerivedData MagnitudeFunc(std::vector inputData, DataType type) return DerivedData(); } -DerivedData Cross3DFunc(const std::vector inputData, DataType type) +DerivedData Cross3DFunc(const std::vector inputData, DataType type, int nproc) { PERFSTUBS_SCOPED_TIMER("derived::Function::Cross3DFunc"); if (inputData.size() != 6) @@ -649,7 +661,7 @@ DerivedData Cross3DFunc(const std::vector inputData, DataType type) * (ex: partial derivatives in x direction at point (0,0,0) * only use data from (1,0,0), etc ) */ -DerivedData Curl3DFunc(const std::vector inputData, DataType type) +DerivedData Curl3DFunc(const std::vector inputData, DataType type, int nproc) { PERFSTUBS_SCOPED_TIMER("derived::Function::Curl3DFunc"); size_t dims[3] = {inputData[0].Count[0], inputData[0].Count[1], inputData[0].Count[2]}; @@ -675,6 +687,166 @@ DerivedData Curl3DFunc(const std::vector inputData, DataType type) return DerivedData(); } +DerivedData MinFunc(std::vector inputData, DataType type, int nproc) +{ + PERFSTUBS_SCOPED_TIMER("derived::Function::MinFunc"); + +#define declare_type_min(T) \ + if (type == helper::GetDataType()) \ + { \ + T *minVal = (T *)malloc(sizeof(T)); \ + *minVal = *((T *)(inputData[0].Data)); \ + for (DerivedData d : inputData) \ + { \ + size_t dataSize = std::accumulate(std::begin(d.Count), std::end(d.Count), 1, \ + std::multiplies()); \ + *minVal = detail::ApplyAllToOne((T *)d.Data, dataSize, \ + [](T a, T b) { return std::min(a, b); }, *minVal); \ + } \ + return DerivedData({(void *)minVal, {0}, {size_t(nproc)}}); \ + } + ADIOS2_FOREACH_ATTRIBUTE_PRIMITIVE_STDTYPE_1ARG(declare_type_min) + helper::Throw("Derived", "Function", "MinFunc", + "Invalid variable types"); + return DerivedData(); +} + +DerivedData MaxFunc(std::vector inputData, DataType type, int nproc) +{ + PERFSTUBS_SCOPED_TIMER("derived::Function::MaxFunc"); + +#define declare_type_max(T) \ + if (type == helper::GetDataType()) \ + { \ + T *maxVal = (T *)malloc(sizeof(T)); \ + *maxVal = *((T *)(inputData[0].Data)); \ + for (DerivedData d : inputData) \ + { \ + size_t dataSize = std::accumulate(std::begin(d.Count), std::end(d.Count), 1, \ + std::multiplies()); \ + *maxVal = detail::ApplyAllToOne((T *)d.Data, dataSize, \ + [](T a, T b) { return std::max(a, b); }, *maxVal); \ + } \ + return DerivedData({(void *)maxVal, {0}, {size_t(nproc)}}); \ + } + ADIOS2_FOREACH_ATTRIBUTE_PRIMITIVE_STDTYPE_1ARG(declare_type_max) + helper::Throw("Derived", "Function", "MaxFunc", + "Invalid variable types"); + return DerivedData(); +} + +DerivedData SumFunc(std::vector inputData, DataType type, int nproc) +{ + PERFSTUBS_SCOPED_TIMER("derived::Function::SumFunc"); + +#define declare_type_sum(T) \ + if (type == helper::GetDataType()) \ + { \ + T *sumVal = (T *)malloc(sizeof(T)); \ + *sumVal = (T)0; \ + for (DerivedData d : inputData) \ + { \ + size_t dataSize = std::accumulate(std::begin(d.Count), std::end(d.Count), 1, \ + std::multiplies()); \ + *sumVal = detail::ApplyAllToOne((T *)inputData[0].Data, dataSize, \ + [](T a, T b) { return a + b; }, *sumVal); \ + } \ + return DerivedData({(void *)sumVal, {0}, {size_t(nproc)}}); \ + } + ADIOS2_FOREACH_ATTRIBUTE_PRIMITIVE_STDTYPE_1ARG(declare_type_sum) + helper::Throw("Derived", "Function", "SumFunc", + "Invalid variable types"); + return DerivedData(); +} + +DerivedData MeanFunc(std::vector inputData, DataType type, int nproc) +{ + PERFSTUBS_SCOPED_TIMER("derived::Function::MeanFunc"); + +#define declare_type_mean(T) \ + if (type == helper::GetDataType()) \ + { \ + T *meanVal = (T *)malloc(sizeof(T)); \ + *meanVal = (T)0; \ + size_t totalSize = 0; \ + for (DerivedData d : inputData) \ + { \ + size_t dataSize = std::accumulate(std::begin(d.Count), std::end(d.Count), 1, \ + std::multiplies()); \ + totalSize += dataSize; \ + *meanVal = detail::ApplyAllToOne((T *)inputData[0].Data, dataSize, \ + [](T a, T b) { return a + b; }, *meanVal); \ + } \ + *meanVal /= (T)totalSize; \ + return DerivedData({(void *)meanVal, {0}, {size_t(nproc)}}); \ + } + ADIOS2_FOREACH_ATTRIBUTE_PRIMITIVE_STDTYPE_1ARG(declare_type_mean) + helper::Throw("Derived", "Function", "MeanFunc", + "Invalid variable types"); + return DerivedData(); +} + +DerivedData VarianceFunc(std::vector inputData, DataType type, int nproc) +{ + PERFSTUBS_SCOPED_TIMER("derived::Function::VarianceFunc"); + DerivedData meanData = MeanFunc(inputData, type, nproc); + size_t totalSize = 0; + for (DerivedData d : inputData) + totalSize += + std::accumulate(std::begin(d.Count), std::end(d.Count), 1, std::multiplies()); + +#define declare_type_variance(T) \ + if (type == helper::GetDataType()) \ + { \ + T meanVal = *((T *)meanData.Data); \ + T *varianceVal = (T *)malloc(sizeof(T)); \ + *varianceVal = (T)0; \ + for (DerivedData d : inputData) \ + { \ + size_t dataSize = std::accumulate(std::begin(d.Count), std::end(d.Count), 1, \ + std::multiplies()); \ + *varianceVal = detail::ApplyAllToOne( \ + (T *)inputData[0].Data, dataSize, \ + [&meanVal, &totalSize](T a, T b) { \ + return a + (((b - meanVal) * (b - meanVal)) / (T)totalSize); \ + }, \ + *varianceVal); \ + } \ + return DerivedData({(void *)varianceVal, {0}, {size_t(nproc)}}); \ + } + ADIOS2_FOREACH_ATTRIBUTE_PRIMITIVE_STDTYPE_1ARG(declare_type_variance) + helper::Throw("Derived", "Function", "VarianceFunc", + "Invalid variable types"); + return DerivedData(); +} + +DerivedData StDevFunc(std::vector inputData, DataType type, int nproc) +{ + PERFSTUBS_SCOPED_TIMER("derived::Function::StdevFunc"); + DataType inputType = inputData[0].Type; + DerivedData varianceData = VarianceFunc(inputData, inputType, nproc); + + if (inputType == DataType::LongDouble) + { + long double varianceVal = *((long double *)varianceData.Data); + long double *stdevVal = (long double *)malloc(sizeof(long double)); + *stdevVal = std::sqrt(varianceVal); + return DerivedData({(void *)stdevVal, {0}, {size_t(nproc)}}); + } +#define declare_type_stdev(T) \ + else if (inputType == helper::GetDataType()) \ + { \ + T varianceVal = *((T *)varianceData.Data); \ + double *stdevVal = (double *)malloc(sizeof(double)); \ + *stdevVal = std::sqrt(varianceVal); \ + return DerivedData({(void *)stdevVal, {0}, {size_t(nproc)}}); \ + } + ADIOS2_FOREACH_ATTRIBUTE_PRIMITIVE_STDTYPE_1ARG(declare_type_stdev) + helper::Throw("Derived", "Function", "StdevFunc", + "Invalid variable types"); + return DerivedData(); +} + /* Functions that return output dimensions * Input: A list of variable dimensions (start, count, shape) * Output: (start, count, shape) of the output operation */ @@ -752,6 +924,12 @@ std::tuple CurlDimsFunc(std::vector ScalarDimsFunc(std::vector> input) +{ + // Start, Count, Shape + return {{0}, {1}, {1}}; +} + DataType SameTypeFunc(DataType input) { return input; } DataType FloatTypeFunc(DataType input) diff --git a/source/adios2/toolkit/derived/Function.h b/source/adios2/toolkit/derived/Function.h index 2f2bed708f..1752ea344d 100644 --- a/source/adios2/toolkit/derived/Function.h +++ b/source/adios2/toolkit/derived/Function.h @@ -7,26 +7,34 @@ namespace adios2 { namespace derived { -DerivedData AddFunc(std::vector input, DataType type); -DerivedData SubtractFunc(std::vector input, DataType type); -DerivedData SinFunc(std::vector input, DataType type); -DerivedData CosFunc(std::vector input, DataType type); -DerivedData TanFunc(std::vector input, DataType type); -DerivedData AsinFunc(std::vector input, DataType type); -DerivedData AcosFunc(std::vector input, DataType type); -DerivedData AtanFunc(std::vector input, DataType type); -DerivedData MultFunc(std::vector input, DataType type); -DerivedData DivFunc(std::vector input, DataType type); -DerivedData SqrtFunc(std::vector input, DataType type); -DerivedData PowFunc(std::vector input, DataType type); -DerivedData MagnitudeFunc(std::vector input, DataType type); -DerivedData Cross3DFunc(std::vector input, DataType type); -DerivedData Curl3DFunc(std::vector input, DataType type); +DerivedData AddFunc(std::vector input, DataType type, int nproc); +DerivedData SubtractFunc(std::vector input, DataType type, int nproc); +DerivedData SinFunc(std::vector input, DataType type, int nproc); +DerivedData CosFunc(std::vector input, DataType type, int nproc); +DerivedData TanFunc(std::vector input, DataType type, int nproc); +DerivedData AsinFunc(std::vector input, DataType type, int nproc); +DerivedData AcosFunc(std::vector input, DataType type, int nproc); +DerivedData AtanFunc(std::vector input, DataType type, int nproc); +DerivedData MultFunc(std::vector input, DataType type, int nproc); +DerivedData DivFunc(std::vector input, DataType type, int nproc); +DerivedData SqrtFunc(std::vector input, DataType type, int nproc); +DerivedData PowFunc(std::vector input, DataType type, int nproc); +DerivedData MagnitudeFunc(std::vector input, DataType type, int nproc); +DerivedData Cross3DFunc(std::vector input, DataType type, int nproc); +DerivedData Curl3DFunc(std::vector input, DataType type, int nproc); +DerivedData MinFunc(std::vector input, DataType type, int nproc); +DerivedData MaxFunc(std::vector input, DataType type, int nproc); +DerivedData SumFunc(std::vector input, DataType type, int nproc); +DerivedData MeanFunc(std::vector input, DataType type, int nproc); +DerivedData MedianFunc(std::vector input, DataType type, int nproc); +DerivedData VarianceFunc(std::vector input, DataType type, int nproc); +DerivedData StDevFunc(std::vector input, DataType type, int nproc); std::tuple SameDimsFunc(std::vector> input); std::tuple SameDimsWithAgrFunc(std::vector> input); std::tuple Cross3DDimsFunc(std::vector> input); std::tuple CurlDimsFunc(std::vector> input); +std::tuple ScalarDimsFunc(std::vector> input); DataType SameTypeFunc(DataType input); DataType FloatTypeFunc(DataType input); diff --git a/testing/adios2/derived/CMakeLists.txt b/testing/adios2/derived/CMakeLists.txt index 097e7cc7d7..2df2938853 100644 --- a/testing/adios2/derived/CMakeLists.txt +++ b/testing/adios2/derived/CMakeLists.txt @@ -3,4 +3,4 @@ #accompanying file Copyright.txt for details. #------------------------------------------------------------------------------# -gtest_add_tests_helper(DerivedCorrectness MPI_NONE BP Derived. "") +gtest_add_tests_helper(DerivedCorrectness MPI_ALLOW BP Derived. "") diff --git a/testing/adios2/derived/TestBPDerivedCorrectness.cpp b/testing/adios2/derived/TestBPDerivedCorrectness.cpp index 4c3fadbb3b..01c7a18f93 100644 --- a/testing/adios2/derived/TestBPDerivedCorrectness.cpp +++ b/testing/adios2/derived/TestBPDerivedCorrectness.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -27,6 +28,13 @@ class DerivedCorrectnessP : public DerivedCorrectness, TEST_P(DerivedCorrectnessP, ScalarFunctionsCorrectnessTest) { + int rank = 0; + int size = 1; +#if ADIOS2_USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); +#endif + const size_t Nx = 10, Ny = 3, Nz = 6; const size_t steps = 2; // Application variable @@ -46,7 +54,11 @@ TEST_P(DerivedCorrectnessP, ScalarFunctionsCorrectnessTest) simArray3[i] = distribution(generator); } +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else adios2::ADIOS adios; +#endif adios2::IO bpOut = adios.DeclareIO("BPWriteAddExpression"); @@ -58,10 +70,22 @@ TEST_P(DerivedCorrectnessP, ScalarFunctionsCorrectnessTest) const std::string derDivName = "derived/div"; const std::string derPowName = "derived/pow"; const std::string derSqrtName = "derived/sqrt"; - - auto Ux = bpOut.DefineVariable(varname[0], {Nx, Ny, Nz}, {0, 0, 0}, {Nx, Ny, Nz}); - auto Uy = bpOut.DefineVariable(varname[1], {Nx, Ny, Nz}, {0, 0, 0}, {Nx, Ny, Nz}); - auto Uz = bpOut.DefineVariable(varname[2], {Nx, Ny, Nz}, {0, 0, 0}, {Nx, Ny, Nz}); + const std::string derMinName = "derived/min"; + const std::string derMaxName = "derived/max"; + const std::string derSumName = "derived/sum"; + const std::string derMeanName = "derived/mean"; + const std::string derVarianceName = "derived/variance"; + const std::string derStdevName = "derived/stdev"; + + const adios2::Dims shape{static_cast(Nx * size), static_cast(Ny * size), + static_cast(Nz * size)}; + const adios2::Dims start{static_cast(Nx * rank), static_cast(Ny * rank), + static_cast(Nz * rank)}; + const adios2::Dims count{Nx, Ny, Nz}; + + auto Ux = bpOut.DefineVariable(varname[0], shape, start, count); + auto Uy = bpOut.DefineVariable(varname[1], shape, start, count); + auto Uz = bpOut.DefineVariable(varname[2], shape, start, count); // clang-format off bpOut.DefineDerivedVariable(derAgrAdd, "x=" + varname[0] + "\n" @@ -99,6 +123,30 @@ TEST_P(DerivedCorrectnessP, ScalarFunctionsCorrectnessTest) "x =" + varname[0] + " \n" "sqrt(x)", mode); + bpOut.DefineDerivedVariable(derMinName, + "x =" + varname[0] + " \n" + "min(x)", + mode); + bpOut.DefineDerivedVariable(derMaxName, + "x =" + varname[0] + " \n" + "max(x)", + mode); + bpOut.DefineDerivedVariable(derSumName, + "x =" + varname[0] + " \n" + "sum(x)", + mode); + bpOut.DefineDerivedVariable(derMeanName, + "x =" + varname[0] + " \n" + "mean(x)", + mode); + bpOut.DefineDerivedVariable(derVarianceName, + "x =" + varname[0] + " \n" + "variance(x)", + mode); + bpOut.DefineDerivedVariable(derStdevName, + "x =" + varname[0] + " \n" + "stdev(x)", + mode); // clang-format on std::string filename = "derivedScalar.bp"; adios2::Engine bpFileWriter = bpOut.Open(filename, adios2::Mode::Write); @@ -126,6 +174,8 @@ TEST_P(DerivedCorrectnessP, ScalarFunctionsCorrectnessTest) std::vector readDiv; std::vector readPow; std::vector readSqrt; + std::vector readMin, readMax, readSum, readMean, readVariance; + std::vector readStdev; float calcFloat; double calcDouble; @@ -143,8 +193,18 @@ TEST_P(DerivedCorrectnessP, ScalarFunctionsCorrectnessTest) bpFileReader.Get(derDivName, readDiv); bpFileReader.Get(derPowName, readPow); bpFileReader.Get(derSqrtName, readSqrt); + bpFileReader.Get(derMinName, readMin); + bpFileReader.Get(derMaxName, readMax); + bpFileReader.Get(derSumName, readSum); + bpFileReader.Get(derMeanName, readMean); + bpFileReader.Get(derVarianceName, readVariance); + bpFileReader.Get(derStdevName, readStdev); bpFileReader.EndStep(); + float min = std::numeric_limits::max(); + float max = std::numeric_limits::min(); + float sum = 0; + for (size_t ind = 0; ind < Nx * Ny * Nz; ++ind) { calcFloat = readUx[ind] + readUy[ind] + readUz[ind]; @@ -164,8 +224,30 @@ TEST_P(DerivedCorrectnessP, ScalarFunctionsCorrectnessTest) calcDouble = std::sqrt(readUx[ind]); EXPECT_TRUE(fabs(calcDouble - readSqrt[ind]) < epsilon); + min = std::min(min, readUx[ind]); + max = std::max(max, readUx[ind]); + sum += readUx[ind]; } + float mean = sum / (Nx * Ny * Nz); + EXPECT_TRUE(fabs(min - readMin[0]) < epsilon); + EXPECT_TRUE(fabs(max - readMax[0]) < epsilon); + EXPECT_TRUE(fabs(sum - readSum[0]) < epsilon); + EXPECT_TRUE(fabs(mean - readMean[0]) < epsilon); + + float variance = 0; + + // Calculate Variance now that mean is computed + for (size_t ind = 0; ind < Nx * Ny * Nz; ++ind) + { + variance += (readUx[ind] - mean) * (readUx[ind] - mean); + } + variance /= (Nx * Ny * Nz); + double stdev = std::sqrt(variance); + + EXPECT_TRUE(fabs(variance - readVariance[0]) < epsilon); + EXPECT_TRUE(fabs(stdev - readStdev[0]) < epsilon); + for (size_t ind = 0; ind < Nx * Ny; ++ind) { size_t start = ind * Nz; @@ -700,12 +782,17 @@ INSTANTIATE_TEST_SUITE_P(DerivedCorrectness, DerivedCorrectnessP, ::testing::Values(adios2::DerivedVarType::StatsOnly, adios2::DerivedVarType::ExpressionString, adios2::DerivedVarType::StoreData)); + int main(int argc, char **argv) { int result; ::testing::InitGoogleTest(&argc, argv); result = RUN_ALL_TESTS(); - + +#if ADIOS2_USE_MPI + MPI_Finalize(); +#endif + return result; }