Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Extend Expressions: Stats funcs #4368

Open
wants to merge 13 commits into
base: master
Choose a base branch
from
Open

Extend Expressions: Stats funcs #4368

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
06678dc
Create stats functions
lizdulac Sep 23, 2024
31b3887
wip shrink failing example
lizdulac Sep 24, 2024
c685331
Merge branch 'ornladios:master' into statsFuncs
lizdulac Oct 11, 2024
87f3693
change stats test to vector
lizdulac Oct 11, 2024
10b392a
Merge branch 'ornladios:master' into statsFuncs
lizdulac Oct 17, 2024
59c8e6b
Fix dimensions bug
lizdulac Oct 17, 2024
98d62f8
formatting
lizdulac Oct 22, 2024
cb889a4
formatting
lizdulac Oct 22, 2024
194ae00
Merge branch 'master' into statsFuncs
eisenhauer Oct 23, 2024
9a732a1
Default DerivedVariables OFF
lizdulac Oct 23, 2024
c4dfae3
Add mpi
lizdulac Dec 5, 2024
a83b798
format
lizdulac Dec 5, 2024
687f731
MPI finalize
lizdulac Dec 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions source/adios2/core/VariableDerived.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ void VariableDerived::UpdateExprDim(std::map<std::string, std::tuple<Dims, Dims,

std::vector<std::tuple<void *, Dims, Dims>>
VariableDerived::ApplyExpression(std::map<std::string, std::unique_ptr<MinVarInfo>> &NameToMVI,
bool DoCompute)
bool DoCompute, int nproc)
{
size_t numBlocks = 0;
// check that all variables have the same number of blocks
Expand Down Expand Up @@ -71,7 +71,7 @@ VariableDerived::ApplyExpression(std::map<std::string, std::unique_ptr<MinVarInf
inputData.insert({variable.first, varData});
}
std::vector<adios2::derived::DerivedData> outputData =
m_Expr.ApplyExpression(m_Type, numBlocks, inputData);
m_Expr.ApplyExpression(m_Type, numBlocks, inputData, nproc);

std::vector<std::tuple<void *, Dims, Dims>> blockData;
for (size_t i = 0; i < numBlocks; i++)
Expand Down
3 changes: 2 additions & 1 deletion source/adios2/core/VariableDerived.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ class VariableDerived : public VariableBase
void UpdateExprDim(std::map<std::string, std::tuple<Dims, Dims, Dims>> NameToDims);

std::vector<std::tuple<void *, Dims, Dims>>
ApplyExpression(std::map<std::string, std::unique_ptr<MinVarInfo>> &mvi, bool DoCompute = true);
ApplyExpression(std::map<std::string, std::unique_ptr<MinVarInfo>> &mvi, bool DoCompute = true,
int nproc = 1);
};

} // end namespace core
Expand Down
14 changes: 9 additions & 5 deletions source/adios2/engine/bp5/BP5Writer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -548,7 +548,7 @@ void BP5Writer::ComputeDerivedVariables()
std::vector<std::tuple<void *, Dims, Dims>> DerivedBlockData;
// for expressionString, just generate the blocksinfo
bool DoCompute = derivedVar->GetDerivedType() != DerivedVarType::ExpressionString;
DerivedBlockData = derivedVar->ApplyExpression(nameToVarInfo, DoCompute);
DerivedBlockData = derivedVar->ApplyExpression(nameToVarInfo, DoCompute, m_Comm.Size());

// Send the derived variable to ADIOS2 internal logic
for (auto derivedBlock : DerivedBlockData)
Expand Down Expand Up @@ -884,7 +884,8 @@ uint64_t BP5Writer::CountStepsInMetadataIndex(format::BufferSTL &bufferSTL)

switch (recordID)
{
case IndexRecord::WriterMapRecord: {
case IndexRecord::WriterMapRecord:
{
m_AppendWriterCount =
(uint32_t)helper::ReadValue<uint64_t>(buffer, position, IsLittleEndian);
m_AppendAggregatorCount =
Expand All @@ -899,7 +900,8 @@ uint64_t BP5Writer::CountStepsInMetadataIndex(format::BufferSTL &bufferSTL)
position += m_AppendWriterCount * sizeof(uint64_t);
break;
}
case IndexRecord::StepRecord: {
case IndexRecord::StepRecord:
{
position += 2 * sizeof(uint64_t); // MetadataPos, MetadataSize
const uint64_t FlushCount =
helper::ReadValue<uint64_t>(buffer, position, IsLittleEndian);
Expand Down Expand Up @@ -968,7 +970,8 @@ uint64_t BP5Writer::CountStepsInMetadataIndex(format::BufferSTL &bufferSTL)

switch (recordID)
{
case IndexRecord::WriterMapRecord: {
case IndexRecord::WriterMapRecord:
{
m_AppendWriterCount =
(uint32_t)helper::ReadValue<uint64_t>(buffer, position, IsLittleEndian);
m_AppendAggregatorCount =
Expand All @@ -986,7 +989,8 @@ uint64_t BP5Writer::CountStepsInMetadataIndex(format::BufferSTL &bufferSTL)
}
break;
}
case IndexRecord::StepRecord: {
case IndexRecord::StepRecord:
{
m_AppendMetadataIndexPos =
position - sizeof(unsigned char) - sizeof(uint64_t); // pos of RecordID
const uint64_t MetadataPos =
Expand Down
84 changes: 62 additions & 22 deletions source/adios2/toolkit/derived/Expression.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,26 +38,58 @@ const std::map<ExpressionOperator, OperatorProperty> op_property = {
{ExpressionOperator::OP_ATAN, {"ATAN", false}},
{ExpressionOperator::OP_MAGN, {"MAGNITUDE", false}},
{ExpressionOperator::OP_CROSS, {"CROSS", false}},
{ExpressionOperator::OP_CURL, {"CURL", false}}};
{ExpressionOperator::OP_CURL, {"CURL", false}},
{ExpressionOperator::OP_MIN, {"MIN", false}},
{ExpressionOperator::OP_MAX, {"MAX", false}},
{ExpressionOperator::OP_SUM, {"SUM", false}},
{ExpressionOperator::OP_MEAN, {"MEAN", false}},
{ExpressionOperator::OP_VARIANCE, {"VARIANCE", false}},
{ExpressionOperator::OP_STDEV, {"STDEV", false}}};

const std::map<std::string, ExpressionOperator> string_to_op = {
{"ALIAS", ExpressionOperator::OP_ALIAS}, /* Parser-use only */
{"PATH", ExpressionOperator::OP_PATH}, /* Parser-use only */
{"NUM", ExpressionOperator::OP_NUM}, /* Parser-use only */
{"INDEX", ExpressionOperator::OP_INDEX}, {"+", ExpressionOperator::OP_ADD},
{"add", ExpressionOperator::OP_ADD}, {"ADD", ExpressionOperator::OP_ADD},
{"-", ExpressionOperator::OP_SUBTRACT}, {"SUBTRACT", ExpressionOperator::OP_SUBTRACT},
{"/", ExpressionOperator::OP_DIV}, {"divide", ExpressionOperator::OP_DIV},
{"DIVIDE", ExpressionOperator::OP_DIV}, {"*", ExpressionOperator::OP_MULT},
{"multiply", ExpressionOperator::OP_MULT}, {"MULTIPLY", ExpressionOperator::OP_MULT},
{"SQRT", ExpressionOperator::OP_SQRT}, {"sqrt", ExpressionOperator::OP_SQRT},
{"pow", ExpressionOperator::OP_POW}, {"POW", ExpressionOperator::OP_POW},
{"sin", ExpressionOperator::OP_SIN}, {"cos", ExpressionOperator::OP_COS},
{"tan", ExpressionOperator::OP_TAN}, {"asin", ExpressionOperator::OP_ASIN},
{"acos", ExpressionOperator::OP_ACOS}, {"atan", ExpressionOperator::OP_ATAN},
{"^", ExpressionOperator::OP_POW}, {"magnitude", ExpressionOperator::OP_MAGN},
{"MAGNITUDE", ExpressionOperator::OP_MAGN}, {"cross", ExpressionOperator::OP_CROSS},
{"curl", ExpressionOperator::OP_CURL}, {"CURL", ExpressionOperator::OP_CURL}};
{"INDEX", ExpressionOperator::OP_INDEX},
{"+", ExpressionOperator::OP_ADD},
{"add", ExpressionOperator::OP_ADD},
{"ADD", ExpressionOperator::OP_ADD},
{"-", ExpressionOperator::OP_SUBTRACT},
{"SUBTRACT", ExpressionOperator::OP_SUBTRACT},
{"/", ExpressionOperator::OP_DIV},
{"divide", ExpressionOperator::OP_DIV},
{"DIVIDE", ExpressionOperator::OP_DIV},
{"*", ExpressionOperator::OP_MULT},
{"multiply", ExpressionOperator::OP_MULT},
{"MULTIPLY", ExpressionOperator::OP_MULT},
{"SQRT", ExpressionOperator::OP_SQRT},
{"sqrt", ExpressionOperator::OP_SQRT},
{"pow", ExpressionOperator::OP_POW},
{"POW", ExpressionOperator::OP_POW},
{"sin", ExpressionOperator::OP_SIN},
{"cos", ExpressionOperator::OP_COS},
{"tan", ExpressionOperator::OP_TAN},
{"asin", ExpressionOperator::OP_ASIN},
{"acos", ExpressionOperator::OP_ACOS},
{"atan", ExpressionOperator::OP_ATAN},
{"^", ExpressionOperator::OP_POW},
{"magnitude", ExpressionOperator::OP_MAGN},
{"MAGNITUDE", ExpressionOperator::OP_MAGN},
{"cross", ExpressionOperator::OP_CROSS},
{"curl", ExpressionOperator::OP_CURL},
{"CURL", ExpressionOperator::OP_CURL},
{"min", ExpressionOperator::OP_MIN},
{"MIN", ExpressionOperator::OP_MIN},
{"max", ExpressionOperator::OP_MAX},
{"MAX", ExpressionOperator::OP_MAX},
{"sum", ExpressionOperator::OP_SUM},
{"SUM", ExpressionOperator::OP_SUM},
{"mean", ExpressionOperator::OP_MEAN},
{"MEAN", ExpressionOperator::OP_MEAN},
{"variance", ExpressionOperator::OP_VARIANCE},
{"VARIANCE", ExpressionOperator::OP_VARIANCE},
{"stdev", ExpressionOperator::OP_STDEV},
{"STDEV", ExpressionOperator::OP_STDEV}};

inline std::string get_op_name(ExpressionOperator op) { return op_property.at(op).name; }

Expand Down Expand Up @@ -134,7 +166,7 @@ namespace derived
{
struct OperatorFunctions
{
std::function<DerivedData(std::vector<DerivedData>, DataType)> ComputeFct;
std::function<DerivedData(std::vector<DerivedData>, DataType, int)> ComputeFct;
std::function<std::tuple<Dims, Dims, Dims>(std::vector<std::tuple<Dims, Dims, Dims>>)> DimsFct;
std::function<DataType(DataType)> TypeFct;
};
Expand All @@ -155,7 +187,13 @@ std::map<adios2::detail::ExpressionOperator, OperatorFunctions> OpFunctions = {
{adios2::detail::ExpressionOperator::OP_MAGN,
{MagnitudeFunc, SameDimsWithAgrFunc, SameTypeFunc}},
{adios2::detail::ExpressionOperator::OP_CROSS, {Cross3DFunc, Cross3DDimsFunc, SameTypeFunc}},
{adios2::detail::ExpressionOperator::OP_CURL, {Curl3DFunc, CurlDimsFunc, SameTypeFunc}}};
{adios2::detail::ExpressionOperator::OP_CURL, {Curl3DFunc, CurlDimsFunc, SameTypeFunc}},
{adios2::detail::ExpressionOperator::OP_MIN, {MinFunc, ScalarDimsFunc, SameTypeFunc}},
{adios2::detail::ExpressionOperator::OP_MAX, {MaxFunc, ScalarDimsFunc, SameTypeFunc}},
{adios2::detail::ExpressionOperator::OP_SUM, {SumFunc, ScalarDimsFunc, SameTypeFunc}},
{adios2::detail::ExpressionOperator::OP_MEAN, {MeanFunc, ScalarDimsFunc, SameTypeFunc}},
{adios2::detail::ExpressionOperator::OP_VARIANCE, {VarianceFunc, ScalarDimsFunc, SameTypeFunc}},
{adios2::detail::ExpressionOperator::OP_STDEV, {StDevFunc, ScalarDimsFunc, FloatTypeFunc}}};

Expression::Expression(std::string string_exp)
: m_Shape({0}), m_Start({0}), m_Count({0}), ExprString(string_exp)
Expand Down Expand Up @@ -195,9 +233,9 @@ DataType Expression::GetType(std::map<std::string, DataType> NameToType)

std::vector<DerivedData>
Expression::ApplyExpression(DataType type, size_t numBlocks,
std::map<std::string, std::vector<DerivedData>> nameToData)
std::map<std::string, std::vector<DerivedData>> nameToData, int nproc)
{
return m_Expr.ApplyExpression(type, numBlocks, nameToData);
return m_Expr.ApplyExpression(type, numBlocks, nameToData, nproc);
}

void ExpressionTree::set_base(double c) { detail.constant = c; }
Expand Down Expand Up @@ -341,7 +379,8 @@ DataType ExpressionTree::GetType(std::map<std::string, DataType> NameToType)

std::vector<DerivedData>
ExpressionTree::ApplyExpression(DataType type, size_t numBlocks,
std::map<std::string, std::vector<DerivedData>> nameToData)
std::map<std::string, std::vector<DerivedData>> nameToData,
int nproc)
{
// create operands for the computation function
// exprData[0] = list of void* data for block 0 for each variable
Expand All @@ -361,7 +400,8 @@ ExpressionTree::ApplyExpression(DataType type, size_t numBlocks,
else
{
deallocate.push_back(true);
auto subexpData = std::get<0>(subexp).ApplyExpression(type, numBlocks, nameToData);
auto subexpData =
std::get<0>(subexp).ApplyExpression(type, numBlocks, nameToData, nproc);
for (size_t blk = 0; blk < numBlocks; blk++)
{
exprData[blk].push_back(subexpData[blk]);
Expand All @@ -373,7 +413,7 @@ ExpressionTree::ApplyExpression(DataType type, size_t numBlocks,
auto op_fct = OpFunctions.at(detail.operation);
for (size_t blk = 0; blk < numBlocks; blk++)
{
outputData[blk] = op_fct.ComputeFct(exprData[blk], type);
outputData[blk] = op_fct.ComputeFct(exprData[blk], type, nproc);
}
// deallocate intermediate data after computing the operation
for (size_t blk = 0; blk < numBlocks; blk++)
Expand Down
13 changes: 10 additions & 3 deletions source/adios2/toolkit/derived/Expression.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,14 @@ enum ExpressionOperator
OP_ATAN,
OP_MAGN,
OP_CROSS,
OP_CURL
OP_CURL,
OP_MIN,
OP_MAX,
OP_SUM,
OP_MEAN,
OP_MEDIAN,
OP_VARIANCE,
OP_STDEV
};
}

Expand Down Expand Up @@ -79,7 +86,7 @@ class ExpressionTree
DataType GetType(std::map<std::string, DataType> NameToType);
std::vector<DerivedData>
ApplyExpression(DataType type, size_t numBlocks,
std::map<std::string, std::vector<DerivedData>> nameToData);
std::map<std::string, std::vector<DerivedData>> nameToData, int nproc);
void print();
std::string toStringExpr();
};
Expand Down Expand Up @@ -109,7 +116,7 @@ class Expression
std::vector<std::string> VariableNameList();
std::vector<DerivedData>
ApplyExpression(DataType type, size_t numBlocks,
std::map<std::string, std::vector<DerivedData>> nameToData);
std::map<std::string, std::vector<DerivedData>> nameToData, int nproc);
};

}
Expand Down
Loading