diff --git a/bindings/Python/CMakeLists.txt b/bindings/Python/CMakeLists.txt index 1c91cb6d19..9c7f249201 100644 --- a/bindings/Python/CMakeLists.txt +++ b/bindings/Python/CMakeLists.txt @@ -3,6 +3,7 @@ Python_add_library(adios2_py MODULE py11ADIOS.cpp py11IO.cpp py11Variable.cpp + py11VariableDerived.cpp py11Attribute.cpp py11Engine.cpp py11Operator.cpp diff --git a/bindings/Python/py11IO.cpp b/bindings/Python/py11IO.cpp index 3d8f03782c..0ace77d169 100644 --- a/bindings/Python/py11IO.cpp +++ b/bindings/Python/py11IO.cpp @@ -146,6 +146,23 @@ Variable IO::DefineVariable(const std::string &name, const pybind11::object &val return Variable(variable); } +VariableDerived IO::DefineDerivedVariable(const std::string &name, const std::string &exp_string, + const DerivedVarType varType) +{ + helper::CheckForNullptr(m_IO, + "for variable " + name + ", in call to IO::DefineDerivedVariable"); + +#ifdef ADIOS2_HAVE_DERIVED_VARIABLE + adios2::core::VariableDerived *dv = &m_IO->DefineDerivedVariable(name, exp_string, varType); + adios2::py11::VariableDerived vd(dv); +#else + adios2::py11::VariableDerived vd; + throw std::invalid_argument("ERROR: Derived Variables are not supported in this adios2 build " + ", in call to DefineDerivedVariable\n"); +#endif + return vd; +} + Variable IO::InquireVariable(const std::string &name) { helper::CheckForNullptr(m_IO, "for variable " + name + ", in call to IO::InquireVariable"); diff --git a/bindings/Python/py11IO.h b/bindings/Python/py11IO.h index 80c7c7599b..2f7a667776 100644 --- a/bindings/Python/py11IO.h +++ b/bindings/Python/py11IO.h @@ -19,6 +19,7 @@ #include "py11Attribute.h" #include "py11Engine.h" #include "py11Variable.h" +#include "py11VariableDerived.h" #include "py11types.h" namespace adios2 @@ -62,6 +63,10 @@ class IO const Dims &shape, const Dims &start, const Dims &count, const bool isConstantDims); + VariableDerived + DefineDerivedVariable(const std::string &name, const std::string &expression, + const DerivedVarType varType = DerivedVarType::MetadataOnly); + Variable InquireVariable(const std::string &name); Attribute DefineAttribute(const std::string &name, const pybind11::array &array, diff --git a/bindings/Python/py11VariableDerived.cpp b/bindings/Python/py11VariableDerived.cpp new file mode 100644 index 0000000000..f020e58813 --- /dev/null +++ b/bindings/Python/py11VariableDerived.cpp @@ -0,0 +1,51 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * py11Variable.cpp + */ + +#include "py11VariableDerived.h" + +#include "adios2/helper/adiosFunctions.h" + +namespace adios2 +{ +namespace py11 +{ + +#ifdef ADIOS2_HAVE_DERIVED_VARIABLE +VariableDerived::VariableDerived(core::VariableDerived *v) : m_VariableDerived(v) {} + +VariableDerived::operator bool() const noexcept +{ + return (m_VariableDerived == nullptr) ? false : true; +} + +std::string VariableDerived::Name() const +{ + helper::CheckForNullptr(m_VariableDerived, "in call to VariableDerived::Name"); + return m_VariableDerived->m_Name; +} + +DerivedVarType VariableDerived::Type() const +{ + helper::CheckForNullptr(m_VariableDerived, "in call to VariableDerived::Type"); + return m_VariableDerived->GetDerivedType(); +} + +#else + +VariableDerived::operator bool() const noexcept { return false; } + +std::string VariableDerived::Name() const +{ + return "DerivedVariables are not supported in this ADIOS2 build"; +} + +DerivedVarType VariableDerived::Type() const { return DerivedVarType::ExpressionString; } + +#endif + +} // end namespace py11 +} // end namespace adios2 diff --git a/bindings/Python/py11VariableDerived.h b/bindings/Python/py11VariableDerived.h new file mode 100644 index 0000000000..7c84b439a6 --- /dev/null +++ b/bindings/Python/py11VariableDerived.h @@ -0,0 +1,54 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * py11VariableDerived.h + * + */ + +#ifndef ADIOS2_BINDINGS_PYTHON_VARIABLEDERIVED_H_ +#define ADIOS2_BINDINGS_PYTHON_VARIABLEDERIVED_H_ + +#include "adios2/common/ADIOSConfig.h" + +#ifdef ADIOS2_HAVE_DERIVED_VARIABLE +#include "adios2/core/VariableDerived.h" +#else +#include "adios2/common/ADIOSTypes.h" +#endif + +namespace adios2 +{ +namespace py11 +{ + +class IO; +class Engine; + +class VariableDerived +{ + friend class IO; + friend class Engine; + +public: + VariableDerived() = default; + + ~VariableDerived() = default; + + explicit operator bool() const noexcept; + + std::string Name() const; + + DerivedVarType Type() const; + +private: +#ifdef ADIOS2_HAVE_DERIVED_VARIABLE + VariableDerived(adios2::core::VariableDerived *v); + adios2::core::VariableDerived *m_VariableDerived = nullptr; +#endif +}; + +} // end namespace py11 +} // end namespace adios2 + +#endif /* ADIOS2_BINDINGS_PYTHON_VARIABLEDERIVED_H_ */ diff --git a/bindings/Python/py11glue.cpp b/bindings/Python/py11glue.cpp index 2c40280f99..0f51248cc2 100644 --- a/bindings/Python/py11glue.cpp +++ b/bindings/Python/py11glue.cpp @@ -28,6 +28,7 @@ #include "py11Operator.h" #include "py11Query.h" #include "py11Variable.h" +#include "py11VariableDerived.h" #if ADIOS2_USE_MPI @@ -124,6 +125,12 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) .value("OtherError", adios2::StepStatus::OtherError) .export_values(); + pybind11::enum_(m, "DerivedVarType") + .value("MetadataOnly", adios2::DerivedVarType::MetadataOnly) + .value("ExpressionString", adios2::DerivedVarType::ExpressionString) + .value("StoreData", adios2::DerivedVarType::StoreData) + .export_values(); + pybind11::class_(m, "ADIOS") // Python 2 .def("__nonzero__", @@ -218,6 +225,14 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) adios2::py11::IO::DefineVariable, pybind11::return_value_policy::move, pybind11::arg("name")) + .def("DefineDerivedVariable", + (adios2::py11::VariableDerived(adios2::py11::IO::*)( + const std::string &, const std::string &, const adios2::DerivedVarType)) & + adios2::py11::IO::DefineDerivedVariable, + pybind11::return_value_policy::move, pybind11::arg("name"), + pybind11::arg("expression"), + pybind11::arg("vartype") = adios2::DerivedVarType::MetadataOnly) + .def("InquireVariable", &adios2::py11::IO::InquireVariable, pybind11::return_value_policy::move) @@ -379,6 +394,22 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) .def("Operations", &adios2::py11::Variable::Operations) .def("RemoveOperations", &adios2::py11::Variable::RemoveOperations); + pybind11::class_(m, "VariableDerived") + // Python 2 + .def("__nonzero__", + [](const adios2::py11::VariableDerived &vd) { + const bool opBool = vd ? true : false; + return opBool; + }) + // Python 3 + .def("__bool__", + [](const adios2::py11::VariableDerived &vd) { + const bool opBool = vd ? true : false; + return opBool; + }) + .def("Name", &adios2::py11::VariableDerived::Name) + .def("Type", &adios2::py11::VariableDerived::Type); + pybind11::class_(m, "Attribute") // Python 2 .def("__nonzero__", diff --git a/python/adios2/derived_variable.py b/python/adios2/derived_variable.py new file mode 100644 index 0000000000..8832b54645 --- /dev/null +++ b/python/adios2/derived_variable.py @@ -0,0 +1,43 @@ +"""License: + Distributed under the OSI-approved Apache License, Version 2.0. See + accompanying file Copyright.txt for details. +""" + + +class DerivedVariable: + """High level representation of the DerivedVariable class in the adios2.bindings""" + + def __init__(self, implementation): + self.impl = implementation + + @property + def impl(self): + """Bindings implementation of the class""" + return self._impl + + @impl.setter + def impl(self, implementation): + self._impl = implementation + + def __eq__(self, other): + if isinstance(other, DerivedVariable): + return self.name() == other.name() + return False + + def type(self): + """ + Type of the DerivedVariable + + Returns: + str: Type of the DerivedVariable. + """ + return self.impl.Type() + + def name(self): + """ + Name of the DerivedVariable + + Returns: + str: Name of the DerivedVariable. + """ + return self.impl.Name() diff --git a/python/adios2/io.py b/python/adios2/io.py index ae0318d0cf..d0f4d42510 100644 --- a/python/adios2/io.py +++ b/python/adios2/io.py @@ -6,6 +6,7 @@ import numpy as np from adios2.attribute import Attribute from adios2.variable import Variable +from adios2.derived_variable import DerivedVariable from adios2.engine import Engine @@ -242,6 +243,31 @@ def remove_all_variables(self): """ self.impl.RemoveAllVariables() + def define_derived_variable(self, name, expression, etype=None): + """ + Define a derived variable with an expression + + Parameters + name + name as it appears in variable list in the output + + expression + expression string using other variable names, operators and functions + + type + DerivedVarType.MetadataOnly : store only the metadata of the derived variable + DerivedVarType.ExpressionString : store only the definition, nothing else + DerivedVarType.StoreData : store as a complete variable (data and metadata) + """ + var_impl = None + + if etype is None: + var_impl = self.impl.DefineDerivedVariable(name, expression) + else: + var_impl = self.impl.DefineDerivedVariable(name, expression, etype) + + return DerivedVariable(var_impl) + def open(self, name, mode, comm=None): """ Open an engine diff --git a/python/adios2/variable.py b/python/adios2/variable.py index 865a550536..37a3836415 100644 --- a/python/adios2/variable.py +++ b/python/adios2/variable.py @@ -5,7 +5,7 @@ class Variable: - """High level representation of the Attribute class in the adios2.bindings""" + """High level representation of the Variable class in the adios2.bindings""" def __init__(self, implementation): self.impl = implementation diff --git a/source/adios2/common/ADIOSTypes.h b/source/adios2/common/ADIOSTypes.h index 0907831959..2df11670a8 100644 --- a/source/adios2/common/ADIOSTypes.h +++ b/source/adios2/common/ADIOSTypes.h @@ -33,7 +33,6 @@ namespace adios2 { -#ifdef ADIOS2_HAVE_DERIVED_VARIABLE /** Type of derived variables */ enum class DerivedVarType { @@ -41,7 +40,6 @@ enum class DerivedVarType ExpressionString, ///< Store only the expression string StoreData ///< Store data and metadata }; -#endif /** Memory space for the user provided buffers */ enum class MemorySpace diff --git a/testing/adios2/python/CMakeLists.txt b/testing/adios2/python/CMakeLists.txt index 84ebb9b49d..a52df04fba 100644 --- a/testing/adios2/python/CMakeLists.txt +++ b/testing/adios2/python/CMakeLists.txt @@ -21,6 +21,10 @@ python_add_test(NAME Api.Python.Attribute SCRIPT TestAttribute.py) python_add_test(NAME Api.Python.Stream SCRIPT TestStream.py) python_add_test(NAME Api.Python.FileReader SCRIPT TestFileReader.py) +if (ADIOS2_HAVE_Derived_Variable) + python_add_test(NAME Api.Python.DerivedVariable SCRIPT TestDerivedVariable.py) +endif() + if(ADIOS2_HAVE_MPI) add_python_mpi_test(BPWriteReadString) add_python_mpi_test(BPWriteTypesHighLevelAPI) diff --git a/testing/adios2/python/TestDerivedVariable.py b/testing/adios2/python/TestDerivedVariable.py new file mode 100644 index 0000000000..133213ab4a --- /dev/null +++ b/testing/adios2/python/TestDerivedVariable.py @@ -0,0 +1,79 @@ +from adios2 import Stream +from adios2.bindings import DerivedVarType +import unittest +import numpy as np + + +class TestDerivedVariable(unittest.TestCase): + FILENAME = "pythontestderivedvariable.bp" + EXPR = "t = temps\nt+t" + TEMP = np.array([35, 40, 30, 45], dtype=np.int64) + + def test_01_create_write(self): + with Stream(self.FILENAME, "w") as f: + temps = f.io.define_variable("temps", self.TEMP, self.TEMP.shape, [0], self.TEMP.shape) + dmo = f.io.define_derived_variable("derived/metadataonly", self.EXPR) + ds = f.io.define_derived_variable( + "derived/storedata", self.EXPR, DerivedVarType.StoreData + ) + de = f.io.define_derived_variable( + "derived/expressionstring", self.EXPR, DerivedVarType.ExpressionString + ) + f.write("temps", self.TEMP) + self.assertEqual(temps.name(), "temps") + self.assertEqual(temps.block_id(), 0) + self.assertEqual(temps.count(), [4]) + self.assertEqual(temps.shape(), [4]) + self.assertEqual(temps.sizeof(), 8) + self.assertEqual(dmo.name(), "derived/metadataonly") + self.assertEqual(dmo.type(), DerivedVarType.MetadataOnly) + self.assertEqual(ds.name(), "derived/storedata") + self.assertEqual(ds.type(), DerivedVarType.StoreData) + self.assertEqual(de.name(), "derived/expressionstring") + self.assertEqual(de.type(), DerivedVarType.ExpressionString) + + def test_02_create_reader(self): + with Stream(self.FILENAME, "r") as f: + for _ in f.steps(): + temps = f.inquire_variable("temps") + temps_ds = f.inquire_variable("derived/storedata") + temps_dm = f.inquire_variable("derived/metadataonly") + + self.assertEqual(temps.name(), "temps") + self.assertEqual(temps.block_id(), 0) + self.assertEqual(temps.count(), [4]) + self.assertEqual(temps.sizeof(), 8) + self.assertEqual(temps.steps(), 1) + self.assertEqual(temps.steps_start(), 0) + + self.assertEqual(temps_ds.name(), "derived/storedata") + self.assertEqual(temps_ds.block_id(), 0) + self.assertEqual(temps_ds.count(), [4]) + self.assertEqual(temps_ds.sizeof(), 8) + self.assertEqual(temps_ds.steps(), 1) + self.assertEqual(temps_ds.steps_start(), 0) + + self.assertEqual(temps_dm.name(), "derived/metadataonly") + self.assertEqual(temps_dm.block_id(), 0) + self.assertEqual(temps_dm.count(), [4]) + self.assertEqual(temps_dm.sizeof(), 8) + self.assertEqual(temps_dm.steps(), 1) + self.assertEqual(temps_dm.steps_start(), 0) + + t = f.read("temps", start=[0], count=temps.count()) + if not (t == self.TEMP).all(): + raise ValueError( + "ERROR: Reading 'temps' failed. " + f"Data does not match original. data = {t}" + ) + + ts = f.read("derived/storedata", start=[0], count=temps_ds.count()) + if not (ts == 2 * self.TEMP).all(): + raise ValueError( + "ERROR: Reading 'derived/storedata' failed. " + f"Data does not match expected values. data = {ts}" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/adios2/python/TestVariable.py b/testing/adios2/python/TestVariable.py index 9a8dccd7eb..28098ad432 100644 --- a/testing/adios2/python/TestVariable.py +++ b/testing/adios2/python/TestVariable.py @@ -1,63 +1,51 @@ -from adios2.adios import Adios - -import adios2.bindings as bindings - +from adios2 import Stream, Adios +from adios2.bindings import DerivedVarType import unittest import numpy as np class TestVariable(unittest.TestCase): - def test_create_write(self): - adios = Adios() - with adios.declare_io("BPWriter") as io: - temps = io.define_variable("temps", np.empty([4], dtype=np.int64)) - with io.open("pythontestvariable.bp", bindings.Mode.Write) as engine: - temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) - engine.put(temps, temps_measures) - self.assertEqual(temps.name(), "temps") - self.assertEqual(temps.block_id(), 0) - self.assertEqual(temps.count(), []) - self.assertEqual(temps.shape(), []) - self.assertEqual(temps.sizeof(), 8) - - def test_create_reader(self): - adios = Adios() - with adios.declare_io("BPWriter") as io: - temps = io.define_variable("temps", np.empty([4], dtype=np.int64)) - with io.open("pythontestvariable.bp", bindings.Mode.Write) as engine: - temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) - engine.put(temps, temps_measures) - - with adios.declare_io("BPReader") as reader: - with reader.open("pythontestvariable.bp", bindings.Mode.Read) as engine: - engine.begin_step() - temps = reader.inquire_variable("temps") - engine.end_step() + FILENAME = "pythontestvariable.bp" + TEMP = np.array([35, 40, 30, 45], dtype=np.int64) + + def test_01_create_write(self): + with Stream(self.FILENAME, "w") as f: + temps = f.io.define_variable("temps", self.TEMP, self.TEMP.shape, [0], self.TEMP.shape) + f.write(temps, self.TEMP) + self.assertEqual(temps.name(), "temps") + self.assertEqual(temps.block_id(), 0) + self.assertEqual(temps.count(), [4]) + self.assertEqual(temps.shape(), [4]) + self.assertEqual(temps.sizeof(), 8) + + def test_02_create_reader(self): + with Stream(self.FILENAME, "r") as f: + for _ in f.steps(): + temps = f.inquire_variable("temps") self.assertEqual(temps.name(), "temps") self.assertEqual(temps.block_id(), 0) - self.assertEqual(temps.count(), []) + self.assertEqual(temps.count(), [4]) self.assertEqual(temps.sizeof(), 8) self.assertEqual(temps.steps(), 1) self.assertEqual(temps.steps_start(), 0) - def test_operators(self): + def test_03_operators(self): adios = Adios() op1 = adios.define_operator("noop", "null") - with adios.declare_io("BPWriter") as io: - temps = io.define_variable("temps", np.empty([4], dtype=np.int64)) + iow = adios.declare_io("BPWriter") + + with Stream(iow, "pythontestvariableoperators.bp", "w") as f: + temps = f.io.define_variable("temps", self.TEMP, self.TEMP.shape, [0], self.TEMP.shape) temps.add_operation(op1) - with io.open("pythontestvariable.bp", bindings.Mode.Write) as engine: - temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) - engine.put(temps, temps_measures) + f.write(temps, self.TEMP) op2 = adios.define_operator("noop2", "null") - with adios.declare_io("BPReader") as reader: - with reader.open("pythontestvariable.bp", bindings.Mode.Read) as engine: - engine.begin_step() - temps = reader.inquire_variable("temps") + ior = adios.declare_io("BPReader") + with Stream(ior, "pythontestvariableoperators.bp", "r") as f: + for _ in f.steps(): + temps = f.inquire_variable("temps") temps.add_operation(op2) - engine.end_step() if __name__ == "__main__":