Add io.define_derived_variable(name, expression, type) function to Py…

…thon API. (#4275)

bindings/Python/CMakeLists.txt

@@ -3,6 +3,7 @@ Python_add_library(adios2_py MODULE
   py11ADIOS.cpp
   py11IO.cpp
   py11Variable.cpp
+  py11VariableDerived.cpp
   py11Attribute.cpp
   py11Engine.cpp
   py11Operator.cpp

bindings/Python/py11IO.cpp

@@ -146,6 +146,23 @@ Variable IO::DefineVariable(const std::string &name, const pybind11::object &val
     return Variable(variable);
 }
 
+VariableDerived IO::DefineDerivedVariable(const std::string &name, const std::string &exp_string,
+                                          const DerivedVarType varType)
+{
+    helper::CheckForNullptr(m_IO,
+                            "for variable " + name + ", in call to IO::DefineDerivedVariable");
+
+#ifdef ADIOS2_HAVE_DERIVED_VARIABLE
+    adios2::core::VariableDerived *dv = &m_IO->DefineDerivedVariable(name, exp_string, varType);
+    adios2::py11::VariableDerived vd(dv);
+#else
+    adios2::py11::VariableDerived vd;
+    throw std::invalid_argument("ERROR: Derived Variables are not supported in this adios2 build "
+                                ", in call to DefineDerivedVariable\n");
+#endif
+    return vd;
+}
+
 Variable IO::InquireVariable(const std::string &name)
 {
     helper::CheckForNullptr(m_IO, "for variable " + name + ", in call to IO::InquireVariable");

bindings/Python/py11IO.h

@@ -19,6 +19,7 @@
 #include "py11Attribute.h"
 #include "py11Engine.h"
 #include "py11Variable.h"
+#include "py11VariableDerived.h"
 #include "py11types.h"
 
 namespace adios2
@@ -62,6 +63,10 @@ class IO
                             const Dims &shape, const Dims &start, const Dims &count,
                             const bool isConstantDims);
 
+    VariableDerived
+    DefineDerivedVariable(const std::string &name, const std::string &expression,
+                          const DerivedVarType varType = DerivedVarType::MetadataOnly);
+
     Variable InquireVariable(const std::string &name);
 
     Attribute DefineAttribute(const std::string &name, const pybind11::array &array,

bindings/Python/py11VariableDerived.cpp

@@ -0,0 +1,51 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * py11Variable.cpp
+ */
+
+#include "py11VariableDerived.h"
+
+#include "adios2/helper/adiosFunctions.h"
+
+namespace adios2
+{
+namespace py11
+{
+
+#ifdef ADIOS2_HAVE_DERIVED_VARIABLE
+VariableDerived::VariableDerived(core::VariableDerived *v) : m_VariableDerived(v) {}
+
+VariableDerived::operator bool() const noexcept
+{
+    return (m_VariableDerived == nullptr) ? false : true;
+}
+
+std::string VariableDerived::Name() const
+{
+    helper::CheckForNullptr(m_VariableDerived, "in call to VariableDerived::Name");
+    return m_VariableDerived->m_Name;
+}
+
+DerivedVarType VariableDerived::Type() const
+{
+    helper::CheckForNullptr(m_VariableDerived, "in call to VariableDerived::Type");
+    return m_VariableDerived->GetDerivedType();
+}
+
+#else
+
+VariableDerived::operator bool() const noexcept { return false; }
+
+std::string VariableDerived::Name() const
+{
+    return "DerivedVariables are not supported in this ADIOS2 build";
+}
+
+DerivedVarType VariableDerived::Type() const { return DerivedVarType::ExpressionString; }
+
+#endif
+
+} // end namespace py11
+} // end namespace adios2

bindings/Python/py11VariableDerived.h

@@ -0,0 +1,54 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * py11VariableDerived.h
+ *
+ */
+
+#ifndef ADIOS2_BINDINGS_PYTHON_VARIABLEDERIVED_H_
+#define ADIOS2_BINDINGS_PYTHON_VARIABLEDERIVED_H_
+
+#include "adios2/common/ADIOSConfig.h"
+
+#ifdef ADIOS2_HAVE_DERIVED_VARIABLE
+#include "adios2/core/VariableDerived.h"
+#else
+#include "adios2/common/ADIOSTypes.h"
+#endif
+
+namespace adios2
+{
+namespace py11
+{
+
+class IO;
+class Engine;
+
+class VariableDerived
+{
+    friend class IO;
+    friend class Engine;
+
+public:
+    VariableDerived() = default;
+
+    ~VariableDerived() = default;
+
+    explicit operator bool() const noexcept;
+
+    std::string Name() const;
+
+    DerivedVarType Type() const;
+
+private:
+#ifdef ADIOS2_HAVE_DERIVED_VARIABLE
+    VariableDerived(adios2::core::VariableDerived *v);
+    adios2::core::VariableDerived *m_VariableDerived = nullptr;
+#endif
+};
+
+} // end namespace py11
+} // end namespace adios2
+
+#endif /* ADIOS2_BINDINGS_PYTHON_VARIABLEDERIVED_H_ */

bindings/Python/py11glue.cpp

@@ -28,6 +28,7 @@
 #include "py11Operator.h"
 #include "py11Query.h"
 #include "py11Variable.h"
+#include "py11VariableDerived.h"
 
 #if ADIOS2_USE_MPI
 
@@ -124,6 +125,12 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m)
         .value("OtherError", adios2::StepStatus::OtherError)
         .export_values();
 
+    pybind11::enum_<adios2::DerivedVarType>(m, "DerivedVarType")
+        .value("MetadataOnly", adios2::DerivedVarType::MetadataOnly)
+        .value("ExpressionString", adios2::DerivedVarType::ExpressionString)
+        .value("StoreData", adios2::DerivedVarType::StoreData)
+        .export_values();
+
     pybind11::class_<adios2::py11::ADIOS>(m, "ADIOS")
         // Python 2
         .def("__nonzero__",
@@ -218,6 +225,14 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m)
                  adios2::py11::IO::DefineVariable,
              pybind11::return_value_policy::move, pybind11::arg("name"))
 
+        .def("DefineDerivedVariable",
+             (adios2::py11::VariableDerived(adios2::py11::IO::*)(
+                 const std::string &, const std::string &, const adios2::DerivedVarType)) &
+                 adios2::py11::IO::DefineDerivedVariable,
+             pybind11::return_value_policy::move, pybind11::arg("name"),
+             pybind11::arg("expression"),
+             pybind11::arg("vartype") = adios2::DerivedVarType::MetadataOnly)
+
         .def("InquireVariable", &adios2::py11::IO::InquireVariable,
              pybind11::return_value_policy::move)
 
@@ -379,6 +394,22 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m)
         .def("Operations", &adios2::py11::Variable::Operations)
         .def("RemoveOperations", &adios2::py11::Variable::RemoveOperations);
 
+    pybind11::class_<adios2::py11::VariableDerived>(m, "VariableDerived")
+        // Python 2
+        .def("__nonzero__",
+             [](const adios2::py11::VariableDerived &vd) {
+                 const bool opBool = vd ? true : false;
+                 return opBool;
+             })
+        // Python 3
+        .def("__bool__",
+             [](const adios2::py11::VariableDerived &vd) {
+                 const bool opBool = vd ? true : false;
+                 return opBool;
+             })
+        .def("Name", &adios2::py11::VariableDerived::Name)
+        .def("Type", &adios2::py11::VariableDerived::Type);
+
     pybind11::class_<adios2::py11::Attribute>(m, "Attribute")
         // Python 2
         .def("__nonzero__",

python/adios2/derived_variable.py

@@ -0,0 +1,43 @@
+"""License:
+  Distributed under the OSI-approved Apache License, Version 2.0.  See
+  accompanying file Copyright.txt for details.
+"""
+
+
+class DerivedVariable:
+    """High level representation of the DerivedVariable class in the adios2.bindings"""
+
+    def __init__(self, implementation):
+        self.impl = implementation
+
+    @property
+    def impl(self):
+        """Bindings implementation of the class"""
+        return self._impl
+
+    @impl.setter
+    def impl(self, implementation):
+        self._impl = implementation
+
+    def __eq__(self, other):
+        if isinstance(other, DerivedVariable):
+            return self.name() == other.name()
+        return False
+
+    def type(self):
+        """
+        Type of the DerivedVariable
+
+        Returns:
+            str: Type of the DerivedVariable.
+        """
+        return self.impl.Type()
+
+    def name(self):
+        """
+        Name of the DerivedVariable
+
+        Returns:
+            str: Name of the DerivedVariable.
+        """
+        return self.impl.Name()

python/adios2/io.py

@@ -6,6 +6,7 @@
 import numpy as np
 from adios2.attribute import Attribute
 from adios2.variable import Variable
+from adios2.derived_variable import DerivedVariable
 from adios2.engine import Engine
 
 
@@ -242,6 +243,31 @@ def remove_all_variables(self):
         """
         self.impl.RemoveAllVariables()
 
+    def define_derived_variable(self, name, expression, etype=None):
+        """
+        Define a derived variable with an expression
+
+        Parameters
+            name
+                name as it appears in variable list in the output
+
+            expression
+                expression string using other variable names, operators and functions
+
+            type
+                DerivedVarType.MetadataOnly     : store only the metadata of the derived variable
+                DerivedVarType.ExpressionString : store only the definition, nothing else
+                DerivedVarType.StoreData        : store as a complete variable (data and metadata)
+        """
+        var_impl = None
+
+        if etype is None:
+            var_impl = self.impl.DefineDerivedVariable(name, expression)
+        else:
+            var_impl = self.impl.DefineDerivedVariable(name, expression, etype)
+
+        return DerivedVariable(var_impl)
+
     def open(self, name, mode, comm=None):
         """
         Open an engine

python/adios2/variable.py

@@ -5,7 +5,7 @@
 
 
 class Variable:
-    """High level representation of the Attribute class in the adios2.bindings"""
+    """High level representation of the Variable class in the adios2.bindings"""
 
     def __init__(self, implementation):
         self.impl = implementation

source/adios2/common/ADIOSTypes.h

@@ -33,15 +33,13 @@
 namespace adios2
 {
 
-#ifdef ADIOS2_HAVE_DERIVED_VARIABLE
 /** Type of derived variables */
 enum class DerivedVarType
 {
     MetadataOnly,     ///< Store only the metadata (default)
     ExpressionString, ///< Store only the expression string
     StoreData         ///< Store data and metadata
 };
-#endif
 
 /** Memory space for the user provided buffers */
 enum class MemorySpace

testing/adios2/python/CMakeLists.txt

@@ -21,6 +21,10 @@ python_add_test(NAME Api.Python.Attribute SCRIPT TestAttribute.py)
 python_add_test(NAME Api.Python.Stream SCRIPT TestStream.py)
 python_add_test(NAME Api.Python.FileReader SCRIPT TestFileReader.py)
 
+if (ADIOS2_HAVE_Derived_Variable)
+  python_add_test(NAME Api.Python.DerivedVariable SCRIPT TestDerivedVariable.py)
+endif()
+
 if(ADIOS2_HAVE_MPI)
   add_python_mpi_test(BPWriteReadString)
   add_python_mpi_test(BPWriteTypesHighLevelAPI)