Merge remote-tracking branch 'upstream/master' into adios-xrootd

* upstream/master:
  Fix links to tutorial materials (ornladios#4086)
  BlockIndex.Evaluate() made branch based on BP5 vs BP4. To support CampaignReader engine, decision is made based on whether MinBlocksInfo is supported by engine.
  Update documentation for 2.10 changes to the GPU-backend (ornladios#4083)
  Add test for single string attribute vs string array attribute with a single element
  Raise an exception if remote open fails (ornladios#4069)
  - Python: fix for scalar reading. If a global value has 1 step (i.e. always in streaming), read returns a 0-dim numpy array (single value). If the variable has multiple steps (only in ReadRandomAccess mode), read returns a 1-dim numpy array even if the step selection is a single step. This way, read of a certain variable always results in the same type of array no matter the number of steps selected. - Python: fix for string attributes: return a string, not a list of one element which is a string, to be consistent with string global values and with other APIs.
  format more
  format
  Python: add the same treatment to attributes as to variables before: return scalars (0-dim ndarray) for single value attributes.

# Conflicts:
#	source/adios2/engine/bp5/BP5Reader.cpp
#	source/adios2/engine/bp5/BP5Reader.h

bindings/Python/py11Attribute.cpp

@@ -36,6 +36,12 @@ std::string Attribute::Type() const
     return ToString(m_Attribute->m_Type);
 }
 
+bool Attribute::SingleValue() const
+{
+    helper::CheckForNullptr(m_Attribute, "in call to Attribute::SingleValue");
+    return m_Attribute->m_IsSingleValue;
+}
+
 std::vector<std::string> Attribute::DataString()
 {
     helper::CheckForNullptr(m_Attribute, "in call to Attribute::DataStrings");

bindings/Python/py11Attribute.h

@@ -37,6 +37,8 @@ class Attribute
 
     std::string Type() const;
 
+    bool SingleValue() const;
+
     pybind11::array Data();
 
     std::vector<std::string> DataString();

bindings/Python/py11IO.cpp

@@ -180,8 +180,15 @@ Attribute IO::DefineAttribute(const std::string &name, const pybind11::array &ar
     else if (pybind11::isinstance<pybind11::array_t<T, pybind11::array::c_style>>(array))          \
     {                                                                                              \
         const T *data = reinterpret_cast<const T *>(array.data());                                 \
-        const size_t size = static_cast<size_t>(array.size());                                     \
-        attribute = &m_IO->DefineAttribute<T>(name, data, size, variableName, separator);          \
+        if (array.ndim())                                                                          \
+        {                                                                                          \
+            const size_t size = static_cast<size_t>(array.size());                                 \
+            attribute = &m_IO->DefineAttribute<T>(name, data, size, variableName, separator);      \
+        }                                                                                          \
+        else                                                                                       \
+        {                                                                                          \
+            attribute = &m_IO->DefineAttribute<T>(name, data[0], variableName, separator);         \
+        }                                                                                          \
     }
     ADIOS2_FOREACH_NUMPY_ATTRIBUTE_TYPE_1ARG(declare_type)
 #undef declare_type

bindings/Python/py11glue.cpp

@@ -395,7 +395,8 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m)
         .def("Name", &adios2::py11::Attribute::Name)
         .def("Type", &adios2::py11::Attribute::Type)
         .def("DataString", &adios2::py11::Attribute::DataString)
-        .def("Data", &adios2::py11::Attribute::Data);
+        .def("Data", &adios2::py11::Attribute::Data)
+        .def("SingleValue", &adios2::py11::Attribute::SingleValue);
 
     pybind11::class_<adios2::py11::Engine>(m, "Engine")
         // Python 2

docs/user_guide/source/advanced/gpu_aware.rst

@@ -2,12 +2,13 @@
  GPU-aware I/O
 #################
 
-The ``Put`` and ``Get`` functions in the BP4 and BP5 engines can receive user buffers allocated on the host or the device in both Sync and Deferred modes.
+The ``Put`` and ``Get`` functions in the default file engine (BP5) and some streaming engines (SST, DataMan) can receive user buffers allocated on the host or the device in both Sync and Deferred modes.
 
 .. note::
-    CUDA, HIP and SYCL allocated buffers are supported for device data.
+    Buffers allocated on the device with CUDA, HIP and SYCL are supported.
 
-If ADIOS2 is built without GPU support, only buffers allocated on the host are supported. If ADIOS2 is built with any GPU support, by default, the library will automatically detect where does the buffer memory physically resides.
+If ADIOS2 is built without GPU support, only buffers allocated on the host are supported.
+When GPU support is enabled, the default behavior is for ADIOS2 to automatically detect where the buffer memory physically resides.
 
 Users can also provide information about where the buffer was allocated by using the ``SetMemorySpace`` function within each variable.
 
@@ -20,17 +21,20 @@ Users can also provide information about where the buffer was allocated by using
         GPU     ///< GPU memory spaces
     };
 
+If ADIOS2 is built without GPU support, the available MemorySpace values are only ``Detect`` and ``Host``.
+
 ADIOS2 can use a CUDA or Kokkos backend for enabling GPU support. Only one backend can be active at a given time based on how ADIOS2 is build.
 
 **********************************
 Building ADIOS2 with a GPU backend
 **********************************
 
+By default both backends are ``OFF`` even if CUDA or Kokkos are installed and available to avoid a possible conflict between if both backends are enabled at the same time.
 
 Building with CUDA enabled
 --------------------------
 
-If there is no CUDA toolkit installed, cmake will turn CUDA off automatically. ADIOS2 default behavior for ``ADIOS2_USE_CUDA`` is to enable CUDA if it can find a CUDA toolkit on the system. In case the system has a CUDA toolkit installed, but it is desired to build ADIOS2 without CUDA enabled ``-DADIOS2_USE_CUDA=OFF`` must be used.
+The ADIOS2 default behavior is to turn ``OFF`` the CUDA backend. Building with the CUDA backend requires ``-DADIOS2_USE_Kokkos=ON`` and an available CUDA toolkit on the system.
 
 When building ADIOS2 with CUDA enabled, the user is responsible with setting the correct ``CMAKE_CUDA_ARCHITECTURES`` (e.g. for Summit the ``CMAKE_CUDA_ARCHITECTURES`` needs to be set to 70 to match the NVIDIA Volta V100).
 
@@ -43,11 +47,23 @@ The Kokkos library can be used to enable GPU within ADIOS2. Based on how Kokkos
     Kokkos version >= 3.7 is required to enable the GPU backend in ADIOS2
 
 
-****************
-Writing GPU code
-****************
+*******************
+Writing GPU buffers
+*******************
+
+The ADIOS2 API for Device pointers is identical to using Host buffers for both the read and write logic.
+Internally each ADIOS2 variable holds a memory space for the data it receives. Once the memory space is set (eithr directly by the user through calls to ``SetMemorySpace`` or after detecting the buffer memory space the first ``Put`` or ``Get`` call) to either Host or Device, it cannot be changed.
+
+The ``examples/hello`` folder contains several codes that use Device buffers:
+ - `bpStepsWriteRead{Cuda|Hip}` show CUDA and HIP codes using BP5 with GPU pointers
+ - `bpStepsWriteReadKokkos contains` Fortran and C++ codes using ``Kokkos::View`` with different memory spaces and a Kokkos code using different layouts on Host buffers
+ - `datamanKokkos` shows an example of streaming a ``Kokkos::View`` with DataMan using different memory spaces
+ - `sstKokkos` shows an example of streaming a ``Kokkos::View`` with SST using different memory spaces
+
+Example using a Device buffer
+-----------------------------
 
-The following is a simple example of writing data to storage directly from a GPU buffer allocated with CUDA relying on the automatic detection of device pointers in ADIOS2. The ADIOS2 API is identical to codes using Host buffers for both the read and write logic.
+The following is a simple example of writing data to storage directly from a GPU buffer allocated with CUDA relying on the automatic detection of device pointers in ADIOS2.
 
 .. code-block:: c++
 
@@ -56,7 +72,7 @@ The following is a simple example of writing data to storage directly from a GPU
     cudaMemset(gpuSimData, 0, N);
     auto data = io.DefineVariable<float>("data", shape, start, count);
 
-    io.SetEngine("BP5"); // or BPFile
+    io.SetEngine("BP5");
     adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write);
     // Simulation steps
     for (size_t step = 0; step < nSteps; ++step)
@@ -82,13 +98,13 @@ If the ``SetMemorySpace`` function is used, the ADIOS2 library will not detect a
 Underneath, ADIOS2 relies on the backend used at build time to transfer the data. If ADIOS2 was build with CUDA, only CUDA buffers can be provided. If ADIOS2 was build with Kokkos (with CUDA enabled) only CUDA buffers can be provided. If ADIOS2 was build with Kokkos (with HIP enabled) only HIP buffers can be provided.
 
 .. note::
-    The SYCL backend in Kokkos can be used to run on Nvida, AMD and Intel GPUs
+    The SYCL backend in Kokkos can be used to run on Nvida, AMD and Intel GPUs, but we recommand using SYCL for Intel, HIP for AMD and CUDA for Nvidia.
 
 
 Kokkos applications
 --------------------
 
-ADIOS2 supports GPU buffers provided in the form of ``Kokkos::View`` directly in the Put/Get calls. The memory space can be automatically detected or provided by the user, in the same way as in the CUDA example.
+ADIOS2 supports GPU buffers provided in the form of ``Kokkos::View`` directly in the Put/Get calls. The memory space is automatically detected from the View information. In addition to the memory space, for ``Kokkos::View`` ADIOS2 also extracts the layout of the array and adjust the variable dimensions to be able to build the global shape (across ranks) of the array.
 
 .. code-block:: c++
 
@@ -97,6 +113,34 @@ ADIOS2 supports GPU buffers provided in the form of ``Kokkos::View`` directly in
 
 If the CUDA backend is being used (and not Kokkos) to enable GPU support in ADIOS2, Kokkos applications can still directly pass ``Kokkos::View`` as long as the correct external header is included: ``#include <adios2/cxx11/KokkosView.h>``.
 
+*******************
+Reading GPU buffers
+*******************
+
+The GPU-aware backend allows different layouts for global arrays without requiring the user to update the code for each case. The user defines the shape of the global array and ADIOS2 adjusts the dimensions for each rank according to the buffer layout and memory space.
+
+The following example shows a global array of shape (4, 3) when running with 2 ranks, each contributing half of it.
+
+.. code-block:: text
+
+   Write on LayoutRight, read on LayoutRight
+   1 1 1  // rank 0
+   2 2 2
+   3 3 3  // rank 1
+   4 4 4
+   Write on LayoutRight, read on LayoutLeft
+   1 2 3 4
+   1 2 3 4
+   1 2 3 4
+
+On the read side, the Shape function can take a memory space or a layout to return the correct dimensions of the variable.
+For the previous example, if a C++ code using two ranks wants to read the data into a GPU buffer, the Shape of the local array should be (3, 2). If the same data will be read on CPU buffers, the shape should be (2, 3). Both of the following code would give acceptable answers:
+
+.. code-block:: c++
+
+   auto dims_host = data.Shape(adios2::MemorySpace::Host);
+   auto dims_device = data.Shape(adios2::ArrayOrdering::ColumnMajor);
+
 ***************
 Build scripts
 ***************

docs/user_guide/source/api_python/adios2-doc-read-filereader.py

@@ -9,14 +9,18 @@
         for key, value in info.items():
             print("\t" + key + ": " + value, end=" ")
         print()
+    print()
 
     nproc = s.read("nproc")
-    print(f"nproc is {nproc} of type {type(nproc)}")
+    print(f"nproc is {nproc} of type {type(nproc)} with ndim {nproc.ndim}")
 
     # read variables return a numpy array with corresponding selection
     steps = int(vars["physical_time"]["AvailableStepsCount"])
     physical_time = s.read("physical_time", step_selection=[0, steps])
-    print(f"physical_time is {physical_time} of type {type(physical_time)}")
+    print(
+        f"physical_time is {physical_time} of type {type(physical_time)} with "
+        f"ndim {physical_time.ndim} shape = {physical_time.shape}"
+    )
 
     steps = int(vars["temperature"]["AvailableStepsCount"])
     temperature = s.read("temperature", step_selection=[0, steps])

docs/user_guide/source/api_python/adios2-doc-read.py → docs/user_guide/source/api_python/adios2-doc-read-stream.py

@@ -16,7 +16,7 @@
 
         if s.current_step() == 0:
             nproc = s.read("nproc")
-            print(f"nproc is {nproc} of type {type(nproc)}")
+            print(f"nproc is {nproc} of type {type(nproc)} with ndim {nproc.ndim}")
 
         # read variables return a numpy array with corresponding selection
         physical_time = s.read("physical_time")
@@ -26,6 +26,6 @@
         print(f"temperature array size is {temperature.size} of shape {temperature.shape}")
         print(f"temperature unit is {temp_unit} of type {type(temp_unit)}")
         pressure = s.read("pressure")
-        press_unit = s.read_attribute("pressure/unit")
+        press_unit = s.read_attribute("unit", "pressure")
         print(f"pressure unit is {press_unit} of type {type(press_unit)}")
         print()

docs/user_guide/source/api_python/python_example.rst

@@ -44,6 +44,7 @@ Python Write example
    count = [nx]
    
    temperature = np.zeros(nx, dtype=np.double)
+   pressure = np.ones(nx, dtype=np.double)
    delta_time = 0.01
    physical_time = 0.0
    nsteps = 5
@@ -125,17 +126,17 @@ Python Read "step-by-step" example
     nproc is 4 of type <class 'numpy.ndarray'>
     physical_time is 0.0 of type <class 'numpy.ndarray'>
     temperature array size is 40 of shape (40,)
-    temperature unit is ['K'] of type <class 'list'>
-    pressure unit is ['Pa'] of type <class 'list'>
+    temperature unit is K of type <class 'str'>
+    pressure unit is Pa of type <class 'str'>
 
     Current step is 1
     variable_name: physical_time    AvailableStepsCount: 1  Max: 0.01       Min: 0.01       Shape:          SingleValue: true   Type: double
     variable_name: pressure         AvailableStepsCount: 1  Max: 1  Min: 1  Shape: 40       SingleValue: false      Type: double
     variable_name: temperature      AvailableStepsCount: 1  Max: 0  Min: 0  Shape: 40       SingleValue: false      Type: double
     physical_time is 0.01 of type <class 'numpy.ndarray'>
     temperature array size is 40 of shape (40,)
-    temperature unit is ['K'] of type <class 'list'>
-    pressure unit is ['Pa'] of type <class 'list'>
+    temperature unit is K of type <class 'str'>
+    pressure unit is Pa of type <class 'str'>
 
     ...
 
@@ -156,14 +157,18 @@ Python Read Random Access example
             for key, value in info.items():
                 print("\t" + key + ": " + value, end=" ")
             print()
+        print()
 
         nproc = s.read("nproc")
-        print(f"nproc is {nproc} of type {type(nproc)}")
+        print(f"nproc is {nproc} of type {type(nproc)} with ndim {nproc.ndim}")
         
         # read variables return a numpy array with corresponding selection
         steps = int(vars['physical_time']['AvailableStepsCount'])
         physical_time = s.read("physical_time", step_selection=[0, steps])
-        print(f"physical_time is {physical_time} of type {type(physical_time)}")
+        print(
+            f"physical_time is {physical_time} of type {type(physical_time)} with "
+            f"ndim {physical_time.ndim} shape = {physical_time.shape}"
+        )
 
         steps = int(vars['temperature']['AvailableStepsCount'])
         temperature = s.read("temperature", step_selection=[0, steps])
@@ -183,8 +188,9 @@ Python Read Random Access example
     variable_name: physical_time    AvailableStepsCount: 5  Max: 0.04       Min: 0  Shape:          SingleValue: true       Type: double
     variable_name: pressure         AvailableStepsCount: 5  Max: 1  Min: 1  Shape: 40       SingleValue: false      Type: double
     variable_name: temperature      AvailableStepsCount: 5  Max: 0  Min: 0  Shape: 40       SingleValue: false      Type: double
-    nproc is 4 of type <class 'numpy.ndarray'>
-    physical_time is [0.   0.01 0.02 0.03 0.04] of type <class 'numpy.ndarray'>
+
+    nproc is 4 of type <class 'numpy.ndarray'> with ndim 0
+    physical_time is [0.   0.01 0.02 0.03 0.04] of type <class 'numpy.ndarray'> with ndim 1 shape = (5,)
     temperature array size is 200 of shape (200,)
-    temperature unit is ['K'] of type <class 'list'>
+    temperature unit is K of type <class 'str'>
 

docs/user_guide/source/tutorials/overview.rst

@@ -16,5 +16,5 @@ More specifically, we will go through the following examples:
 
 More advance tutorials that cover more information related to:
 
-- Running ADIOS2 at scale (through files or streaming) with hands-on excercises: `Exascale I/O tutorial <http://tinyurl.com/adios-eied>`
-- Using ADIOS2 with Paraview, TAU, Catalyst, FIDES, VTK-M: `ADIOS2 tutorial at SC23 <http://tinyurl.com/adios-sc2023>`
+- Running ADIOS2 at scale (through files or streaming) with hands-on excercises: `Exascale I/O tutorial <http://tinyurl.com/adios-eied>`_
+- Using ADIOS2 with Paraview, TAU, Catalyst, FIDES, VTK-M: `ADIOS2 tutorial at SC23 <http://tinyurl.com/adios-sc2023>`_

examples/hello/bpReader/bpReaderHeatMap2D.py

@@ -35,27 +35,40 @@
     for j in range(0, Ny):
         value = iGlobal * shape[1] + j
         temperatures[i * Nx + j] = value
-#        print(str(i) + "," + str(j) + " " + str(value))
 
 with Stream("HeatMap2D_py.bp", "w", comm) as obpStream:
     obpStream.write("temperature2D", temperatures, shape, start, count)
     if not rank:
         obpStream.write("N", [size, Nx, Ny])  # will be an array in output
         obpStream.write("Nx", numpy.array(Nx))  # will be a scalar in output
         obpStream.write("Ny", Ny)  # will be a scalar in output
+        obpStream.write_attribute("nproc", size)  # will be a single value attribute in output
         obpStream.write_attribute("dimensions", [size * Nx, Ny], "temperature2D")
 
 if not rank:
     with FileReader("HeatMap2D_py.bp", MPI.COMM_SELF) as ibpFile:
+        # scalar variables are read as a numpy array with 0 dimension
+        in_nx = ibpFile.read("Nx")
+        in_ny = ibpFile.read("Ny")
+        print(f"Incoming nx, ny = {in_nx}, {in_ny}")
+
+        # single value attribute is read as a numpy array with 0 dimension
+        in_nproc = ibpFile.read_attribute("nproc")
+        print(f"Incoming nproc = {in_nproc}")
+        # array attribute is read as a numpy array or string list
+        in_dims = ibpFile.read_attribute("temperature2D/dimensions")
+        print(f"Incoming diumensions = {in_dims}")
+
+        # On option is to inquire a variable to know its type, shape
+        # directly, not as strings, and then we can use the variable
+        # object to set selection and/or set steps to read
         var_inTemperature = ibpFile.inquire_variable("temperature2D")
         if var_inTemperature is not None:
             var_inTemperature.set_selection([[2, 2], [4, 4]])
             inTemperatures = ibpFile.read(var_inTemperature)
-
-        in_nx = ibpFile.read("Nx")  # scalar is read as a numpy array with 1 element
-        in_ny = ibpFile.read("Ny")  # scalar is read as a numpy array with 1 element
-        print(f"Incoming nx, ny = {in_nx}, {in_ny}")
-
-        print("Incoming temperature map")
-        for i in range(0, inTemperatures.shape[1]):
-            print(str(inTemperatures[i]))
+            print(
+                f"Incoming temperature map with selection "
+                f"start = {var_inTemperature.start()}, count = {var_inTemperature.count()}"
+            )
+            for i in range(0, inTemperatures.shape[1]):
+                print(str(inTemperatures[i]))

python/adios2/attribute.py

@@ -42,13 +42,24 @@ def type(self):
         """
         return self.impl.Type()
 
+    def single_value(self):
+        """
+        True if the attribute is a single value, False if it is an array
+
+        Returns:
+            True or False.
+        """
+        return self.impl.SingleValue()
+
     def data(self):
         """
         Content of the Attribute
 
         Returns:
             Content of the Attribute as a non string.
         """
+        if self.single_value():
+            return self.impl.Data()[0]
         return self.impl.Data()
 
     def data_string(self):
@@ -58,4 +69,6 @@ def data_string(self):
         Returns:
             Content of the Attribute as a str.
         """
+        if self.single_value():
+            return self.impl.DataString()[0]
         return self.impl.DataString()

python/adios2/file_reader.py

@@ -25,7 +25,3 @@ def _(self, io: IO, path, comm=None):
         super().__init__(io, path, "rra", comm)
 
     # pylint: enable=E1121
-
-    def variables(self):
-        """Returns the list of variables contained in the opened file"""
-        return [self._io.inquire_variable(var) for var in self.available_variables()]