Merge pull request #4042 from vicentebolea/backport-master

Backport master PR into release210

.github/workflows/everything.yml

@@ -270,9 +270,9 @@ jobs:
 
     runs-on: ${{ matrix.image }}
     env:
-      GH_YML_JOBNAME: ${{ matrix.os }}-${{ matrix.compiler }}-${{ matrix.parallel }}
+      GH_YML_JOBNAME: ${{ matrix.image }}-${{ matrix.compiler }}-${{ matrix.shared }}-${{ matrix.parallel }}
       GH_YML_BASE_OS: macOS
-      GH_YML_MATRIX_OS: ${{ matrix.os }}
+      GH_YML_MATRIX_OS: ${{ matrix.image }}
       GH_YML_MATRIX_COMPILER: ${{ matrix.compiler }}
       GH_YML_MATRIX_PARALLEL: ${{ matrix.parallel }}
       CCACHE_BASEDIR: "${GITHUB_WORKSPACE}"
@@ -283,15 +283,17 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos11, macos12]
-        parallel: [serial]
+        image: [macos-11, macos-12]
+        shared: [static, shared]
         include:
-          - os: macos11
-            image: macos-11
+          - parallel: serial
+          - image: macos-11
             compiler: xcode13_0
-          - os: macos12
-            image: macos-12
+          - image: macos-12
             compiler: xcode13_4_1
+        exclude:
+          - image: macos-11
+            shared: static
 
     steps:
       - uses: actions/checkout@v4
@@ -308,9 +310,9 @@ jobs:
         id: restore-cache
         with:
           path: .ccache
-          key: ccache-${{ matrix.os }}-${{ matrix.compiler }}-${{ matrix.parallel }}-${{ github.sha }}
+          key: ccache-${{ matrix.image}}-${{ matrix.compiler }}-${{ matrix.parallel }}-${{ github.sha }}
           restore-keys: |
-            ccache-${{ matrix.os }}-${{ matrix.compiler }}-${{ matrix.parallel }}
+            ccache-${{ matrix.image }}-${{ matrix.compiler }}-${{ matrix.parallel }}
       - name: Configure cache
         run: ccache -z
       - name: Update
@@ -327,7 +329,7 @@ jobs:
         id: save-cache
         with:
           path: .ccache
-          key: ccache-${{ matrix.os }}-${{ matrix.compiler }}-${{ matrix.parallel }}-${{ github.sha }}
+          key: ccache-${{ matrix.image }}-${{ matrix.compiler }}-${{ matrix.parallel }}-${{ github.sha }}
       - name: Test
         run: gha/scripts/ci/gh-actions/run.sh test
 

cmake/ADIOSFunctions.cmake

@@ -91,7 +91,7 @@ function(python_add_test)
     set_property(TEST ${ARGS_NAME} PROPERTY
       ENVIRONMENT
         "PYTHONPATH=${ADIOS2_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR};$ENV{PYTHONPATH}"
-        "PATH=$<TARGET_FILE_DIR:adios2>;$ENV{PATH}"
+        "PATH=$<TARGET_FILE_DIR:adios2_py>;$ENV{PATH}"
     )
   endif()
 endfunction()

docs/user_guide/source/api_full/python.rst

@@ -37,3 +37,8 @@ Operator class
 --------------
 .. autoclass:: adios2.bindings.adios2_bindings::Operator
     :members:
+
+Query class
+--------------
+.. autoclass:: adios2.bindings.adios2_bindings::Query
+    :members:

examples/hello/sstReader/sstReader-bindings.py

@@ -36,7 +36,15 @@
     sstReader.EndStep()
 
     print(
-        "Rank=", rank, "loop index =", loopStep, "stream step =", currentStep, "data =", floatArray
+        "Rank=",
+        rank,
+        "loop index =",
+        loopStep,
+        "stream step =",
+        currentStep,
+        "data =",
+        floatArray,
+        flush=True,
     )
     loopStep = loopStep + 1
 

examples/hello/sstReader/sstReader.py

@@ -34,4 +34,5 @@
             currentStep,
             "data =",
             floatArray,
+            flush=True,
         )

examples/hello/sstWriter/sstWriter-bindings.py

@@ -33,7 +33,7 @@
 
 sstWriter = sstIO.Open("helloSst", adios2.Mode.Write)
 for i in range(4):
-    print("Rank=", rank, "loop index =", i, "data =", myArray)
+    print("Rank=", rank, "loop index =", i, "data =", myArray, flush=True)
     sstWriter.BeginStep()
     sstWriter.Put(ioArray, myArray, adios2.Mode.Sync)
     myArray += increment

examples/hello/sstWriter/sstWriter.py

@@ -34,7 +34,7 @@
         sleep(1.0)
 
         stream.write("bpFloats", myArray, [size * nx], [rank * nx], [nx])
-        print("Rank=", rank, "loop index =", currentStep, "data =", myArray)
+        print("Rank=", rank, "loop index =", currentStep, "data =", myArray, flush=True)
         myArray += increment
         # Warning: the data of the current step is not published until
         # the next loop entry or the exit of the loop

pyproject.toml

@@ -48,6 +48,9 @@ Documentation = "https://adios2.readthedocs.io/"
 Discussions = "https://github.com/ornladios/ADIOS2/discussions"
 Changelog = "https://github.com/ornladios/ADIOS2/releases"
 
+[project.entry-points."xarray.backends"]
+adios = "adios2.xarraybackend:AdiosBackendEntrypoint"
+
 [tool.cibuildwheel]
 # Trigger an install of the package, and run a basic test
 test-command = "python -m unittest adios2.test.simple_read_write.TestSimpleReadWrite"

python/adios2/file_reader.py

@@ -5,7 +5,7 @@
 from functools import singledispatchmethod
 from adios2 import Stream, IO
 
-
+# pylint: disable=W0221
 class FileReader(Stream):
     """High level implementation of the FileReader class for read Random access mode"""
 
@@ -19,7 +19,7 @@ def __init__(self, path, comm=None):
     # e.g. FileReader(io: adios2.IO, path, mode)
     # pylint: disable=E1121
     @__init__.register(IO)
-    def _(self, io: IO, path, mode, comm=None):
+    def _(self, io: IO, path, comm=None):
         super().__init__(io, path, "rra", comm)
 
     # pylint: enable=E1121

python/adios2/io.py

@@ -66,7 +66,15 @@ def define_attribute(
                 e.g. variable_name + separator + name ("var/attr")
                 Not used if variable_name is empty
         """
-        return Attribute(self.impl, name, content, variable_name, separator)
+
+        # string or list of strings passed on as is
+        if isinstance(content, list) and len(content) > 0 and isinstance(content[0], str):
+            return Attribute(self.impl, name, content, variable_name, separator)
+        if isinstance(content, str):
+            return Attribute(self.impl, name, content, variable_name, separator)
+
+        # python values (single or list) needs to be passed as a numpy array
+        return Attribute(self.impl, name, np.asarray(content), variable_name, separator)
 
     def inquire_attribute(self, name, variable_name="", separator="/"):
         """
@@ -103,10 +111,7 @@ def available_attributes(self):
                 value
                     attribute information dictionary
         """
-        attributes = {}
-        for name, attr in self.impl.AvailableAttributes():
-            attributes[name] = Attribute(attr, name)
-        return attributes
+        return self.impl.AvailableAttributes()
 
     def remove_attribute(self, name):
         """

python/adios2/stream.py

@@ -202,6 +202,27 @@ def inquire_variable(self, name):
         """
         return self._io.inquire_variable(name)
 
+    def inquire_attribute(self, name, variable_name="", separator="/"):
+        """
+        Inquire an attribute
+
+        Parameters
+            name
+                attribute name
+
+            variable_name
+                if attribute is associated with a variable
+
+            separator
+                concatenation string between variable_name and attribute
+                e.g. variable_name + separator + name ("var/attr")
+                Not used if variable_name is empty
+
+        Returns
+            The attribute if it is defined, otherwise None
+        """
+        return self._io.inquire_attribute(name, variable_name, separator)
+
     @singledispatchmethod
     def write(self, variable: Variable, content):
         """
@@ -311,10 +332,6 @@ def read(self, variable: Variable):
 
             output_shape = np.array(count)
             output_shape[0] *= steps
-            print(
-                f"Stream.read variable {variable.name()} dtype = {dtype} "
-                f"shape = {output_shape}, steps = {variable.steps()}"
-            )
         else:
             # scalar
             output_shape = (variable.selection_size(),)

scripts/ci/cmake/ci-macos-12-xcode13_4_1-static-serial.cmake

@@ -0,0 +1,52 @@
+# Client maintainer: [email protected]
+set(ENV{CC}  clang)
+set(ENV{CXX} clang++)
+set(ENV{FC}  gfortran-11)
+
+set(dashboard_cache "
+BUILD_SHARED_LIBS=OFF
+CMAKE_BUILD_TYPE:STRING=Release
+BUILD_TESTING:BOOL=ON
+ADIOS2_BUILD_EXAMPLES:BOOL=OFF
+
+ADIOS2_USE_AWSSDK:STRING=OFF
+ADIOS2_USE_Blosc2:STRING=OFF
+ADIOS2_USE_Blosc:BOOL=OFF
+ADIOS2_USE_BZip2:BOOL=OFF
+ADIOS2_USE_Catalyst:STRING=OFF
+ADIOS2_USE_CUDA:STRING=OFF
+ADIOS2_USE_DAOS:STRING=OFF
+ADIOS2_USE_DataMan:BOOL=OFF
+ADIOS2_USE_DataSpaces:STRING=OFF
+ADIOS2_USE_Fortran:BOOL=OFF
+ADIOS2_USE_HDF5:BOOL=OFF
+ADIOS2_USE_HDF5_VOL:STRING=OFF
+ADIOS2_USE_IME:STRING=OFF
+ADIOS2_USE_Kokkos:STRING=OFF
+ADIOS2_USE_LIBPRESSIO:STRING=OFF
+ADIOS2_USE_MGARD:STRING=OFF
+ADIOS2_USE_MHS:STRING=OFF
+ADIOS2_USE_MPI:BOOL=OFF
+ADIOS2_USE_PNG:BOOL=OFF
+ADIOS2_USE_Python:BOOL=OFF
+ADIOS2_USE_SSC:BOOL=OFF
+ADIOS2_USE_SST:BOOL=OFF
+ADIOS2_USE_SZ:BOOL=OFF
+ADIOS2_USE_ZeroMQ:STRING=OFF
+ADIOS2_USE_ZFP:BOOL=OFF
+
+CMAKE_C_COMPILER_LAUNCHER=ccache
+CMAKE_CXX_COMPILER_LAUNCHER=ccache
+CMAKE_C_FLAGS:STRING=-Wall
+CMAKE_CXX_FLAGS:STRING=-Wall
+CMAKE_Fortran_FLAGS:STRING=-Wall
+")
+
+set(ENV{MACOSX_DEPLOYMENT_TARGET} "12.1")
+set(CTEST_CMAKE_GENERATOR "Ninja")
+list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
+set(CTEST_TEST_ARGS
+  # Unclear why these tests currently die.  Disabling until it can be addressed.
+  EXCLUDE "Install.Make.*"
+)
+include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)

scripts/ci/images/Dockerfile.ci-el8-intel

@@ -27,7 +27,7 @@ RUN dnf install -y \
         patch \
         patchelf \
         python39-devel \
-        python3-pip \
+        python39-pip \
         tar \
         tcl \
         unzip \

source/adios2/CMakeLists.txt

@@ -80,6 +80,7 @@ add_library(adios2_core
   toolkit/format/buffer/chunk/ChunkV.cpp
   toolkit/format/buffer/ffs/BufferFFS.cpp
   toolkit/format/buffer/heap/BufferSTL.cpp
+  toolkit/format/buffer/heap/BufferMalloc.cpp
   toolkit/format/buffer/malloc/MallocV.cpp
 
   toolkit/format/bp/BPBase.cpp toolkit/format/bp/BPBase.tcc

source/adios2/engine/bp5/BP5Reader.cpp

@@ -70,8 +70,8 @@ void BP5Reader::InstallMetadataForTimestep(size_t Step)
     {
         // variable metadata for timestep
         size_t ThisMDSize =
-            helper::ReadValue<uint64_t>(m_Metadata.m_Buffer, Position, m_Minifooter.IsLittleEndian);
-        char *ThisMD = m_Metadata.m_Buffer.data() + MDPosition;
+            helper::ReadValue<uint64_t>(m_Metadata.Data(), Position, m_Minifooter.IsLittleEndian);
+        char *ThisMD = m_Metadata.Data() + MDPosition;
         if (m_OpenMode == Mode::ReadRandomAccess)
         {
             m_BP5Deserializer->InstallMetaData(ThisMD, ThisMDSize, WriterRank, Step);
@@ -86,8 +86,8 @@ void BP5Reader::InstallMetadataForTimestep(size_t Step)
     {
         // attribute metadata for timestep
         size_t ThisADSize =
-            helper::ReadValue<uint64_t>(m_Metadata.m_Buffer, Position, m_Minifooter.IsLittleEndian);
-        char *ThisAD = m_Metadata.m_Buffer.data() + MDPosition;
+            helper::ReadValue<uint64_t>(m_Metadata.Data(), Position, m_Minifooter.IsLittleEndian);
+        char *ThisAD = m_Metadata.Data() + MDPosition;
         if (ThisADSize > 0)
             m_BP5Deserializer->InstallAttributeData(ThisAD, ThisADSize);
         MDPosition += ThisADSize;
@@ -820,8 +820,7 @@ void BP5Reader::UpdateBuffer(const TimePoint &timeoutInstant, const Seconds &pol
                 for (auto p : m_FilteredMetadataInfo)
                 {
                     m_JSONProfiler.AddBytes("metadataread", p.second);
-                    m_MDFileManager.ReadFile(m_Metadata.m_Buffer.data() + mempos, p.second,
-                                             p.first);
+                    m_MDFileManager.ReadFile(m_Metadata.Data() + mempos, p.second, p.first);
                     mempos += p.second;
                 }
                 m_MDFileAlreadyReadSize = expectedMinFileSize;
@@ -863,14 +862,20 @@ void BP5Reader::UpdateBuffer(const TimePoint &timeoutInstant, const Seconds &pol
             }
         }
 
-        // broadcast buffer to all ranks from zero
-        m_Comm.BroadcastVector(m_Metadata.m_Buffer);
-
         // broadcast metadata index buffer to all ranks from zero
         m_Comm.BroadcastVector(m_MetaMetadata.m_Buffer);
 
         InstallMetaMetaData(m_MetaMetadata);
 
+        size_t inputSize = m_Comm.BroadcastValue(m_Metadata.Size(), 0);
+
+        if (m_Comm.Rank() != 0)
+        {
+            m_Metadata.Resize(inputSize, "metadata broadcast");
+        }
+
+        m_Comm.Bcast(m_Metadata.Data(), inputSize, 0);
+
         if (m_OpenMode == Mode::ReadRandomAccess)
         {
             for (size_t Step = 0; Step < m_MetadataIndexTable.size(); Step++)