Merge pull request #3078 from eisenhauer/PerformDataWrite

Add PerformDataWrite, modify docs for that and PerformPuts

bindings/C/adios2/c/adios2_c_engine.cpp

@@ -406,6 +406,31 @@ adios2_error adios2_perform_puts(adios2_engine *engine)
     }
 }
 
+adios2_error adios2_perform_data_write(adios2_engine *engine)
+{
+    try
+    {
+        adios2::helper::CheckForNullptr(
+            engine, "for adios2_engine, in call to adios2_perform_data_write");
+
+        adios2::core::Engine *engineCpp =
+            reinterpret_cast<adios2::core::Engine *>(engine);
+
+        if (engineCpp->m_EngineType == "NULL")
+        {
+            return adios2_error_none;
+        }
+
+        engineCpp->PerformDataWrite();
+        return adios2_error_none;
+    }
+    catch (...)
+    {
+        return static_cast<adios2_error>(
+            adios2::helper::ExceptionToError("adios2_perform_data_write"));
+    }
+}
+
 adios2_error adios2_get(adios2_engine *engine, adios2_variable *variable,
                         void *values, const adios2_mode mode)
 {

bindings/C/adios2/c/adios2_c_engine.h

@@ -134,13 +134,21 @@ adios2_error adios2_put_by_name(adios2_engine *engine,
 
 /**
  * Performs all the adios2_put and adios2_put_by_name called with mode
- * adios2_mode_deferred, up to this point, by putting the data in the Engine.
- * User data can be reused after this point.
+ * adios2_mode_deferred, up to this point, by copying user data into internal
+ * ADIOS buffers. User data can be reused after this point.
  * @param engine handler for a particular engine where data will be put
  * @return adios2_error 0: success, see enum adios2_error for errors
  */
 adios2_error adios2_perform_puts(adios2_engine *engine);
 
+/**
+ * Write array data to disk.  This may relieve memory pressure by clearing ADIOS
+ * buffers.  It is a collective call. User data can be reused after this point.
+ * @param engine handler for a particular engine where data will be put
+ * @return adios2_error 0: success, see enum adios2_error for errors
+ */
+adios2_error adios2_perform_data_write(adios2_engine *engine);
+
 //***************** GET *****************
 /**
  * Gets data associated with a Variable from an engine, used for engines with

bindings/CXX11/adios2/cxx11/Engine.cpp

@@ -87,6 +87,16 @@ void Engine::PerformPuts()
     m_Engine->PerformPuts();
 }
 
+void Engine::PerformDataWrite()
+{
+    helper::CheckForNullptr(m_Engine, "in call to Engine::PerformDataWrite");
+    if (m_Engine->m_EngineType == "NULL")
+    {
+        return;
+    }
+    m_Engine->PerformDataWrite();
+}
+
 void Engine::PerformGets()
 {
     helper::CheckForNullptr(m_Engine, "in call to Engine::PerformGets");

bindings/CXX11/adios2/cxx11/Engine.h

@@ -191,9 +191,16 @@ class Engine
     void Put(const std::string &variableName, const T &datum,
              const Mode launch = Mode::Deferred);
 
-    /** Perform all Put calls in Deferred mode up to this point */
+    /** Perform all Put calls in Deferred mode up to this point.  Specifically,
+     * this causes Deferred data to be copied into ADIOS internal buffers as if
+     * the Put had been done in Sync mode. */
     void PerformPuts();
 
+    /** Write already-Put() array data to disk.  If supported by the engine,
+     * this may relieve memory pressure by clearing ADIOS buffers.  It is a
+     * collective call and can only be called between Begin/EndStep pairs. */
+    void PerformDataWrite();
+
     /**
      * Get data associated with a Variable from the Engine
      * @param variable contains variable metadata information

bindings/Fortran/f2c/adios2_f2c_engine.cpp

@@ -100,6 +100,12 @@ void FC_GLOBAL(adios2_perform_puts_f2c,
     *ierr = static_cast<int>(adios2_perform_puts(*engine));
 }
 
+void FC_GLOBAL(adios2_perform_data_write_f2c,
+               ADIOS2_PERFORM_DATA_WRITE_F2C)(adios2_engine **engine, int *ierr)
+{
+    *ierr = static_cast<int>(adios2_perform_data_write(*engine));
+}
+
 // ******** GETS */
 void FC_GLOBAL(adios2_get_f2c, ADIOS2_get_F2C)(adios2_engine **engine,
                                                adios2_variable **variable,

bindings/Fortran/modules/adios2_engine_mod.f90

@@ -21,6 +21,7 @@ module adios2_engine_mod
     external adios2_lock_writer_definitions_f2c
     external adios2_perform_gets_f2c
     external adios2_perform_puts_f2c
+    external adios2_perform_data_write_f2c
     external adios2_steps_f2c
 
 contains
@@ -34,6 +35,15 @@ subroutine adios2_perform_puts(engine, ierr)
 
     end subroutine
 
+    subroutine adios2_perform_data_write(engine, ierr)
+        type(adios2_engine), intent(in) :: engine
+        integer, intent(out) :: ierr
+
+        if(trim(engine%type) == "NULL") return
+        call adios2_perform_data_write_f2c(engine%f2c, ierr)
+
+    end subroutine
+
     subroutine adios2_perform_gets(engine, ierr)
         type(adios2_engine), intent(in) :: engine
         integer, intent(out) :: ierr

bindings/Python/py11Engine.cpp

@@ -127,6 +127,16 @@ void Engine::PerformPuts()
     m_Engine->PerformPuts();
 }
 
+void Engine::PerformDataWrite()
+{
+    helper::CheckForNullptr(m_Engine, "in call to PerformDataWrite");
+    if (m_Engine->m_EngineType == "NULL")
+    {
+        return;
+    }
+    m_Engine->PerformDataWrite();
+}
+
 void Engine::Get(Variable variable, pybind11::array &array, const Mode launch)
 {
     helper::CheckForNullptr(m_Engine,

bindings/Python/py11Engine.h

@@ -53,6 +53,7 @@ class Engine
              const Mode launch = Mode::Deferred);
     void Put(Variable variable, const std::string &string);
     void PerformPuts();
+    void PerformDataWrite();
 
     void Get(Variable variable, pybind11::array &array,
              const Mode launch = Mode::Deferred);

bindings/Python/py11glue.cpp

@@ -474,6 +474,8 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m)
 
         .def("PerformPuts", &adios2::py11::Engine::PerformPuts)
 
+        .def("PerformDataWrite", &adios2::py11::Engine::PerformDataWrite)
+
         .def("Get",
              (void (adios2::py11::Engine::*)(adios2::py11::Variable,
                                              pybind11::array &,

docs/user_guide/source/advice/advice.rst

@@ -61,9 +61,11 @@ The goal is to provide specific advice and good practices about the use of ADIOS
 
 15. ``Put Span``: create all spans in a step before populating them. Spans follow the same iterator invalidation rules as ``std::vector``, so use ``span.data()`` to always keep the span pointer up-to-date 
 
-16. Prefer populating data before calling ``Put`` in deferred mode, rather than manipulating it between ``Put`` and ``PerformPuts``, ``EndStep``, or ``Close``
+16. Always populate data before calling ``Put`` in deferred mode,
+    and do not change it between ``Put`` and ``PerformPuts``, ``EndStep``, or ``Close``
 
-17. Use ``BeginStep`` and ``EndStep`` to write code that is portable across all ADIOS 2 Engines: file and streaming.
+17. Use ``BeginStep`` and ``EndStep`` to write code that is portable
+    across all ADIOS 2 Engine types: file and streaming.
 
 18. Always use ``Close`` for every call to ``Open``.
 

docs/user_guide/source/components/engine.rst

@@ -69,7 +69,10 @@ Put: modes and memory contracts
 ``Put`` publishes data in ADIOS2.
 It is unavailable unless the ``Engine`` is created in ``Write`` or ``Append`` mode.
 
-The most common signature is the one that passes a ``Variable<T>`` object for the metadata, a ``const`` piece of contiguous memory for the data, and a mode for either ``Deferred`` (data is collected until EndStep/PerformPuts/Close) or ``Sync`` (data is reusable immediately).
+The most common signature is the one that passes a ``Variable<T>``
+object for the metadata, a ``const`` piece of contiguous memory for
+the data, and a mode for either ``Deferred`` (data may be collected at
+Put() or not until EndStep/PerformPuts/Close) or ``Sync`` (data is reusable immediately).
 This is the most common use case in applications.
 
 1. Deferred (default) or Sync mode, data is contiguous memory 
@@ -133,7 +136,7 @@ The following table summarizes the memory contracts required by ADIOS2 engines b
 +----------+-------------+----------------------------------------------------+
 |          | Pointer     | do not modify until PerformPuts/EndStep/Close      |
 | Deferred |             |                                                    |
-|          | Contents    | consumed at PerformPuts/EndStep/Close              |
+|          | Contents    | consumed at Put or PerformPuts/EndStep/Close       |
 +----------+-------------+----------------------------------------------------+
 |          | Pointer     | modify after Put                                   |
 | Sync     |             |                                                    |
@@ -164,7 +167,8 @@ Each ``Engine`` will give a concrete meaning to  each functions signatures, but
 
    - "data pointer" do not modify (e.g. resize) until first call to ``PerformPuts``, ``EndStep`` or ``Close``.
 
-   - "data contents" consumed at first call to ``PerformPuts``, ``EndStep`` or ``Close``. It's recommended practice to set all data contents before Put.
+   - "data contents" may be consumed immediately or at first call to
+     ``PerformPuts``, ``EndStep`` or ``Close``.  Do not modify data contents after Put.
 
 
    Usage:
@@ -180,11 +184,13 @@ Each ``Engine`` will give a concrete meaning to  each functions signatures, but
          // associated with current variable metadata
          engine.Put(variable, data);
 
-         // valid but not recommended
-         // risk of changing "data pointer" (e.g. resize) 
+         // Modifying data after Put(Deferred) may result in different
+	 // results with different engines
+         // Any resize of data after Put(Deferred) may result in
+	 // memory corruption or segmentation faults
          data[1] = 10; 
 
-         // "data contents" must be ready
+         // "data contents" must not have been changed
          // "data pointer" must be the same as in Put
          engine.EndStep();   
          //engine.PerformPuts();  
@@ -306,7 +312,25 @@ The ``data`` fed to the ``Put`` function is assumed to be allocated on the Host
 PerformsPuts
 ------------
 
-   Executes all pending ``Put`` calls in deferred mode ad collect spans data
+   Executes all pending ``Put`` calls in deferred mode and collects
+   span data.  Specifically this call copies Put(Deferred) data into
+   internal ADIOS buffers, as if Put(Sync) had been used instead.
+
+.. note::
+
+   This call allows the reuse of user buffers, but may negatively
+   impact performance on some engines.
+
+
+PerformsDataWrite
+------------
+
+   If supported by the engine, moves data from prior ``Put`` calls to disk
+
+.. note::
+
+   Currently only supported by the BP5 file engine.
+
 
 
 Get: modes and memory contracts
@@ -339,7 +363,7 @@ The following table summarizes the memory contracts required by ADIOS2 engines b
 +----------+-------------+-----------------------------------------------+
 |          | Pointer     | do not modify until PerformPuts/EndStep/Close |
 | Deferred |             |                                               |
-|          | Contents    | populated at PerformPuts/EndStep/Close        |
+|          | Contents    | populated at Put or PerformPuts/EndStep/Close |
 +----------+-------------+-----------------------------------------------+
 |          | Pointer     | modify after Put                              |
 | Sync     |             |                                               |
@@ -359,9 +383,9 @@ The following table summarizes the memory contracts required by ADIOS2 engines b
 
    - "data pointer": do not modify (e.g. resize) until first call to ``PerformPuts``, ``EndStep`` or ``Close``.
 
-   - "data contents": populated at first call to ``PerformPuts``, ``EndStep`` or ``Close``.
+   - "data contents": populated at ``Put``, or at first call to ``PerformPuts``, ``EndStep`` or ``Close``.
 
-   Usage:
+   Usage:`
 
       .. code-block:: c++
 
@@ -380,9 +404,9 @@ The following table summarizes the memory contracts required by ADIOS2 engines b
          // leave resize to adios2
          //engine.Get(variable, data);
 
-         // "data contents" must be ready
          // "data pointer" must be the same as in Get
          engine.EndStep();   
+         // "data contents" are now ready
          //engine.PerformPuts();  
          //engine.Close();
 
@@ -454,7 +478,7 @@ The following example illustrates the basic API usage in write mode for data gen
       // Application can modify dataT address and contents
 
       // deferred functions return immediately (lazy evaluation),
-      // dataU, dataV and dataW pointers must not be modified
+      // dataU, dataV and dataW pointers and contents must not be modified
       // until PerformPuts, EndStep or Close.
       // 1st batch
       engine.Put(varU, dataU);
@@ -465,23 +489,26 @@ The following example illustrates the basic API usage in write mode for data gen
       engine.Put(varW, dataW, adios2::Mode::Deferred);
 
       // effectively dataU, dataV, dataW are "deferred"
-      // until the first call to PerformPuts, EndStep or Close.
-      // Application MUST NOT modify the data pointer (e.g. resize memory).
+      // possibly until the first call to PerformPuts, EndStep or Close.
+      // Application MUST NOT modify the data pointer (e.g. resize
+      // memory) or change data contents.
       engine.PerformPuts();
 
       // dataU, dataV, dataW pointers/values can now be reused
 
       // ... Application modifies dataU, dataV, dataW 
 
       //2nd batch
+      dataU[0] = 10
+      dataV[0] = 10
+      dataW[0] = 10 
       engine.Put(varU, dataU);
       engine.Put(varV, dataV);
       engine.Put(varW, dataW);
       // Application MUST NOT modify dataU, dataV and dataW pointers (e.g. resize),
-      // optionally data can be modified, but not recommended
-      dataU[0] = 10
-      dataV[0] = 10
-      dataW[0] = 10 
+      // Contents should also not be modified after Put() and before
+      // PerformPuts() because ADIOS may access the data immediately
+      // or not until PerformPuts(), depending upon the engine
       engine.PerformPuts();
 
       // dataU, dataV, dataW pointers/values can now be reused

source/adios2/core/Engine.cpp

@@ -63,6 +63,7 @@ size_t Engine::CurrentStep() const
 void Engine::EndStep() { ThrowUp("EndStep"); }
 void Engine::PerformPuts() { ThrowUp("PerformPuts"); }
 void Engine::PerformGets() { ThrowUp("PerformGets"); }
+void Engine::PerformDataWrite() { return; }
 
 void Engine::Close(const int transportIndex)
 {

source/adios2/core/Engine.h

@@ -364,6 +364,10 @@ class Engine
      * PerformGets, BeginStep or Open */
     virtual void PerformGets();
 
+    /** Write array data to disk.  This may relieve memory pressure by clearing
+     * ADIOS buffers.  It is a collective call. */
+    virtual void PerformDataWrite();
+
     /**
      * Closes a particular transport, or all if transportIndex = -1 (default).
      * @param transportIndex index returned from IO AddTransport, default (-1) =

source/adios2/engine/bp5/BP5Writer.cpp

@@ -1393,7 +1393,9 @@ void BP5Writer::FlushData(const bool isFinal)
     }
 }
 
-void BP5Writer::Flush(const int transportIndex) { FlushData(false); }
+void BP5Writer::Flush(const int transportIndex) {}
+
+void BP5Writer::PerformDataWrite() { FlushData(false); }
 
 void BP5Writer::DoClose(const int transportIndex)
 {
@@ -1539,26 +1541,6 @@ void BP5Writer::FlushProfiler()
     }
 }
 
-/*write the content of metadata index file*/
-void BP5Writer::PopulateMetadataIndexFileContent(
-    format::BufferSTL &b, const uint64_t currentStep, const uint64_t mpirank,
-    const uint64_t pgIndexStart, const uint64_t variablesIndexStart,
-    const uint64_t attributesIndexStart, const uint64_t currentStepEndPos,
-    const uint64_t currentTimeStamp)
-{
-    PERFSTUBS_SCOPED_TIMER("BP5Writer::PopulateMetadataIndexFileContent");
-    auto &buffer = b.m_Buffer;
-    auto &position = b.m_Position;
-    helper::CopyToBuffer(buffer, position, &currentStep);
-    helper::CopyToBuffer(buffer, position, &mpirank);
-    helper::CopyToBuffer(buffer, position, &pgIndexStart);
-    helper::CopyToBuffer(buffer, position, &variablesIndexStart);
-    helper::CopyToBuffer(buffer, position, &attributesIndexStart);
-    helper::CopyToBuffer(buffer, position, &currentStepEndPos);
-    helper::CopyToBuffer(buffer, position, &currentTimeStamp);
-    position += 8;
-}
-
 size_t BP5Writer::DebugGetDataBufferSize() const
 {
     return m_BP5Serializer.DebugGetDataBufferSize();