Merge pull request #1659 from JasonRuonanWang/table-engine

Add Table engine for non-step based extremely large global arrays, and table based data

CMakeLists.txt

@@ -118,6 +118,7 @@ adios_option(MPI       "Enable support for MPI" AUTO)
 adios_option(DataMan   "Enable support for DataMan" AUTO)
 adios_option(DataSpaces "Enable support for DATASPACES" AUTO)
 adios_option(SSC       "Enable support for SSC" AUTO)
+adios_option(Table     "Enable support for Table" AUTO)
 adios_option(SST       "Enable support for SST" AUTO)
 adios_option(ZeroMQ    "Enable support for ZeroMQ" AUTO)
 adios_option(HDF5      "Enable support for the HDF5 engine" AUTO)
@@ -139,7 +140,7 @@ if(ADIOS2_HAVE_MPI)
 endif()
 
 set(ADIOS2_CONFIG_OPTS
-    Blosc BZip2 ZFP SZ MGARD PNG MPI DataMan SSC SST DataSpaces ZeroMQ HDF5 Python Fortran SysVShMem Profiling Endian_Reverse
+    Blosc BZip2 ZFP SZ MGARD PNG MPI DataMan Table SSC SST DataSpaces ZeroMQ HDF5 Python Fortran SysVShMem Profiling Endian_Reverse
 )
 GenerateADIOSHeaderConfig(${ADIOS2_CONFIG_OPTS})
 configure_file(

cmake/DetectOptions.cmake

@@ -131,6 +131,17 @@ if(NOT (CMAKE_CXX_COMPILER_ID STREQUAL "PGI") AND NOT MSVC)
     endif()
 endif()
 
+# Table
+if(NOT (CMAKE_CXX_COMPILER_ID STREQUAL "PGI") AND NOT MSVC)
+    if(ZeroMQ_FOUND AND ADIOS2_HAVE_MPI)
+        if(ADIOS2_USE_Table STREQUAL AUTO)
+            set(ADIOS2_HAVE_Table TRUE)
+        elseif(ADIOS2_USE_Table)
+            set(ADIOS2_HAVE_Table TRUE)
+        endif()
+    endif()
+endif()
+
 # DataSpaces
 if(ADIOS2_USE_DataSpaces STREQUAL AUTO)
   find_package(DataSpaces 1.8)

source/adios2/CMakeLists.txt

@@ -121,17 +121,23 @@ if(ADIOS2_HAVE_ZeroMQ)
     target_link_libraries(adios2 PRIVATE ZeroMQ::ZMQ)
 endif()
 
-if(ADIOS2_HAVE_DataMan)
+if(ADIOS2_HAVE_DataMan OR ADIOS2_HAVE_WDM OR ADIOS2_HAVE_Table)
     target_sources(adios2 PRIVATE
         toolkit/format/dataman/DataManSerializer.cpp
         toolkit/format/dataman/DataManSerializer.tcc
+        )
+    target_link_libraries(adios2 PRIVATE nlohmann_json)
+endif()
+
+
+if(ADIOS2_HAVE_DataMan)
+    target_sources(adios2 PRIVATE
         engine/dataman/DataManCommon.cpp
         engine/dataman/DataManReader.cpp
         engine/dataman/DataManReader.tcc
         engine/dataman/DataManWriter.cpp
         engine/dataman/DataManWriter.tcc
         )
-    target_link_libraries(adios2 PRIVATE nlohmann_json)
 endif()
 
 if(ADIOS2_HAVE_SSC)
@@ -143,7 +149,13 @@ if(ADIOS2_HAVE_SSC)
         engine/ssc/SscWriter.cpp
         engine/ssc/SscWriter.tcc
         )
-    target_link_libraries(adios2 PRIVATE nlohmann_json)
+endif()
+
+if(ADIOS2_HAVE_Table)
+    target_sources(adios2 PRIVATE
+        engine/table/TableWriter.cpp
+        engine/table/TableWriter.tcc
+        )
 endif()
 
 if(ADIOS2_HAVE_SST)

source/adios2/core/IO.cpp

@@ -41,6 +41,10 @@
 #include "adios2/engine/ssc/SscWriter.h"
 #endif
 
+#ifdef ADIOS2_HAVE_TABLE // external dependencies
+#include "adios2/engine/table/TableWriter.h"
+#endif
+
 #ifdef ADIOS2_HAVE_SST // external dependencies
 #include "adios2/engine/sst/SstReader.h"
 #include "adios2/engine/sst/SstWriter.h"
@@ -559,6 +563,22 @@ Engine &IO::Open(const std::string &name, const Mode mode,
 #else
         throw std::invalid_argument("ERROR: this version didn't compile with "
                                     "SSC library, can't use SSC engine\n");
+#endif
+    }
+    else if (engineTypeLC == "table")
+    {
+#ifdef ADIOS2_HAVE_TABLE
+        if (mode == Mode::Write)
+            engine = std::make_shared<engine::TableWriter>(*this, name, mode,
+                                                           mpiComm);
+        else
+            throw std::invalid_argument(
+                "ERROR: Table engine only supports Write. It uses other "
+                "engines as backend. Please use corresponding engines for "
+                "Read\n");
+#else
+        throw std::invalid_argument("ERROR: this version didn't compile with "
+                                    "Table library, can't use Table engine\n");
 #endif
     }
     else if (engineTypeLC == "sst" || engineTypeLC == "effis")

source/adios2/engine/dataman/DataManReader.cpp

@@ -25,8 +25,7 @@ namespace engine
 DataManReader::DataManReader(IO &io, const std::string &name, const Mode mode,
                              MPI_Comm mpiComm)
 : DataManCommon("DataManReader", io, name, mode, mpiComm),
-  m_DataManSerializer(m_IsRowMajor, m_ContiguousMajor, m_IsLittleEndian,
-                      mpiComm)
+  m_DataManSerializer(mpiComm, 0, m_IsRowMajor)
 {
     m_EndMessage = " in call to IO Open DataManReader " + m_Name + "\n";
     Init();

source/adios2/engine/dataman/DataManWriter.cpp

@@ -42,7 +42,7 @@ StepStatus DataManWriter::BeginStep(StepMode mode, const float timeout_sec)
 
     for (size_t i = 0; i < m_Channels; ++i)
     {
-        m_DataManSerializer[i]->New(m_BufferSize);
+        m_DataManSerializer[i]->NewWriterBuffer(m_BufferSize);
     }
 
     if (m_Verbosity >= 5)
@@ -115,8 +115,8 @@ void DataManWriter::Init()
     for (size_t i = 0; i < m_Channels; ++i)
     {
         m_DataManSerializer.push_back(
-            std::make_shared<format::DataManSerializer>(
-                m_IsRowMajor, m_ContiguousMajor, m_IsLittleEndian, m_MPIComm));
+            std::make_shared<format::DataManSerializer>(m_MPIComm, m_BufferSize,
+                                                        m_IsRowMajor));
     }
 }
 

source/adios2/engine/ssc/SscReader.cpp

@@ -28,8 +28,7 @@ namespace engine
 SscReader::SscReader(IO &io, const std::string &name, const Mode mode,
                      MPI_Comm mpiComm)
 : Engine("SscReader", io, name, mode, mpiComm),
-  m_DataManSerializer(helper::IsRowMajor(io.m_HostLanguage), true,
-                      helper::IsLittleEndian(), mpiComm),
+  m_DataManSerializer(mpiComm, 0, helper::IsRowMajor(io.m_HostLanguage)),
   m_RepliedMetadata(std::make_shared<std::vector<char>>())
 {
     TAU_SCOPED_TIMER_FUNC();

source/adios2/engine/ssc/SscWriter.cpp

@@ -28,8 +28,8 @@ namespace engine
 SscWriter::SscWriter(IO &io, const std::string &name, const Mode mode,
                      MPI_Comm mpiComm)
 : Engine("SscWriter", io, name, mode, mpiComm),
-  m_DataManSerializer(helper::IsRowMajor(io.m_HostLanguage), true,
-                      helper::IsLittleEndian(), mpiComm)
+  m_DataManSerializer(mpiComm, m_DefaultBufferSize,
+                      helper::IsRowMajor(io.m_HostLanguage))
 {
     TAU_SCOPED_TIMER_FUNC();
     Init();
@@ -48,7 +48,7 @@ StepStatus SscWriter::BeginStep(StepMode mode, const float timeoutSeconds)
     if (m_CurrentStep % m_StepsPerAggregation == 0)
     {
         m_CurrentStepActive = true;
-        m_DataManSerializer.New(m_DefaultBufferSize);
+        m_DataManSerializer.NewWriterBuffer(m_DefaultBufferSize);
         if (not m_AttributesSet)
         {
             m_DataManSerializer.PutAttributes(m_IO);