Merge pull request #1660 from eisenhauer/NVStreamDP

Shared memory data plane based on GT NVStream
ornladios · Aug 7, 2019 · 8008a10 · 8008a10
2 parents 2bc027a + 7e086ff
commit 8008a10
Show file tree

Hide file tree

Showing 8 changed files with 1,035 additions and 3 deletions.
diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake
@@ -214,6 +214,11 @@ if(ADIOS2_USE_SST AND NOT MSVC)
       set(ADIOS2_SST_HAVE_CRAY_DRC TRUE)
     endif()
   endif()
+  find_package(NVSTREAM)
+  if(NVSTREAM_FOUND)
+    find_package(Boost OPTIONAL_COMPONENTS thread log filesystem system)
+    set(ADIOS2_SST_HAVE_NVSTREAM TRUE)
+  endif()
 endif()
 
 #SysV IPC

diff --git a/cmake/FindNVSTREAM.cmake b/cmake/FindNVSTREAM.cmake
@@ -0,0 +1,63 @@
+#------------------------------------------------------------------------------#
+# Distributed under the OSI-approved Apache License, Version 2.0.  See
+# accompanying file Copyright.txt for details.
+#------------------------------------------------------------------------------#
+#
+# FindNVSTREAM
+# -----------
+#
+# Try to find the NVSTREAM library
+#
+# This module defines the following variables:
+#
+#   NVSTREAM_FOUND        - System has NVSTREAM
+#   NVSTREAM_INCLUDE_DIRS - The NVSTREAM include directory
+#   NVSTREAM_LIBRARIES    - Link these to use NVSTREAM
+#   NVSTREAM_VERSION      - Version of the NVSTREAM library to support
+#
+# and the following imported targets:
+#   NVStream::NVStream - The core NVSTREAM library
+#
+# You can also set the following variable to help guide the search:
+#   NVSTREAM_ROOT - The install prefix for NVSTREAM containing the
+#                     include and lib folders
+#                     Note: this can be set as a CMake variable or an
+#                           environment variable.  If specified as a CMake
+#                           variable, it will override any setting specified
+#                           as an environment variable.
+
+if(NOT NVSTREAM_FOUND)
+  if((NOT NVSTREAM_ROOT) AND (NOT (ENV{NVSTREAM_ROOT} STREQUAL "")))
+    set(NVSTREAM_ROOT "$ENV{NVSTREAM_ROOT}")
+  endif()
+  if(NVSTREAM_ROOT)
+    set(NVSTREAM_INCLUDE_OPTS HINTS ${NVSTREAM_ROOT}/include NO_DEFAULT_PATHS)
+    set(NVSTREAM_LIBRARY_OPTS
+      HINTS ${NVSTREAM_ROOT}/lib ${NVSTREAM_ROOT}/lib64
+      NO_DEFAULT_PATHS
+    )
+  endif()
+
+  find_path(NVSTREAM_INCLUDE_DIR nvs/store.h ${NVSTREAM_INCLUDE_OPTS})
+  find_library(NVSTREAM_LIBRARY libyuma.a ${NVSTREAM_LIBRARY_OPTS})
+
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(NVStream
+    FOUND_VAR NVSTREAM_FOUND
+    VERSION_VAR NVSTREAM_VERSION
+    REQUIRED_VARS NVSTREAM_LIBRARY NVSTREAM_INCLUDE_DIR
+  )
+    message(STATUS "NVSTREAM_FOUND is ${NVSTREAM_FOUND}, LIB is ${NVSTREAM_LIBRARY}")
+  if(NVSTREAM_FOUND)
+    set(NVSTREAM_INCLUDE_DIRS ${NVSTREAM_INCLUDE_DIR})
+    set(NVSTREAM_LIBRARIES ${NVSTREAM_LIBRARY})
+    if(NVSTREAM_FOUND AND NOT TARGET NVStream::NVStream)
+        message(STATUS "ADDING LIBRARY NVSTREAM_FOUND is ${NVSTREAM_FOUND}, LIB is ${NVSTREAM_LIBRARY}")
+      add_library(NVStream::NVStream UNKNOWN IMPORTED)
+      set_target_properties(NVStream::NVStream PROPERTIES
+        IMPORTED_LOCATION             "${NVSTREAM_LIBRARY}"
+        INTERFACE_INCLUDE_DIRECTORIES "${NVSTREAM_INCLUDE_DIR}"
+      )
+    endif()
+  endif()
+endif()
diff --git a/source/adios2/toolkit/sst/CMakeLists.txt b/source/adios2/toolkit/sst/CMakeLists.txt
@@ -25,6 +25,12 @@ if(ADIOS2_SST_HAVE_LIBFABRIC)
   endif()
 endif()
 
+if(ADIOS2_SST_HAVE_NVSTREAM)
+  target_sources(sst PRIVATE dp/nvstream_dp.c dp/nvswrapper.cpp)
+  target_link_libraries(sst PRIVATE NVStream::NVStream ${Boost_LIBRARIES})
+  set(CMAKE_REQUIRED_INCLUDES ${NVSTREAM_INCLUDE_DIRS})
+endif()
+
 if(ADIOS2_HAVE_ZFP)
   target_sources(sst PRIVATE cp/ffs_zfp.c)
   target_link_libraries(sst PRIVATE zfp::zfp)
@@ -44,6 +50,7 @@ set(SST_CONFIG_OPTS
   LIBFABRIC
   FI_GNI
   CRAY_DRC
+  NVSTREAM
 )
 include(SSTFunctions)
 GenerateSSTHeaderConfig(${SST_CONFIG_OPTS})

diff --git a/source/adios2/toolkit/sst/cp/cp_reader.c b/source/adios2/toolkit/sst/cp/cp_reader.c
@@ -79,7 +79,13 @@ static char *readContactInfoFile(const char *Name, SstStream Stream,
     else
     {
         char Tmp[strlen(SSTMAGICV0)];
-        fread(Tmp, strlen(SSTMAGICV0), 1, WriterInfo);
+        if (fread(Tmp, strlen(SSTMAGICV0), 1, WriterInfo) != 1)
+        {
+            fprintf(stderr,
+                    "Filesystem read failed in SST Open, failing operation\n");
+            fclose(WriterInfo);
+            Badfile++;
+        }
         Size -= strlen(SSTMAGICV0);
         if (strncmp(Tmp, SSTMAGICV0, strlen(SSTMAGICV0)) != 0)
         {
@@ -99,8 +105,11 @@ static char *readContactInfoFile(const char *Name, SstStream Stream,
     char *Buffer = calloc(1, Size + 1);
     if (fread(Buffer, Size, 1, WriterInfo) != 1)
     {
-        fprintf(stderr, "Filesystem read failed, exiting\n");
-        exit(1);
+        fprintf(stderr,
+                "Filesystem read failed in SST Open, failing operation\n");
+        free(Buffer);
+        fclose(WriterInfo);
+        return NULL;
     }
     fclose(WriterInfo);
     return Buffer;

diff --git a/source/adios2/toolkit/sst/dp/dp.c b/source/adios2/toolkit/sst/dp/dp.c
@@ -13,6 +13,9 @@
 #ifdef SST_HAVE_LIBFABRIC
 extern CP_DP_Interface LoadRdmaDP();
 #endif /* SST_HAVE_LIBFABRIC */
+#ifdef SST_HAVE_NVSTREAM
+extern CP_DP_Interface LoadNvstreamDP();
+#endif /* SST_HAVE_LIBFABRIC */
 extern CP_DP_Interface LoadEVpathDP();
 
 typedef struct _DPElement
@@ -60,6 +63,11 @@ CP_DP_Interface SelectDP(CP_Services Svcs, void *CP_Stream,
         AddDPPossibility(Svcs, CP_Stream, List, LoadRdmaDP(), "rdma", Params);
 #endif /* SST_HAVE_LIBFABRIC */
 
+#ifdef SST_HAVE_NVSTREAM
+    List = AddDPPossibility(Svcs, CP_Stream, List, LoadNvstreamDP(), "nvstream",
+                            Params);
+#endif /* SST_HAVE_LIBFABRIC */
+
     int SelectedDP = -1;
     int BestPriority = -1;
     int BestPrioDP = -1;