diff --git a/CMakeLists.txt b/CMakeLists.txt index dc706eaa17..200c1093d4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -225,7 +225,7 @@ endif() set(ADIOS2_CONFIG_OPTS - BP5 DataMan DataSpaces HDF5 HDF5_VOL MHS SST CUDA Fortran MPI Python Blosc Blosc2 BZip2 LIBPRESSIO MGARD PNG SZ ZFP DAOS IME O_DIRECT Sodium Catalyst SysVShMem ZeroMQ Profiling Endian_Reverse + BP5 DataMan DataSpaces HDF5 HDF5_VOL MHS SST CUDA Fortran MPI Python Blosc Blosc2 BZip2 LIBPRESSIO MGARD PNG SZ ZFP DAOS IME O_DIRECT Sodium Catalyst SysVShMem ZeroMQ Profiling Endian_Reverse GPU_Support ) GenerateADIOSHeaderConfig(${ADIOS2_CONFIG_OPTS}) diff --git a/bindings/CXX11/adios2/cxx11/Engine.cpp b/bindings/CXX11/adios2/cxx11/Engine.cpp index 1b140309e8..748af4e041 100644 --- a/bindings/CXX11/adios2/cxx11/Engine.cpp +++ b/bindings/CXX11/adios2/cxx11/Engine.cpp @@ -12,12 +12,20 @@ #include "Engine.tcc" #include "adios2/core/Engine.h" - #include "adios2/helper/adiosFunctions.h" namespace adios2 { +#ifdef ADIOS2_HAVE_GPU_SUPPORT +void Engine::CheckMemorySpace(MemorySpace variableMem, MemorySpace bufferMem) +{ + if (variableMem != MemorySpace::Detect && variableMem != bufferMem) + helper::Throw("CXX-Bindings", "Engine", "Put", + "Memory space mismatch"); +} +#endif + Engine::operator bool() const noexcept { if (m_Engine == nullptr) diff --git a/bindings/CXX11/adios2/cxx11/Engine.h b/bindings/CXX11/adios2/cxx11/Engine.h index ccb4e68751..1043315e91 100644 --- a/bindings/CXX11/adios2/cxx11/Engine.h +++ b/bindings/CXX11/adios2/cxx11/Engine.h @@ -38,6 +38,10 @@ class Engine friend class IO; friend class QueryWorker; +#ifdef ADIOS2_HAVE_GPU_SUPPORT + void CheckMemorySpace(MemorySpace variableMem, MemorySpace bufferMem); +#endif + public: /** * Empty (default) constructor, use it as a placeholder for future @@ -217,10 +221,13 @@ class Engine void Put(Variable variable, U const &data, const Mode launch = Mode::Deferred) { - auto adios_data = static_cast>(data); - auto mem_space = adios_data.memory_space(); - variable.SetMemorySpace(mem_space); - Put(variable, adios_data.data(), launch); + auto bufferView = static_cast>(data); +#ifdef ADIOS2_HAVE_GPU_SUPPORT + auto bufferMem = bufferView.memory_space(); + auto variableMem = variable.GetMemorySpace(); + CheckMemorySpace(variableMem, bufferMem); +#endif + Put(variable, bufferView.data(), launch); } /** Perform all Put calls in Deferred mode up to this point. Specifically, diff --git a/bindings/CXX11/adios2/cxx11/KokkosView.h b/bindings/CXX11/adios2/cxx11/KokkosView.h index 1083d03bb2..8b6397b0da 100644 --- a/bindings/CXX11/adios2/cxx11/KokkosView.h +++ b/bindings/CXX11/adios2/cxx11/KokkosView.h @@ -17,7 +17,7 @@ struct memspace_kokkos_to_adios2 static constexpr adios2::MemorySpace value = adios2::MemorySpace::Host; }; -#ifdef KOKKOS_ENABLE_CUDA +#if defined(KOKKOS_ENABLE_CUDA) && defined(ADIOS2_HAVE_CUDA) template <> struct memspace_kokkos_to_adios2 @@ -25,6 +25,18 @@ struct memspace_kokkos_to_adios2 static constexpr adios2::MemorySpace value = adios2::MemorySpace::CUDA; }; +template <> +struct memspace_kokkos_to_adios2 +{ + static constexpr adios2::MemorySpace value = adios2::MemorySpace::CUDA; +}; + +template <> +struct memspace_kokkos_to_adios2 +{ + static constexpr adios2::MemorySpace value = adios2::MemorySpace::CUDA; +}; + #endif } // namespace detail diff --git a/bindings/CXX11/adios2/cxx11/Variable.cpp b/bindings/CXX11/adios2/cxx11/Variable.cpp index 5e3ca08569..b58d92a517 100644 --- a/bindings/CXX11/adios2/cxx11/Variable.cpp +++ b/bindings/CXX11/adios2/cxx11/Variable.cpp @@ -38,6 +38,12 @@ namespace adios2 } \ \ template <> \ + MemorySpace Variable::GetMemorySpace() \ + { \ + return m_Variable->m_MemSpace; \ + } \ + \ + template <> \ void Variable::SetShape(const Dims &shape) \ { \ helper::CheckForNullptr(m_Variable, \ diff --git a/bindings/CXX11/adios2/cxx11/Variable.h b/bindings/CXX11/adios2/cxx11/Variable.h index 77aa3a9e0f..cbe5c1af6c 100644 --- a/bindings/CXX11/adios2/cxx11/Variable.h +++ b/bindings/CXX11/adios2/cxx11/Variable.h @@ -154,6 +154,12 @@ class Variable */ void SetMemorySpace(const MemorySpace mem); + /** + * Get the memory space that was set by the application + * @return the memory space stored in the Variable object + */ + MemorySpace GetMemorySpace(); + /** * Set new shape, care must be taken when reading back the variable for * different steps. Only applies to Global arrays. diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index fa8324c431..625b1da781 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -193,6 +193,7 @@ endif() if(CMAKE_CUDA_COMPILER AND CUDAToolkit_FOUND) enable_language(CUDA) set(ADIOS2_HAVE_CUDA TRUE) + set(ADIOS2_HAVE_GPU_Support TRUE) endif() # Fortran diff --git a/examples/kokkos/kokkosWriteRead.cpp b/examples/kokkos/kokkosWriteRead.cpp index 7ac475bbdc..d92f58253f 100644 --- a/examples/kokkos/kokkosWriteRead.cpp +++ b/examples/kokkos/kokkosWriteRead.cpp @@ -23,7 +23,7 @@ } \ } while (false) -int BPWrite(const std::string fname, const size_t N, int nSteps, +int BPWrite(const std::string fname, const size_t N, size_t nSteps, const std::string engine) { int rank, size; @@ -86,18 +86,16 @@ int BPWrite(const std::string fname, const size_t N, int nSteps, return 0; } -int BPRead(const std::string fname, const size_t N, int nSteps, +int BPRead(const std::string fname, const size_t N, size_t nSteps, std::string engine) { - int rank, size; + int rank; #if ADIOS2_USE_MPI int provided; MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &size); #else rank = 0; - size = 1; #endif // Create ADIOS structures #if ADIOS2_USE_MPI @@ -148,7 +146,8 @@ int main(int argc, char **argv) { const std::vector list_of_engines = {"BPFile", "BP5"}; const size_t N = 6000; - int nSteps = 10, ret = 0, rank = 0; + size_t nSteps = 10; + int ret = 0, rank = 0; #if ADIOS2_USE_MPI int provided; MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); diff --git a/source/adios2/common/ADIOSTypes.h b/source/adios2/common/ADIOSTypes.h index 86b51626b1..36f5fbeb3f 100644 --- a/source/adios2/common/ADIOSTypes.h +++ b/source/adios2/common/ADIOSTypes.h @@ -35,9 +35,13 @@ namespace adios2 /** Memory space for the user provided buffers */ enum class MemorySpace { +#ifdef ADIOS2_HAVE_GPU_SUPPORT Detect, ///< Detect the memory space automatically - Host, ///< Host memory space (default) - CUDA ///< CUDA memory spaces +#endif + Host, ///< Host memory space +#ifdef ADIOS2_HAVE_CUDA + CUDA ///< CUDA memory spaces +#endif }; /** Variable shape type identifier, assigned automatically from the signature of diff --git a/source/adios2/core/Variable.cpp b/source/adios2/core/Variable.cpp index fd01fe628c..011c957a25 100644 --- a/source/adios2/core/Variable.cpp +++ b/source/adios2/core/Variable.cpp @@ -49,7 +49,7 @@ namespace core info.StepsCount = stepsCount; \ info.Data = const_cast(data); \ info.Operations = m_Operations; \ - info.MemSpace = DetectMemorySpace((void *)data); \ + info.MemSpace = GetMemorySpace((void *)data); \ m_BlocksInfo.push_back(info); \ return m_BlocksInfo.back(); \ } \ diff --git a/source/adios2/core/VariableBase.cpp b/source/adios2/core/VariableBase.cpp index 3822507049..c12e017859 100644 --- a/source/adios2/core/VariableBase.cpp +++ b/source/adios2/core/VariableBase.cpp @@ -48,41 +48,27 @@ size_t VariableBase::TotalSize() const noexcept return helper::GetTotalSize(m_Count); } -MemorySpace VariableBase::DetectMemorySpace(const void *ptr) +MemorySpace VariableBase::GetMemorySpace(const void *ptr) { - if (m_MemSpaceRequested != MemorySpace::Detect) +#ifdef ADIOS2_HAVE_GPU_SUPPORT + if (m_MemSpace != MemorySpace::Detect) { - m_MemSpaceDetected = m_MemSpaceRequested; - return m_MemSpaceRequested; + return m_MemSpace; } +#endif - // if the user requested MemorySpace::Detect #ifdef ADIOS2_HAVE_CUDA cudaPointerAttributes attr; cudaPointerGetAttributes(&attr, ptr); if (attr.type == cudaMemoryTypeDevice) { - m_MemSpaceDetected = MemorySpace::CUDA; return MemorySpace::CUDA; } #endif - m_MemSpaceDetected = MemorySpace::Host; return MemorySpace::Host; } -MemorySpace VariableBase::GetMemorySpace(const void *ptr) -{ - if (m_MemSpaceDetected == MemorySpace::Detect) - m_MemSpaceDetected = DetectMemorySpace(ptr); - return m_MemSpaceDetected; -} - -void VariableBase::SetMemorySpace(const MemorySpace mem) -{ - m_MemSpaceRequested = mem; - // reset the detected memory space - m_MemSpaceDetected = MemorySpace::Detect; -} +void VariableBase::SetMemorySpace(const MemorySpace mem) { m_MemSpace = mem; } void VariableBase::SetShape(const adios2::Dims &shape) { diff --git a/source/adios2/core/VariableBase.h b/source/adios2/core/VariableBase.h index ac92482895..1b29e958ff 100644 --- a/source/adios2/core/VariableBase.h +++ b/source/adios2/core/VariableBase.h @@ -50,13 +50,13 @@ class VariableBase /** Variable -> sizeof(T), * VariableStruct -> from constructor sizeof(struct) */ const size_t m_ElementSize; - /* User requested memory space and the detected memory space */ -#ifdef ADIOS2_HAVE_CUDA - MemorySpace m_MemSpaceRequested = MemorySpace::Detect; + + /* User requested memory space */ +#ifdef ADIOS2_HAVE_GPU_SUPPORT + MemorySpace m_MemSpace = MemorySpace::Detect; #else - MemorySpace m_MemSpaceRequested = MemorySpace::Host; + MemorySpace m_MemSpace = MemorySpace::Host; #endif - MemorySpace m_MemSpaceDetected = MemorySpace::Detect; ShapeID m_ShapeID = ShapeID::Unknown; ///< see shape types in ADIOSTypes.h size_t m_BlockID = 0; ///< current block ID for local variables, global = 0 @@ -122,12 +122,6 @@ class VariableBase */ size_t TotalSize() const noexcept; - /** - * Detect the memory space where a buffer was allocated and return it - * @param pointer to the user data - */ - MemorySpace DetectMemorySpace(const void *ptr); - /** * Get the memory space where a given buffers was allocated * @param pointer to the user data diff --git a/source/adios2/engine/bp5/BP5Writer.cpp b/source/adios2/engine/bp5/BP5Writer.cpp index 9cf1956631..71bc982cc3 100644 --- a/source/adios2/engine/bp5/BP5Writer.cpp +++ b/source/adios2/engine/bp5/BP5Writer.cpp @@ -1737,7 +1737,7 @@ void BP5Writer::PutCommon(VariableBase &variable, const void *values, bool sync) } // if the user buffer is allocated on the GPU always use sync mode - if (variable.DetectMemorySpace(values) != MemorySpace::Host) + if (variable.GetMemorySpace(values) != MemorySpace::Host) sync = true; size_t *Shape = NULL;