Merge pull request #1480 from germasch/pr/more_unified_mpi

final mpiwrap / mpidummy bits

bindings/Python/py11File.cpp

@@ -43,13 +43,13 @@ File::File(const std::string &name, const std::string mode, MPI_Comm comm,
 
 File::File(const std::string &name, const std::string mode,
            const std::string engineType)
-: File(name, mode, MPI_COMM_SELF, engineType)
+: File(name, mode, MPI_COMM_NULL, engineType)
 {
 }
 
 File::File(const std::string &name, const std::string mode,
            const std::string &configFile, const std::string ioInConfigFile)
-: File(name, mode, MPI_COMM_SELF, configFile, ioInConfigFile)
+: File(name, mode, MPI_COMM_NULL, configFile, ioInConfigFile)
 {
 }
 

source/adios2/core/ADIOS.cpp

@@ -61,25 +61,7 @@ ADIOS::ADIOS(const std::string configFile, MPI_Comm mpiComm,
              const bool debugMode, const std::string hostLanguage)
 : m_ConfigFile(configFile), m_DebugMode(debugMode), m_HostLanguage(hostLanguage)
 {
-    if (m_DebugMode && mpiComm == MPI_COMM_NULL)
-    {
-        throw std::ios_base::failure(
-            "ERROR: MPI communicator is MPI_COMM_NULL, "
-            " in call to ADIOS constructor\n");
-    }
-
-    int flag;
-    MPI_Initialized(&flag);
-    if (flag)
-    {
-        MPI_Comm_dup(mpiComm, &m_MPIComm);
-        m_NeedMPICommFree = true;
-    }
-    else
-    {
-        m_MPIComm = mpiComm;
-        m_NeedMPICommFree = false;
-    }
+    SMPI_Comm_dup(mpiComm, &m_MPIComm);
 
     if (!configFile.empty())
     {
@@ -93,7 +75,7 @@ ADIOS::ADIOS(const std::string configFile, MPI_Comm mpiComm,
 
 ADIOS::ADIOS(const std::string configFile, const bool debugMode,
              const std::string hostLanguage)
-: ADIOS(configFile, MPI_COMM_SELF, debugMode, hostLanguage)
+: ADIOS(configFile, MPI_COMM_NULL, debugMode, hostLanguage)
 {
 }
 
@@ -104,17 +86,19 @@ ADIOS::ADIOS(MPI_Comm mpiComm, const bool debugMode,
 }
 
 ADIOS::ADIOS(const bool debugMode, const std::string hostLanguage)
-: ADIOS("", MPI_COMM_SELF, debugMode, hostLanguage)
+: ADIOS("", MPI_COMM_NULL, debugMode, hostLanguage)
 {
 }
 
 ADIOS::~ADIOS()
 {
+    // Handle the case where MPI is finalized before the ADIOS destructor is
+    // called, which happens, e.g., with global / static ADIOS objects
     int flag;
     MPI_Finalized(&flag);
-    if (!flag && m_NeedMPICommFree)
+    if (!flag)
     {
-        MPI_Comm_free(&m_MPIComm);
+        SMPI_Comm_free(&m_MPIComm);
     }
 }
 

source/adios2/core/ADIOS.h

@@ -184,8 +184,6 @@ class ADIOS
     void RemoveAllIOs() noexcept;
 
 private:
-    bool m_NeedMPICommFree;
-
     /** XML File to be read containing configuration information */
     const std::string m_ConfigFile;
 

source/adios2/core/Stream.cpp

@@ -31,7 +31,7 @@ Stream::Stream(const std::string &name, const Mode mode, MPI_Comm comm,
 
 Stream::Stream(const std::string &name, const Mode mode,
                const std::string engineType, const std::string hostLanguage)
-: Stream(name, mode, MPI_COMM_SELF, engineType, hostLanguage)
+: Stream(name, mode, MPI_COMM_NULL, engineType, hostLanguage)
 {
 }
 
@@ -51,7 +51,7 @@ Stream::Stream(const std::string &name, const Mode mode, MPI_Comm comm,
 Stream::Stream(const std::string &name, const Mode mode,
                const std::string configFile, const std::string ioInConfigFile,
                const std::string hostLanguage)
-: Stream(name, mode, MPI_COMM_SELF, configFile, ioInConfigFile, hostLanguage)
+: Stream(name, mode, MPI_COMM_NULL, configFile, ioInConfigFile, hostLanguage)
 {
 }
 

source/adios2/helper/mpidummy.cpp

@@ -140,10 +140,6 @@ int MPI_Gather(const void *sendbuf, int sendcnt, MPI_Datatype sendtype,
     {
         RETURN_CHECK(MPI_ERR_ROOT);
     }
-    if (comm == MPI_COMM_NULL)
-    {
-        RETURN_CHECK(MPI_ERR_COMM);
-    }
 
     ier = MPIDUMMY::MPI_Type_size(sendtype, &n);
     if (ier != MPI_SUCCESS)
@@ -223,10 +219,6 @@ int MPI_Scatter(const void *sendbuf, int sendcnt, MPI_Datatype sendtype,
     {
         RETURN_CHECK(MPI_ERR_ROOT);
     }
-    if (comm == MPI_COMM_NULL)
-    {
-        RETURN_CHECK(MPI_ERR_COMM);
-    }
 
     ier = MPIDUMMY::MPI_Type_size(sendtype, &n);
     if (ier != MPI_SUCCESS)
@@ -307,77 +299,6 @@ int MPI_Wait(MPI_Request * /*request*/, MPI_Status * /*status*/)
     RETURN_CHECK(MPI_SUCCESS);
 }
 
-#ifndef ADIOS2_HAVE_MPI
-
-int MPI_File_open(MPI_Comm /*comm*/, const char *filename, int amode,
-                  MPI_Info /*info*/, MPI_File *fh)
-{
-    std::string mode;
-    if (amode | MPI_MODE_RDONLY)
-    {
-        mode += "r";
-    }
-    if (amode | MPI_MODE_WRONLY)
-    {
-        mode += "w";
-    }
-    if (amode | MPI_MODE_APPEND)
-    {
-        mode += "a";
-    }
-    mode += "b";
-
-    *fh = std::fopen(filename, mode.c_str());
-    if (!*fh)
-    {
-        return -1;
-    }
-    RETURN_CHECK(MPI_SUCCESS);
-}
-
-int MPI_File_close(MPI_File *fh) { return fclose(*fh); }
-
-int MPI_File_get_size(MPI_File fh, MPI_Offset *size)
-{
-    long curpos = std::ftell(fh);
-    fseek(fh, 0, SEEK_END); // go to end, returned is the size in bytes
-    long endpos = std::ftell(fh);
-    std::fseek(fh, curpos, SEEK_SET); // go back where we were
-    *size = static_cast<MPI_Offset>(endpos);
-    // printf("MPI_File_get_size: fh=%d, size=%lld\n", fh, *size);
-    RETURN_CHECK(MPI_SUCCESS);
-}
-
-int MPI_File_read(MPI_File fh, void *buf, int count, MPI_Datatype datatype,
-                  MPI_Status *status)
-{
-    // FIXME: int count can read only 2GB (*datatype size) array at max
-    size_t bytes_to_read = static_cast<size_t>(count) * datatype;
-    size_t bytes_read;
-    bytes_read = std::fread(buf, 1, bytes_to_read, fh);
-    if (bytes_read != bytes_to_read)
-    {
-        return -2;
-    }
-    *status = bytes_read;
-    // printf("MPI_File_read: fh=%d, count=%d, typesize=%d, bytes read=%lld\n",
-    // fh, count, datatype, *status);
-    RETURN_CHECK(MPI_SUCCESS);
-}
-
-int MPI_File_seek(MPI_File fh, MPI_Offset offset, int whence)
-{
-    return std::fseek(fh, offset, whence) == MPI_SUCCESS;
-}
-
-int MPI_Get_count(const MPI_Status *status, MPI_Datatype, int *count)
-{
-    *count = static_cast<int>(*status);
-    RETURN_CHECK(MPI_SUCCESS);
-}
-
-#endif
-
 double MPI_Wtime()
 {
     std::chrono::duration<double> now =

source/adios2/helper/mpidummy.h

@@ -24,10 +24,8 @@
 typedef int MPI_Comm;
 typedef uint64_t MPI_Status;
 typedef uint64_t MPI_Request;
-typedef FILE *MPI_File;
 typedef int MPI_Info;
 typedef int MPI_Datatype;
-typedef long int MPI_Offset;
 typedef int MPI_Fint;
 typedef int MPI_Op;
 
@@ -168,15 +166,6 @@ int MPI_Wait(MPI_Request *request, MPI_Status *status);
 
 int MPI_Type_size(MPI_Datatype datatype, int *size);
 
-int MPI_File_open(MPI_Comm comm, const char *filename, int amode, MPI_Info info,
-                  MPI_File *fh);
-int MPI_File_close(MPI_File *fh);
-int MPI_File_get_size(MPI_File fh, MPI_Offset *size);
-int MPI_File_read(MPI_File fh, void *buf, int count, MPI_Datatype datatype,
-                  MPI_Status *status);
-int MPI_File_seek(MPI_File fh, MPI_Offset offset, int whence);
-
-int MPI_Get_count(const MPI_Status *status, MPI_Datatype datatype, int *count);
 int MPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm *comm_out);
 
 int MPI_Get_processor_name(char *name, int *resultlen);

source/adios2/helper/mpiwrap.cpp

@@ -23,10 +23,10 @@ static inline bool SMPI_Available()
 #ifdef ADIOS2_HAVE_MPI
 
 // if MPI is available use it when possible, otherwise fall back to mpidummy
-#define DISPATCH(FUNC, ...)                                                    \
+#define DISPATCH(COMM, FUNC, ...)                                              \
     do                                                                         \
     {                                                                          \
-        if (SMPI_Available())                                                  \
+        if (COMM != MPI_COMM_NULL)                                             \
         {                                                                      \
             return ::FUNC(__VA_ARGS__);                                        \
         }                                                                      \
@@ -36,75 +36,76 @@ static inline bool SMPI_Available()
 #else
 
 // if compiled without MPI, always dispatch to mpidummy
-#define DISPATCH(FUNC, ...) return ::FUNC(__VA_ARGS__)
+#define DISPATCH(COMM, FUNC, ...) return ::FUNC(__VA_ARGS__)
 
 #endif
 
 int SMPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm)
 {
-    DISPATCH(MPI_Comm_dup, comm, newcomm);
+    DISPATCH(comm, MPI_Comm_dup, comm, newcomm);
 }
 
-int SMPI_Comm_free(MPI_Comm *comm) { DISPATCH(MPI_Comm_free, comm); }
+int SMPI_Comm_free(MPI_Comm *comm) { DISPATCH(*comm, MPI_Comm_free, comm); }
 
 int SMPI_Comm_rank(MPI_Comm comm, int *rank)
 {
-    DISPATCH(MPI_Comm_rank, comm, rank);
+    DISPATCH(comm, MPI_Comm_rank, comm, rank);
 }
 
 int SMPI_Comm_size(MPI_Comm comm, int *size)
 {
-    DISPATCH(MPI_Comm_size, comm, size);
+    DISPATCH(comm, MPI_Comm_size, comm, size);
 }
 
-int SMPI_Barrier(MPI_Comm comm) { DISPATCH(MPI_Barrier, comm); }
+int SMPI_Barrier(MPI_Comm comm) { DISPATCH(comm, MPI_Barrier, comm); }
 
 int SMPI_Bcast(void *buffer, int count, MPI_Datatype datatype, int root,
                MPI_Comm comm)
 {
-    DISPATCH(MPI_Bcast, buffer, count, datatype, root, comm);
+    DISPATCH(comm, MPI_Bcast, buffer, count, datatype, root, comm);
 }
 
 int SMPI_Gather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                 void *recvbuf, int recvcount, MPI_Datatype recvtype, int root,
                 MPI_Comm comm)
 {
-    DISPATCH(MPI_Gather, sendbuf, sendcount, sendtype, recvbuf, recvcount,
+    DISPATCH(comm, MPI_Gather, sendbuf, sendcount, sendtype, recvbuf, recvcount,
              recvtype, root, comm);
 }
 
 int SMPI_Gatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                  void *recvbuf, const int *recvcounts, const int *displs,
                  MPI_Datatype recvtype, int root, MPI_Comm comm)
 {
-    DISPATCH(MPI_Gatherv, sendbuf, sendcount, sendtype, recvbuf, recvcounts,
-             displs, recvtype, root, comm);
+    DISPATCH(comm, MPI_Gatherv, sendbuf, sendcount, sendtype, recvbuf,
+             recvcounts, displs, recvtype, root, comm);
 }
 
 int SMPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                    void *recvbuf, int recvcount, MPI_Datatype recvtype,
                    MPI_Comm comm)
 {
-    DISPATCH(MPI_Allgather, sendbuf, sendcount, sendtype, recvbuf, recvcount,
-             recvtype, comm);
+    DISPATCH(comm, MPI_Allgather, sendbuf, sendcount, sendtype, recvbuf,
+             recvcount, recvtype, comm);
 }
 
 int SMPI_Allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
                     void *recvbuf, int *recvcounts, int *displs,
                     MPI_Datatype recvtype, MPI_Comm comm)
 {
-    DISPATCH(MPI_Allgatherv, sendbuf, sendcount, sendtype, recvbuf, recvcounts,
-             displs, recvtype, comm);
+    DISPATCH(comm, MPI_Allgatherv, sendbuf, sendcount, sendtype, recvbuf,
+             recvcounts, displs, recvtype, comm);
 }
 
 int SMPI_Reduce(const void *sendbuf, void *recvbuf, int count,
                 MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm)
 {
-    DISPATCH(MPI_Reduce, sendbuf, recvbuf, count, datatype, op, root, comm);
+    DISPATCH(comm, MPI_Reduce, sendbuf, recvbuf, count, datatype, op, root,
+             comm);
 }
 
 int SMPI_Allreduce(const void *sendbuf, void *recvbuf, int count,
                    MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
 {
-    DISPATCH(MPI_Allreduce, sendbuf, recvbuf, count, datatype, op, comm);
+    DISPATCH(comm, MPI_Allreduce, sendbuf, recvbuf, count, datatype, op, comm);
 }

source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp

@@ -17,7 +17,7 @@ namespace adios2
 namespace aggregator
 {
 
-MPIAggregator::MPIAggregator() : m_Comm(MPI_COMM_SELF) {}
+MPIAggregator::MPIAggregator() : m_Comm(MPI_COMM_NULL) {}
 
 MPIAggregator::~MPIAggregator()
 {

source/adios2/toolkit/transport/Transport.cpp

@@ -20,8 +20,6 @@ Transport::Transport(const std::string type, const std::string library,
                      MPI_Comm mpiComm, const bool debugMode)
 : m_Type(type), m_Library(library), m_MPIComm(mpiComm), m_DebugMode(debugMode)
 {
-    SMPI_Comm_rank(m_MPIComm, &m_RankMPI);
-    SMPI_Comm_size(m_MPIComm, &m_SizeMPI);
 }
 
 void Transport::IWrite(const char *buffer, size_t size, Status &status,

source/adios2/toolkit/transport/Transport.h

@@ -34,8 +34,6 @@ class Transport
     Mode m_OpenMode = Mode::Undefined; ///< at Open from ADIOSTypes.h
     bool m_IsOpen = false; ///< true: open for communication, false: unreachable
     MPI_Comm m_MPIComm;    ///< current MPI communicator
-    int m_RankMPI = 0;     ///< from MPI_Comm_Rank
-    int m_SizeMPI = 1;     ///< from MPI_Comm_Size
     profiling::IOChrono m_Profiler; ///< profiles Open, Write/Read, Close
 
     struct Status