add MPI_Init and MPI_Finalize PMPI wrappers

LLNL · Apr 2, 2019 · 0be4071 · 0be4071
1 parent e6894ca
commit 0be4071
Show file tree

Hide file tree

Showing 7 changed files with 187 additions and 89 deletions.
diff --git a/client/src/Makefile.am b/client/src/Makefile.am
@@ -7,70 +7,64 @@ AM_CFLAGS = -Wall -Wno-strict-aliasing
 
 include_HEADERS = unifycr.h
 
-libunifycr_la_SOURCES = \
-  unifycr.h \
-  unifycr-fixed.c \
-  unifycr-fixed.h \
-  unifycr-stack.c \
-  unifycr-stack.h \
-  unifycr-stdio.c \
-  unifycr-stdio.h \
-  unifycr-sysio.c \
-  unifycr-sysio.h \
-  unifycr-dirops.h \
-  unifycr-dirops.c \
-  unifycr.c \
-  unifycr.h \
-  unifycr-internal.h \
-  unifycr_client.c \
-  flatbuffers_common_builder.h \
-  flatbuffers_common_reader.h \
-  ucr_read_builder.h \
-  ucr_read_reader.h \
-  uthash.h \
-  utlist.h
-
-libunifycr_la_CPPFLAGS = \
+CLIENT_COMMON_CPPFLAGS = \
   -I$(top_builddir)/client \
   -I$(top_srcdir)/common/src
 
-libunifycr_la_CFLAGS = $(MPI_CFLAGS) $(MERCURY_CFLAGS) $(ARGOBOTS_CFLAGS) $(MARGO_CFLAGS) $(FLATCC_CFLAGS)
-libunifycr_la_LDFLAGS = -version-info $(LIBUNIFYCR_LT_VERSION) $(MPI_CLDFLAGS) $(MERCURY_LDFLAGS) $(MERCURY_LIBS) $(ARGOBOTS_LDFLAGS) $(ARGOBOTS_LIBS) $(MARGO_LDFLAGS) $(MARGO_LIBS) $(FLATCC_LDFLAGS) $(FLATCC_LIBS)
-libunifycr_la_LIBADD = $(top_builddir)/common/src/libunifycr_common.la \
-					   -lcrypto -lrt -lpthread
+CLIENT_COMMON_CFLAGS = \
+  $(MPI_CFLAGS) \
+  $(MERCURY_CFLAGS) $(ARGOBOTS_CFLAGS) $(MARGO_CFLAGS) \
+  $(FLATCC_CFLAGS)
 
-libunifycr_gotcha_la_SOURCES = \
+CLIENT_COMMON_LDFLAGS = \
+  -version-info $(LIBUNIFYCR_LT_VERSION) \
+  $(MPI_CLDFLAGS) \
+  $(MERCURY_LDFLAGS) $(MERCURY_LIBS) \
+  $(ARGOBOTS_LDFLAGS) $(ARGOBOTS_LIBS) \
+  $(MARGO_LDFLAGS) $(MARGO_LIBS) \
+  $(FLATCC_LDFLAGS) $(FLATCC_LIBS)
+
+CLIENT_COMMON_LIBADD = \
+  $(top_builddir)/common/src/libunifycr_common.la \
+  -lcrypto -lrt -lpthread
+
+CLIENT_COMMON_SOURCES = \
+  flatbuffers_common_builder.h \
+  flatbuffers_common_reader.h \
+  ucr_read_builder.h \
+  ucr_read_reader.h \
+  unifycr.c \
   unifycr.h \
+  unifycr-dirops.h \
+  unifycr-dirops.c \
   unifycr-fixed.c \
   unifycr-fixed.h \
+  unifycr-internal.h \
   unifycr-stack.c \
   unifycr-stack.h \
   unifycr-stdio.c \
   unifycr-stdio.h \
   unifycr-sysio.c \
   unifycr-sysio.h \
-  unifycr-dirops.h \
-  unifycr-dirops.c \
-  unifycr.c \
-  unifycr.h \
-  gotcha_map_unifycr_list.h \
-  unifycr-internal.h \
   unifycr_client.c \
-  flatbuffers_common_builder.h \
-  flatbuffers_common_reader.h \
-  ucr_read_builder.h \
-  ucr_read_reader.h \
   uthash.h \
   utlist.h
 
-libunifycr_gotcha_la_CPPFLAGS = \
-  -DUNIFYCR_GOTCHA \
-  -I$(top_builddir)/client \
-  -I$(top_srcdir)/common/src
+if USE_PMPI_WRAPPERS
+CLIENT_COMMON_SOURCES += \
+  pmpi_wrappers.c \
+  pmpi_wrappers.h
+endif
 
-libunifycr_gotcha_la_CFLAGS = $(GOTCHA_CFLAGS) $(MPI_CFLAGS) $(MERCURY_CFLAGS) $(ARGOBOTS_CFLAGS) $(MARGO_CFLAGS) $(FLATCC_CFLAGS)
-libunifycr_gotcha_la_LDFLAGS = -version-info $(LIBUNIFYCR_LT_VERSION) $(GOTCHA_LDFLAGS) $(MPI_CLDFLAGS) $(MERCURY_LDFLAGS) $(MERCURY_LIBS) $(ARGOBOTS_LDFLAGS) $(ARGOBOTS_LIBS) $(MARGO_LDFLAGS) $(MARGO_LIBS) $(FLATCC_LDFLAGS) $(FLATCC_LIBS)
-libunifycr_gotcha_la_LIBADD = $(top_builddir)/common/src/libunifycr_common.la \
-                              -lgotcha -lcrypto -lrt -lpthread
+libunifycr_la_SOURCES  = $(CLIENT_COMMON_SOURCES)
+libunifycr_la_CPPFLAGS = $(CLIENT_COMMON_CPPFLAGS)
+libunifycr_la_CFLAGS   = $(CLIENT_COMMON_CFLAGS)
+libunifycr_la_LDFLAGS  = $(CLIENT_COMMON_LDFLAGS)
+libunifycr_la_LIBADD   = $(CLIENT_COMMON_LIBADD)
 
+libunifycr_gotcha_la_SOURCES  = $(CLIENT_COMMON_SOURCES) gotcha_map_unifycr_list.h
+libunifycr_gotcha_la_CPPFLAGS = $(CLIENT_COMMON_CPPFLAGS) -DUNIFYCR_GOTCHA
+libunifycr_gotcha_la_CFLAGS   = $(CLIENT_COMMON_CFLAGS) $(GOTCHA_CFLAGS)
+libunifycr_gotcha_la_LDFLAGS  = $(CLIENT_COMMON_LDFLAGS) $(GOTCHA_LDFLAGS)
+libunifycr_gotcha_la_LIBADD   = $(CLIENT_COMMON_LIBADD) -lgotcha
 
diff --git a/client/src/pmpi_wrappers.c b/client/src/pmpi_wrappers.c
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2019, Lawrence Livermore National Security, LLC.
+ * Produced at the Lawrence Livermore National Laboratory.
+ *
+ * Copyright 2019, UT-Battelle, LLC.
+ *
+ * LLNL-CODE-741539
+ * All rights reserved.
+ *
+ * This is the license for UnifyCR.
+ * For details, see https://github.com/LLNL/UnifyCR.
+ * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text.
+ */
+
+#include "pmpi_wrappers.h"
+#include "unifycr.h"
+#include <mpi.h>
+#include <stdio.h>
+
+int unifycr_mpi_init(int* argc, char*** argv)
+{
+    int rc, ret;
+    int rank;
+    int world_sz = 0;
+    int app_id = 0;
+
+    //fprintf(stderr, "DEBUG: %s - before PMPI_Init()\n", __func__);
+
+    ret = PMPI_Init(argc, argv);
+
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &world_sz);
+
+    //fprintf(stderr, "DEBUG: %s - after PMPI_Init(), rank=%d ret=%d\n",
+    //        __func__, rank, ret);
+
+    rc = unifycr_mount("/unifycr", rank, (size_t)world_sz, app_id);
+    if (UNIFYCR_SUCCESS != rc) {
+        fprintf(stderr, "UNIFYCR ERROR: unifycr_mount() failed with '%s'\n",
+                unifycr_error_enum_description((unifycr_error_e)rc));
+    }
+
+    return ret;
+}
+
+int MPI_Init(int* argc, char*** argv)
+{
+    return unifycr_mpi_init(argc, argv);
+}
+
+int unifycr_mpi_finalize(void)
+{
+    int rc, ret;
+
+    rc = unifycr_unmount();
+    if (UNIFYCR_SUCCESS != rc) {
+        fprintf(stderr, "UNIFYCR ERROR: unifycr_unmount() failed with '%s'\n",
+                unifycr_error_enum_description((unifycr_error_e)rc));
+    }
+
+    //fprintf(stderr, "DEBUG: %s - before PMPI_Finalize()\n", __func__);
+
+    ret = PMPI_Finalize();
+
+    //fprintf(stderr, "DEBUG: %s - after PMPI_Finalize(), ret=%d\n",
+    //        __func__, ret);
+
+    return ret;
+}
+
+int MPI_Finalize(void)
+{
+    return unifycr_mpi_finalize();
+}
diff --git a/client/src/pmpi_wrappers.h b/client/src/pmpi_wrappers.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2019, Lawrence Livermore National Security, LLC.
+ * Produced at the Lawrence Livermore National Laboratory.
+ *
+ * Copyright 2019, UT-Battelle, LLC.
+ *
+ * LLNL-CODE-741539
+ * All rights reserved.
+ *
+ * This is the license for UnifyCR.
+ * For details, see https://github.com/LLNL/UnifyCR.
+ * Please read https://github.com/LLNL/UnifyCR/LICENSE for full license text.
+ */
+
+#ifndef UNIFYCR_PMPI_WRAPPERS_H
+#define UNIFYCR_PMPI_WRAPPERS_H
+
+/* MPI_Init PMPI wrapper */
+int unifycr_mpi_init(int* argc, char*** argv);
+int MPI_Init(int* argc, char*** argv);
+
+/* MPI_Finalize PMPI wrapper */
+int unifycr_mpi_finalize(void);
+int MPI_Finalize(void);
+
+#endif /* UNIFYCR_PMPI_WRAPPERS_H */
diff --git a/client/src/unifycr.c b/client/src/unifycr.c
@@ -1,8 +1,8 @@
 /*
- * Copyright (c) 2017, Lawrence Livermore National Security, LLC.
+ * Copyright (c) 2019, Lawrence Livermore National Security, LLC.
  * Produced at the Lawrence Livermore National Laboratory.
  *
- * Copyright 2017, UT-Battelle, LLC.
+ * Copyright 2019, UT-Battelle, LLC.
  *
  * LLNL-CODE-741539
  * All rights reserved.
@@ -69,6 +69,9 @@
 
 #include "unifycr_log.h"
 
+/* avoid duplicate mounts (for now) */
+static int unifycr_mounted = -1;
+
 /* global rpc context (probably should find a better spot for this) */
 unifycr_client_rpc_context_t* unifycr_rpc_context = NULL;
 
@@ -2639,14 +2642,23 @@ int unifycr_mount(const char prefix[], int rank, size_t size,
     bool b;
     char* cfgval;
 
-    glb_rank = rank;
-
-    /* print log messages to stderr */
-    unifycr_log_open(NULL);
+    if (-1 != unifycr_mounted) {
+        if (l_app_id != unifycr_mounted) {
+            LOGERR("multiple mount support not yet implemented");
+            return UNIFYCR_FAILURE;
+        } else {
+            LOGDBG("already mounted");
+            return UNIFYCR_SUCCESS;
+        }
+    }
 
     /* record our rank for debugging messages,
      * record the value we should use for an app_id */
     app_id = l_app_id;
+    glb_rank = rank;
+
+    /* print log messages to stderr */
+    unifycr_log_open(NULL);
 
     /************************
      * read configuration values
@@ -2713,10 +2725,6 @@ int unifycr_mount(const char prefix[], int rank, size_t size,
         return ret;
     }
 
-    /************************
-     * establish connection to server
-     ************************/
-
     /* open rpc connection to server */
     char* addr_string = rpc_lookup_server_addr();
     if (addr_string == NULL) {
@@ -2754,23 +2762,22 @@ int unifycr_mount(const char prefix[], int rank, size_t size,
         return UNIFYCR_FAILURE;
     }
 
-    /************************
-     * create a local entry for our mount point directory
-     ************************/
-
     /* add mount point as a new directory in the file list */
     if (unifycr_get_fid_from_path(prefix) < 0) {
         /* no entry exists for mount point, so create one */
         int fid = unifycr_fid_create_directory(prefix);
         if (fid < 0) {
             /* if there was an error, return it */
             LOGERR("failed to create directory entry for mount point: `%s'",
-                prefix);
+                   prefix);
             return UNIFYCR_FAILURE;
         }
     }
 
-    return rc;
+    /* record client state as mounted for specific app_id */
+    unifycr_mounted = app_id;
+
+    return UNIFYCR_SUCCESS;
 }
 
 /* free resources allocated in corresponding call
@@ -2859,6 +2866,10 @@ int unifycr_unmount(void)
     int rc;
     int ret = UNIFYCR_SUCCESS;
 
+    if (-1 == unifycr_mounted) {
+        return UNIFYCR_SUCCESS;
+    }
+
     /************************
      * tear down connection to server
      ************************/
@@ -2917,5 +2928,7 @@ int unifycr_unmount(void)
     /* shut down our logging */
     unifycr_log_close();
 
+    unifycr_mounted = -1;
+
     return ret;
 }
diff --git a/client/src/unifycr.h b/client/src/unifycr.h
@@ -73,10 +73,6 @@ typedef struct {
     int rank;
 } name_rank_pair_t;
 
-#if 0
-int unifycr_mount(const char prefix[], int rank, size_t size,
-                  int l_app_id, int subtype);
-#endif
 int unifycr_mount(const char prefix[], int rank, size_t size,
                   int l_app_id);
 int unifycr_unmount(void);