Merge branch 'master' into pmi-config

src/mpid/ch4/netmod/ofi/ofi_init.h

@@ -14,6 +14,7 @@
 #include "ofi_impl.h"
 #include "mpir_cvars.h"
 #include "pmi.h"
+#include "mpidu_shm.h"
 
 /*
 === BEGIN_MPI_T_CVAR_INFO_BLOCK ===
@@ -714,6 +715,10 @@ static inline int MPIDI_NM_mpi_init_hook(int rank,
                                                      &MPIDI_Global.ep, 0));
 
     if (do_av_insert) {
+        int local_rank, num_local = 0, local_rank_0 = -1;
+        MPIDU_shm_seg_t memory;
+        MPIDU_shm_barrier_t *barrier;
+        int start, end;
 
         /* ---------------------------------- */
         /* Get our endpoint name and publish  */
@@ -741,17 +746,32 @@ static inline int MPIDI_NM_mpi_init_hook(int rank,
 #else
         MPIDI_OFI_PMI_CALL_POP(PMI_KVS_Put(MPIDI_Global.kvsname, keyS, val), pmi);
         MPIDI_OFI_PMI_CALL_POP(PMI_KVS_Commit(MPIDI_Global.kvsname), pmi);
-        MPIDI_OFI_PMI_CALL_POP(PMI_Barrier(), pmi);
 #endif
 
+        for (i = 0; i < size; i++) {
+            if (MPIDI_CH4_rank_is_local(i, MPIR_Process.comm_world)) {
+                if (i == rank)
+                    local_rank = num_local;
+
+                if (local_rank_0 == -1)
+                    local_rank_0 = i;
+
+                num_local++;
+            }
+        }
+
+        MPIDU_shm_seg_alloc(size * MPIDI_Global.addrnamelen, (void **)&table);
+        MPIDU_shm_seg_commit(&memory, &barrier, num_local, local_rank, local_rank_0, rank);
+
         /* -------------------------------- */
         /* Create our address table from    */
         /* encoded KVS values               */
         /* -------------------------------- */
-        table = (char *) MPL_malloc(size * MPIDI_Global.addrnamelen);
-        maxlen = MPIDI_KVSAPPSTRLEN;
-
-        for (i = 0; i < size; i++) {
+        start = local_rank * (size / num_local);
+        end = start + (size / num_local);
+        if (local_rank == num_local - 1)
+            end += size % num_local;
+        for (i = start; i < end; i++) {
             sprintf(keyS, "OFI-%d", i);
 #ifdef USE_PMI2_API
             MPIDI_OFI_PMI_CALL_POP(PMI2_KVS_Get
@@ -765,6 +785,9 @@ static inline int MPIDI_NM_mpi_init_hook(int rank,
                                (valS, "OFI", (char *) &table[i * MPIDI_Global.addrnamelen],
                                 MPIDI_Global.addrnamelen, &maxlen), buscard_len);
         }
+        mpi_errno = MPIDU_shm_barrier(barrier, num_local);
+        if (mpi_errno)
+            MPIR_ERR_POP(mpi_errno);
 
         /* -------------------------------- */
         /* Table is constructed.  Map it    */
@@ -776,6 +799,11 @@ static inline int MPIDI_NM_mpi_init_hook(int rank,
             MPIDI_OFI_AV(&MPIDIU_get_av(0, i)).dest = mapped_table[i];
         }
         MPL_free(mapped_table);
+
+        mpi_errno = MPIDU_shm_barrier(barrier, num_local);
+        if (mpi_errno)
+            MPIR_ERR_POP(mpi_errno);
+        MPIDU_shm_seg_destroy(&memory, num_local);
     }
 
     /* -------------------------------- */
@@ -871,9 +899,6 @@ static inline int MPIDI_NM_mpi_init_hook(int rank,
 
     fi_freeinfo(hints);
 
-    if (table)
-        MPL_free(table);
-
     MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_NETMOD_OFI_INIT);
     return mpi_errno;
   fn_fail:

src/mpid/ch4/netmod/ofi/util.c

@@ -700,6 +700,10 @@ static inline void create_dt_map()
     dtsize[FI_LONG_DOUBLE] = sizeof(long double);
     dtsize[FI_LONG_DOUBLE_COMPLEX] = sizeof(long double complex);
 
+    /* when atomics are disabled and atomics capability are not
+     * enabled call fo fi_atomic*** may crash */
+    MPIR_Assert(MPIDI_OFI_ENABLE_ATOMICS);
+
     for (i = 0; i < MPIDI_OFI_DT_SIZES; i++)
         for (j = 0; j < MPIDI_OFI_OP_SIZES; j++) {
             enum fi_datatype fi_dt = (enum fi_datatype) -1;
@@ -817,5 +821,8 @@ void MPIDI_OFI_index_datatypes()
     add_index(MPI_LONG_INT, &index);
     add_index(MPI_SHORT_INT, &index);   /* 60 */
     add_index(MPI_LONG_DOUBLE_INT, &index);
-    create_dt_map();
+
+    /* do not generate map when atomics are not enabled */
+    if(MPIDI_OFI_ENABLE_ATOMICS)
+        create_dt_map();
 }

src/mpid/ch4/src/ch4_init.h

@@ -213,13 +213,6 @@ MPL_STATIC_INLINE_PREFIX int MPID_Init(int *argc,
     MPIR_Datatype_init();
     MPIR_Group_init();
 
-    mpi_errno = MPIDI_NM_mpi_init_hook(rank, size, appnum, &MPIR_Process.attrs.tag_ub,
-                                       MPIR_Process.comm_world,
-                                       MPIR_Process.comm_self, has_parent, 1, &netmod_contexts);
-    if (mpi_errno != MPI_SUCCESS) {
-        MPIR_ERR_POPFATAL(mpi_errno);
-    }
-
 #ifdef MPIDI_BUILD_CH4_LOCALITY_INFO
     int i;
     for (i = 0; i < MPIR_Process.comm_world->local_size; i++) {
@@ -245,6 +238,13 @@ MPL_STATIC_INLINE_PREFIX int MPID_Init(int *argc,
     }
 #endif
 
+    mpi_errno = MPIDI_NM_mpi_init_hook(rank, size, appnum, &MPIR_Process.attrs.tag_ub,
+                                       MPIR_Process.comm_world,
+                                       MPIR_Process.comm_self, has_parent, 1, &netmod_contexts);
+    if (mpi_errno != MPI_SUCCESS) {
+        MPIR_ERR_POPFATAL(mpi_errno);
+    }
+
 #ifdef MPIDI_BUILD_CH4_SHM
     mpi_errno = MPIDI_SHM_mpi_init_hook(rank, size);
 

src/mpid/ch4/subconfigure.m4

@@ -371,19 +371,8 @@ if test "$enable_ch4r_per_comm_msg_queue" = "yes" ; then
         [Define if CH4U will use per-communicator message queues])
 fi
 
-PAC_ARG_SHARED_MEMORY
-
-AC_CONFIG_FILES([
-src/mpid/ch4/src/mpid_ch4_net_array.c
-src/mpid/ch4/include/netmodpre.h
-src/mpid/ch4/include/shmpre.h
-])
-PAC_ARG_SHARED_MEMORY
-])dnl end AM_COND_IF(BUILD_CH4,...)
-
-AM_CONDITIONAL([BUILD_CH4_SHM],[test "$ch4_shm_level" = "yes" -o "$ch4_shm_level" = "exclusive"])
-
 AC_CHECK_HEADERS(sys/mman.h sys/stat.h fcntl.h)
+AC_CHECK_FUNC(mmap, [], [AC_MSG_ERROR(mmap is required to build CH4)])
 
 gl_FUNC_RANDOM_R
 if test "$HAVE_RANDOM_R" = "1" -a "$HAVE_STRUCT_RANDOM_DATA" = "1" ; then
@@ -393,6 +382,20 @@ else
     AC_MSG_NOTICE([Using a non-symmetric heap])
 fi
 
+AC_CHECK_FUNCS(gethostname)
+if test "$ac_cv_func_gethostname" = "yes" ; then
+    # Do we need to declare gethostname?
+    PAC_FUNC_NEEDS_DECL([#include <unistd.h>],gethostname)
+fi
+
+AC_CONFIG_FILES([
+src/mpid/ch4/src/mpid_ch4_net_array.c
+src/mpid/ch4/include/netmodpre.h
+src/mpid/ch4/include/shmpre.h
+])
+])dnl end AM_COND_IF(BUILD_CH4,...)
+
+AM_CONDITIONAL([BUILD_CH4_SHM],[test "$ch4_shm_level" = "yes" -o "$ch4_shm_level" = "exclusive"])
 
 ])dnl end _BODY
 

src/mpid/common/shm/mpidu_shm_alloc.c

@@ -50,6 +50,8 @@ static int check_alloc(MPIDU_shm_seg_t *memory, MPIDU_shm_barrier_t *barrier,
 
 static size_t segment_len = 0;
 
+static int num_segments = 0;
+
 typedef struct asym_check_region 
 {
     void *base_ptr;
@@ -337,7 +339,7 @@ int MPIDU_shm_seg_commit(MPIDU_shm_seg_t *memory, MPIDU_shm_barrier_t **barrier,
 
             /* post name of shared file */
             MPIR_Assert(local_procs_0 == rank);
-            MPL_snprintf(key, key_max_sz, "sharedFilename[%i]", rank);
+            MPL_snprintf(key, key_max_sz, "sharedFilename[%i]-%i", rank, num_segments);
 
             mpi_errno = MPL_shm_hnd_get_serialized_by_ref(memory->hnd, &serialized_hnd);
             if (mpi_errno != MPI_SUCCESS) MPIR_ERR_POP (mpi_errno);
@@ -367,7 +369,7 @@ int MPIDU_shm_seg_commit(MPIDU_shm_seg_t *memory, MPIDU_shm_barrier_t **barrier,
             MPIR_ERR_CHKANDJUMP1 (pmi_errno != PMI_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**pmi_barrier", "**pmi_barrier %d", pmi_errno);
 
             /* get name of shared file */
-            MPL_snprintf(key, key_max_sz, "sharedFilename[%i]", local_procs_0);
+            MPL_snprintf(key, key_max_sz, "sharedFilename[%i]-%i", local_procs_0, num_segments);
             pmi_errno = PMI_KVS_Get(kvs_name, key, val, val_max_sz);
             MPIR_ERR_CHKANDJUMP1(pmi_errno != PMI_SUCCESS, mpi_errno, MPI_ERR_OTHER,
                                  "**pmi_kvs_get", "**pmi_kvs_get %d", pmi_errno);
@@ -401,6 +403,8 @@ int MPIDU_shm_seg_commit(MPIDU_shm_seg_t *memory, MPIDU_shm_barrier_t **barrier,
         memory->symmetrical = 0 ;
     }
 #endif
+    num_segments++;
+
     /* assign sections of the shared memory segment to their pointers */
 
     start_addr = current_addr;

src/pm/hydra/configure.ac

@@ -658,6 +658,7 @@ for option in $enable_g ; do
         all)
 		PAC_APPEND_FLAG(-g, CFLAGS)
 		AC_DEFINE(USE_MEMORY_TRACING,1,[Define if memory tracing is enabled])
+		AC_DEFINE(PMI_KEY_CHECK,1,[Define if we should check for PMI key collisions])
 		;;
         *)
 		;;

src/pm/hydra/include/hydra.h

@@ -15,6 +15,7 @@
 #include "hydra_config.h"
 
 #include "mpl.h"
+#include "mpl_uthash.h"
 
 extern char *HYD_dbg_prefix;
 
@@ -315,6 +316,7 @@ struct HYD_exec {
 struct HYD_pg {
     int pgid;
     struct HYD_proxy *proxy_list;
+    int proxy_count;
     int pg_process_count;
     int barrier_count;
 
@@ -369,6 +371,8 @@ struct HYD_proxy {
     int control_fd;
 
     struct HYD_proxy *next;
+
+    MPL_UT_hash_handle hh;
 };
 
 /* Global user parameters */

src/pm/hydra/include/hydra_server.h

@@ -38,6 +38,9 @@ struct HYD_server_info_s {
     /* Process groups */
     struct HYD_pg pg_list;
 
+    /* Hash for fast proxy lookup */
+    struct HYD_proxy *proxy_hash;
+
     /* Cleanup */
     int cmd_pipe[2];
 

src/pm/hydra/pm/pmiserv/common.c

@@ -128,6 +128,7 @@ HYD_status HYD_pmcd_pmi_allocate_kvs(struct HYD_pmcd_pmi_kvs ** kvs, int pgid)
     HYDU_MALLOC_OR_JUMP(*kvs, struct HYD_pmcd_pmi_kvs *, sizeof(struct HYD_pmcd_pmi_kvs), status);
     MPL_snprintf((*kvs)->kvsname, PMI_MAXKVSLEN, "kvs_%d_%d", (int) getpid(), pgid);
     (*kvs)->key_pair = NULL;
+    (*kvs)->tail = NULL;
 
   fn_exit:
     HYDU_FUNC_EXIT();
@@ -156,7 +157,7 @@ void HYD_pmcd_free_pmi_kvs_list(struct HYD_pmcd_pmi_kvs *kvs_list)
 
 HYD_status HYD_pmcd_pmi_add_kvs(const char *key, char *val, struct HYD_pmcd_pmi_kvs *kvs, int *ret)
 {
-    struct HYD_pmcd_pmi_kvs_pair *key_pair, *run, *last;
+    struct HYD_pmcd_pmi_kvs_pair *key_pair;
     HYD_status status = HYD_SUCCESS;
 
     HYDU_FUNC_ENTER();
@@ -171,8 +172,12 @@ HYD_status HYD_pmcd_pmi_add_kvs(const char *key, char *val, struct HYD_pmcd_pmi_
 
     if (kvs->key_pair == NULL) {
         kvs->key_pair = key_pair;
+        kvs->tail = key_pair;
     }
     else {
+#ifdef PMI_KEY_CHECK
+        struct HYD_pmcd_pmi_kvs_pair *run, *last;
+
         for (run = kvs->key_pair; run; run = run->next) {
             if (!strcmp(run->key, key_pair->key)) {
                 /* duplicate key found */
@@ -183,6 +188,10 @@ HYD_status HYD_pmcd_pmi_add_kvs(const char *key, char *val, struct HYD_pmcd_pmi_
         }
         /* Add key_pair to end of list. */
         last->next = key_pair;
+#else
+        kvs->tail->next = key_pair;
+        kvs->tail = key_pair;
+#endif
     }
 
   fn_exit:

src/pm/hydra/pm/pmiserv/common.h

@@ -26,6 +26,7 @@ struct HYD_pmcd_pmi_kvs_pair {
 struct HYD_pmcd_pmi_kvs {
     char kvsname[PMI_MAXKVSLEN];        /* Name of this kvs */
     struct HYD_pmcd_pmi_kvs_pair *key_pair;
+    struct HYD_pmcd_pmi_kvs_pair *tail;
 };
 
 struct HYD_pmcd_hdr {