src: Experiment with changes to shmem_transport_probe

Sandia-OpenSHMEM · Jun 11, 2024 · f7e20dc · f7e20dc
1 parent 4e08aef
commit f7e20dc
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 10 deletions.
diff --git a/src/shmem_comm.h b/src/shmem_comm.h
@@ -251,11 +251,12 @@ shmem_internal_atomic(shmem_ctx_t ctx, void *target, const void *source, size_t
            the CXI provider */
         unsigned long long tmp_fetch = 0;
         shmem_transport_fetch_atomic((shmem_transport_ctx_t *)ctx, target,
-                                     source, &tmp_fetch, len, pe, op, datatype);
-        shmem_transport_get_wait((shmem_transport_ctx_t *)ctx);
+                                     source, &tmp_fetch, len, pe, op, datatype, nic_idx);
+        shmem_transport_get_wait((shmem_transport_ctx_t *)ctx, nic_idx);
 #else
         shmem_transport_atomic((shmem_transport_ctx_t *)ctx, target, source,
                                len, pe, op, datatype, nic_idx);
+#endif
     }
 }
 
@@ -291,11 +292,12 @@ shmem_internal_atomic_set(shmem_ctx_t ctx, void *target, const void *source, siz
            the CXI provider */
         unsigned long long tmp_fetch = 0;
         shmem_transport_fetch_atomic((shmem_transport_ctx_t *)ctx, target,
-                                     source, &tmp_fetch, len, pe, FI_ATOMIC_WRITE, datatype);
-        shmem_transport_get_wait((shmem_transport_ctx_t *)ctx);
+                                     source, &tmp_fetch, len, pe, FI_ATOMIC_WRITE, datatype, nic_idx);
+        shmem_transport_get_wait((shmem_transport_ctx_t *)ctx, nic_idx);
 #else
         shmem_transport_atomic_set((shmem_transport_ctx_t *)ctx, target,
                                    source, len, pe, datatype, nic_idx);
+#endif
     }
 }
 

diff --git a/src/transport_ofi.c b/src/transport_ofi.c
@@ -2137,8 +2137,8 @@ int shmem_transport_ctx_create(struct shmem_internal_team_t *team, long options,
     ctxp->stx_idx = malloc(shmem_transport_ofi_num_nics * sizeof(int));
     for (size_t idx = 0; idx < shmem_transport_ofi_num_nics; idx++) {
 #ifndef USE_CTX_LOCK
-        shmem_internal_cntr_write(&ctxp->pending_put_cntr, 0);
-        shmem_internal_cntr_write(&ctxp->pending_get_cntr, 0);
+        shmem_internal_cntr_write(&ctxp->pending_put_cntr[idx], 0);
+        shmem_internal_cntr_write(&ctxp->pending_get_cntr[idx], 0);
 #else
         ctxp->pending_put_cntr[idx] = 0;
         ctxp->pending_get_cntr[idx] = 0;

diff --git a/src/transport_ofi.h b/src/transport_ofi.h
@@ -389,10 +389,16 @@ void shmem_transport_probe(void)
 #  ifdef USE_THREAD_COMPLETION
     if (0 == pthread_mutex_trylock(&shmem_transport_ofi_progress_lock)) {
 #  endif
-        struct fi_cq_entry buf;
-        int ret = fi_cq_read(shmem_transport_ofi_target_cq, &buf, 1);
-        if (ret == 1)
-            RAISE_WARN_STR("Unexpected event");
+//        struct fi_cq_entry buf;
+//        int ret = fi_cq_read(shmem_transport_ofi_target_cq, &buf, 1);
+//        if (ret == 1)
+//            RAISE_WARN_STR("Unexpected event");
+        for (size_t i = 0; i < shmem_transport_ofi_num_nics; i++) {
+            struct fi_cq_entry buf;
+            int ret = fi_cq_read(shmem_transport_ctx_default.cq[i], &buf, 1);
+            if (ret == 1)
+                RAISE_WARN_STR("Unexpected event");
+        }
 #  ifdef USE_THREAD_COMPLETION
         pthread_mutex_unlock(&shmem_transport_ofi_progress_lock);
     }