ch4: fix vci locking in partitioned communication

In general, we only enter critical sections once at entrance functions, e.g. MPIDIG_mpi_psend_init, MPIDIG_mpi_precv_init. We assume already in critical section with secondary functions including active message handlers.
pmodels · Jun 8, 2021 · a12a700 · a12a700
1 parent ee9a8e2
commit a12a700
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 6 deletions.
diff --git a/src/mpid/ch4/generic/am/mpidig_am_part.c b/src/mpid/ch4/generic/am/mpidig_am_part.c
@@ -17,8 +17,6 @@ static int part_req_create(void *buf, int partitions, MPI_Aint count,
     MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_PART_REQ_CREATE);
     MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_PART_REQ_CREATE);
 
-    MPID_THREAD_CS_ENTER(VCI, MPIDI_VCI(0).lock);
-
     /* Set refcnt=1 for user-defined partitioned pattern; decrease at request_free. */
     MPIDI_CH4_REQUEST_CREATE(req, kind, 0, 1);
     MPIR_ERR_CHKANDSTMT((req) == NULL, mpi_errno, MPIX_ERR_NOREQ, goto fn_fail, "**nomemreq");
@@ -45,7 +43,6 @@ static int part_req_create(void *buf, int partitions, MPI_Aint count,
     *req_ptr = req;
 
   fn_exit:
-    MPID_THREAD_CS_EXIT(VCI, MPIDI_VCI(0).lock);
     MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_PART_REQ_CREATE);
     return mpi_errno;
   fn_fail:
@@ -116,6 +113,7 @@ int MPIDIG_mpi_precv_init(void *buf, int partitions, int count,
     MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_MPI_PRECV_INIT);
     MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_MPI_PRECV_INIT);
 
+    MPID_THREAD_CS_ENTER(VCI, MPIDI_VCI(0).lock);
 
     /* Create and initialize device-layer partitioned request */
     mpi_errno = part_req_create(buf, partitions, count, datatype, source, tag, comm,
@@ -148,6 +146,7 @@ int MPIDIG_mpi_precv_init(void *buf, int partitions, int count,
 #endif
 
   fn_exit:
+    MPID_THREAD_CS_EXIT(VCI, MPIDI_VCI(0).lock);
     MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_MPI_PRECV_INIT);
     return mpi_errno;
   fn_fail:

diff --git a/src/mpid/ch4/generic/am/mpidig_am_part.h b/src/mpid/ch4/generic/am/mpidig_am_part.h
@@ -57,7 +57,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDIG_post_pready(MPIR_Request * part_sreq, int is
     MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_POST_PREADY);
     MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_POST_PREADY);
 
-    MPID_THREAD_CS_ENTER(VCI, MPIDI_VCI(0).lock);
+    /* note: already in critical section */
 
     /* Send data when all partitions are ready */
     if (MPIR_cc_get(MPIDIG_PART_REQUEST(part_sreq, u.send).ready_cntr) ==
@@ -68,7 +68,6 @@ MPL_STATIC_INLINE_PREFIX int MPIDIG_post_pready(MPIR_Request * part_sreq, int is
     }
 
   fn_exit:
-    MPID_THREAD_CS_EXIT(VCI, MPIDI_VCI(0).lock);
     MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_POST_PREADY);
     return mpi_errno;
   fn_fail:
@@ -129,7 +128,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDIG_mpi_parrived(MPIR_Request * request, int par
         *flag = FALSE;
 
         /* Trigger progress to process AM packages in case wait with parrived in a loop. */
-        mpi_errno = MPID_Progress_test(NULL);
+        mpi_errno = MPIDI_progress_test_vci(0);
         MPIR_ERR_CHECK(mpi_errno);
     }