Merge pull request openshmem-org#43 from manjugv/ticket-218

Thread Safety Proposal (RM Ticket openshmem-org#218)

content/backmatter.tex

@@ -572,6 +572,30 @@ \section{Version 1.4}
 \item Added the \FUNC{shmem\_calloc} function.
 \\ See Section \ref{subsec:shmem_calloc}.
 %
+\item Introduced the thread safe semantics that define the interaction between
+    \openshmem routines and user threads.
+\\See Section \ref{subsec:thread_support}.
+%
+\item Added the new routine \FUNC{shmem\_init\_thread} to initialize the
+    \openshmem library with one of the defined thread levels.
+\\See Section \ref{subsec:shmem_init_thread}.
+%
+\item Added the new routine \FUNC{shmem\_query\_thread} to query the thread
+    level provided by the \openshmem implementation.
+\\See Section \ref{subsec:shmem_query_thread}.
+%
+\item Clarified the semantics of \FUNC{shmem\_quiet} for a multithreaded
+    \openshmem \ac{PE}.
+\\See Section \ref{subsec:shmem_quiet}
+%
+\item Revised the description of \FUNC{shmem\_barrier\_all} for a multithreaded
+    \openshmem \ac{PE}.
+\\See Section \ref{subsec:shmem_barrier_all}
+%
+\item Revised the description of \FUNC{shmem\_wait} for a multithreaded
+    \openshmem \ac{PE}.
+\\See Section \ref{subsec:shmem_wait}
+%
 \item Clarified description for \CONST{SHMEM\_VENDOR\_STRING}.
 \\See Section \ref{subsec:library_constants}.
 %

content/shmem_barrier_all.tex

@@ -1,7 +1,7 @@
 \apisummary{
-    Registers the arrival of a \ac{PE} at a barrier and suspends \ac{PE} execution
-    until all other \acp{PE} arrive at the barrier and all local and remote memory
-    updates are completed.
+    Registers the arrival of a \ac{PE} at a barrier and blocks the \ac{PE}
+    until all other \acp{PE} arrive at the barrier and all local and remote
+    memory updates are completed.
 }
 
 \begin{apidefinition}
@@ -23,8 +23,9 @@
 \apidescription{   
     The \FUNC{shmem\_barrier\_all} routine registers the arrival of a \ac{PE} at
     a barrier. Barriers are a fast mechanism for synchronizing all \acp{PE} at
-    once.  This routine causes a \ac{PE} to suspend execution until all \acp{PE}
-    have called \FUNC{shmem\_barrier\_all}.  This routine must be used with
+    once.  This routine blocks the \ac{PE} until all \acp{PE} have called
+    \FUNC{shmem\_barrier\_all}. In a multithreaded \openshmem
+    program, only the calling thread is blocked. This routine must be used with
     \acp{PE} started by \FUNC{shmem\_init}.
 
     Prior to synchronizing with other \acp{PE}, \FUNC{shmem\_barrier\_all}

content/shmem_init_thread.tex

@@ -0,0 +1,47 @@
+\apisummary{
+Initializes the OpenSHMEM library, similar to \FUNC{shmem\_init}, and performs any
+initialization required for supporting the provided thread level.
+}
+
+\begin{apidefinition}
+
+\begin{Csynopsis}
+int shmem_init_thread(int requested, int *provided);
+\end{Csynopsis}
+
+\begin{apiarguments}
+\apiargument{IN}{requested}{The thread level support requested by the user.}
+\apiargument{OUT}{provided}{The thread level support provided by the \openshmem implementation.}
+\end{apiarguments}
+
+\apidescription{
+\FUNC{shmem\_init\_thread} initializes the \openshmem library in the same way as 
+\FUNC{shmem\_init}. In addition, \FUNC{shmem\_init\_thread} also performs 
+the initialization required for supporting the provided thread level. 
+The argument \VAR{requested} is used to specify the desired level of 
+thread support. The argument \VAR{provided} returns the support level 
+provided by the library. The allowed values for \VAR{provided} and 
+\VAR{requested} are \CONST{SHMEM\_THREAD\_SINGLE}, \CONST{SHMEM\_THREAD\_FUNNELED},
+\CONST{SHMEM\_THREAD\_SERIALIZED}, or \CONST{SHMEM\_THREAD\_MULTIPLE}.
+
+An \openshmem program is initialized either by \FUNC{shmem\_init} or \FUNC{shmem\_init\_thread}. 
+Similar to \FUNC{shmem\_init}, the \FUNC{shmem\_init\_thread} routine may not 
+be called multiple times in an \openshmem program. If the call to \FUNC{shmem\_init\_thread} 
+is unsuccessful in allocating and initializing resources for the 
+\openshmem library, then the behavior of any subsequent call 
+to the \openshmem library is undefined.
+}
+
+\apireturnvalues{
+\FUNC{shmem\_init\_thread} returns 0 upon success; otherwise, it returns a
+non-zero value.
+}
+
+\apinotes{
+The \openshmem library can be initialized either by \FUNC{shmem\_init} 
+or \FUNC{shmem\_init\_thread}. If the \openshmem library is initialized 
+by \FUNC{shmem\_init}, the library implementation can choose to 
+support one of the defined thread levels.
+}
+
+\end{apidefinition}

content/shmem_query_thread.tex

@@ -0,0 +1,31 @@
+\apisummary{
+Returns the level of thread support provided by the library.
+}
+
+\begin{apidefinition}
+
+\begin{Csynopsis}
+void shmem_query_thread(int *provided);
+\end{Csynopsis}
+
+\begin{apiarguments}
+\apiargument{OUT}{provided}{The thread level support provided by the \openshmem implementation.}
+\end{apiarguments}
+
+\apidescription{
+The \FUNC{shmem\_query\_thread} call returns the level of thread support
+currently being provided. The value returned will be same as \VAR{provided}
+returned in the \FUNC{shmem\_init\_thread}, if the \openshmem library was
+initialized by \FUNC{shmem\_init\_thread}. If the library was initialized by
+\FUNC{shmem\_init}, the implementation can choose to provide one of the defined
+thread levels, and \FUNC{shmem\_query\_thread} returns this thread level.
+}
+
+\apireturnvalues{
+None.
+}
+
+\apinotes{
+None.
+}
+\end{apidefinition}

content/shmem_quiet.tex

@@ -45,11 +45,30 @@
     memory stores, and nonblocking \PUT{} and \GET{} routines) to
     symmetric data objects on all \acp{PE}.
 
-    \FUNC{shmem\_quiet} does not have an effect on the ordering between memory 
-    accesses issued by the target PE.
-    \FUNC{shmem\_wait}, \FUNC{shmem\_wait\_until}, \FUNC{shmem\_test}, \FUNC{shmem\_barrier},
-    \FUNC{shmem\_barrier\_all} routines can be called by the target PE to guarantee 
-    ordering of its memory accesses.
+    In an \openshmem program with multithreaded \acp{PE}, it is the
+    user's responsibility to ensure ordering between operations issued by the threads
+    in a \ac{PE} that target symmetric memory (e.g. \PUT{}, \ac{AMO}, memory stores,
+    and nonblocking routines) and calls by threads in that \ac{PE} to
+    \FUNC{shmem\_quiet}. The \FUNC{shmem\_quiet} routine can enforce memory store ordering only for the
+    calling thread. Thus, to ensure ordering for memory stores performed by a thread that is
+    not the thread calling \FUNC{shmem\_quiet}, the update must be made visible to the
+    calling thread according to the rules of the memory model associated with
+    the threading environment.
+
+     A call to \FUNC{shmem\_quiet} by a thread completes the operations posted prior
+     to calling \FUNC{shmem\_quiet}. If the user intends to also complete operations
+     issued by a thread that is not the thread calling \FUNC{shmem\_quiet}, the
+     user must ensure that the operations are performed prior to the call to
+     \FUNC{shmem\_quiet}. This may require the use of a synchronization
+     operation provided by the threading package. For example, when using POSIX
+     Threads, the user may call the \FUNC{pthread\_barrier\_wait} routine to
+     ensure that all threads have issued operations before a thread calls
+     \FUNC{shmem\_quiet}.
+
+    \FUNC{shmem\_quiet} does not have an effect on the ordering between memory
+    accesses issued by the target PE. \FUNC{shmem\_wait}, \FUNC{shmem\_wait\_until},
+    \FUNC{shmem\_test}, \FUNC{shmem\_barrier}, \FUNC{shmem\_barrier\_all} routines
+    can be called by the target PE to guarantee ordering of its memory accesses.
 }
 
 \begin{apiexamples}

content/shmem_wait.tex

@@ -37,9 +37,8 @@
 
 \begin{apiarguments}
 
-\apiargument{OUT}{ivar}{A remotely accessible integer variable that is being updated
-    by another \ac{PE}.  When using \CorCpp, the type of \VAR{ivar} should
-    match that implied in the SYNOPSIS section.} 
+\apiargument{OUT}{ivar}{A remotely accessible integer variable. When using \CorCpp,
+    the type of \VAR{ivar} should match that implied in the SYNOPSIS section.} 
 \apiargument{IN}{cmp}{The compare operator that compares \VAR{ivar} with
   \VAR{cmp\_value}.  When using \Fortran, it must  be of default kind.
   When using \CorCpp, it must be of type \CTYPE{shmem\_cmp\_t}.}
@@ -52,15 +51,13 @@
 
 \apidescription{ 
     \FUNC{shmem\_wait} and \FUNC{shmem\_wait\_until} wait for \VAR{ivar} to be
-    changed by a remote write or an atomic operation issued by a different \ac{PE}.
+    changed by a write or an atomic operation issued by a \ac{PE}.
     These  routines can be used for point-to-point direct synchronization.  A call
-    to \VAR{shmem\_wait} does not return until some other \ac{PE} writes  a value,
+    to \FUNC{shmem\_wait} does not return until a \ac{PE} writes a value,
     not equal to \VAR{cmp\_value}, into \VAR{ivar} on the waiting \ac{PE}.  A call
-    to \FUNC{shmem\_wait\_until} does not return until some  other \ac{PE} changes
+    to \FUNC{shmem\_wait\_until} does not return until a \ac{PE} changes
     \VAR{ivar} to satisfy the condition implied by \VAR{cmp} and \VAR{cmp\_value}.
-    This mechanism is useful when a \ac{PE} needs to tell another \ac{PE} that it
-    has completed some action.  The \FUNC{shmem\_wait}  routines return when
-    \VAR{ivar} is no  longer  equal  to \VAR{cmp\_value}. The
+    The \FUNC{shmem\_wait} routines return when \VAR{ivar} is no longer equal to \VAR{cmp\_value}. The
     \FUNC{shmem\_wait\_until} routines return when the compare condition is true.
     The compare condition is defined by the \VAR{ivar}  argument  compared with the
     \VAR{cmp\_value} using the comparison operator, \VAR{cmp}. 

content/synchronization_model.tex

@@ -19,9 +19,9 @@
 \begin{tabular}{p{0.2\textwidth} | p{0.7\textwidth}}
 {}
 &
-{ Waits for a symmetric variable to be updated by a remote \ac{PE}. Should be
+Waits for a symmetric variable to be updated by a remote \ac{PE}. Should be
 used when computation on the local \ac{PE} cannot proceed without the value that
-the remote \ac{PE} is to update.} \tabularnewline
+the remote \ac{PE} is to update. \tabularnewline
 \hline 
 \end{tabular}
 

content/threads_intro.tex

@@ -0,0 +1,63 @@
+This section specifies the interaction between \openshmem interfaces and the
+user threads, and also describes the routines that can be used for initializing and
+querying the thread environment. There are four levels of threading supported by
+the \openshmem implementation.
+
+\begin{itemize}
+\item {\bf \CONST{SHMEM\_THREAD\_SINGLE}} The \openshmem program may not be multithreaded.
+
+\item {\bf \CONST{SHMEM\_THREAD\_FUNNELED}}
+The \openshmem program may be multithreaded. However, the program must ensure
+that only the main thread invokes the \openshmem interfaces. The main thread
+is the thread that invokes either \FUNC{shmem\_init} or \FUNC{shmem\_init\_thread}.
+
+\item {\bf \CONST{SHMEM\_THREAD\_SERIALIZED}}
+The \openshmem program may be multithreaded. However, the program must ensure
+that the \openshmem interfaces are not invoked concurrently by multiple threads.
+
+\item {\bf \CONST{SHMEM\_THREAD\_MULTIPLE}}
+The \openshmem program may be multithreaded and any thread may invoke the \openshmem
+interfaces.
+\end{itemize}
+
+\hspace{-15pt}
+{\bf Semantics}
+
+\begin{itemize}
+\item
+In \CONST{SHMEM\_THREAD\_FUNNELED}, \CONST{SHMEM\_THREAD\_SERIALIZED}, and
+\CONST{SHMEM\_THREAD\_MULTIPLE} thread levels, the \FUNC{shmem\_init} and
+\FUNC{shmem\_finalize} calls may only be invoked by the same thread.
+
+\item
+Any \openshmem operation initiated by a thread is considered an action of the
+\ac{PE} as a whole. The symmetric heap and symmetric variables scope are not
+impacted by multiple threads invoking the \openshmem interfaces, i.e.,
+each \ac{PE} has a single symmetric data segment and symmetric heap that is shared by
+all threads within that \ac{PE}.  For example, a thread invoking a memory allocation
+routine such as \FUNC{shmem\_malloc} allocates memory that is accessible by
+all threads of the \ac{PE}. The requirement that the same symmetric heap operations
+must be executed by all \acp{PE} in the same order also applies in a threaded
+environment. Similarly, the completion of collective operations is not impacted
+by multiple threads. For example, \FUNC{shmem\_barrier\_all} is completed when
+all \acp{PE} enter and exit the \FUNC{shmem\_barrier\_all} call, even though
+only one thread in the \ac{PE} is participating in the collective call.
+
+\item Blocking \openshmem calls will only block the calling thread, allowing
+other threads, if available, to continue the execution. The calling thread will
+be blocked until the event on which it is waiting occurs. Once the blocked communication is
+completed, the thread is ready for continuing the execution. A blocked thread
+will not prevent progress of other threads on the same \ac{PE}, and will not
+prevent them from executing other \openshmem calls when the thread level permits.
+Also, a blocked thread will not prevent the progress of \openshmem calls on other \acp{PE}.
+
+\item In the \CONST{SHMEM\_THREAD\_MULTIPLE} thread level, all \openshmem calls are thread-safe,
+i.e., two concurrently running threads may make \openshmem calls and the outcome
+will be as if the calls executed in some order, even if their execution is interleaved.
+
+\item In the \CONST{SHMEM\_THREAD\_SERIALIZED} and \CONST{SHMEM\_THREAD\_MULTIPLE} thread levels,
+if multiple threads call the collective calls, it is the programmer's responsibility
+to ensure the correct ordering of collective calls, including the symmetric
+heap management functions.
+
+\end{itemize}

main_spec.tex

@@ -79,11 +79,21 @@ \subsubsection{\textbf{SHMEM\_INFO\_GET\_NAME}}\label{subsec:shmem_info_get_name
 \subsubsection{\textbf{START\_PES}}\label{subsec:start_pes}
 \input{content/start_pes}
 
+\subsection{Thread Support}
+\label{subsec:thread_support}
+\input{content/threads_intro.tex}
 
+\subsubsection{\textbf{SHMEM\_INIT\_THREAD}}
+\label{subsec:shmem_init_thread}
+\input{content/shmem_init_thread}
 
+\subsubsection{\textbf{SHMEM\_QUERY\_THREAD}}
+\label{subsec:shmem_query_thread}
+\input{content/shmem_query_thread}
 
 
 \subsection{Memory Management Routines}
+\label{sec:mem_routines}
 \openshmem provides a set of \acp{API} for managing the symmetric heap. The
 \acp{API} allow one to dynamically allocate, deallocate, reallocate and align
 symmetric data objects in the symmetric heap, in \Cstd and \Fortran.
@@ -103,11 +113,6 @@ \subsubsection{\textbf{SHPCLMOVE}}\label{subsec:shpclmove}
 \subsubsection{\textbf{SHPDEALLOC}}\label{subsec:shpdealloc}
 \input{content/shpdealloc.tex}
 
-
-
-
-
-
 \subsection{Remote Memory Access Routines}\label{sec:rma}
 \input{content/rma_intro.tex}