Merge pull request #204 from jdinan/pr/atomicity-clarification

Update atomic memory model text

content/backmatter.tex

@@ -558,6 +558,8 @@ \section{Version 1.5}
 \item This item is a template for changelist entries and should be deleted
     before this document is published.
     \\See Annex~\ref{sec:changelog}.
+\item Clarified the atomicity guarantees of the \openshmem memory model.
+    \\See Section~\ref{subsec:amo_guarantees}.
 \end{itemize}
 
 \section{Version 1.4}

content/memory_model.tex

@@ -54,16 +54,53 @@
 
 \subsection{Atomicity Guarantees}\label{subsec:amo_guarantees}
 
-\openshmem contains a number of routines that operate on symmetric data
-atomically (Section \ref{sec:amo}).  These routines guarantee that accesses by
-\openshmem's atomic operations with the same datatype will be exclusive, but do not guarantee
-exclusivity in combination with other routines, either inside \openshmem or
-outside.
-
-For example: during the execution of an atomic remote integer increment
+\openshmem contains a number of routines that perform atomic operations on
+symmetric data objects, which are defined in Section \ref{sec:amo}.
+The atomic routines
+guarantee that concurrent accesses by any of these routines to the same
+location and using the same datatype (specified in Tables~\ref{stdamotypes} and
+\ref{extamotypes}) will be exclusive.
+\openshmem atomic operations do not guarantee exclusivity in the following
+scenarios, all of which result in undefined behavior.
+\begin{enumerate}
+    \item When concurrent accesses to the same location are performed using
+        \openshmem atomic operations using different datatypes.
+    \item When atomic and non-atomic \openshmem operations are used to access
+        the same location concurrently.
+    \item When \openshmem atomic operations and non-\openshmem operations (e.g.
+        load and store operations) are used to access the same location
+        concurrently.
+\end{enumerate}
+For example, during the execution of an atomic remote integer increment, i.e. \FUNC{shmem\_atomic\_inc},
 operation on a symmetric variable \VAR{X}, no other \openshmem atomic operation
 may access \VAR{X}.  After the increment, \VAR{X} will have increased its value
 by \CONST{1} on the destination \ac{PE}, at which point other atomic operations
 may then modify that \VAR{X}.  However, access to the symmetric object \VAR{X}
 with non-atomic operations, such as one-sided \OPR{put} or \OPR{get} operations,
 will invalidate the atomicity guarantees.
+
+\cexample
+    {The following \CorCpp example illustrates scenario 1.  In this example,
+    different datatypes are used to access the same location concurrently,
+    resulting in undefined behavior.  The undefined behavior can be resolved by
+    using the same datatype in all concurrent operations.  For example, the
+    32-bit value can be left-shifted and a 64-bit atomic OR operation can be
+    used.}
+    {./example_code/amo_scenario_1.c}
+
+\cexample
+    {The following \CorCpp example illustrates scenario 2.  In this example,
+    atomic increment operations are concurrent with a non-atomic reduction
+    operation, resulting in undefined behavior.  The undefined behavior can be
+    resolved by inserting a barrier operation before the reduction.  The
+    barrier ensures that all local and remote AMOs have completed before the
+    reduction operation accesses $x$.}
+    {./example_code/amo_scenario_2.c}
+
+\cexample
+    {The following \CorCpp example illustrates scenario 3.  In this example, an
+    \openshmem atomic increment operation is concurrent with a local increment
+    operation, resulting in undefined behavior.  The undefined behavior can be
+    resolved by replacing the local increment operation with an \openshmem
+    atomic increment.}
+    {./example_code/amo_scenario_3.c}

example_code/amo_scenario_1.c

@@ -0,0 +1,16 @@
+#include <shmem.h>
+
+int main(void) {
+    static uint64_t x = 0;
+
+    shmem_init();
+    /* Undefined behavior: The following AMOs access the same location concurrently using
+     * different types. */
+    if (shmem_my_pe() > 0)
+      shmem_uint32_atomic_or((uint32_t*)&x, shmem_my_pe()+1, 0);
+    else
+      shmem_uint64_atomic_or(&x, shmem_my_pe()+1, 0);
+
+    shmem_finalize();
+    return 0;
+}

example_code/amo_scenario_2.c

@@ -0,0 +1,21 @@
+#include <shmem.h>
+
+int main(void) {
+    static long psync[SHMEM_REDUCE_SYNC_SIZE];
+    static int pwrk[SHMEM_REDUCE_MIN_WRKDATA_SIZE];
+    static int x = 0, y = 0;
+
+    for (int i = 0; i < SHMEM_REDUCE_SYNC_SIZE; i++)
+        psync[i] = SHMEM_SYNC_VALUE;
+
+    shmem_init();
+    shmem_int_atomic_inc(&x, (shmem_my_pe()+1) % shmem_n_pes());
+    /* Undefined behavior: The following reduction operation performs accesses to symmetric
+     * variable 'x' that are concurrent with previously issued atomic increment operations
+     * on the same variable. */
+    shmem_int_sum_to_all(&y, &x, 1, 0, 0, shmem_n_pes(), pwrk, psync);
+
+    shmem_finalize();
+    return 0;
+}
+

example_code/amo_scenario_3.c

@@ -0,0 +1,16 @@
+#include <shmem.h>
+
+int main(void) {
+    static int x = 0;
+
+    shmem_init();
+    /* Undefined behavior: OpenSHMEM atomic increment operations are concurrent with the local
+     * increment of symmetric variable 'x'. */
+    if (shmem_my_pe() > 0)
+      shmem_int_atomic_inc(&x, 0);
+    else
+      x++;
+
+    shmem_finalize();
+    return 0;
+}

utils/defs.tex

@@ -530,6 +530,11 @@
 {
 }
 
+\newcommand{\cexample}[2]{
+    #1
+    \lstinputlisting[language={C}, tabsize=2,
+      basicstyle=\ttfamily\footnotesize,
+      morekeywords={size_t, ptrdiff_t, shmem_ctx_t}]{#2}}
 %
 % End library API description template commands
 %