diff --git a/content/backmatter.tex b/content/backmatter.tex index 69b6970ad..b7c6eea75 100644 --- a/content/backmatter.tex +++ b/content/backmatter.tex @@ -558,6 +558,8 @@ \section{Version 1.5} \item This item is a template for changelist entries and should be deleted before this document is published. \\See Annex~\ref{sec:changelog}. +\item Clarified the atomicity guarantees of the \openshmem memory model. + \\See Section~\ref{subsec:amo_guarantees}. \end{itemize} \section{Version 1.4} diff --git a/content/memory_model.tex b/content/memory_model.tex index 6dae1efb6..206f2b70c 100644 --- a/content/memory_model.tex +++ b/content/memory_model.tex @@ -54,16 +54,53 @@ \subsection{Atomicity Guarantees}\label{subsec:amo_guarantees} -\openshmem contains a number of routines that operate on symmetric data -atomically (Section \ref{sec:amo}). These routines guarantee that accesses by -\openshmem's atomic operations with the same datatype will be exclusive, but do not guarantee -exclusivity in combination with other routines, either inside \openshmem or -outside. - -For example: during the execution of an atomic remote integer increment +\openshmem contains a number of routines that perform atomic operations on +symmetric data objects, which are defined in Section \ref{sec:amo}. +The atomic routines +guarantee that concurrent accesses by any of these routines to the same +location and using the same datatype (specified in Tables~\ref{stdamotypes} and +\ref{extamotypes}) will be exclusive. +\openshmem atomic operations do not guarantee exclusivity in the following +scenarios, all of which result in undefined behavior. +\begin{enumerate} + \item When concurrent accesses to the same location are performed using + \openshmem atomic operations using different datatypes. + \item When atomic and non-atomic \openshmem operations are used to access + the same location concurrently. + \item When \openshmem atomic operations and non-\openshmem operations (e.g. + load and store operations) are used to access the same location + concurrently. +\end{enumerate} +For example, during the execution of an atomic remote integer increment, i.e. \FUNC{shmem\_atomic\_inc}, operation on a symmetric variable \VAR{X}, no other \openshmem atomic operation may access \VAR{X}. After the increment, \VAR{X} will have increased its value by \CONST{1} on the destination \ac{PE}, at which point other atomic operations may then modify that \VAR{X}. However, access to the symmetric object \VAR{X} with non-atomic operations, such as one-sided \OPR{put} or \OPR{get} operations, will invalidate the atomicity guarantees. + +\cexample + {The following \CorCpp example illustrates scenario 1. In this example, + different datatypes are used to access the same location concurrently, + resulting in undefined behavior. The undefined behavior can be resolved by + using the same datatype in all concurrent operations. For example, the + 32-bit value can be left-shifted and a 64-bit atomic OR operation can be + used.} + {./example_code/amo_scenario_1.c} + +\cexample + {The following \CorCpp example illustrates scenario 2. In this example, + atomic increment operations are concurrent with a non-atomic reduction + operation, resulting in undefined behavior. The undefined behavior can be + resolved by inserting a barrier operation before the reduction. The + barrier ensures that all local and remote AMOs have completed before the + reduction operation accesses $x$.} + {./example_code/amo_scenario_2.c} + +\cexample + {The following \CorCpp example illustrates scenario 3. In this example, an + \openshmem atomic increment operation is concurrent with a local increment + operation, resulting in undefined behavior. The undefined behavior can be + resolved by replacing the local increment operation with an \openshmem + atomic increment.} + {./example_code/amo_scenario_3.c} diff --git a/example_code/amo_scenario_1.c b/example_code/amo_scenario_1.c new file mode 100644 index 000000000..0ab058cb8 --- /dev/null +++ b/example_code/amo_scenario_1.c @@ -0,0 +1,16 @@ +#include + +int main(void) { + static uint64_t x = 0; + + shmem_init(); + /* Undefined behavior: The following AMOs access the same location concurrently using + * different types. */ + if (shmem_my_pe() > 0) + shmem_uint32_atomic_or((uint32_t*)&x, shmem_my_pe()+1, 0); + else + shmem_uint64_atomic_or(&x, shmem_my_pe()+1, 0); + + shmem_finalize(); + return 0; +} diff --git a/example_code/amo_scenario_2.c b/example_code/amo_scenario_2.c new file mode 100644 index 000000000..d89b7696f --- /dev/null +++ b/example_code/amo_scenario_2.c @@ -0,0 +1,21 @@ +#include + +int main(void) { + static long psync[SHMEM_REDUCE_SYNC_SIZE]; + static int pwrk[SHMEM_REDUCE_MIN_WRKDATA_SIZE]; + static int x = 0, y = 0; + + for (int i = 0; i < SHMEM_REDUCE_SYNC_SIZE; i++) + psync[i] = SHMEM_SYNC_VALUE; + + shmem_init(); + shmem_int_atomic_inc(&x, (shmem_my_pe()+1) % shmem_n_pes()); + /* Undefined behavior: The following reduction operation performs accesses to symmetric + * variable 'x' that are concurrent with previously issued atomic increment operations + * on the same variable. */ + shmem_int_sum_to_all(&y, &x, 1, 0, 0, shmem_n_pes(), pwrk, psync); + + shmem_finalize(); + return 0; +} + diff --git a/example_code/amo_scenario_3.c b/example_code/amo_scenario_3.c new file mode 100644 index 000000000..9c5f16e97 --- /dev/null +++ b/example_code/amo_scenario_3.c @@ -0,0 +1,16 @@ +#include + +int main(void) { + static int x = 0; + + shmem_init(); + /* Undefined behavior: OpenSHMEM atomic increment operations are concurrent with the local + * increment of symmetric variable 'x'. */ + if (shmem_my_pe() > 0) + shmem_int_atomic_inc(&x, 0); + else + x++; + + shmem_finalize(); + return 0; +} diff --git a/utils/defs.tex b/utils/defs.tex index dbfdffdcf..aef8b0f31 100644 --- a/utils/defs.tex +++ b/utils/defs.tex @@ -530,6 +530,11 @@ { } +\newcommand{\cexample}[2]{ + #1 + \lstinputlisting[language={C}, tabsize=2, + basicstyle=\ttfamily\footnotesize, + morekeywords={size_t, ptrdiff_t, shmem_ctx_t}]{#2}} % % End library API description template commands %