From 7f7d0949c9aadedd099df59456ce467a76925a0a Mon Sep 17 00:00:00 2001 From: James Dinan Date: Thu, 26 Sep 2019 15:49:10 -0400 Subject: [PATCH 1/8] Edits from September f2f --- content/shmem_collect.tex | 12 ------------ content/shmem_ctx_create.tex | 2 +- content/shmem_sync.tex | 7 +++---- content/shmem_team_create_ctx.tex | 5 +++-- content/teams_intro.tex | 4 ++-- example_code/shmem_sync_example.c | 10 +++++++--- example_code/shmem_team_context.c | 12 ++++++------ 7 files changed, 22 insertions(+), 30 deletions(-) diff --git a/content/shmem_collect.tex b/content/shmem_collect.tex index 6c031a259..f49d165d0 100644 --- a/content/shmem_collect.tex +++ b/content/shmem_collect.tex @@ -122,18 +122,6 @@ } {\color{Green} -\apidesctable{ -The \dest{} and \source{} data objects must conform to certain typing -constraints, which are as follows: -}{Routine}{Data type of \VAR{dest} and \VAR{source}} -\apitablerow{\FUNC{shmem\_collectmem}, \FUNC{shmem\_fcollectmem}}{\Cstd: Any data type. \VAR{nelems} is scaled in bytes.}% -\apitablerow{\FUNC{shmem\_collect64}, \FUNC{shmem\_fcollect64}}% - {Any noncharacter type that has an element size of \CONST{64} bits. No \Fortran derived types nor - \CorCpp{} structures are allowed.} -\apitablerow{\FUNC{shmem\_collect32}, \FUNC{shmem\_fcollect32}}% - {Any noncharacter type that has an element size of \CONST{32} bits. No \Fortran derived types nor - \CorCpp{} structures are allowed.} -} \apireturnvalues{ \newtext{Zero on successful local completion. Nonzero otherwise.} diff --git a/content/shmem_ctx_create.tex b/content/shmem_ctx_create.tex index 62ebf120c..41633f98e 100644 --- a/content/shmem_ctx_create.tex +++ b/content/shmem_ctx_create.tex @@ -1,5 +1,5 @@ \apisummary{ - Create a communication context \newtext{locally}. + Create a communication context. } \begin{apidefinition} diff --git a/content/shmem_sync.tex b/content/shmem_sync.tex index 0c8c7ae58..db159b70e 100644 --- a/content/shmem_sync.tex +++ b/content/shmem_sync.tex @@ -1,8 +1,7 @@ \apisummary{ - \newtext{Registers the arrival of a \ac{PE} at a synchronization point and suspends - execution until all other \acp{PE} in a given \openshmem team or active set - arrive at a synchronization point. For multithreaded programs, execution is suspended - as specified by the threading model (Section \ref{subsec:thread_support}).} + \newtext{Registers the arrival of a \ac{PE} at a synchronization point. + This routine does not return until all other PEs in a given OpenSHMEM team + or active set arrive at the synchronization point.} \oldtext{% Performs all operations described in the \FUNC{shmem\_sync\_all} interface but with respect to a subset of \acp{PE} defined by the active set. diff --git a/content/shmem_team_create_ctx.tex b/content/shmem_team_create_ctx.tex index dce63be92..b992cb6ce 100644 --- a/content/shmem_team_create_ctx.tex +++ b/content/shmem_team_create_ctx.tex @@ -1,5 +1,5 @@ \apisummary{ - Create a communication context from a team locally. + Create a communication context from a team. } \begin{apidefinition} @@ -20,7 +20,8 @@ \apidescription{ The \FUNC{shmem\_team\_create\_ctx} routine creates a new communication context and returns its handle through the \VAR{ctx} argument. - This context is created from the team specified by the \VAR{team} argument. + This context is created from the team specified by the \VAR{team} argument; + however, the context creation operation is not collective In addition to the team, the \FUNC{shmem\_team\_create\_ctx} routine accepts the same arguments and provides all the same return conditions as the diff --git a/content/teams_intro.tex b/content/teams_intro.tex index ddca69dcc..107b4aa66 100644 --- a/content/teams_intro.tex +++ b/content/teams_intro.tex @@ -81,8 +81,8 @@ \subsubsection*{Team Creation} is detailed further in Section~\ref{subsec:shmem_team_config_t}. \acp{PE} in a newly created teams are consecutively numbered with starting with -\ac{PE} number 0. \acp{PE} are always ordered by the existing global \ac{PE} number that -would be returned by the \FUNC{shmem\_my\_pe} routine. Team relative \ac{PE} +\ac{PE} number 0. \acp{PE} are always ordered by the existing \ac{PE} number that +would be returned by the \FUNC{shmem\_team\_my\_pe} routine called on the parent team. Team relative \ac{PE} numbers can be used for point-to-point operations through team-based contexts (see Section~\ref{sec:ctx}) or using the translation routine \FUNC{shmem\_team\_translate\_pe}. diff --git a/example_code/shmem_sync_example.c b/example_code/shmem_sync_example.c index 2e367a428..e1fc54832 100644 --- a/example_code/shmem_sync_example.c +++ b/example_code/shmem_sync_example.c @@ -21,19 +21,23 @@ int main(void) shmem_team_split_strided(SHMEM_TEAM_WORLD, 0, 3, npes / 3 + odd_npes, config, 0, &threes_team); - int my_pe_twos = shmem_team_my_pe(twos_team); + int my_pe_twos = shmem_team_my_pe(twos_team); int my_pe_threes = shmem_team_my_pe(threes_team); + int npes_twos = shmem_team_n_pes(twos_team); + int npes_threes = shmem_team_n_pes(threes_team); if (twos_team != SHMEM_TEAM_INVALID) { /* put the value 2 to the next team member in a circular fashion */ - shmem_p(&x, 2, (me + 2) % npes); + shmem_p(&x, 2, shmem_team_translate_pe(twos_team, (my_pe_twos + 1) % + npes_twos, SHMEM_TEAM_WORLD)); shmem_quiet(); shmem_sync(twos_team); } if (threes_team != SHMEM_TEAM_INVALID) { /* put the value 3 to the next team member in a circular fashion */ - shmem_p(&x, 3, (me + 3) % npes); + shmem_p(&x, 3, shmem_team_translate_pe(threes_team, (my_pe_threes + 1) % + npes_threes, SHMEM_TEAM_WORLD)); shmem_quiet(); shmem_sync(threes_team); } diff --git a/example_code/shmem_team_context.c b/example_code/shmem_team_context.c index 57d8c9621..5b1e07ffc 100644 --- a/example_code/shmem_team_context.c +++ b/example_code/shmem_team_context.c @@ -11,7 +11,7 @@ int my_ctx_translate_pe(shmem_ctx_t src_ctx, int src_pe, shmem_ctx_t dest_ctx) if (dest_ctx == SHMEM_CTX_INVALID) { return -1; } - + shmem_team_t src_team, dest_team; shmem_ctx_get_team(src_ctx, &src_team); shmem_ctx_get_team(dest_ctx, &dest_team); @@ -37,7 +37,7 @@ void my_send_to_neighbor(shmem_ctx_t ctx, int *val) fprintf (stderr, "Send to neighbor fail due to invalid context\n"); return; } - + shmem_team_t team; shmem_ctx_get_team(ctx, &team); int pe = shmem_team_my_pe(team); @@ -56,7 +56,7 @@ int main() int npes = shmem_n_pes(); isum = 0; - + shmem_team_t team_2s, team_3s; shmem_ctx_t ctx_2s, ctx_3s; shmem_team_config_t conf; @@ -76,7 +76,7 @@ int main() my_send_to_neighbor(ctx_3s, &ival3); // Quiet all contexts and synchronize all PEs to complete the data transfers - shmem_ctx_quiet(ctx_2s); + shmem_ctx_quiet(ctx_2s); shmem_ctx_quiet(ctx_3s); shmem_team_sync(SHMEM_TEAM_WORLD); @@ -89,10 +89,10 @@ int main() } else { // Add up the results on pe 4 of the 3s team, using the 2s team context - shmem_ctx_int_atomic_add(ctx_2s, &isum, ival2 + ival3, _pe4_of_3s_in_2s); + shmem_ctx_int_atomic_add(ctx_2s, &isum, ival2 + ival3, pe4_of_3s_in_2s); } } - + // Quiet the context and synchronize PEs to complete the operation shmem_ctx_quiet(ctx_2s); shmem_team_sync(SHMEM_TEAM_WORLD); From fe6e1e31b18d57bc110b908da1a3720be0a95b0d Mon Sep 17 00:00:00 2001 From: James Dinan Date: Tue, 29 Oct 2019 15:33:06 -0400 Subject: [PATCH 2/8] Put back apidesctable in shmem_collect.tex --- content/shmem_collect.tex | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/content/shmem_collect.tex b/content/shmem_collect.tex index 4d5e2234e..5f645c368 100644 --- a/content/shmem_collect.tex +++ b/content/shmem_collect.tex @@ -110,6 +110,17 @@ \end{itemize} } +\apidesctable{ +The \dest{} and \source{} data objects must conform to certain typing +constraints, which are as follows: +}{Routine}{Data type of \VAR{dest} and \VAR{source}} +\apitablerow{\FUNC{shmem\_collectmem}, \FUNC{shmem\_fcollectmem}}{\Cstd: Any data type. \VAR{nelems} is scaled in bytes.}% +\apitablerow{\FUNC{shmem\_collect64}, \FUNC{shmem\_fcollect64}}% + {Any noncharacter type that has an element size of \CONST{64} bits. No \Fortran derived types nor + \CorCpp{} structures are allowed.} +\apitablerow{\FUNC{shmem\_collect32}, \FUNC{shmem\_fcollect32}}% + {Any noncharacter type that has an element size of \CONST{32} bits. No \Fortran derived types nor + \CorCpp{} structures are allowed.} \apireturnvalues{ Zero on successful local completion. Nonzero otherwise. From 7d9a9b3cf0832e488316883308a7a126c4e76c1d Mon Sep 17 00:00:00 2001 From: James Dinan Date: Tue, 5 Nov 2019 13:55:43 -0500 Subject: [PATCH 3/8] Add missing full stop. --- content/shmem_team_create_ctx.tex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/shmem_team_create_ctx.tex b/content/shmem_team_create_ctx.tex index b992cb6ce..f92eb45d3 100644 --- a/content/shmem_team_create_ctx.tex +++ b/content/shmem_team_create_ctx.tex @@ -21,7 +21,7 @@ The \FUNC{shmem\_team\_create\_ctx} routine creates a new communication context and returns its handle through the \VAR{ctx} argument. This context is created from the team specified by the \VAR{team} argument; - however, the context creation operation is not collective + however, the context creation operation is not collective. In addition to the team, the \FUNC{shmem\_team\_create\_ctx} routine accepts the same arguments and provides all the same return conditions as the From fe0ad7b6133bab7121e92ac4f30b60da6b1272a9 Mon Sep 17 00:00:00 2001 From: James Dinan Date: Fri, 15 Nov 2019 10:01:34 -0500 Subject: [PATCH 4/8] Updates from @gmegan --- content/shmem_sync.tex | 2 +- content/teams_intro.tex | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/content/shmem_sync.tex b/content/shmem_sync.tex index e2e40f5c8..93027e5e9 100644 --- a/content/shmem_sync.tex +++ b/content/shmem_sync.tex @@ -1,7 +1,7 @@ \apisummary{ Registers the arrival of a \ac{PE} at a synchronization point. This routine does not return until all other PEs in a given OpenSHMEM team - or active set arrive at the synchronization point. + or active set arrive at this synchronization point. } \begin{apidefinition} diff --git a/content/teams_intro.tex b/content/teams_intro.tex index 107b4aa66..f48489eaa 100644 --- a/content/teams_intro.tex +++ b/content/teams_intro.tex @@ -80,9 +80,9 @@ \subsubsection*{Team Creation} This configuration argument is of type \CTYPE{shmem\_team\_config\_t}, which is detailed further in Section~\ref{subsec:shmem_team_config_t}. -\acp{PE} in a newly created teams are consecutively numbered with starting with -\ac{PE} number 0. \acp{PE} are always ordered by the existing \ac{PE} number that -would be returned by the \FUNC{shmem\_team\_my\_pe} routine called on the parent team. Team relative \ac{PE} +\acp{PE} in a newly created teams are consecutively numbered starting with +\ac{PE} number 0. \acp{PE} are ordered by their \ac{PE} number in +the parent team. Team relative \ac{PE} numbers can be used for point-to-point operations through team-based contexts (see Section~\ref{sec:ctx}) or using the translation routine \FUNC{shmem\_team\_translate\_pe}. From e5fe070abaa12090afcc9c5bb7ff31098ca5746a Mon Sep 17 00:00:00 2001 From: James Dinan Date: Fri, 15 Nov 2019 13:23:06 -0500 Subject: [PATCH 5/8] Fix shmem_sync_example This example was broken in several ways. The team parameters were incorrect (e.g. odd_npes led to broken behavior) and also PE 0 was included in both the twos and threes, leading to data corruption. --- example_code/shmem_sync_example.c | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/example_code/shmem_sync_example.c b/example_code/shmem_sync_example.c index e1fc54832..302ec90fd 100644 --- a/example_code/shmem_sync_example.c +++ b/example_code/shmem_sync_example.c @@ -5,7 +5,8 @@ int main(void) { static int x = 10101; - shmem_team_t twos_team, threes_team; + shmem_team_t twos_team = SHMEMX_TEAM_INVALID; + shmem_team_t threes_team = SHMEMX_TEAM_INVALID; shmem_team_config_t *config; shmem_init(); @@ -13,39 +14,39 @@ int main(void) int me = shmem_my_pe(); int npes = shmem_n_pes(); - int odd_npes = npes % 2; + if (npes > 2) + shmem_team_split_strided(SHMEMX_TEAM_WORLD, 2, 2, (npes-1) / 2, config, + 0, &twos_team); - shmem_team_split_strided(SHMEM_TEAM_WORLD, 0, 2, npes / 2, config, 0, - &twos_team); - - shmem_team_split_strided(SHMEM_TEAM_WORLD, 0, 3, npes / 3 + odd_npes, - config, 0, &threes_team); + if (npes > 3) + shmem_team_split_strided(SHMEMX_TEAM_WORLD, 3, 3, (npes-1) / 3, config, + 0, &threes_team); int my_pe_twos = shmem_team_my_pe(twos_team); int my_pe_threes = shmem_team_my_pe(threes_team); int npes_twos = shmem_team_n_pes(twos_team); int npes_threes = shmem_team_n_pes(threes_team); - if (twos_team != SHMEM_TEAM_INVALID) { + if (twos_team != SHMEMX_TEAM_INVALID) { /* put the value 2 to the next team member in a circular fashion */ shmem_p(&x, 2, shmem_team_translate_pe(twos_team, (my_pe_twos + 1) % - npes_twos, SHMEM_TEAM_WORLD)); + npes_twos, SHMEMX_TEAM_WORLD)); shmem_quiet(); shmem_sync(twos_team); } - if (threes_team != SHMEM_TEAM_INVALID) { + if (threes_team != SHMEMX_TEAM_INVALID) { /* put the value 3 to the next team member in a circular fashion */ - shmem_p(&x, 3, shmem_team_translate_pe(threes_team, (my_pe_threes + 1) % - npes_threes, SHMEM_TEAM_WORLD)); + shmem_p(&x, 3, shmem_team_translate_pe(threes_team, (my_pe_threes + 1) % + npes_threes, SHMEMX_TEAM_WORLD)); shmem_quiet(); shmem_sync(threes_team); } - if (me % 3 == 0 && x != 3) { + if (me && me % 3 == 0 && x != 3) { shmem_global_exit(3); } - else if (me % 2 == 0 && x != 2) { + else if (me && me % 2 == 0 && x != 2) { shmem_global_exit(2); } else if (x != 10101) { From 2e5bb54bf1e1657b2b6c1715df78e3f604540d01 Mon Sep 17 00:00:00 2001 From: James Dinan Date: Fri, 15 Nov 2019 13:26:42 -0500 Subject: [PATCH 6/8] Replace PEs with \acp{PE} --- content/shmem_sync.tex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/shmem_sync.tex b/content/shmem_sync.tex index 93027e5e9..310e3b674 100644 --- a/content/shmem_sync.tex +++ b/content/shmem_sync.tex @@ -1,6 +1,6 @@ \apisummary{ Registers the arrival of a \ac{PE} at a synchronization point. - This routine does not return until all other PEs in a given OpenSHMEM team + This routine does not return until all other \acp{PE} in a given OpenSHMEM team or active set arrive at this synchronization point. } From 78d735b2b7ebb0057a5024bf7894afdcc6945b0e Mon Sep 17 00:00:00 2001 From: James Dinan Date: Wed, 4 Dec 2019 16:50:37 -0500 Subject: [PATCH 7/8] Fix shmem_sync_example Signed-off-by: James Dinan --- example_code/shmem_sync_example.c | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/example_code/shmem_sync_example.c b/example_code/shmem_sync_example.c index 302ec90fd..9f8f7fcf5 100644 --- a/example_code/shmem_sync_example.c +++ b/example_code/shmem_sync_example.c @@ -5,8 +5,8 @@ int main(void) { static int x = 10101; - shmem_team_t twos_team = SHMEMX_TEAM_INVALID; - shmem_team_t threes_team = SHMEMX_TEAM_INVALID; + shmem_team_t twos_team = SHMEM_TEAM_INVALID; + shmem_team_t threes_team = SHMEM_TEAM_INVALID; shmem_team_config_t *config; shmem_init(); @@ -15,39 +15,41 @@ int main(void) int npes = shmem_n_pes(); if (npes > 2) - shmem_team_split_strided(SHMEMX_TEAM_WORLD, 2, 2, (npes-1) / 2, config, - 0, &twos_team); + shmem_team_split_strided(SHMEM_TEAM_WORLD, 2, 2, (npes-1) / 2, config, + 0, &twos_team); if (npes > 3) - shmem_team_split_strided(SHMEMX_TEAM_WORLD, 3, 3, (npes-1) / 3, config, - 0, &threes_team); + shmem_team_split_strided(SHMEM_TEAM_WORLD, 3, 3, (npes-1) / 3, config, + 0, &threes_team); int my_pe_twos = shmem_team_my_pe(twos_team); int my_pe_threes = shmem_team_my_pe(threes_team); int npes_twos = shmem_team_n_pes(twos_team); int npes_threes = shmem_team_n_pes(threes_team); - if (twos_team != SHMEMX_TEAM_INVALID) { + if (twos_team != SHMEM_TEAM_INVALID) { /* put the value 2 to the next team member in a circular fashion */ shmem_p(&x, 2, shmem_team_translate_pe(twos_team, (my_pe_twos + 1) % - npes_twos, SHMEMX_TEAM_WORLD)); + npes_twos, SHMEM_TEAM_WORLD)); shmem_quiet(); shmem_sync(twos_team); } - if (threes_team != SHMEMX_TEAM_INVALID) { + shmem_sync(SHMEM_TEAM_WORLD); + + if (threes_team != SHMEM_TEAM_INVALID) { /* put the value 3 to the next team member in a circular fashion */ shmem_p(&x, 3, shmem_team_translate_pe(threes_team, (my_pe_threes + 1) % - npes_threes, SHMEMX_TEAM_WORLD)); + npes_threes, SHMEM_TEAM_WORLD)); shmem_quiet(); shmem_sync(threes_team); } - if (me && me % 3 == 0 && x != 3) { - shmem_global_exit(3); + if (me && me % 3 == 0) { + if (x != 3) shmem_global_exit(3); } - else if (me && me % 2 == 0 && x != 2) { - shmem_global_exit(2); + else if (me && me % 2 == 0) { + if (x != 2) shmem_global_exit(2); } else if (x != 10101) { shmem_global_exit(1); From 0408da8aeb109e2c8f1d63a76efb602c6cee219e Mon Sep 17 00:00:00 2001 From: James Dinan Date: Thu, 5 Dec 2019 09:42:49 -0500 Subject: [PATCH 8/8] Minor edit from @agrippa Signed-off-by: James Dinan --- content/teams_intro.tex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/teams_intro.tex b/content/teams_intro.tex index f48489eaa..82ce2d4a7 100644 --- a/content/teams_intro.tex +++ b/content/teams_intro.tex @@ -80,7 +80,7 @@ \subsubsection*{Team Creation} This configuration argument is of type \CTYPE{shmem\_team\_config\_t}, which is detailed further in Section~\ref{subsec:shmem_team_config_t}. -\acp{PE} in a newly created teams are consecutively numbered starting with +\acp{PE} in a newly created team are consecutively numbered starting with \ac{PE} number 0. \acp{PE} are ordered by their \ac{PE} number in the parent team. Team relative \ac{PE} numbers can be used for point-to-point operations through team-based