From f7cd4a6ae83e498e127ececf9bc05ea4c6fba27d Mon Sep 17 00:00:00 2001 From: David Ozog Date: Fri, 28 Feb 2020 17:15:19 -0500 Subject: [PATCH 1/7] Simplify & cleanup the teams example programs Signed-off-by: David Ozog --- example_code/shmem_reduce_example.c | 45 +++------ example_code/shmem_team_context.c | 139 +++++++++++----------------- 2 files changed, 69 insertions(+), 115 deletions(-) diff --git a/example_code/shmem_reduce_example.c b/example_code/shmem_reduce_example.c index 7d5b28b16..1a0fe336e 100644 --- a/example_code/shmem_reduce_example.c +++ b/example_code/shmem_reduce_example.c @@ -2,53 +2,34 @@ #include #include -/* As if we receive some value from external source */ -long recv_a_value(unsigned seed, int npes) { - srand(seed); - return rand() % npes; -} - -/* Validate the value we recieved */ -unsigned char is_valid(long value, int npes) { - if (value == (npes - 1)) - return 0; - return 1; -} +#define NELEMS 32 int main(void) { shmem_init(); int mype = shmem_my_pe(); int npes = shmem_n_pes(); - size_t num = 32; - long *values = shmem_malloc(num * sizeof(long)); - long *sums = shmem_malloc(num * sizeof(long)); + int *values = shmem_malloc(NELEMS * sizeof(int)); - unsigned char *valid_me = shmem_malloc(num * sizeof(unsigned char)); - unsigned char *valid_all = shmem_malloc(num * sizeof(unsigned char)); + static int my_maximal_values = 0; + static int all_maximal_values; - values[0] = recv_a_value((unsigned)mype, npes); - valid_me[0] = is_valid(values[0], npes); + srand((unsigned)mype); - for (int i = 1; i < num; i++) { - values[i] = recv_a_value((unsigned)values[i - 1], npes); - valid_me[i] = is_valid(values[i], npes); + for (int i = 0; i < NELEMS ; i++) { + values[i] = rand() % npes; + + /* Count the instances of maximal values (i.e., equal to (npes-1)) */ + my_maximal_values += (values[i] == (npes - 1)) ? 1 : 0; } /* Wait for all PEs to initialize reductions arrays */ shmem_sync(SHMEM_TEAM_WORLD); - shmem_and_reduce(SHMEM_TEAM_WORLD, valid_all, valid_me, num); - shmem_sum_reduce(SHMEM_TEAM_WORLD, sums, values, num); + shmem_sum_reduce(SHMEM_TEAM_WORLD, &all_maximal_values, &my_maximal_values, 1); - for (int i = 0; i < num; i++) { - if (valid_all[i]) { - printf("[%d] = %ld\n", i, sums[i]); - } - else { - printf("[%d] = invalid on one or more pe\n", i); - } - } + if (mype == 0) + printf("found a total of %d maximal random numbers\n", all_maximal_values); shmem_finalize(); return 0; diff --git a/example_code/shmem_team_context.c b/example_code/shmem_team_context.c index 0e3559946..c7d93af0f 100644 --- a/example_code/shmem_team_context.c +++ b/example_code/shmem_team_context.c @@ -1,107 +1,80 @@ #include #include -int isum, ival; -int my_ctx_translate_pe(shmem_ctx_t src_ctx, int src_pe, shmem_ctx_t dest_ctx) { - if (src_ctx == SHMEM_CTX_INVALID) { - return -1; - } - if (dest_ctx == SHMEM_CTX_INVALID) { - return -1; - } - - shmem_team_t src_team, dest_team; - shmem_ctx_get_team(src_ctx, &src_team); - shmem_ctx_get_team(dest_ctx, &dest_team); - return shmem_team_translate_pe(src_team, src_pe, dest_pe); -} - -shmem_ctx_t my_team_create_ctx(shmem_team_t team) { - if (team == SHMEM_TEAM_INVALID) { - return SHMEM_CTX_INVALID; - } +int main(void) +{ + static int sum = 0, val_2, val_3; + shmem_team_t team_2, team_3; + shmem_ctx_t ctx_2, ctx_3; + shmem_team_config_t conf; - shmem_ctx_t ctx; - if (shmem_team_create_ctx(team, 0, &ctx) != 0) { - fprintf(stderr, "Failed to create context for PE team\n"); - return SHMEM_CTX_INVALID; - } - return ctx; -} - -void my_send_to_neighbor(shmem_ctx_t ctx, int *val) { - if (ctx == SHMEM_CTX_INVALID) { - fprintf(stderr, "Send to neighbor fail due to invalid context\n"); - return; - } + shmem_init(); - shmem_team_t team; - shmem_ctx_get_team(ctx, &team); - int team_mype = shmem_team_my_pe(team); - int team_npes = shmem_team_n_pes(team); - int rpe = (team_mype + 1) % team_npes; + int npes = shmem_n_pes(); + int my_pe = shmem_my_pe(); + conf.num_contexts = 1; + long cmask = SHMEM_TEAM_NUM_CONTEXTS; - // put my pe number in the buffer on my right hand neighbor - shmem_ctx_int_put(ctx, val, &pe, 1, rpe); -} + /* Create team_2 with PEs numbered 0, 2, 4, ... */ + shmem_team_split_strided(SHMEM_TEAM_WORLD, 0, 2, (npes + 1) / 2, &conf, cmask, &team_2); -int main() { - shmem_init(); + /* Create team_3 with PEs numbered 0, 3, 6, ... */ + shmem_team_split_strided(SHMEM_TEAM_WORLD, 0, 3, (npes + 2) / 3, &conf, cmask, &team_3); - int npes = shmem_n_pes(); - isum = 0; + /* Create a context on team_2. */ + int ret = shmem_team_create_ctx(team_2, 0, &ctx_2); - shmem_team_t team_2s, team_3s; - shmem_ctx_t ctx_2s, ctx_3s; - shmem_team_config_t conf = {.num_contexts = 1}; - long cmask = SHMEM_TEAM_NUM_CONTEXTS; + /* Create a context on team_3. */ + ret = shmem_team_create_ctx(team_3, 0, &ctx_3); - // Create team with PEs numbered 0, 2, 4, ... - shmem_team_spit_strided(SHMEM_TEAM_WORLD, 0, 2, npes / 2, &conf, cmask, &team_2s); - // Create team with PEs numbered 0, 3, 6, ... - shmem_team_split_strided(SHMEM_TEAM_WORLD, 0, 3, npes / 3, &conf, cmask, &team_3s); - - ctx_2s = my_team_create_ctx(team_2s); - ctx_3s = my_team_create_ctx(team_3s); + /* Within each team, put my PE number to my neighbor in a ring-based manner. */ + if (ctx_2 != SHMEM_CTX_INVALID) { + int pe = shmem_team_my_pe(team_2); + shmem_ctx_int_put(ctx_2, &val_2, &pe, 1, (pe + 1) % shmem_team_n_pes(team_2)); + } - // Send some values using the two team contexts contexts - my_send_to_neighbor(ctx_2s, &ival2); - my_send_to_neighbor(ctx_3s, &ival3); + if (ctx_3 != SHMEM_CTX_INVALID) { + int pe = shmem_team_my_pe(team_3); + shmem_ctx_int_put(ctx_3, &val_3, &pe, 1, (pe + 1) % shmem_team_n_pes(team_3)); + } - // Quiet all contexts and synchronize all PEs to complete the data transfers - shmem_ctx_quiet(ctx_2s); - shmem_ctx_quiet(ctx_3s); + /* Quiet both contexts and synchronize all PEs to complete the data transfers. */ + shmem_ctx_quiet(ctx_2); + shmem_ctx_quiet(ctx_3); shmem_team_sync(SHMEM_TEAM_WORLD); - // We will add up some results on pe 4 of team_3s using ctx_2s - if ((team_3s != SHMEM_TEAM_INVALID) && (team_2s != SHMEM_TEAM_INVALID)) { - int pe4_of_3s_in_2s = my_ctx_translate_pe(ctx_3s, 4, ctx_2s); - - if (pe4_of_3s_in_2s < 0) { - fprintf(stderr, "Fail to translate pe 4 from 3s context to 2s context\n"); - } - else { - // Add up the results on pe 4 of the 3s team, using the 2s team context - shmem_ctx_int_atomic_add(ctx_2s, &isum, ival2 + ival3, pe4_of_3s_in_2s); - } - } + /* Sum the values among PEs that are in both team_2 and team_3 on PE 0 with ctx_2. */ + if (team_3 != SHMEM_TEAM_INVALID && team_2 != SHMEM_TEAM_INVALID) + shmem_ctx_int_atomic_add(ctx_2, &sum, val_2 + val_3, 0); - // Quiet the context and synchronize PEs to complete the operation - shmem_ctx_quiet(ctx_2s); + /* Quiet the context and synchronize PEs to complete the operation. */ + shmem_ctx_quiet(ctx_2); shmem_team_sync(SHMEM_TEAM_WORLD); - if (shmem_team_my_pe(team_3s) == 4) { - printf("The total value on PE 4 of the 3s team is %d\n", isum); + /* Validate the result. */ + if (shmem_my_pe() == 0) { + int vsum = 0; + for (int i = 0; i < npes; i ++) { + if (i % 2 == 0 && i % 3 == 0) { + vsum += ((i - 2) < 0) ? shmem_team_n_pes(team_2) - 1 : + shmem_team_translate_pe(SHMEM_TEAM_WORLD, i - 2, team_2); + vsum += ((i - 3) < 0) ? shmem_team_n_pes(team_3) - 1 : + shmem_team_translate_pe(SHMEM_TEAM_WORLD, i - 3, team_3); + } + } + if (sum != vsum) { + fprintf(stderr, "Unexpected result, npes = %d, vsum = %d, sum = %d\n", shmem_n_pes(), vsum, sum); + shmem_global_exit(1); + } } - // Destroy contexts before teams - shmem_ctx_destroy(ctx_2s); - shmem_team_destroy(team_2s); + /* Destroy contexts before teams. */ + shmem_ctx_destroy(ctx_2); + shmem_team_destroy(team_2); - shmem_ctx_destroy(ctx_3s); - shmem_team_destroy(team_3s); + shmem_ctx_destroy(ctx_3); + shmem_team_destroy(team_3); shmem_finalize(); - return 0; } From f16e72be85bd3e9907bfd4f6df3c63713ba32e87 Mon Sep 17 00:00:00 2001 From: "David M. Ozog" Date: Tue, 3 Mar 2020 16:41:30 -0500 Subject: [PATCH 2/7] Update reduction example text, add an OR reduction Signed-off-by: David M. Ozog --- content/shmem_reductions.tex | 10 ++++++---- example_code/shmem_reduce_example.c | 30 ++++++++++++++++++++--------- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/content/shmem_reductions.tex b/content/shmem_reductions.tex index 7c738a0bb..bacf39fa6 100644 --- a/content/shmem_reductions.tex +++ b/content/shmem_reductions.tex @@ -315,10 +315,12 @@ \begin{apiexamples} \apicexample - {This \Cstd[11] reduction example gets integers from an external - source (random generator in this example), tests to see if the \ac{PE} got a valid - value, and outputs the sum of values for which all \acp{PE} got a valid - value.} + {In the following \Cstd[11] example, each \ac{PE} intializes an array of + random integers with values between $0$ and $npes-1$, inclusively. An OR + reduction then tracks the array indices where maximal values occur (maximal + values equal $npes - 1$), and a SUM reduction counts the total number of + maximal values across all PEs. + } {./example_code/shmem_reduce_example.c} {} diff --git a/example_code/shmem_reduce_example.c b/example_code/shmem_reduce_example.c index 1a0fe336e..c00d9e304 100644 --- a/example_code/shmem_reduce_example.c +++ b/example_code/shmem_reduce_example.c @@ -11,25 +11,37 @@ int main(void) { int *values = shmem_malloc(NELEMS * sizeof(int)); - static int my_maximal_values = 0; - static int all_maximal_values; + unsigned char *value_is_maximal = shmem_malloc(NELEMS * sizeof(unsigned char)); + unsigned char *value_is_maximal_all = shmem_malloc(NELEMS * sizeof(unsigned char)); + + static int maximal_values_count = 0; + static int maximal_values_total; srand((unsigned)mype); - for (int i = 0; i < NELEMS ; i++) { + for (int i = 0; i < NELEMS; i++) { values[i] = rand() % npes; - /* Count the instances of maximal values (i.e., equal to (npes-1)) */ - my_maximal_values += (values[i] == (npes - 1)) ? 1 : 0; + /* Track and count instances of maximal values (i.e., values equal to (npes-1)) */ + value_is_maximal[i] = (values[i] == (npes - 1)) ? 1 : 0; + maximal_values_count += value_is_maximal[i]; } /* Wait for all PEs to initialize reductions arrays */ shmem_sync(SHMEM_TEAM_WORLD); - shmem_sum_reduce(SHMEM_TEAM_WORLD, &all_maximal_values, &my_maximal_values, 1); - - if (mype == 0) - printf("found a total of %d maximal random numbers\n", all_maximal_values); + shmem_or_reduce(SHMEM_TEAM_WORLD, value_is_maximal_all, value_is_maximal, NELEMS); + shmem_sum_reduce(SHMEM_TEAM_WORLD, &maximal_values_total, &maximal_values_count, 1); + + if (mype == 0) { + printf("Found %d maximal random numbers across all PEs.\n", maximal_values_total); + printf("A maximal number occured (at least once) at the following indices:\n"); + for (int i = 0; i < NELEMS; i++) { + if (value_is_maximal_all[i] == 1) { + printf("%d ", i); + } + } printf("\n"); + } shmem_finalize(); return 0; From 621399b410e6194b5e1613b5c0236bb2ee839bea Mon Sep 17 00:00:00 2001 From: "David M. Ozog" Date: Thu, 5 Mar 2020 10:12:34 -0500 Subject: [PATCH 3/7] team_create_ctx example: my_pe->mype, check return Signed-off-by: David M. Ozog --- example_code/shmem_team_context.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/example_code/shmem_team_context.c b/example_code/shmem_team_context.c index c7d93af0f..2ba545328 100644 --- a/example_code/shmem_team_context.c +++ b/example_code/shmem_team_context.c @@ -11,8 +11,8 @@ int main(void) shmem_init(); - int npes = shmem_n_pes(); - int my_pe = shmem_my_pe(); + int npes = shmem_n_pes(); + int mype = shmem_my_pe(); conf.num_contexts = 1; long cmask = SHMEM_TEAM_NUM_CONTEXTS; @@ -25,9 +25,19 @@ int main(void) /* Create a context on team_2. */ int ret = shmem_team_create_ctx(team_2, 0, &ctx_2); + if (ret != 0) { + printf("%d: Error creating context ctx_2 (%d)\n", mype, ret); + shmem_global_exit(ret); + } + /* Create a context on team_3. */ ret = shmem_team_create_ctx(team_3, 0, &ctx_3); + if (ret != 0) { + printf("%d: Error creating context ctx_3 (%d)\n", mype, ret); + shmem_global_exit(ret); + } + /* Within each team, put my PE number to my neighbor in a ring-based manner. */ if (ctx_2 != SHMEM_CTX_INVALID) { int pe = shmem_team_my_pe(team_2); @@ -53,7 +63,7 @@ int main(void) shmem_team_sync(SHMEM_TEAM_WORLD); /* Validate the result. */ - if (shmem_my_pe() == 0) { + if (mype == 0) { int vsum = 0; for (int i = 0; i < npes; i ++) { if (i % 2 == 0 && i % 3 == 0) { From 1f9af0ec0f6340b40c6facc1a41437cfd89709ed Mon Sep 17 00:00:00 2001 From: David Ozog Date: Fri, 6 Mar 2020 14:26:59 -0500 Subject: [PATCH 4/7] Restore "return 0" in shmem_team_context example. Signed-off-by: David Ozog --- example_code/shmem_team_context.c | 1 + 1 file changed, 1 insertion(+) diff --git a/example_code/shmem_team_context.c b/example_code/shmem_team_context.c index 2ba545328..94202a620 100644 --- a/example_code/shmem_team_context.c +++ b/example_code/shmem_team_context.c @@ -87,4 +87,5 @@ int main(void) shmem_team_destroy(team_3); shmem_finalize(); + return 0; } From 4ce980507b44cffbd4f96cfbe84f05fd1f3e977e Mon Sep 17 00:00:00 2001 From: David Ozog Date: Wed, 11 Mar 2020 17:27:12 -0400 Subject: [PATCH 5/7] Update example_code/shmem_reduce_example.c Co-Authored-By: James Dinan --- example_code/shmem_reduce_example.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/example_code/shmem_reduce_example.c b/example_code/shmem_reduce_example.c index c00d9e304..9e025c9d7 100644 --- a/example_code/shmem_reduce_example.c +++ b/example_code/shmem_reduce_example.c @@ -40,7 +40,8 @@ int main(void) { if (value_is_maximal_all[i] == 1) { printf("%d ", i); } - } printf("\n"); + } + printf("\n"); } shmem_finalize(); From d3b8242c314579672c7f3d26af13e1aa4e8be17d Mon Sep 17 00:00:00 2001 From: "David M. Ozog" Date: Mon, 16 Mar 2020 15:50:31 -0400 Subject: [PATCH 6/7] team_context example: improve return value checks Signed-off-by: David M. Ozog --- example_code/shmem_team_context.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/example_code/shmem_team_context.c b/example_code/shmem_team_context.c index 94202a620..331b323df 100644 --- a/example_code/shmem_team_context.c +++ b/example_code/shmem_team_context.c @@ -17,15 +17,25 @@ int main(void) long cmask = SHMEM_TEAM_NUM_CONTEXTS; /* Create team_2 with PEs numbered 0, 2, 4, ... */ - shmem_team_split_strided(SHMEM_TEAM_WORLD, 0, 2, (npes + 1) / 2, &conf, cmask, &team_2); + int ret = shmem_team_split_strided(SHMEM_TEAM_WORLD, 0, 2, (npes + 1) / 2, &conf, cmask, &team_2); + + if (ret != 0) { + printf("%d: Error creating team team_2 (%d)\n", mype, ret); + shmem_global_exit(ret); + } /* Create team_3 with PEs numbered 0, 3, 6, ... */ - shmem_team_split_strided(SHMEM_TEAM_WORLD, 0, 3, (npes + 2) / 3, &conf, cmask, &team_3); + ret = shmem_team_split_strided(SHMEM_TEAM_WORLD, 0, 3, (npes + 2) / 3, &conf, cmask, &team_3); + + if (ret != 0) { + printf("%d: Error creating team team_3 (%d)\n", mype, ret); + shmem_global_exit(ret); + } /* Create a context on team_2. */ - int ret = shmem_team_create_ctx(team_2, 0, &ctx_2); + ret = shmem_team_create_ctx(team_2, 0, &ctx_2); - if (ret != 0) { + if (ret != 0 && team_2 != SHMEM_TEAM_INVALID) { printf("%d: Error creating context ctx_2 (%d)\n", mype, ret); shmem_global_exit(ret); } @@ -33,7 +43,7 @@ int main(void) /* Create a context on team_3. */ ret = shmem_team_create_ctx(team_3, 0, &ctx_3); - if (ret != 0) { + if (ret != 0 && team_3 != SHMEM_TEAM_INVALID) { printf("%d: Error creating context ctx_3 (%d)\n", mype, ret); shmem_global_exit(ret); } From 18888103cf7ed27357b87ac3f5a3c1ec2161fa6c Mon Sep 17 00:00:00 2001 From: "David M. Ozog" Date: Thu, 2 Apr 2020 11:20:43 -0400 Subject: [PATCH 7/7] Move ctx_get_team example to team_create_ctx Signed-off-by: David M. Ozog --- content/shmem_ctx_get_team.tex | 11 ----------- content/shmem_team_create_ctx.tex | 7 ++++++- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/content/shmem_ctx_get_team.tex b/content/shmem_ctx_get_team.tex index a378a0e34..7e77bbaff 100644 --- a/content/shmem_ctx_get_team.tex +++ b/content/shmem_ctx_get_team.tex @@ -38,15 +38,4 @@ Zero on success; otherwise, nonzero. } -\begin{apiexamples} - - \apicexample - {The following example demonstrates the use of contexts for multiple teams in a - \Cstd[11] program. This example shows contexts being used to communicate within - a team using team \ac{PE} numbers, and across teams using translated \ac{PE} numbers.} - {./example_code/shmem_team_context.c} - {} - -\end{apiexamples} - \end{apidefinition} diff --git a/content/shmem_team_create_ctx.tex b/content/shmem_team_create_ctx.tex index 6f626a6e6..7ebc49c69 100644 --- a/content/shmem_team_create_ctx.tex +++ b/content/shmem_team_create_ctx.tex @@ -59,7 +59,12 @@ } \begin{apiexamples} - See example in Section \ref{subsec:shmem_ctx_get_team} + \apicexample + {The following example demonstrates the use of contexts for multiple teams in a + \Cstd[11] program. This example shows contexts being used to communicate within + a team using team \ac{PE} numbers, and across teams using translated \ac{PE} numbers.} + {./example_code/shmem_team_context.c} + {} \end{apiexamples} \end{apidefinition}