From 97ae44c4596856b1461ece75fe7700ef30248806 Mon Sep 17 00:00:00 2001 From: "David M. Ozog" Date: Thu, 12 Dec 2019 14:34:37 -0500 Subject: [PATCH 1/6] Readd teams spec tests to "spec-example" directory Signed-off-by: David M. Ozog --- test/spec-example/Makefile.am | 3 + test/spec-example/shmem_reduce_example.c | 63 +++++++++++ test/spec-example/shmem_team_context.c | 131 +++++++++++++++++++++++ test/spec-example/shmem_team_split_2D.c | 50 +++++++++ 4 files changed, 247 insertions(+) create mode 100644 test/spec-example/shmem_reduce_example.c create mode 100644 test/spec-example/shmem_team_context.c create mode 100644 test/spec-example/shmem_team_split_2D.c diff --git a/test/spec-example/Makefile.am b/test/spec-example/Makefile.am index cc8de332b..c0807a0ca 100644 --- a/test/spec-example/Makefile.am +++ b/test/spec-example/Makefile.am @@ -27,10 +27,13 @@ check_PROGRAMS += \ shmemx_test_any \ shmemx_test_some \ shmemx_team_split_strided \ + shmemx_team_split_2D \ shmemx_team_translate \ + shmemx_team_context \ shmemx_team_sync \ shmemx_team_broadcast \ shmemx_team_collect \ + shmemx_reduce \ shmemx_team_alltoall \ shmemx_team_alltoalls diff --git a/test/spec-example/shmem_reduce_example.c b/test/spec-example/shmem_reduce_example.c new file mode 100644 index 000000000..c7ba1a52f --- /dev/null +++ b/test/spec-example/shmem_reduce_example.c @@ -0,0 +1,63 @@ +/* + * This test program is derived from an example program in the + * OpenSHMEM specification. + */ + +#include +#include +#include +#include + +/* As if we receive some value from external source */ +static long recv_a_value(unsigned seed, int npes) { + srand(seed); + return rand() % npes; +} + +/* Validate the value we recieved */ +static unsigned char is_valid(long value, int npes) { + if (value > (npes-1)) + return 0; + return 1; +} + +int main(void) +{ + + shmem_init(); + int me = shmem_my_pe(); + int npes = shmem_n_pes(); + size_t num = 32; + + long *values = shmem_malloc(num * sizeof(long)); + long *sums = shmem_malloc(num * sizeof(long)); + + unsigned char *valid_me = shmem_malloc(num * sizeof(unsigned char)); + unsigned char *valid_all = shmem_malloc(num * sizeof(unsigned char)); + + values[0] = recv_a_value((unsigned)me, npes); + valid_me[0] = is_valid(values[0], npes); + + for (int i=1; i < num; i++) { + values[i] = recv_a_value((unsigned)values[i-1], npes); + valid_me[i] = is_valid(values[i], npes); + } + + /* Wait for all PEs to initialize reductions arrays */ + shmemx_sync(SHMEMX_TEAM_WORLD); + + shmemx_and_reduce(SHMEMX_TEAM_WORLD, valid_all, valid_me, num); + shmemx_sum_reduce(SHMEMX_TEAM_WORLD, sums, values, num); + + for (int i=0; i < num; i++) { + if (valid_all[i]) { + printf ("[%d] = %ld\n", i, sums[i]); + } + else { + printf ("[%d] = invalid on one or more pe\n", i); + } + } + + shmem_finalize(); + return 0; +} diff --git a/test/spec-example/shmem_team_context.c b/test/spec-example/shmem_team_context.c new file mode 100644 index 000000000..2e14446fd --- /dev/null +++ b/test/spec-example/shmem_team_context.c @@ -0,0 +1,131 @@ +/* + * This test program is derived from an example program in the + * OpenSHMEM specification. + */ + +#include +#include +#include + +int isum, ival2, ival3; + +int my_ctx_translate_pe(shmem_ctx_t, int, shmem_ctx_t); +shmem_ctx_t my_team_create_ctx(shmemx_team_t); +void my_send_to_neighbor(shmem_ctx_t, int *); + +int my_ctx_translate_pe(shmem_ctx_t src_ctx, int src_pe, shmem_ctx_t dest_ctx) +{ + if (src_ctx == SHMEMX_CTX_INVALID) { + return -1; + } + if (dest_ctx == SHMEMX_CTX_INVALID) { + return -1; + } + + shmemx_team_t src_team, dest_team; + shmemx_ctx_get_team(src_ctx, &src_team); + shmemx_ctx_get_team(dest_ctx, &dest_team); + return shmemx_team_translate_pe(src_team, src_pe, dest_team); +} + +shmem_ctx_t my_team_create_ctx(shmemx_team_t team) { + if (team == SHMEMX_TEAM_INVALID) { + return SHMEMX_CTX_INVALID; + } + + shmem_ctx_t ctx; + if (shmemx_team_create_ctx(team, 0, &ctx) != 0) { + fprintf (stderr, "Failed to create context for PE team\n"); + return SHMEMX_CTX_INVALID; + } + return ctx; +} + +void my_send_to_neighbor(shmem_ctx_t ctx, int *val) +{ + if (ctx == SHMEMX_CTX_INVALID) { + fprintf (stderr, "Send to neighbor fail due to invalid context\n"); + return; + } + + shmemx_team_t team; + shmemx_ctx_get_team(ctx, &team); + int pe = shmemx_team_my_pe(team); + int npes = shmemx_team_n_pes(team); + int rpe = (pe + 1) % npes; + + // put my pe number in the buffer on my right hand neighbor + shmem_ctx_int_put(ctx, val, &pe, 1, rpe); +} + + + +int main() +{ + shmem_init(); + + int npes = shmem_n_pes(); + isum = 0; + + shmemx_team_t team_2s, team_3s; + shmem_ctx_t ctx_2s, ctx_3s; + shmemx_team_config_t conf; + conf.num_contexts = 1; + long cmask = SHMEMX_TEAM_NUM_CONTEXTS; + + if (npes < 4) { + fprintf(stderr, "Requires at least 4 PEs\n"); + shmem_global_exit(1); + } + + // Create team with PEs numbered 0, 2, 4, ... + shmemx_team_split_strided(SHMEMX_TEAM_WORLD, 0, 2, npes / 2, &conf, cmask, &team_2s); + // Sync between splits from same parent team into teams with overlapping membership + shmemx_team_sync(SHMEMX_TEAM_WORLD); + // Create team with PEs numbered 0, 3, 6, ... + shmemx_team_split_strided(SHMEMX_TEAM_WORLD, 0, 3, npes / 3, &conf, cmask, &team_3s); + + ctx_2s = my_team_create_ctx(team_2s); + ctx_3s = my_team_create_ctx(team_3s); + + // Send some values using the two team contexts contexts + ival2 = 2; + ival3 = 3; + my_send_to_neighbor(ctx_2s, &ival2); + my_send_to_neighbor(ctx_3s, &ival3); + + // Quiet all contexts and synchronize all PEs to complete the data transfers + shmem_ctx_quiet(ctx_2s); + shmem_ctx_quiet(ctx_3s); + shmemx_team_sync(SHMEMX_TEAM_WORLD); + + // We will add up some results on pe 4 of team_3s using ctx_2s + if ((team_3s != SHMEMX_TEAM_INVALID) && (team_2s != SHMEMX_TEAM_INVALID)) { + int _pe4_of_3s_in_2s = my_ctx_translate_pe(ctx_3s, 4, ctx_2s); + + if (_pe4_of_3s_in_2s < 0) { + fprintf (stderr, "Fail to translate pe 4 from 3s context to 2s context\n"); + } + else { + // Add up the results on pe 4 of the 3s team, using the 2s team context + shmem_ctx_int_atomic_add(ctx_2s, &isum, ival2 + ival3, _pe4_of_3s_in_2s); + } + } + + // Quiet the context and synchronize PEs to complete the operation + shmem_ctx_quiet(ctx_2s); + shmemx_team_sync(SHMEMX_TEAM_WORLD); + + if (shmemx_team_my_pe(team_3s) == 4) { + printf ("The total value on PE 4 of the 3s team is %d\n", isum); + } + + // Destroy contexts before teams + shmem_ctx_destroy(ctx_2s); + shmemx_team_destroy(team_2s); + + shmem_ctx_destroy(ctx_3s); + shmemx_team_destroy(team_3s); + + shmem_finalize(); +} diff --git a/test/spec-example/shmem_team_split_2D.c b/test/spec-example/shmem_team_split_2D.c new file mode 100644 index 000000000..aaba69c0e --- /dev/null +++ b/test/spec-example/shmem_team_split_2D.c @@ -0,0 +1,50 @@ +/* + * This test program is derived from an example program in the + * OpenSHMEM specification. + */ + +#include +#include +#include + +int main(void) +{ + int xdim = 3; + int ydim = 4; + + shmem_init(); + int pe = shmem_my_pe(); + int npes = shmem_n_pes(); + + if (npes < (xdim*ydim)) { + fprintf(stderr, "Not enough PEs to create 4x3xN layout\n"); + shmem_global_exit(1); + } + + int zdim = (npes / (xdim*ydim)) + ( ((npes % (xdim*ydim)) > 0) ? 1 : 0 ); + shmemx_team_t xteam, yzteam, yteam, zteam; + + + shmemx_team_split_2d(SHMEMX_TEAM_WORLD, xdim, NULL, 0, &xteam, NULL, 0, &yzteam); + // No synchronization is needed between these split operations + // yzteam is immediately ready to be used in collectives + shmemx_team_split_2d(yzteam, ydim, NULL, 0, &yteam, NULL, 0, &zteam); + + // We don't need the yzteam anymore + shmemx_team_destroy(yzteam); + + int my_x = shmemx_team_my_pe(xteam); + int my_y = shmemx_team_my_pe(yteam); + int my_z = shmemx_team_my_pe(zteam); + + for (int zdx = 0; zdx < zdim; zdx++) + for (int ydx = 0; ydx < ydim; ydx++) + for (int xdx = 0; xdx < xdim; xdx++) { + if ((my_x == xdx) && (my_y == ydx) && (my_z == zdx)) { + printf ("(%d, %d, %d) is me = %d\n", my_x, my_y, my_z, pe); + } + shmemx_team_sync(SHMEMX_TEAM_WORLD); + } + + shmem_finalize(); +} From d454ea04cd31ae311061d737172fbd0cdf40c359 Mon Sep 17 00:00:00 2001 From: "David M. Ozog" Date: Fri, 13 Dec 2019 17:27:07 -0500 Subject: [PATCH 2/6] Clean shmem_team_context example and fix Makefile Signed-off-by: David M. Ozog --- test/spec-example/Makefile.am | 6 +- test/spec-example/shmem_team_context.c | 143 +++++++++---------------- 2 files changed, 55 insertions(+), 94 deletions(-) diff --git a/test/spec-example/Makefile.am b/test/spec-example/Makefile.am index c0807a0ca..c39816466 100644 --- a/test/spec-example/Makefile.am +++ b/test/spec-example/Makefile.am @@ -27,13 +27,13 @@ check_PROGRAMS += \ shmemx_test_any \ shmemx_test_some \ shmemx_team_split_strided \ - shmemx_team_split_2D \ + shmem_team_split_2D \ shmemx_team_translate \ - shmemx_team_context \ + shmem_team_context \ shmemx_team_sync \ shmemx_team_broadcast \ shmemx_team_collect \ - shmemx_reduce \ + shmem_reduce_example \ shmemx_team_alltoall \ shmemx_team_alltoalls diff --git a/test/spec-example/shmem_team_context.c b/test/spec-example/shmem_team_context.c index 2e14446fd..bd39c944c 100644 --- a/test/spec-example/shmem_team_context.c +++ b/test/spec-example/shmem_team_context.c @@ -7,120 +7,81 @@ #include #include -int isum, ival2, ival3; -int my_ctx_translate_pe(shmem_ctx_t, int, shmem_ctx_t); -shmem_ctx_t my_team_create_ctx(shmemx_team_t); -void my_send_to_neighbor(shmem_ctx_t, int *); - -int my_ctx_translate_pe(shmem_ctx_t src_ctx, int src_pe, shmem_ctx_t dest_ctx) +int main(void) { - if (src_ctx == SHMEMX_CTX_INVALID) { - return -1; - } - if (dest_ctx == SHMEMX_CTX_INVALID) { - return -1; - } - - shmemx_team_t src_team, dest_team; - shmemx_ctx_get_team(src_ctx, &src_team); - shmemx_ctx_get_team(dest_ctx, &dest_team); - return shmemx_team_translate_pe(src_team, src_pe, dest_team); -} - -shmem_ctx_t my_team_create_ctx(shmemx_team_t team) { - if (team == SHMEMX_TEAM_INVALID) { - return SHMEMX_CTX_INVALID; - } - - shmem_ctx_t ctx; - if (shmemx_team_create_ctx(team, 0, &ctx) != 0) { - fprintf (stderr, "Failed to create context for PE team\n"); - return SHMEMX_CTX_INVALID; - } - return ctx; -} - -void my_send_to_neighbor(shmem_ctx_t ctx, int *val) -{ - if (ctx == SHMEMX_CTX_INVALID) { - fprintf (stderr, "Send to neighbor fail due to invalid context\n"); - return; - } - - shmemx_team_t team; - shmemx_ctx_get_team(ctx, &team); - int pe = shmemx_team_my_pe(team); - int npes = shmemx_team_n_pes(team); - int rpe = (pe + 1) % npes; - - // put my pe number in the buffer on my right hand neighbor - shmem_ctx_int_put(ctx, val, &pe, 1, rpe); -} - - + static int sum = 0, val2, val3; + shmemx_team_t team_2s, team_3s; + shmem_ctx_t ctx_2s, ctx_3s; + shmemx_team_config_t conf; -int main() -{ shmem_init(); int npes = shmem_n_pes(); - isum = 0; - - shmemx_team_t team_2s, team_3s; - shmem_ctx_t ctx_2s, ctx_3s; - shmemx_team_config_t conf; conf.num_contexts = 1; long cmask = SHMEMX_TEAM_NUM_CONTEXTS; - if (npes < 4) { - fprintf(stderr, "Requires at least 4 PEs\n"); - shmem_global_exit(1); + /* Create team with PEs numbered 0, 2, 4, ... */ + shmemx_team_split_strided(SHMEMX_TEAM_WORLD, 0, 2, (npes+1) / 2, &conf, cmask, &team_2s); + /* Sync between splits from same parent team into teams with overlapping membership */ + shmemx_team_sync(SHMEMX_TEAM_WORLD); + /* Create team with PEs numbered 0, 3, 6, ... */ + shmemx_team_split_strided(SHMEMX_TEAM_WORLD, 0, 3, (npes+2) / 3, &conf, cmask, &team_3s); + + /* Create a context on the 2's team. */ + if (team_2s != SHMEMX_TEAM_INVALID) { + int ret = shmemx_team_create_ctx(team_2s, 0, &ctx_2s); + if (ret != 0) ctx_2s = SHMEMX_CTX_INVALID; } - // Create team with PEs numbered 0, 2, 4, ... - shmemx_team_split_strided(SHMEMX_TEAM_WORLD, 0, 2, npes / 2, &conf, cmask, &team_2s); - // Sync between splits from same parent team into teams with overlapping membership - shmemx_team_sync(SHMEMX_TEAM_WORLD); - // Create team with PEs numbered 0, 3, 6, ... - shmemx_team_split_strided(SHMEMX_TEAM_WORLD, 0, 3, npes / 3, &conf, cmask, &team_3s); + /* Create a context on the 3's team. */ + if (team_3s != SHMEMX_TEAM_INVALID) { + int ret = shmemx_team_create_ctx(team_3s, 0, &ctx_3s); + if (ret != 0) ctx_3s = SHMEMX_CTX_INVALID; + } - ctx_2s = my_team_create_ctx(team_2s); - ctx_3s = my_team_create_ctx(team_3s); + /* Within each team, put my PE number to my neighbor in a ring-based manner. */ + if (ctx_2s != SHMEMX_CTX_INVALID) { + int pe = shmemx_team_my_pe(team_2s); + shmem_ctx_int_put(ctx_2s, &val2, &pe, 1, (pe + 1) % shmemx_team_n_pes(team_2s)); + } - // Send some values using the two team contexts contexts - ival2 = 2; - ival3 = 3; - my_send_to_neighbor(ctx_2s, &ival2); - my_send_to_neighbor(ctx_3s, &ival3); + if (ctx_3s != SHMEMX_CTX_INVALID) { + int pe = shmemx_team_my_pe(team_3s); + shmem_ctx_int_put(ctx_3s, &val3, &pe, 1, (pe + 1) % shmemx_team_n_pes(team_3s)); + } - // Quiet all contexts and synchronize all PEs to complete the data transfers + /* Quiet all contexts and synchronize all PEs to complete the data transfers. */ shmem_ctx_quiet(ctx_2s); shmem_ctx_quiet(ctx_3s); shmemx_team_sync(SHMEMX_TEAM_WORLD); - // We will add up some results on pe 4 of team_3s using ctx_2s - if ((team_3s != SHMEMX_TEAM_INVALID) && (team_2s != SHMEMX_TEAM_INVALID)) { - int _pe4_of_3s_in_2s = my_ctx_translate_pe(ctx_3s, 4, ctx_2s); - - if (_pe4_of_3s_in_2s < 0) { - fprintf (stderr, "Fail to translate pe 4 from 3s context to 2s context\n"); - } - else { - // Add up the results on pe 4 of the 3s team, using the 2s team context - shmem_ctx_int_atomic_add(ctx_2s, &isum, ival2 + ival3, _pe4_of_3s_in_2s); - } - } + /* Sum the values among PE's that are in both the 2's and 3's teams on PE 0. */ + if (team_3s != SHMEMX_TEAM_INVALID && team_2s != SHMEMX_TEAM_INVALID) + shmem_int_atomic_add(&sum, val2 + val3, 0); - // Quiet the context and synchronize PEs to complete the operation - shmem_ctx_quiet(ctx_2s); + /* Quiet the context and synchronize PEs to complete the operation */ + shmem_quiet(); shmemx_team_sync(SHMEMX_TEAM_WORLD); - if (shmemx_team_my_pe(team_3s) == 4) { - printf ("The total value on PE 4 of the 3s team is %d\n", isum); + /* Validate result */ + if (shmem_my_pe() == 0) { + int vsum = 0; + for (int i = 0; i < npes; i ++) { + if (i % 2 == 0 && i % 3 == 0) { + vsum += ((i - 2) < 0) ? shmemx_team_n_pes(team_2s) - 1 : + shmemx_team_translate_pe(SHMEMX_TEAM_WORLD, i - 2, team_2s); + vsum += ((i - 3) < 0) ? shmemx_team_n_pes(team_3s) - 1 : + shmemx_team_translate_pe(SHMEMX_TEAM_WORLD, i - 3, team_3s); + } + } + if (sum != vsum) { + fprintf(stderr, "Unexpected result\n"); + shmem_global_exit(1); + } } - // Destroy contexts before teams + /* Destroy contexts before teams */ shmem_ctx_destroy(ctx_2s); shmemx_team_destroy(team_2s); From 674785cbf660a607dba2e5e758e22b103add6547 Mon Sep 17 00:00:00 2001 From: David Ozog Date: Fri, 13 Dec 2019 20:08:49 -0500 Subject: [PATCH 3/6] Cleaning up spec-example/shmem_team_context.c more Signed-off-by: David Ozog --- test/spec-example/shmem_team_context.c | 86 +++++++++++++------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/test/spec-example/shmem_team_context.c b/test/spec-example/shmem_team_context.c index bd39c944c..3ac400f35 100644 --- a/test/spec-example/shmem_team_context.c +++ b/test/spec-example/shmem_team_context.c @@ -10,83 +10,83 @@ int main(void) { - static int sum = 0, val2, val3; - shmemx_team_t team_2s, team_3s; - shmem_ctx_t ctx_2s, ctx_3s; + static int sum = 0, val_2, val_3; + shmemx_team_t team_2, team_3; + shmem_ctx_t ctx_2, ctx_3; shmemx_team_config_t conf; shmem_init(); - int npes = shmem_n_pes(); + int npes = shmem_n_pes(); + int my_pe = shmem_my_pe(); conf.num_contexts = 1; long cmask = SHMEMX_TEAM_NUM_CONTEXTS; - /* Create team with PEs numbered 0, 2, 4, ... */ - shmemx_team_split_strided(SHMEMX_TEAM_WORLD, 0, 2, (npes+1) / 2, &conf, cmask, &team_2s); - /* Sync between splits from same parent team into teams with overlapping membership */ - shmemx_team_sync(SHMEMX_TEAM_WORLD); - /* Create team with PEs numbered 0, 3, 6, ... */ - shmemx_team_split_strided(SHMEMX_TEAM_WORLD, 0, 3, (npes+2) / 3, &conf, cmask, &team_3s); + /* Create team_2 with PEs numbered 0, 2, 4, ... */ + shmemx_team_split_strided(SHMEMX_TEAM_WORLD, 0, 2, (npes + 1) / 2, &conf, cmask, &team_2); + + /* Create team_3 with PEs numbered 0, 3, 6, ... */ + shmemx_team_split_strided(SHMEMX_TEAM_WORLD, 0, 3, (npes + 2) / 3, &conf, cmask, &team_3); - /* Create a context on the 2's team. */ - if (team_2s != SHMEMX_TEAM_INVALID) { - int ret = shmemx_team_create_ctx(team_2s, 0, &ctx_2s); - if (ret != 0) ctx_2s = SHMEMX_CTX_INVALID; + /* Create a context on team_2. */ + if (team_2 != SHMEMX_TEAM_INVALID) { + int ret = shmemx_team_create_ctx(team_2, 0, &ctx_2); + if (ret != 0) ctx_2 = SHMEMX_CTX_INVALID; } - /* Create a context on the 3's team. */ - if (team_3s != SHMEMX_TEAM_INVALID) { - int ret = shmemx_team_create_ctx(team_3s, 0, &ctx_3s); - if (ret != 0) ctx_3s = SHMEMX_CTX_INVALID; + /* Create a context on team_3. */ + if (team_3 != SHMEMX_TEAM_INVALID) { + int ret = shmemx_team_create_ctx(team_3, 0, &ctx_3); + if (ret != 0) ctx_3 = SHMEMX_CTX_INVALID; } /* Within each team, put my PE number to my neighbor in a ring-based manner. */ - if (ctx_2s != SHMEMX_CTX_INVALID) { - int pe = shmemx_team_my_pe(team_2s); - shmem_ctx_int_put(ctx_2s, &val2, &pe, 1, (pe + 1) % shmemx_team_n_pes(team_2s)); + if (ctx_2 != SHMEMX_CTX_INVALID) { + int pe = shmemx_team_my_pe(team_2); + shmem_ctx_int_put(ctx_2, &val_2, &pe, 1, (pe + 1) % shmemx_team_n_pes(team_2)); } - if (ctx_3s != SHMEMX_CTX_INVALID) { - int pe = shmemx_team_my_pe(team_3s); - shmem_ctx_int_put(ctx_3s, &val3, &pe, 1, (pe + 1) % shmemx_team_n_pes(team_3s)); + if (ctx_3 != SHMEMX_CTX_INVALID) { + int pe = shmemx_team_my_pe(team_3); + shmem_ctx_int_put(ctx_3, &val_3, &pe, 1, (pe + 1) % shmemx_team_n_pes(team_3)); } - /* Quiet all contexts and synchronize all PEs to complete the data transfers. */ - shmem_ctx_quiet(ctx_2s); - shmem_ctx_quiet(ctx_3s); + /* Quiet both contexts and synchronize all PEs to complete the data transfers. */ + shmem_ctx_quiet(ctx_2); + shmem_ctx_quiet(ctx_3); shmemx_team_sync(SHMEMX_TEAM_WORLD); - /* Sum the values among PE's that are in both the 2's and 3's teams on PE 0. */ - if (team_3s != SHMEMX_TEAM_INVALID && team_2s != SHMEMX_TEAM_INVALID) - shmem_int_atomic_add(&sum, val2 + val3, 0); + /* Sum the values among PEs that are in both team_2 and team_3 on PE 0 with ctx_2. */ + if (team_3 != SHMEMX_TEAM_INVALID && team_2 != SHMEMX_TEAM_INVALID) + shmem_ctx_int_atomic_add(ctx_2, &sum, val_2 + val_3, 0); - /* Quiet the context and synchronize PEs to complete the operation */ - shmem_quiet(); + /* Quiet the context and synchronize PEs to complete the operation. */ + shmem_ctx_quiet(ctx_2); shmemx_team_sync(SHMEMX_TEAM_WORLD); - /* Validate result */ + /* Validate the result. */ if (shmem_my_pe() == 0) { int vsum = 0; for (int i = 0; i < npes; i ++) { if (i % 2 == 0 && i % 3 == 0) { - vsum += ((i - 2) < 0) ? shmemx_team_n_pes(team_2s) - 1 : - shmemx_team_translate_pe(SHMEMX_TEAM_WORLD, i - 2, team_2s); - vsum += ((i - 3) < 0) ? shmemx_team_n_pes(team_3s) - 1 : - shmemx_team_translate_pe(SHMEMX_TEAM_WORLD, i - 3, team_3s); + vsum += ((i - 2) < 0) ? shmemx_team_n_pes(team_2) - 1 : + shmemx_team_translate_pe(SHMEMX_TEAM_WORLD, i - 2, team_2); + vsum += ((i - 3) < 0) ? shmemx_team_n_pes(team_3) - 1 : + shmemx_team_translate_pe(SHMEMX_TEAM_WORLD, i - 3, team_3); } } if (sum != vsum) { - fprintf(stderr, "Unexpected result\n"); + fprintf(stderr, "Unexpected result, npes = %d, vsum = %d, sum = %d\n", shmem_n_pes(), vsum, sum); shmem_global_exit(1); } } - /* Destroy contexts before teams */ - shmem_ctx_destroy(ctx_2s); - shmemx_team_destroy(team_2s); + /* Destroy contexts before teams. */ + shmem_ctx_destroy(ctx_2); + shmemx_team_destroy(team_2); - shmem_ctx_destroy(ctx_3s); - shmemx_team_destroy(team_3s); + shmem_ctx_destroy(ctx_3); + shmemx_team_destroy(team_3); shmem_finalize(); } From 44a0694a75f395157e310e3cdbfa0037ebd4f114 Mon Sep 17 00:00:00 2001 From: "David M. Ozog" Date: Mon, 16 Dec 2019 17:51:25 -0500 Subject: [PATCH 4/6] Make minor corrections to shmem_team_context test Signed-off-by: David M. Ozog --- test/spec-example/shmem_team_context.c | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/test/spec-example/shmem_team_context.c b/test/spec-example/shmem_team_context.c index 3ac400f35..798954f33 100644 --- a/test/spec-example/shmem_team_context.c +++ b/test/spec-example/shmem_team_context.c @@ -10,10 +10,10 @@ int main(void) { - static int sum = 0, val_2, val_3; - shmemx_team_t team_2, team_3; - shmem_ctx_t ctx_2, ctx_3; - shmemx_team_config_t conf; + static int sum = 0, val_2, val_3; + shmemx_team_t team_2, team_3; + shmem_ctx_t ctx_2, ctx_3; + shmemx_team_config_t conf; shmem_init(); @@ -29,16 +29,12 @@ int main(void) shmemx_team_split_strided(SHMEMX_TEAM_WORLD, 0, 3, (npes + 2) / 3, &conf, cmask, &team_3); /* Create a context on team_2. */ - if (team_2 != SHMEMX_TEAM_INVALID) { - int ret = shmemx_team_create_ctx(team_2, 0, &ctx_2); - if (ret != 0) ctx_2 = SHMEMX_CTX_INVALID; - } + int ret = shmemx_team_create_ctx(team_2, 0, &ctx_2); + if (ret != 0) ctx_2 = SHMEMX_CTX_INVALID; /* Create a context on team_3. */ - if (team_3 != SHMEMX_TEAM_INVALID) { - int ret = shmemx_team_create_ctx(team_3, 0, &ctx_3); - if (ret != 0) ctx_3 = SHMEMX_CTX_INVALID; - } + ret = shmemx_team_create_ctx(team_3, 0, &ctx_3); + if (ret != 0) ctx_3 = SHMEMX_CTX_INVALID; /* Within each team, put my PE number to my neighbor in a ring-based manner. */ if (ctx_2 != SHMEMX_CTX_INVALID) { From 077ee5812d4316d6177eee605d0dd851a7685772 Mon Sep 17 00:00:00 2001 From: "David M. Ozog" Date: Wed, 18 Dec 2019 11:25:42 -0500 Subject: [PATCH 5/6] Alter shmem_team_split_2D test to support any NPES Signed-off-by: David M. Ozog --- test/spec-example/shmem_team_split_2D.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/test/spec-example/shmem_team_split_2D.c b/test/spec-example/shmem_team_split_2D.c index aaba69c0e..7ba79231d 100644 --- a/test/spec-example/shmem_team_split_2D.c +++ b/test/spec-example/shmem_team_split_2D.c @@ -6,25 +6,33 @@ #include #include #include +#include + +/* Find x and y such that x * y == npes and abs(x - y) is minimized. */ +void find_x_and_y_dims(int npes, int *x, int *y) { + for(int divider = ceil(sqrt(npes)); divider >= 1; divider--) + if (npes % divider == 0) { + *x = divider; + *y = npes / divider; + return; + } +} int main(void) { - int xdim = 3; - int ydim = 4; + int xdim, ydim; shmem_init(); int pe = shmem_my_pe(); int npes = shmem_n_pes(); - if (npes < (xdim*ydim)) { - fprintf(stderr, "Not enough PEs to create 4x3xN layout\n"); - shmem_global_exit(1); - } + find_x_and_y_dims(npes, &ydim, &xdim); + + if (shmem_my_pe() == 0) printf("xdim = %d, ydim = %d\n", xdim, ydim); int zdim = (npes / (xdim*ydim)) + ( ((npes % (xdim*ydim)) > 0) ? 1 : 0 ); shmemx_team_t xteam, yzteam, yteam, zteam; - shmemx_team_split_2d(SHMEMX_TEAM_WORLD, xdim, NULL, 0, &xteam, NULL, 0, &yzteam); // No synchronization is needed between these split operations // yzteam is immediately ready to be used in collectives From e72393782403736d587622b127e49a8a8cba2f64 Mon Sep 17 00:00:00 2001 From: "David M. Ozog" Date: Fri, 10 Apr 2020 14:24:00 -0400 Subject: [PATCH 6/6] Merge upstream changes to teams API spec examples Signed-off-by: David M. Ozog --- test/spec-example/shmem_reduce_example.c | 64 +++++++++++------------- test/spec-example/shmem_team_context.c | 37 ++++++++++---- test/spec-example/shmem_team_split_2D.c | 53 ++++++++++++-------- 3 files changed, 88 insertions(+), 66 deletions(-) diff --git a/test/spec-example/shmem_reduce_example.c b/test/spec-example/shmem_reduce_example.c index c7ba1a52f..dd536a72e 100644 --- a/test/spec-example/shmem_reduce_example.c +++ b/test/spec-example/shmem_reduce_example.c @@ -3,59 +3,51 @@ * OpenSHMEM specification. */ +#include +#include #include #include -#include -#include -/* As if we receive some value from external source */ -static long recv_a_value(unsigned seed, int npes) { - srand(seed); - return rand() % npes; -} - -/* Validate the value we recieved */ -static unsigned char is_valid(long value, int npes) { - if (value > (npes-1)) - return 0; - return 1; -} - -int main(void) -{ +#define NELEMS 32 +int main(void) { shmem_init(); - int me = shmem_my_pe(); + int mype = shmem_my_pe(); int npes = shmem_n_pes(); - size_t num = 32; - long *values = shmem_malloc(num * sizeof(long)); - long *sums = shmem_malloc(num * sizeof(long)); + int *values = shmem_malloc(NELEMS * sizeof(int)); + + unsigned char *value_is_maximal = shmem_malloc(NELEMS * sizeof(unsigned char)); + unsigned char *value_is_maximal_all = shmem_malloc(NELEMS * sizeof(unsigned char)); - unsigned char *valid_me = shmem_malloc(num * sizeof(unsigned char)); - unsigned char *valid_all = shmem_malloc(num * sizeof(unsigned char)); + static int maximal_values_count = 0; + static int maximal_values_total; - values[0] = recv_a_value((unsigned)me, npes); - valid_me[0] = is_valid(values[0], npes); + srand((unsigned)mype); - for (int i=1; i < num; i++) { - values[i] = recv_a_value((unsigned)values[i-1], npes); - valid_me[i] = is_valid(values[i], npes); + for (int i = 0; i < NELEMS; i++) { + values[i] = rand() % npes; + + /* Track and count instances of maximal values (i.e., values equal to (npes-1)) */ + value_is_maximal[i] = (values[i] == (npes - 1)) ? 1 : 0; + maximal_values_count += value_is_maximal[i]; } /* Wait for all PEs to initialize reductions arrays */ shmemx_sync(SHMEMX_TEAM_WORLD); - shmemx_and_reduce(SHMEMX_TEAM_WORLD, valid_all, valid_me, num); - shmemx_sum_reduce(SHMEMX_TEAM_WORLD, sums, values, num); + shmemx_or_reduce(SHMEMX_TEAM_WORLD, value_is_maximal_all, value_is_maximal, NELEMS); + shmemx_sum_reduce(SHMEMX_TEAM_WORLD, &maximal_values_total, &maximal_values_count, 1); - for (int i=0; i < num; i++) { - if (valid_all[i]) { - printf ("[%d] = %ld\n", i, sums[i]); - } - else { - printf ("[%d] = invalid on one or more pe\n", i); + if (mype == 0) { + printf("Found %d maximal random numbers across all PEs.\n", maximal_values_total); + printf("A maximal number occured (at least once) at the following indices:\n"); + for (int i = 0; i < NELEMS; i++) { + if (value_is_maximal_all[i] == 1) { + printf("%d ", i); + } } + printf("\n"); } shmem_finalize(); diff --git a/test/spec-example/shmem_team_context.c b/test/spec-example/shmem_team_context.c index 798954f33..91c923d25 100644 --- a/test/spec-example/shmem_team_context.c +++ b/test/spec-example/shmem_team_context.c @@ -3,8 +3,8 @@ * OpenSHMEM specification. */ -#include #include +#include #include @@ -17,24 +17,42 @@ int main(void) shmem_init(); - int npes = shmem_n_pes(); - int my_pe = shmem_my_pe(); + int npes = shmem_n_pes(); + int mype = shmem_my_pe(); conf.num_contexts = 1; long cmask = SHMEMX_TEAM_NUM_CONTEXTS; /* Create team_2 with PEs numbered 0, 2, 4, ... */ - shmemx_team_split_strided(SHMEMX_TEAM_WORLD, 0, 2, (npes + 1) / 2, &conf, cmask, &team_2); + int ret = shmemx_team_split_strided(SHMEMX_TEAM_WORLD, 0, 2, (npes + 1) / 2, &conf, cmask, &team_2); + + if (ret != 0) { + printf("%d: Error creating team team_2 (%d)\n", mype, ret); + shmem_global_exit(ret); + } /* Create team_3 with PEs numbered 0, 3, 6, ... */ - shmemx_team_split_strided(SHMEMX_TEAM_WORLD, 0, 3, (npes + 2) / 3, &conf, cmask, &team_3); + ret = shmemx_team_split_strided(SHMEMX_TEAM_WORLD, 0, 3, (npes + 2) / 3, &conf, cmask, &team_3); + + if (ret != 0) { + printf("%d: Error creating team team_3 (%d)\n", mype, ret); + shmem_global_exit(ret); + } /* Create a context on team_2. */ - int ret = shmemx_team_create_ctx(team_2, 0, &ctx_2); - if (ret != 0) ctx_2 = SHMEMX_CTX_INVALID; + ret = shmemx_team_create_ctx(team_2, 0, &ctx_2); + + if (ret != 0 && team_2 != SHMEMX_TEAM_INVALID) { + printf("%d: Error creating context ctx_2 (%d)\n", mype, ret); + shmem_global_exit(ret); + } /* Create a context on team_3. */ ret = shmemx_team_create_ctx(team_3, 0, &ctx_3); - if (ret != 0) ctx_3 = SHMEMX_CTX_INVALID; + + if (ret != 0 && team_3 != SHMEMX_TEAM_INVALID) { + printf("%d: Error creating context ctx_3 (%d)\n", mype, ret); + shmem_global_exit(ret); + } /* Within each team, put my PE number to my neighbor in a ring-based manner. */ if (ctx_2 != SHMEMX_CTX_INVALID) { @@ -61,7 +79,7 @@ int main(void) shmemx_team_sync(SHMEMX_TEAM_WORLD); /* Validate the result. */ - if (shmem_my_pe() == 0) { + if (mype == 0) { int vsum = 0; for (int i = 0; i < npes; i ++) { if (i % 2 == 0 && i % 3 == 0) { @@ -85,4 +103,5 @@ int main(void) shmemx_team_destroy(team_3); shmem_finalize(); + return 0; } diff --git a/test/spec-example/shmem_team_split_2D.c b/test/spec-example/shmem_team_split_2D.c index 7ba79231d..4c10cb037 100644 --- a/test/spec-example/shmem_team_split_2D.c +++ b/test/spec-example/shmem_team_split_2D.c @@ -3,38 +3,46 @@ * OpenSHMEM specification. */ -#include -#include #include +#include +#include #include -/* Find x and y such that x * y == npes and abs(x - y) is minimized. */ -void find_x_and_y_dims(int npes, int *x, int *y) { - for(int divider = ceil(sqrt(npes)); divider >= 1; divider--) - if (npes % divider == 0) { - *x = divider; - *y = npes / divider; - return; - } +/* Find x and y such that x * y == npes and abs(x - y) is minimized. */ +static void find_xy_dims(int npes, int *x, int *y) { + for(int divider = ceil(sqrt(npes)); divider >= 1; divider--) + if (npes % divider == 0) { + *x = divider; + *y = npes / divider; + return; + } +} + +/* Find x, y, and z such that x * y * z == npes and + * abs(x - y) + abs(x - z) + abs(y - z) is minimized. */ +static void find_xyz_dims(int npes, int *x, int *y, int *z) { + for(int divider = ceil(cbrt(npes)); divider >= 1; divider--) + if (npes % divider == 0) { + *x = divider; + find_xy_dims(npes / divider, y, z); + return; + } } -int main(void) -{ - int xdim, ydim; +int main(void) { + int xdim, ydim, zdim; shmem_init(); - int pe = shmem_my_pe(); + int mype = shmem_my_pe(); int npes = shmem_n_pes(); - find_x_and_y_dims(npes, &ydim, &xdim); + find_xyz_dims(npes, &xdim, &ydim, &zdim); - if (shmem_my_pe() == 0) printf("xdim = %d, ydim = %d\n", xdim, ydim); + if (shmem_my_pe() == 0) printf("xdim = %d, ydim = %d, zdim = %d\n", xdim, ydim, zdim); - int zdim = (npes / (xdim*ydim)) + ( ((npes % (xdim*ydim)) > 0) ? 1 : 0 ); shmemx_team_t xteam, yzteam, yteam, zteam; shmemx_team_split_2d(SHMEMX_TEAM_WORLD, xdim, NULL, 0, &xteam, NULL, 0, &yzteam); - // No synchronization is needed between these split operations // yzteam is immediately ready to be used in collectives shmemx_team_split_2d(yzteam, ydim, NULL, 0, &yteam, NULL, 0, &zteam); @@ -45,14 +53,17 @@ int main(void) int my_y = shmemx_team_my_pe(yteam); int my_z = shmemx_team_my_pe(zteam); - for (int zdx = 0; zdx < zdim; zdx++) - for (int ydx = 0; ydx < ydim; ydx++) + for (int zdx = 0; zdx < zdim; zdx++) { + for (int ydx = 0; ydx < ydim; ydx++) { for (int xdx = 0; xdx < xdim; xdx++) { if ((my_x == xdx) && (my_y == ydx) && (my_z == zdx)) { - printf ("(%d, %d, %d) is me = %d\n", my_x, my_y, my_z, pe); + printf("(%d, %d, %d) is mype = %d\n", my_x, my_y, my_z, mype); } shmemx_team_sync(SHMEMX_TEAM_WORLD); } + } + } shmem_finalize(); + return 0; }