Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Simplify & cleanup the teams example programs #11

Merged
Merged
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions content/shmem_reductions.tex
Original file line number Diff line number Diff line change
@@ -315,10 +315,12 @@
\begin{apiexamples}

\apicexample
{This \Cstd[11] reduction example gets integers from an external
source (random generator in this example), tests to see if the \ac{PE} got a valid
value, and outputs the sum of values for which all \acp{PE} got a valid
value.}
{In the following \Cstd[11] example, each \ac{PE} intializes an array of
random integers with values between $0$ and $npes-1$, inclusively. An OR
reduction then tracks the array indices where maximal values occur (maximal
values equal $npes - 1$), and a SUM reduction counts the total number of
maximal values across all PEs.
}
{./example_code/shmem_reduce_example.c}
{}

54 changes: 24 additions & 30 deletions example_code/shmem_reduce_example.c
Original file line number Diff line number Diff line change
@@ -2,52 +2,46 @@
#include <stdio.h>
#include <stdlib.h>

/* As if we receive some value from external source */
long recv_a_value(unsigned seed, int npes) {
srand(seed);
return rand() % npes;
}

/* Validate the value we recieved */
unsigned char is_valid(long value, int npes) {
if (value == (npes - 1))
return 0;
return 1;
}
#define NELEMS 32

int main(void) {
shmem_init();
int mype = shmem_my_pe();
int npes = shmem_n_pes();
size_t num = 32;

long *values = shmem_malloc(num * sizeof(long));
long *sums = shmem_malloc(num * sizeof(long));
int *values = shmem_malloc(NELEMS * sizeof(int));

unsigned char *valid_me = shmem_malloc(num * sizeof(unsigned char));
unsigned char *valid_all = shmem_malloc(num * sizeof(unsigned char));
unsigned char *value_is_maximal = shmem_malloc(NELEMS * sizeof(unsigned char));
unsigned char *value_is_maximal_all = shmem_malloc(NELEMS * sizeof(unsigned char));

values[0] = recv_a_value((unsigned)mype, npes);
valid_me[0] = is_valid(values[0], npes);
static int maximal_values_count = 0;
static int maximal_values_total;

for (int i = 1; i < num; i++) {
values[i] = recv_a_value((unsigned)values[i - 1], npes);
valid_me[i] = is_valid(values[i], npes);
srand((unsigned)mype);

for (int i = 0; i < NELEMS; i++) {
values[i] = rand() % npes;

/* Track and count instances of maximal values (i.e., values equal to (npes-1)) */
value_is_maximal[i] = (values[i] == (npes - 1)) ? 1 : 0;
maximal_values_count += value_is_maximal[i];
}

/* Wait for all PEs to initialize reductions arrays */
shmem_sync(SHMEM_TEAM_WORLD);

shmem_and_reduce(SHMEM_TEAM_WORLD, valid_all, valid_me, num);
shmem_sum_reduce(SHMEM_TEAM_WORLD, sums, values, num);
shmem_or_reduce(SHMEM_TEAM_WORLD, value_is_maximal_all, value_is_maximal, NELEMS);
shmem_sum_reduce(SHMEM_TEAM_WORLD, &maximal_values_total, &maximal_values_count, 1);

for (int i = 0; i < num; i++) {
if (valid_all[i]) {
printf("[%d] = %ld\n", i, sums[i]);
}
else {
printf("[%d] = invalid on one or more pe\n", i);
if (mype == 0) {
printf("Found %d maximal random numbers across all PEs.\n", maximal_values_total);
jdinan marked this conversation as resolved.
Show resolved Hide resolved
printf("A maximal number occured (at least once) at the following indices:\n");
for (int i = 0; i < NELEMS; i++) {
if (value_is_maximal_all[i] == 1) {
printf("%d ", i);
}
}
printf("\n");
}

shmem_finalize();
144 changes: 64 additions & 80 deletions example_code/shmem_team_context.c
Original file line number Diff line number Diff line change
@@ -1,106 +1,90 @@
#include <shmem.h>
#include <stdio.h>

int isum, ival;

int my_ctx_translate_pe(shmem_ctx_t src_ctx, int src_pe, shmem_ctx_t dest_ctx) {
if (src_ctx == SHMEM_CTX_INVALID) {
return -1;
}
if (dest_ctx == SHMEM_CTX_INVALID) {
return -1;
}

shmem_team_t src_team, dest_team;
shmem_ctx_get_team(src_ctx, &src_team);
shmem_ctx_get_team(dest_ctx, &dest_team);
return shmem_team_translate_pe(src_team, src_pe, dest_pe);
}

shmem_ctx_t my_team_create_ctx(shmem_team_t team) {
if (team == SHMEM_TEAM_INVALID) {
return SHMEM_CTX_INVALID;
}
int main(void)

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This whole example is bit convoluted for a simple team-based context and team-based PE numbering in an RMA operation. Not sure, if we can change this now:

  1. Can we just create one team and use the team world instead of the team_3
  2. Am I missing some sort of a common usage model that the current example tries to provide?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not entirely sure we can make such changes either, but if so, I'm ok with removing team_3. Translating to the world team is probably a more common use case. @jdinan, would this change be permissible?

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The existing teams examples sorely need attention and I fully support using section committee power to fix them. The hitch is that we need 100% consensus for any changes at this point to clear upcoming votes. Unfortunately, we're only going to get one shot at that, since these won't be read until the RC2 meeting. Here's what I propose -- let's do the best job that we can with these in advance of the RC1 meeting. @manjugv can we get some time at the end of that meeting to read these changes for feedback and ensure that they don't cause us to fail the RC2 vote?

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@manjugv can we get some time at the end of that meeting to read these changes for feedback and ensure that they don't cause us to fail the RC2 vote?

👍

{
static int sum = 0, val_2, val_3;
shmem_team_t team_2, team_3;
shmem_ctx_t ctx_2, ctx_3;
shmem_team_config_t conf;

shmem_ctx_t ctx;
if (shmem_team_create_ctx(team, 0, &ctx) != 0) {
fprintf(stderr, "Failed to create context for PE team\n");
return SHMEM_CTX_INVALID;
}
return ctx;
}
shmem_init();

void my_send_to_neighbor(shmem_ctx_t ctx, int *val) {
if (ctx == SHMEM_CTX_INVALID) {
fprintf(stderr, "Send to neighbor fail due to invalid context\n");
return;
}
int npes = shmem_n_pes();
int mype = shmem_my_pe();
conf.num_contexts = 1;
long cmask = SHMEM_TEAM_NUM_CONTEXTS;

shmem_team_t team;
shmem_ctx_get_team(ctx, &team);
int team_mype = shmem_team_my_pe(team);
int team_npes = shmem_team_n_pes(team);
int rpe = (team_mype + 1) % team_npes;
/* Create team_2 with PEs numbered 0, 2, 4, ... */
shmem_team_split_strided(SHMEM_TEAM_WORLD, 0, 2, (npes + 1) / 2, &conf, cmask, &team_2);
jdinan marked this conversation as resolved.
Show resolved Hide resolved

// put my pe number in the buffer on my right hand neighbor
shmem_ctx_int_put(ctx, val, &pe, 1, rpe);
}
/* Create team_3 with PEs numbered 0, 3, 6, ... */
shmem_team_split_strided(SHMEM_TEAM_WORLD, 0, 3, (npes + 2) / 3, &conf, cmask, &team_3);

int main() {
shmem_init();
/* Create a context on team_2. */
int ret = shmem_team_create_ctx(team_2, 0, &ctx_2);
jdinan marked this conversation as resolved.
Show resolved Hide resolved

int npes = shmem_n_pes();
isum = 0;
if (ret != 0) {
printf("%d: Error creating context ctx_2 (%d)\n", mype, ret);
shmem_global_exit(ret);
}

shmem_team_t team_2s, team_3s;
shmem_ctx_t ctx_2s, ctx_3s;
shmem_team_config_t conf = {.num_contexts = 1};
long cmask = SHMEM_TEAM_NUM_CONTEXTS;
/* Create a context on team_3. */
ret = shmem_team_create_ctx(team_3, 0, &ctx_3);
davidozog marked this conversation as resolved.
Show resolved Hide resolved

// Create team with PEs numbered 0, 2, 4, ...
shmem_team_spit_strided(SHMEM_TEAM_WORLD, 0, 2, npes / 2, &conf, cmask, &team_2s);
// Create team with PEs numbered 0, 3, 6, ...
shmem_team_split_strided(SHMEM_TEAM_WORLD, 0, 3, npes / 3, &conf, cmask, &team_3s);
if (ret != 0) {
printf("%d: Error creating context ctx_3 (%d)\n", mype, ret);
shmem_global_exit(ret);
}

ctx_2s = my_team_create_ctx(team_2s);
ctx_3s = my_team_create_ctx(team_3s);
/* Within each team, put my PE number to my neighbor in a ring-based manner. */
if (ctx_2 != SHMEM_CTX_INVALID) {
jdinan marked this conversation as resolved.
Show resolved Hide resolved
int pe = shmem_team_my_pe(team_2);
shmem_ctx_int_put(ctx_2, &val_2, &pe, 1, (pe + 1) % shmem_team_n_pes(team_2));
}

// Send some values using the two team contexts contexts
my_send_to_neighbor(ctx_2s, &ival2);
my_send_to_neighbor(ctx_3s, &ival3);
if (ctx_3 != SHMEM_CTX_INVALID) {
int pe = shmem_team_my_pe(team_3);
shmem_ctx_int_put(ctx_3, &val_3, &pe, 1, (pe + 1) % shmem_team_n_pes(team_3));
}

// Quiet all contexts and synchronize all PEs to complete the data transfers
shmem_ctx_quiet(ctx_2s);
shmem_ctx_quiet(ctx_3s);
/* Quiet both contexts and synchronize all PEs to complete the data transfers. */
shmem_ctx_quiet(ctx_2);
shmem_ctx_quiet(ctx_3);
shmem_team_sync(SHMEM_TEAM_WORLD);

// We will add up some results on pe 4 of team_3s using ctx_2s
if ((team_3s != SHMEM_TEAM_INVALID) && (team_2s != SHMEM_TEAM_INVALID)) {
int pe4_of_3s_in_2s = my_ctx_translate_pe(ctx_3s, 4, ctx_2s);

if (pe4_of_3s_in_2s < 0) {
fprintf(stderr, "Fail to translate pe 4 from 3s context to 2s context\n");
}
else {
// Add up the results on pe 4 of the 3s team, using the 2s team context
shmem_ctx_int_atomic_add(ctx_2s, &isum, ival2 + ival3, pe4_of_3s_in_2s);
}
}
/* Sum the values among PEs that are in both team_2 and team_3 on PE 0 with ctx_2. */
if (team_3 != SHMEM_TEAM_INVALID && team_2 != SHMEM_TEAM_INVALID)
shmem_ctx_int_atomic_add(ctx_2, &sum, val_2 + val_3, 0);

// Quiet the context and synchronize PEs to complete the operation
shmem_ctx_quiet(ctx_2s);
/* Quiet the context and synchronize PEs to complete the operation. */
shmem_ctx_quiet(ctx_2);
shmem_team_sync(SHMEM_TEAM_WORLD);

if (shmem_team_my_pe(team_3s) == 4) {
printf("The total value on PE 4 of the 3s team is %d\n", isum);
/* Validate the result. */
if (mype == 0) {
int vsum = 0;
for (int i = 0; i < npes; i ++) {
if (i % 2 == 0 && i % 3 == 0) {
vsum += ((i - 2) < 0) ? shmem_team_n_pes(team_2) - 1 :
shmem_team_translate_pe(SHMEM_TEAM_WORLD, i - 2, team_2);
vsum += ((i - 3) < 0) ? shmem_team_n_pes(team_3) - 1 :
shmem_team_translate_pe(SHMEM_TEAM_WORLD, i - 3, team_3);
}
}
if (sum != vsum) {
fprintf(stderr, "Unexpected result, npes = %d, vsum = %d, sum = %d\n", shmem_n_pes(), vsum, sum);
shmem_global_exit(1);
}
}

// Destroy contexts before teams
shmem_ctx_destroy(ctx_2s);
shmem_team_destroy(team_2s);
/* Destroy contexts before teams. */
shmem_ctx_destroy(ctx_2);
shmem_team_destroy(team_2);

shmem_ctx_destroy(ctx_3s);
shmem_team_destroy(team_3s);
shmem_ctx_destroy(ctx_3);
shmem_team_destroy(team_3);

shmem_finalize();
return 0;
davidozog marked this conversation as resolved.
Show resolved Hide resolved