From 05eef3e92830ab43c3804ebd33cdc119aef354f3 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 19 Jun 2019 10:27:30 -0600 Subject: [PATCH] mpe logging works --- examples/c/Makefile.am | 2 +- examples/c/darray_no_async.c | 5 - examples/c/example1.c | 5 - examples/c/examplePio.c | 7 - src/clib/pio_darray.c | 24 +-- src/clib/pio_file.c | 8 +- src/clib/pio_internal.h | 6 +- src/clib/pio_msg.c | 8 +- src/clib/pioc.c | 26 ++- src/clib/pioc_support.c | 70 +++++-- tests/cunit/Makefile.am | 2 +- tests/cunit/test_async_multicomp.c | 4 +- tests/cunit/test_darray.c | 2 + tests/cunit/test_darray_async_simple.c | 4 +- tests/cunit/test_perf2.c | 266 +++++++++++++------------ 15 files changed, 241 insertions(+), 198 deletions(-) diff --git a/examples/c/Makefile.am b/examples/c/Makefile.am index 50750dd9132..e3b18a6dd1d 100644 --- a/examples/c/Makefile.am +++ b/examples/c/Makefile.am @@ -19,4 +19,4 @@ endif # RUN_TESTS EXTRA_DIST = run_tests.sh # Clean up files produced during testing. -CLEANFILES = *.nc *.log +CLEANFILES = *.nc *.log *.clog2 *.slog2 diff --git a/examples/c/darray_no_async.c b/examples/c/darray_no_async.c index 05cbc8a4c42..1fdc1252786 100644 --- a/examples/c/darray_no_async.c +++ b/examples/c/darray_no_async.c @@ -377,11 +377,6 @@ int main(int argc, char* argv[]) if ((ret = PIOc_free_iosystem(iosysid))) ERR(ret); -#ifdef USE_MPE - if ((ret = MPE_Finish_log(TEST_NAME))) - MPIERR(ret); -#endif /* USE_MPE */ - /* Finalize the MPI library. */ MPI_Finalize(); diff --git a/examples/c/example1.c b/examples/c/example1.c index 4d4b6042718..036b7f43529 100644 --- a/examples/c/example1.c +++ b/examples/c/example1.c @@ -427,11 +427,6 @@ int check_file(int ntasks, char *filename) { ERR(ret); } -#ifdef USE_MPE - if ((ret = MPE_Finish_log(TEST_NAME))) - MPIERR(ret); -#endif /* USE_MPE */ - /* Finalize the MPI library. */ MPI_Finalize(); diff --git a/examples/c/examplePio.c b/examples/c/examplePio.c index c693474c822..4ad8c95515f 100644 --- a/examples/c/examplePio.c +++ b/examples/c/examplePio.c @@ -473,13 +473,6 @@ struct examplePioClass* epc_cleanUp( struct examplePioClass* this ) PIOc_freedecomp(this->pioIoSystem, this->iodescNCells); PIOc_free_iosystem(this->pioIoSystem); -#ifdef USE_MPE - { - int ret; - if ((ret = MPE_Finish_log(TEST_NAME))) - return NULL; - } -#endif /* USE_MPE */ MPI_Finalize(); diff --git a/src/clib/pio_darray.c b/src/clib/pio_darray.c index c1d94e9f3cf..7e939c1f222 100644 --- a/src/clib/pio_darray.c +++ b/src/clib/pio_darray.c @@ -134,6 +134,10 @@ PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, int ierr; /* Return code. */ void *tmparray; +/* #ifdef USE_MPE */ +/* pio_start_mpe_log(DARRAY_WRITE); */ +/* #endif /\* USE_MPE *\/ */ + /* Get the file info. */ if ((ierr = pio_get_file(ncid, &file))) return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); @@ -411,6 +415,10 @@ PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, if ((ierr = flush_output_buffer(file, flushtodisk, 0))) return pio_err(ios, file, ierr, __FILE__, __LINE__); +/* #ifdef USE_MPE */ +/* pio_stop_mpe_log(DARRAY_WRITE, __func__); */ +/* #endif /\* USE_MPE *\/ */ + return PIO_NOERR; } @@ -649,9 +657,7 @@ PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *arra LOG((1, "PIOc_write_darray ncid = %d varid = %d ioid = %d arraylen = %d", ncid, varid, ioid, arraylen)); #ifdef USE_MPE - if ((ierr = MPE_Log_event(event_num[START][DARRAY_WRITE], 0, - "PIOc_write_darray"))) - return pio_err(NULL, NULL, PIO_EIO, __FILE__, __LINE__); + pio_start_mpe_log(DARRAY_WRITE); #endif /* USE_MPE */ /* Get the file info. */ @@ -852,9 +858,7 @@ PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *arra wmb->num_arrays++; #ifdef USE_MPE - if ((ierr = MPE_Log_event(event_num[END][DARRAY_WRITE], 0, - "PIOc_write_darray"))) - return pio_err(NULL, NULL, PIO_EIO, __FILE__, __LINE__); + pio_stop_mpe_log(DARRAY_WRITE, __func__); #endif /* USE_MPE */ LOG((2, "wmb->num_arrays = %d iodesc->maxbytes / iodesc->mpitype_size = %d " @@ -896,9 +900,7 @@ PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *tmparray; /* unsorted copy of array buf if required */ #ifdef USE_MPE - if ((ierr = MPE_Log_event(event_num[START][DARRAY_READ], 0, - "PIOc_read_darray"))) - return pio_err(NULL, NULL, PIO_EIO, __FILE__, __LINE__); + pio_start_mpe_log(DARRAY_READ); #endif /* USE_MPE */ /* Get the file info. */ @@ -965,9 +967,7 @@ PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, brel(iobuf); #ifdef USE_MPE - if ((ierr = MPE_Log_event(event_num[END][DARRAY_READ], 0, - "PIOc_read_darray"))) - return pio_err(ios, file, PIO_EIO, __FILE__, __LINE__); + pio_stop_mpe_log(DARRAY_READ, __func__); #endif /* USE_MPE */ return PIO_NOERR; diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index a663fe70804..b54ed2834fa 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -227,9 +227,7 @@ int PIOc_closefile(int ncid) int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ #ifdef USE_MPE - if ((ierr = MPE_Log_event(event_num[START][CLOSE], 0, - "PIOc_closefile"))) - return pio_err(NULL, NULL, PIO_EIO, __FILE__, __LINE__); + pio_start_mpe_log(CLOSE); #endif /* USE_MPE */ LOG((1, "PIOc_closefile ncid = %d", ncid)); @@ -306,9 +304,7 @@ int PIOc_closefile(int ncid) return pio_err(ios, file, ierr, __FILE__, __LINE__); #ifdef USE_MPE - if ((ierr = MPE_Log_event(event_num[END][CLOSE], 0, - "PIOc_closefile"))) - return pio_err(ios, NULL, PIO_EIO, __FILE__, __LINE__); + pio_stop_mpe_log(INIT, __func__); #endif /* USE_MPE */ return ierr; diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index 6957298d80d..a368c0b3485 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -364,10 +364,14 @@ extern "C" { /* Handle end and re-defs. */ int pioc_change_def(int ncid, int is_enddef); - /* Initialize and finalize logging. */ + /* Initialize and finalize logging, use --enable-logging at configure. */ int pio_init_logging(void); void pio_finalize_logging(void ); + /* Logging with the MPE library, use --enable-mpe at configure. */ + void pio_start_mpe_log(int state); + void pio_stop_mpe_log(int state, const char *msg); + /* Write a netCDF decomp file. */ int pioc_write_nc_decomp_int(iosystem_desc_t *ios, const char *filename, int cmode, int ndims, int *global_dimlen, int num_tasks, int *task_maplen, int *map, diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 7e4b5e7dc56..cb146cc47e7 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -1,4 +1,5 @@ -/** @file +/** + * @file * * PIO async message handling. This file contains the code which * runs on the IO nodes when async is in use. This code waits for @@ -22,6 +23,11 @@ extern int my_rank; extern int pio_log_level; #endif /* PIO_ENABLE_LOGGING */ +#ifdef USE_MPE +/* The event numbers for MPE logging. */ +extern int event_num[2][NUM_EVENTS]; +#endif /* USE_MPE */ + /** * This function is run on the IO tasks to handle nc_inq_type*() * functions. diff --git a/src/clib/pioc.c b/src/clib/pioc.c index 2d2e21004d1..a3f2d8206b0 100644 --- a/src/clib/pioc.c +++ b/src/clib/pioc.c @@ -501,6 +501,10 @@ PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int ma LOG((1, "PIOc_InitDecomp iosysid = %d pio_type = %d ndims = %d maplen = %d", iosysid, pio_type, ndims, maplen)); +#ifdef USE_MPE + pio_start_mpe_log(DECOMP); +#endif /* USE_MPE */ + /* Get IO system info. */ if (!(ios = pio_get_iosystem_from_id(iosysid))) return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); @@ -620,6 +624,7 @@ PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int ma iodesc->map[m] = compmap[m]; } } + /* Remember the dim sizes. */ if (!(iodesc->dimlen = malloc(sizeof(int) * ndims))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); @@ -722,6 +727,10 @@ PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int ma * PERFTUNE is set. */ performance_tune_rearranger(ios, iodesc); +#ifdef USE_MPE + pio_stop_mpe_log(DECOMP, __func__); +#endif /* USE_MPE */ + return PIO_NOERR; } @@ -925,6 +934,10 @@ PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int base, if ((ret = pio_init_logging())) return pio_err(NULL, NULL, ret, __FILE__, __LINE__); +#ifdef USE_MPE + pio_start_mpe_log(INIT); +#endif /* USE_MPE */ + /* Find the number of computation tasks. */ if ((mpierr = MPI_Comm_size(comp_comm, &num_comptasks))) return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); @@ -1047,6 +1060,9 @@ PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int base, if ((ret = compute_buffer_init(ios))) return ret; +#ifdef USE_MPE + pio_stop_mpe_log(INIT, __func__); +#endif /* USE_MPE */ LOG((2, "Init_Intracomm complete iosysid = %d", *iosysidp)); return PIO_NOERR; @@ -1432,8 +1448,7 @@ PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list, component_count)); #ifdef USE_MPE - if ((ret = MPE_Log_event(event_num[START][INIT], 0, "PIOc_init_async"))) - return pio_err(NULL, NULL, PIO_EIO, __FILE__, __LINE__); + pio_start_mpe_log(INIT); #endif /* USE_MPE */ /* Determine which tasks to use for IO. */ @@ -1716,9 +1731,7 @@ PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list, LOG((2, "Starting message handler io_rank = %d component_count = %d", io_rank, component_count)); #ifdef USE_MPE - if ((ret = MPE_Log_event(event_num[END][INIT], 0, - "about to start processing messages"))) - return pio_err(NULL, NULL, PIO_EIO, __FILE__, __LINE__); + pio_stop_mpe_log(INIT, __func__); #endif /* USE_MPE */ /* Start the message handler loop. This will not return until @@ -1756,8 +1769,7 @@ PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list, #ifdef USE_MPE if (!in_io) - if ((ret = MPE_Log_event(event_num[END][INIT], 0, "end of PIOc_init_async"))) - return pio_err(NULL, NULL, PIO_EIO, __FILE__, __LINE__); + pio_stop_mpe_log(INIT, __func__); #endif /* USE_MPE */ LOG((2, "successfully done with PIOc_init_async")); diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c index 1c2f0b3f745..bebdd3402ed 100644 --- a/src/clib/pioc_support.c +++ b/src/clib/pioc_support.c @@ -199,16 +199,57 @@ init_mpe(int my_rank) * communications. */ if (!my_rank) { - MPE_Describe_state(event_num[START][INIT], event_num[END][INIT], "init", "red"); - MPE_Describe_state(event_num[START][DECOMP], event_num[END][DECOMP], "decomposition", "green"); - MPE_Describe_state(event_num[START][CREATE], event_num[END][CREATE], "create file", "purple"); - MPE_Describe_state(event_num[START][OPEN], event_num[END][OPEN], "open file", "orange"); - MPE_Describe_state(event_num[START][DARRAY_WRITE], event_num[END][DARRAY_WRITE], "darray write", "pink"); - MPE_Describe_state(event_num[START][DARRAY_READ], event_num[END][DARRAY_WRITE], "darray read", "brown"); - MPE_Describe_state(event_num[START][CLOSE], event_num[END][CLOSE], "close file", "blue"); + MPE_Describe_info_state(event_num[START][INIT], event_num[END][INIT], + "PIO init", "red", "%s"); + MPE_Describe_info_state(event_num[START][DECOMP], + event_num[END][DECOMP], "PIO decomposition", + "green", "%s"); + MPE_Describe_info_state(event_num[START][CREATE], event_num[END][CREATE], + "PIO create file", "purple", "%s"); + MPE_Describe_info_state(event_num[START][OPEN], event_num[END][OPEN], + "PIO open file", "orange", "%s"); + MPE_Describe_info_state(event_num[START][DARRAY_WRITE], + event_num[END][DARRAY_WRITE], "PIO darray write", + "pink", "%s"); + MPE_Describe_info_state(event_num[START][DARRAY_READ], + event_num[END][DARRAY_WRITE], "PIO darray read", + "brown", "%s"); + MPE_Describe_info_state(event_num[START][CLOSE], event_num[END][CLOSE], + "PIO close file", "blue", "%s"); } return 0; } + +/** + * Start MPE logging. + * + * @param state_num the MPE event state number to START (ex. INIT). + * @author Ed Hartnett + */ +void +pio_start_mpe_log(int state) +{ + if (MPE_Log_event(event_num[START][state], 0, NULL)) + pio_err(NULL, NULL, PIO_EIO, __FILE__, __LINE__); +} + +/** + * End MPE logging. + * + * @author Ed Hartnett + */ +void +pio_stop_mpe_log(int state, const char *msg) +{ + MPE_LOG_BYTES bytebuf; + int pos = 0; + int ret; + + MPE_Log_pack(bytebuf, &pos, 's', strlen(msg), msg); + if ((ret = MPE_Log_event(event_num[END][state], 0, bytebuf))) + pio_err(NULL, NULL, PIO_EIO, __FILE__, __LINE__); +} + #endif /* USE_MPE */ /** @@ -1889,9 +1930,7 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, int ierr; /* Return code from function calls. */ #ifdef USE_MPE - if ((ierr = MPE_Log_event(event_num[START][CREATE], 0, - "PIOc_createfile_int"))) - return pio_err(NULL, NULL, PIO_EIO, __FILE__, __LINE__); + pio_start_mpe_log(CREATE); #endif /* USE_MPE */ /* Get the IO system info from the iosysid. */ @@ -2036,9 +2075,7 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, pio_add_to_file_list(file); #ifdef USE_MPE - if ((ierr = MPE_Log_event(event_num[END][CREATE], 0, - "PIOc_createfile_int"))) - return pio_err(NULL, file, PIO_EIO, __FILE__, __LINE__); + pio_stop_mpe_log(CREATE, __func__); #endif /* USE_MPE */ LOG((2, "Created file %s file->fh = %d file->pio_ncid = %d", filename, file->fh, file->pio_ncid)); @@ -2350,9 +2387,7 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, int ierr = PIO_NOERR; /* Return code from function calls. */ #ifdef USE_MPE - if ((ierr = MPE_Log_event(event_num[START][OPEN], 0, - "PIOc_openfile_retry"))) - return pio_err(NULL, NULL, PIO_EIO, __FILE__, __LINE__); + pio_start_mpe_log(OPEN); #endif /* USE_MPE */ /* Get the IO system info from the iosysid. */ @@ -2613,8 +2648,7 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, } #ifdef USE_MPE - if ((ierr = MPE_Log_event(event_num[END][OPEN], 0, "PIOc_openfile_retry"))) - return pio_err(ios, file, PIO_EIO, __FILE__, __LINE__); + pio_stop_mpe_log(OPEN, __func__); #endif /* USE_MPE */ LOG((2, "Opened file %s file->pio_ncid = %d file->fh = %d ierr = %d", filename, file->pio_ncid, file->fh, ierr)); diff --git a/tests/cunit/Makefile.am b/tests/cunit/Makefile.am index 0b7042883b4..ebaa75b3b32 100644 --- a/tests/cunit/Makefile.am +++ b/tests/cunit/Makefile.am @@ -70,4 +70,4 @@ test_darray_vard_SOURCES = test_darray_vard.c test_common.c pio_tests.h EXTRA_DIST = run_tests.sh # Clean up files produced during testing. -CLEANFILES = *.nc *.log decomp*.txt +CLEANFILES = *.nc *.log decomp*.txt *.clog2 *.slog2 diff --git a/tests/cunit/test_async_multicomp.c b/tests/cunit/test_async_multicomp.c index 9f1819b9fe3..a34c034fad7 100644 --- a/tests/cunit/test_async_multicomp.c +++ b/tests/cunit/test_async_multicomp.c @@ -83,7 +83,6 @@ int main(int argc, char **argv) { for (int i = 0; i < num_iotypes; i++) { - char filename[NC_MAX_NAME + 1]; /* Test filename. */ int my_comp_idx = my_rank - 1; /* Index in iosysid array. */ int dim_len_2d[NDIM2] = {DIM_LEN2, DIM_LEN3}; int ioid = 0; @@ -92,9 +91,11 @@ int main(int argc, char **argv) &ioid, PIO_SHORT))) ERR(ret); +#ifndef USE_MPE /* For some reason MPE logging breaks this test! */ /* Test with and without darrays. */ for (int use_darray = 0; use_darray < 2; use_darray++) { + char filename[NC_MAX_NAME + 1]; /* Test filename. */ /* Create sample file. */ if ((ret = create_nc_sample_3(iosysid[my_comp_idx], iotype[i], my_rank, my_comp_idx, @@ -106,6 +107,7 @@ int main(int argc, char **argv) filename, 0, 0, ioid))) ERR(ret); } /* next use_darray */ +#endif /* USE_MPE */ /* Free the decomposition. */ if ((ret = PIOc_freedecomp(iosysid[my_comp_idx], ioid))) diff --git a/tests/cunit/test_darray.c b/tests/cunit/test_darray.c index 40c4f7e289f..3c5317ff73b 100644 --- a/tests/cunit/test_darray.c +++ b/tests/cunit/test_darray.c @@ -423,6 +423,8 @@ int main(int argc, char **argv) /* Finalize the MPI library. */ if ((ret = pio_test_finalize(&test_comm))) return ret; + /* if ((ret = pio_test_finalize2(&test_comm, TEST_NAME))) */ + /* return ret; */ printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME); return 0; diff --git a/tests/cunit/test_darray_async_simple.c b/tests/cunit/test_darray_async_simple.c index 6ea74917c72..86aee1d5fef 100644 --- a/tests/cunit/test_darray_async_simple.c +++ b/tests/cunit/test_darray_async_simple.c @@ -208,8 +208,10 @@ int main(int argc, char **argv) } /* endif my_rank < TARGET_NTASKS */ /* Finalize the MPI library. */ - if ((ret = pio_test_finalize2(&test_comm, TEST_NAME))) + if ((ret = pio_test_finalize(&test_comm))) return ret; + /* if ((ret = pio_test_finalize2(&test_comm, TEST_NAME))) */ + /* return ret; */ printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME); diff --git a/tests/cunit/test_perf2.c b/tests/cunit/test_perf2.c index aa651dda582..02a55a1852c 100644 --- a/tests/cunit/test_perf2.c +++ b/tests/cunit/test_perf2.c @@ -9,9 +9,9 @@ #include #include #include -#ifdef USE_MPE -#include -#endif /* USE_MPE */ +/* #ifdef USE_MPE */ +/* #include */ +/* #endif /\* USE_MPE *\/ */ /* The number of tasks this test should run on. */ #define TARGET_NTASKS 16 @@ -65,63 +65,63 @@ int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN, Z_DIM_LEN}; /* Run test for each of the rearrangers. */ #define NUM_REARRANGERS_TO_TEST 2 -#ifdef USE_MPE -/* These are for the event numbers array used to log various events in - * the program with the MPE library, which produces output for the - * Jumpshot program. */ -#define NUM_EVENTS 7 -#define START 0 -#define END 1 -#define INIT 0 -#define DECOMP 1 -#define CREATE 2 -#define DARRAY_WRITE 3 -#define CLOSE 4 -#define CALCULATE 5 -#define INGEST 6 - -#define ERR_LOGGING 99 - -/* This array holds even numbers for MPE. */ -int event_num[2][NUM_EVENTS]; - -/* This will set up the MPE logging event numbers. */ -int -init_logging(int my_rank, int event_num[][NUM_EVENTS]) -{ - /* Get a bunch of event numbers. */ - event_num[START][INIT] = MPE_Log_get_event_number(); - event_num[END][INIT] = MPE_Log_get_event_number(); - event_num[START][DECOMP] = MPE_Log_get_event_number(); - event_num[END][DECOMP] = MPE_Log_get_event_number(); - event_num[START][INGEST] = MPE_Log_get_event_number(); - event_num[END][INGEST] = MPE_Log_get_event_number(); - event_num[START][CLOSE] = MPE_Log_get_event_number(); - event_num[END][CLOSE] = MPE_Log_get_event_number(); - event_num[START][CALCULATE] = MPE_Log_get_event_number(); - event_num[END][CALCULATE] = MPE_Log_get_event_number(); - event_num[START][CREATE] = MPE_Log_get_event_number(); - event_num[END][CREATE] = MPE_Log_get_event_number(); - event_num[START][DARRAY_WRITE] = MPE_Log_get_event_number(); - event_num[END][DARRAY_WRITE] = MPE_Log_get_event_number(); - - /* You should track at least initialization and partitioning, data - * ingest, update computation, all communications, any memory - * copies (if you do that), any output rendering, and any global - * communications. */ - if (!my_rank) - { - MPE_Describe_state(event_num[START][INIT], event_num[END][INIT], "init", "yellow"); - MPE_Describe_state(event_num[START][INGEST], event_num[END][INGEST], "ingest", "red"); - MPE_Describe_state(event_num[START][DECOMP], event_num[END][DECOMP], "decomposition", "green"); - MPE_Describe_state(event_num[START][CALCULATE], event_num[END][CALCULATE], "calculate", "orange"); - MPE_Describe_state(event_num[START][CREATE], event_num[END][CREATE], "create", "purple"); - MPE_Describe_state(event_num[START][CLOSE], event_num[END][CLOSE], "close file", "blue"); - MPE_Describe_state(event_num[START][DARRAY_WRITE], event_num[END][DARRAY_WRITE], "darray write", "pink"); - } - return 0; -} -#endif /* USE_MPE */ +/* #ifdef USE_MPE */ +/* /\* These are for the event numbers array used to log various events in */ +/* * the program with the MPE library, which produces output for the */ +/* * Jumpshot program. *\/ */ +/* #define NUM_EVENTS 7 */ +/* #define START 0 */ +/* #define END 1 */ +/* #define INIT 0 */ +/* #define DECOMP 1 */ +/* #define CREATE 2 */ +/* #define DARRAY_WRITE 3 */ +/* #define CLOSE 4 */ +/* #define CALCULATE 5 */ +/* #define INGEST 6 */ + +/* #define ERR_LOGGING 99 */ + +/* /\* This array holds even numbers for MPE. *\/ */ +/* int event_num[2][NUM_EVENTS]; */ + +/* /\* This will set up the MPE logging event numbers. *\/ */ +/* int */ +/* init_logging(int my_rank, int event_num[][NUM_EVENTS]) */ +/* { */ +/* /\* Get a bunch of event numbers. *\/ */ +/* event_num[START][INIT] = MPE_Log_get_event_number(); */ +/* event_num[END][INIT] = MPE_Log_get_event_number(); */ +/* event_num[START][DECOMP] = MPE_Log_get_event_number(); */ +/* event_num[END][DECOMP] = MPE_Log_get_event_number(); */ +/* event_num[START][INGEST] = MPE_Log_get_event_number(); */ +/* event_num[END][INGEST] = MPE_Log_get_event_number(); */ +/* event_num[START][CLOSE] = MPE_Log_get_event_number(); */ +/* event_num[END][CLOSE] = MPE_Log_get_event_number(); */ +/* event_num[START][CALCULATE] = MPE_Log_get_event_number(); */ +/* event_num[END][CALCULATE] = MPE_Log_get_event_number(); */ +/* event_num[START][CREATE] = MPE_Log_get_event_number(); */ +/* event_num[END][CREATE] = MPE_Log_get_event_number(); */ +/* event_num[START][DARRAY_WRITE] = MPE_Log_get_event_number(); */ +/* event_num[END][DARRAY_WRITE] = MPE_Log_get_event_number(); */ + +/* /\* You should track at least initialization and partitioning, data */ +/* * ingest, update computation, all communications, any memory */ +/* * copies (if you do that), any output rendering, and any global */ +/* * communications. *\/ */ +/* if (!my_rank) */ +/* { */ +/* MPE_Describe_state(event_num[START][INIT], event_num[END][INIT], "init", "yellow"); */ +/* MPE_Describe_state(event_num[START][INGEST], event_num[END][INGEST], "ingest", "red"); */ +/* MPE_Describe_state(event_num[START][DECOMP], event_num[END][DECOMP], "decomposition", "green"); */ +/* MPE_Describe_state(event_num[START][CALCULATE], event_num[END][CALCULATE], "calculate", "orange"); */ +/* MPE_Describe_state(event_num[START][CREATE], event_num[END][CREATE], "create", "purple"); */ +/* MPE_Describe_state(event_num[START][CLOSE], event_num[END][CLOSE], "close file", "blue"); */ +/* MPE_Describe_state(event_num[START][DARRAY_WRITE], event_num[END][DARRAY_WRITE], "darray write", "pink"); */ +/* } */ +/* return 0; */ +/* } */ +/* #endif /\* USE_MPE *\/ */ /* Create the decomposition to divide the 4-dimensional sample data * between the 4 tasks. For the purposes of decomposition we are only @@ -204,7 +204,8 @@ test_darray(int iosysid, int ioid, int num_flavors, int *flavor, /* Use PIO to create the example file in each of the four * available ways. */ - for (int fmt = 0; fmt < num_flavors; fmt++) + /* for (int fmt = 0; fmt < num_flavors; fmt++) */ + for (int fmt = 0; fmt < 1; fmt++) { struct timeval starttime, endtime; long long startt, endt; @@ -213,10 +214,10 @@ test_darray(int iosysid, int ioid, int num_flavors, int *flavor, float delta_in_sec; float mb_per_sec; -#ifdef USE_MPE - if ((ret = MPE_Log_event(event_num[START][CREATE], 0, "start init"))) - return ERR_MPI; -#endif /* USE_MPE */ +/* #ifdef USE_MPE */ +/* if ((ret = MPE_Log_event(event_num[START][CREATE], 0, "start init"))) */ +/* return ERR_MPI; */ +/* #endif /\* USE_MPE *\/ */ /* Create the filename. Use the same filename for all, so we * don't waste disk space. */ @@ -245,24 +246,25 @@ test_darray(int iosysid, int ioid, int num_flavors, int *flavor, if ((ret = PIOc_enddef(ncid))) ERR(ret); -#ifdef USE_MPE - if ((ret = MPE_Log_event(event_num[END][CREATE], 0, "end init"))) - MPIERR(ret); -#endif /* USE_MPE */ +/* #ifdef USE_MPE */ +/* if ((ret = MPE_Log_event(event_num[END][CREATE], 0, "end init"))) */ +/* MPIERR(ret); */ +/* #endif /\* USE_MPE *\/ */ /* Start the clock. */ gettimeofday(&starttime, NULL); - for (int t = 0; t < NUM_TIMESTEPS; t++) + /* for (int t = 0; t < NUM_TIMESTEPS; t++) */ + for (int t = 0; t < 1; t++) { /* Initialize some data. */ for (int f = 0; f < arraylen; f++) test_data[f] = (my_rank * 10 + f) + t * 1000; -#ifdef USE_MPE - if ((ret = MPE_Log_event(event_num[START][DARRAY_WRITE], 0, "start init"))) - return ERR_MPI; -#endif /* USE_MPE */ +/* #ifdef USE_MPE */ +/* if ((ret = MPE_Log_event(event_num[START][DARRAY_WRITE], 0, "start init"))) */ +/* return ERR_MPI; */ +/* #endif /\* USE_MPE *\/ */ /* Set the value of the record dimension. */ if ((ret = PIOc_setframe(ncid, varid, t))) @@ -272,27 +274,27 @@ test_darray(int iosysid, int ioid, int num_flavors, int *flavor, if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue))) ERR(ret); -#ifdef USE_MPE - if ((ret = MPE_Log_event(event_num[END][DARRAY_WRITE], 0, "end init"))) - MPIERR(ret); -#endif /* USE_MPE */ +/* #ifdef USE_MPE */ +/* if ((ret = MPE_Log_event(event_num[END][DARRAY_WRITE], 0, "end init"))) */ +/* MPIERR(ret); */ +/* #endif /\* USE_MPE *\/ */ num_megabytes += (X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN * sizeof(int))/(1024*1024); } -#ifdef USE_MPE - if ((ret = MPE_Log_event(event_num[START][CLOSE], 0, "start init"))) - return ERR_MPI; -#endif /* USE_MPE */ +/* #ifdef USE_MPE */ +/* if ((ret = MPE_Log_event(event_num[START][CLOSE], 0, "start init"))) */ +/* return ERR_MPI; */ +/* #endif /\* USE_MPE *\/ */ /* Close the netCDF file. */ if ((ret = PIOc_closefile(ncid))) ERR(ret); -#ifdef USE_MPE - if ((ret = MPE_Log_event(event_num[END][CLOSE], 0, "end init"))) - MPIERR(ret); -#endif /* USE_MPE */ +/* #ifdef USE_MPE */ +/* if ((ret = MPE_Log_event(event_num[END][CLOSE], 0, "end init"))) */ +/* MPIERR(ret); */ +/* #endif /\* USE_MPE *\/ */ /* Stop the clock. */ @@ -428,27 +430,28 @@ test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, if ((ret = MPI_Comm_size(test_comm, &my_test_size))) MPIERR(ret); -#ifdef USE_MPE - if ((ret = MPE_Log_event(event_num[START][DECOMP], 0, "start init"))) - return ERR_MPI; -#endif /* USE_MPE */ +/* #ifdef USE_MPE */ +/* if ((ret = MPE_Log_event(event_num[START][DECOMP], 0, "start init"))) */ +/* return ERR_MPI; */ +/* #endif /\* USE_MPE *\/ */ /* Decompose the data over the tasks. */ if ((ret = create_decomposition_3d(ntasks, my_rank, iosysid, &ioid))) return ret; - /* Test decomposition read/write. */ - if ((ret = test_decomp_read_write(iosysid, ioid, num_flavors, flavor, my_rank, - ntasks, rearranger, test_comm))) - return ret; + /* /\* Test decomposition read/write. *\/ */ + /* if ((ret = test_decomp_read_write(iosysid, ioid, num_flavors, flavor, my_rank, */ + /* ntasks, rearranger, test_comm))) */ + /* return ret; */ -#ifdef USE_MPE - if ((ret = MPE_Log_event(event_num[END][DECOMP], 0, "end init"))) - MPIERR(ret); -#endif /* USE_MPE */ +/* #ifdef USE_MPE */ +/* if ((ret = MPE_Log_event(event_num[END][DECOMP], 0, "end init"))) */ +/* MPIERR(ret); */ +/* #endif /\* USE_MPE *\/ */ /* Test with/without providing a fill value to PIOc_write_darray(). */ - for (int provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++) + /* for (int provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++) */ + for (int provide_fill = 0; provide_fill < 1; provide_fill++) { /* Run a simple darray test. */ if ((ret = test_darray(iosysid, ioid, num_flavors, flavor, my_rank, @@ -456,19 +459,19 @@ test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, return ret; } -#ifdef USE_MPE - if ((ret = MPE_Log_event(event_num[START][DECOMP], 0, "start init"))) - return ERR_MPI; -#endif /* USE_MPE */ +/* #ifdef USE_MPE */ +/* if ((ret = MPE_Log_event(event_num[START][DECOMP], 0, "start init"))) */ +/* return ERR_MPI; */ +/* #endif /\* USE_MPE *\/ */ /* Free the PIO decomposition. */ if ((ret = PIOc_freedecomp(iosysid, ioid))) ERR(ret); -#ifdef USE_MPE - if ((ret = MPE_Log_event(event_num[END][DECOMP], 0, "end init"))) - MPIERR(ret); -#endif /* USE_MPE */ +/* #ifdef USE_MPE */ +/* if ((ret = MPE_Log_event(event_num[END][DECOMP], 0, "end init"))) */ +/* MPIERR(ret); */ +/* #endif /\* USE_MPE *\/ */ return PIO_NOERR; } @@ -496,13 +499,12 @@ main(int argc, char **argv) 0, -1, &test_comm))) ERR(ERR_INIT); -#ifdef USE_MPE - - if ((ret = MPE_Init_log())) - return ret; - if (init_logging(my_rank, event_num)) - return ERR_LOGGING; -#endif /* USE_MPE */ +/* #ifdef USE_MPE */ +/* if ((ret = MPE_Init_log())) */ +/* return ret; */ +/* if (init_logging(my_rank, event_num)) */ +/* return ERR_LOGGING; */ +/* #endif /\* USE_MPE *\/ */ if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) @@ -532,10 +534,10 @@ main(int argc, char **argv) /* for (r = 0; r < NUM_REARRANGERS_TO_TEST; r++) */ for (r = 0; r < 1; r++) { -#ifdef USE_MPE - if ((ret = MPE_Log_event(event_num[START][INIT], 0, "start init"))) - return ERR_MPI; -#endif /* USE_MPE */ +/* #ifdef USE_MPE */ +/* if ((ret = MPE_Log_event(event_num[START][INIT], 0, "start init"))) */ +/* return ERR_MPI; */ +/* #endif /\* USE_MPE *\/ */ /* Initialize the PIO IO system. This specifies how * many and which processors are involved in I/O. */ @@ -543,10 +545,10 @@ main(int argc, char **argv) ioproc_start, rearranger[r], &iosysid))) return ret; -#ifdef USE_MPE - if ((ret = MPE_Log_event(event_num[END][INIT], 0, "end init"))) - MPIERR(ret); -#endif /* USE_MPE */ +/* #ifdef USE_MPE */ +/* if ((ret = MPE_Log_event(event_num[END][INIT], 0, "end init"))) */ +/* MPIERR(ret); */ +/* #endif /\* USE_MPE *\/ */ /* Run tests. */ if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, @@ -560,17 +562,17 @@ main(int argc, char **argv) } /* next num io procs */ -#ifdef USE_MPE - { - /* This causes problems on my MPICH2 library on Linux, but seems to be - * required for frost. */ - char file_name[128]; - sprintf(file_name, "chart_%d", 1); - if ((ret = MPE_Finish_log(file_name))) - MPIERR(ret); - } +/* #ifdef USE_MPE */ +/* { */ +/* /\* This causes problems on my MPICH2 library on Linux, but seems to be */ +/* * required for frost. *\/ */ +/* char file_name[128]; */ +/* sprintf(file_name, "chart_%d", 1); */ +/* if ((ret = MPE_Finish_log(file_name))) */ +/* MPIERR(ret); */ +/* } */ -#endif /* USE_MPE */ +/* #endif /\* USE_MPE *\/ */ if (!my_rank)