diff --git a/.gitignore b/.gitignore index e27b57e6069..0a47d771bac 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ .project html/ +*~ + diff --git a/cmake/FindMPE.cmake b/cmake/FindMPE.cmake new file mode 100644 index 00000000000..5a964172da7 --- /dev/null +++ b/cmake/FindMPE.cmake @@ -0,0 +1,50 @@ +# - Try to find MPE +# +# This can be controlled by setting the MPE_PATH (or, equivalently, +# the MPE environment variable), or MPE__PATH CMake variables, +# where is the COMPONENT language one needs. +# +# Once done, this will define: +# +# MPE__FOUND (BOOL) - system has MPE +# MPE__IS_SHARED (BOOL) - whether library is shared/dynamic +# MPE__INCLUDE_DIR (PATH) - Location of the C header file +# MPE__INCLUDE_DIRS (LIST) - the MPE include directories +# MPE__LIBRARY (FILE) - Path to the C library file +# MPE__LIBRARIES (LIST) - link these to use MPE +# +# The available COMPONENTS are: C +include (LibFind) +include (LibCheck) + +# Define MPE C Component +define_package_component (MPE DEFAULT + COMPONENT C + INCLUDE_NAMES mpe.h + LIBRARY_NAMES mpe) + +# Search for list of valid components requested +find_valid_components (MPE) + +#============================================================================== +# SEARCH FOR VALIDATED COMPONENTS +foreach (NCDFcomp IN LISTS MPE_FIND_VALID_COMPONENTS) + + # If not found already, search... + if (NOT MPE_${NCDFcomp}_FOUND) + + # Manually add the MPI include and library dirs to search paths + # and search for the package component + if (MPI_${NCDFcomp}_FOUND) + initialize_paths (MPE_${NCDFcomp}_PATHS + INCLUDE_DIRECTORIES ${MPI_${NCDFcomp}_INCLUDE_PATH} + LIBRARIES ${MPI_${NCDFcomp}_LIBRARIES}) + find_package_component(MPE COMPONENT ${NCDFcomp} + PATHS ${MPE_${NCDFcomp}_PATHS}) + else () + find_package_component(MPE COMPONENT ${NCDFcomp}) + endif () + + endif () + +endforeach () diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index da4e308d0b2..31dc3ffd113 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -187,7 +187,7 @@ SHORT_NAMES = NO # description.) # The default value is: NO. -JAVADOC_AUTOBRIEF = NO +JAVADOC_AUTOBRIEF = YES # If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first # line (until the first dot) of a Qt-style comment as the brief description. If diff --git a/doc/source/Examples.txt b/doc/source/Examples.txt index 5596c487708..f2af4ebcf66 100644 --- a/doc/source/Examples.txt +++ b/doc/source/Examples.txt @@ -28,7 +28,12 @@ The C examples are in the examples/c subdirectory. - examplePio.c A simple example showing a write, then read, of a 1D variable. - example1.c Another simple example showing a write, then read, of a - 1D variable. + 1D variable. This produces the same output as examplePio.c. + +- example2.c This example writes a 3D floating point variable, with an + unlimited dimension. The data are written in slices along the + unlimited dimension. This example also incorporates (if installed) + the MPE profiling system to generate graphical output. ### Fortran Examples diff --git a/doc/source/api.txt b/doc/source/api.txt index 16ec04881ec..275224e160f 100644 --- a/doc/source/api.txt +++ b/doc/source/api.txt @@ -41,6 +41,7 @@ - \ref PIO_inq_attname - \ref PIO_inq_att - \ref PIO_inq_attlen + - \ref PIO_inq_var - \ref PIO_inq_varid - \ref PIO_inq_varname - \ref PIO_inq_vartype @@ -55,5 +56,6 @@ - \ref PIO_inq_natts - \ref PIO_inquire_variable - \ref PIO_inquire_dimension + - \ref PIO_inquire_dimension */ diff --git a/examples/c/CMakeLists.txt b/examples/c/CMakeLists.txt index 56bf1c756d1..911611286e1 100644 --- a/examples/c/CMakeLists.txt +++ b/examples/c/CMakeLists.txt @@ -12,8 +12,25 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0") SET(SRC examplePio.c) ADD_EXECUTABLE(examplePio_c ${SRC}) -TARGET_LINK_LIBRARIES(examplePio_c piof pioc) +TARGET_LINK_LIBRARIES(examplePio_c pioc) SET(SRC example1.c) ADD_EXECUTABLE(example1 ${SRC}) -TARGET_LINK_LIBRARIES(example1 piof pioc) \ No newline at end of file +TARGET_LINK_LIBRARIES(example1 pioc) + +SET(SRC example2.c) +ADD_EXECUTABLE(example2 ${SRC}) +TARGET_LINK_LIBRARIES(example2 pioc) + +#===== MPE ===== +find_package (MPE "2.4.8" COMPONENTS C) +if (MPE_C_FOUND) + target_include_directories (example2 + PUBLIC ${MPE_C_INCLUDE_DIRS}) + target_compile_definitions (example2 + PUBLIC HAVE_MPE) + target_link_libraries (example2 pioc lmpe mpe pthread m) +else () + target_compile_definitions (example2 + PUBLIC _NOMPE) +endif () diff --git a/examples/c/example2.c b/examples/c/example2.c new file mode 100644 index 00000000000..b270e864cee --- /dev/null +++ b/examples/c/example2.c @@ -0,0 +1,687 @@ +/** + * @file + * A simple C example for the ParallelIO Library. + * + * This example creates a netCDF output file with one 3D variable. One + * of the dimensions will be unlimited. The example first writes the + * sample file using the ParallelIO library, then reads it with the + * plain old netCDF library to check that it is correct. + * + * This example can be run in parallel for 1, 2, 4, 8, or 16 + * processors. + * + * This example uses the MPE performace profiling library, if it is + * present on the build machine. After the program is run, MPE will + * produce a file called example2.clog2. In order to see the nice + * graphs, execute the commands: + * + * 
+ * clog2ToSlog2 example2.clog2
+ * jumpshot example2.slog2 
+ *
+ */ + +#include +#include +#include +#include +#include +#include +#ifdef TIMING +#include +#endif +#ifdef HAVE_MPE +#include +#endif + +/** The number of possible output netCDF output flavors available to + * the ParallelIO library. */ +#define NUM_NETCDF_FLAVORS 4 + +/** The number of dimensions in the example data. In this example, we + * are using three-dimensional data. */ +#define NDIM 3 + +/** The length of our sample data along each dimension. There will be + * a total of 16 integers in each timestep of our data, and + * responsibilty for writing and reading them will be spread between + * all the processors used to run this example. */ +/**@{*/ +#define X_DIM_LEN 400 +#define Y_DIM_LEN 400 +/**@}*/ + +/** The number of timesteps of data to write. */ +#define NUM_TIMESTEPS 6 + +/** The name of the variable in the netCDF output file. */ +#define VAR_NAME "foo" + +/** Return code when netCDF output file does not match + * expectations. */ +#define ERR_BAD 1001 + +/** The meaning of life, the universe, and everything. */ +#define START_DATA_VAL 42 + +/** Handle MPI errors. This should only be used with MPI library + * function calls. */ +#define MPIERR(e) do { \ + MPI_Error_string(e, err_buffer, &resultlen); \ + fprintf(stderr, "MPI error, line %d, file %s: %s\n", __LINE__, __FILE__, err_buffer); \ + MPI_Finalize(); \ + return 2; \ + } while (0) + +/** Handle non-MPI errors by finalizing the MPI library and exiting + * with an exit code. */ +#define ERR(e) do { \ + fprintf(stderr, "Error %d in %s, line %d\n", e, __FILE__, __LINE__); \ + MPI_Finalize(); \ + return e; \ + } while (0) + +/** Global err buffer for MPI. When there is an MPI error, this buffer + * is used to store the error message that is associated with the MPI + * error. */ +char err_buffer[MPI_MAX_ERROR_STRING]; + +/** This is the length of the most recent MPI error message, stored + * int the global error string. */ +int resultlen; + +/** The dimension names. */ +char dim_name[NDIM][NC_MAX_NAME + 1] = {"timestep", "x", "y"}; + +/** Length of the dimensions in the sample data. */ +int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN}; + +/** Length of chunksizes to use in netCDF-4 files. */ +size_t chunksize[NDIM] = {2, X_DIM_LEN/2, Y_DIM_LEN/2}; + +#ifdef HAVE_MPE +/** Number of MPE events. The start and stop of each event will be + * tracked, and graphed. */ +#define NUM_EVENTS 10 + +/** Start and end for each MPE event. */ +/**@{*/ +#define START 0 +#define END 1 +/**@}*/ + +/** These are for the event numbers array used to log various events + * in the program with the MPE library, which produces output for the + * Jumpshot program. */ +/**@{*/ +#define INIT 0 +#define CREATE_PNETCDF 1 +#define CREATE_CLASSIC 2 +#define CREATE_SERIAL4 3 +#define CREATE_PARALLEL4 4 +#define CALCULATE 5 +#define WRITE 6 +#define CLOSE 7 +#define FREE 8 +#define READ 9 +/**@}*/ +#endif /* HAVE_MPE */ + +/** Some error codes for when things go wrong. */ +/**@{*/ +#define ERR_FILE 1 +#define ERR_DUMB 2 +#define ERR_ARG 3 +#define ERR_MPI 4 +#define ERR_MPITYPE 5 +#define ERR_LOGGING 6 +#define ERR_UPDATE 7 +#define ERR_CALC 8 +#define ERR_COUNT 9 +#define ERR_WRITE 10 +#define ERR_SWAP 11 +#define ERR_INIT 12 +/**@}*/ + +/** This will set up the MPE logging event numbers. + * + * @param my_rank the rank of the processor running the code. + * @param event_num array of MPE event numbers. + * + * @return 0 for success, non-zero for failure. + */ +int +init_logging(int my_rank, int event_num[][NUM_EVENTS]) +{ +#ifdef HAVE_MPE +/* Get a bunch of event numbers. */ + event_num[START][INIT] = MPE_Log_get_event_number(); + event_num[END][INIT] = MPE_Log_get_event_number(); + event_num[START][CREATE_PNETCDF] = MPE_Log_get_event_number(); + event_num[END][CREATE_PNETCDF] = MPE_Log_get_event_number(); + event_num[START][CREATE_CLASSIC] = MPE_Log_get_event_number(); + event_num[END][CREATE_CLASSIC] = MPE_Log_get_event_number(); + event_num[START][CREATE_SERIAL4] = MPE_Log_get_event_number(); + event_num[END][CREATE_SERIAL4] = MPE_Log_get_event_number(); + event_num[START][CREATE_PARALLEL4] = MPE_Log_get_event_number(); + event_num[END][CREATE_PARALLEL4] = MPE_Log_get_event_number(); + event_num[START][CALCULATE] = MPE_Log_get_event_number(); + event_num[END][CALCULATE] = MPE_Log_get_event_number(); + event_num[START][WRITE] = MPE_Log_get_event_number(); + event_num[END][WRITE] = MPE_Log_get_event_number(); + event_num[START][CLOSE] = MPE_Log_get_event_number(); + event_num[END][CLOSE] = MPE_Log_get_event_number(); + event_num[START][FREE] = MPE_Log_get_event_number(); + event_num[END][FREE] = MPE_Log_get_event_number(); + event_num[START][READ] = MPE_Log_get_event_number(); + event_num[END][READ] = MPE_Log_get_event_number(); + +/* You should track at least initialization and partitioning, data + * ingest, update computation, all communications, any memory + * copies (if you do that), any output rendering, and any global + * communications. */ + if (!my_rank) + { + MPE_Describe_state(event_num[START][INIT], event_num[END][INIT], + "init", "yellow"); + MPE_Describe_state(event_num[START][CREATE_PNETCDF], event_num[END][CREATE_PNETCDF], + "create pnetcdf", "red"); + MPE_Describe_state(event_num[START][CREATE_CLASSIC], event_num[END][CREATE_CLASSIC], + "create classic", "red"); + MPE_Describe_state(event_num[START][CREATE_SERIAL4], event_num[END][CREATE_SERIAL4], + "create netcdf-4 serial", "red"); + MPE_Describe_state(event_num[START][CREATE_PARALLEL4], event_num[END][CREATE_PARALLEL4], + "create netcdf-4 parallel", "red"); + MPE_Describe_state(event_num[START][CALCULATE], event_num[END][CALCULATE], + "calculate", "orange"); + MPE_Describe_state(event_num[START][WRITE], event_num[END][WRITE], + "write", "green"); + MPE_Describe_state(event_num[START][CLOSE], event_num[END][CLOSE], + "close", "purple"); + MPE_Describe_state(event_num[START][FREE], event_num[END][FREE], + "free", "blue"); + MPE_Describe_state(event_num[START][READ], event_num[END][READ], + "read", "pink"); + } +#endif /* HAVE_MPE */ + return 0; +} + +/** Check the output file. + * + * Use netCDF to check that the output is as expected. + * + * @param ntasks The number of processors running the example. + * @param filename The name of the example file to check. + * + * @return 0 if example file is correct, non-zero otherwise. */ +int check_file(int ntasks, char *filename) { + + int ncid; /**< File ID from netCDF. */ + int ndims; /**< Number of dimensions. */ + int nvars; /**< Number of variables. */ + int ngatts; /**< Number of global attributes. */ + int unlimdimid; /**< ID of unlimited dimension. */ + size_t dimlen; /**< Length of the dimension. */ + int natts; /**< Number of variable attributes. */ + nc_type xtype; /**< NetCDF data type of this variable. */ + int ret; /**< Return code for function calls. */ + int dimids[NDIM]; /**< Dimension ids for this variable. */ + char my_dim_name[NC_MAX_NAME + 1]; /**< Name of the dimension. */ + char var_name[NC_MAX_NAME + 1]; /**< Name of the variable. */ + size_t start[NDIM]; /**< Zero-based index to start read. */ + size_t count[NDIM]; /**< Number of elements to read. */ + int buffer[X_DIM_LEN]; /**< Buffer to read in data. */ + int expected[X_DIM_LEN]; /**< Data values we expect to find. */ + +/* Open the file. */ + if ((ret = nc_open(filename, 0, &ncid))) + return ret; + +/* Check the metadata. */ + if ((ret = nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid))) + return ret; + if (ndims != NDIM || nvars != 1 || ngatts != 0 || unlimdimid != -1) + return ERR_BAD; + for (int d = 0; d < ndims; d++) + { + if ((ret = nc_inq_dim(ncid, d, my_dim_name, &dimlen))) + return ret; + if (dimlen != X_DIM_LEN || strcmp(my_dim_name, dim_name[d])) + return ERR_BAD; + } + if ((ret = nc_inq_var(ncid, 0, var_name, &xtype, &ndims, dimids, &natts))) + return ret; + if (xtype != NC_FLOAT || ndims != NDIM || dimids[0] != 0 || natts != 0) + return ERR_BAD; + +/* Use the number of processors to figure out what the data in the + * file should look like. */ + int div = X_DIM_LEN * Y_DIM_LEN / ntasks; + for (int d = 0; d < X_DIM_LEN; d++) + expected[d] = START_DATA_VAL + d/div; + +/* Check the data. */ + start[0] = 0; + count[0] = X_DIM_LEN; + if ((ret = nc_get_vara(ncid, 0, start, count, buffer))) + return ret; + for (int d = 0; d < X_DIM_LEN; d++) + if (buffer[d] != expected[d]) + return ERR_BAD; + +/* Close the file. */ + if ((ret = nc_close(ncid))) + return ret; + +/* Everything looks good! */ + return 0; +} + +/** Calculate sample data. This function is deliberately slow in order to take up some time calculating. + * @param my_rank the rank of the processor running the code. + * @param timestep the timestep. + * @param datap pointer where we should write datum. + * + * @return zero for success, non-zero otherwise. + */ +int calculate_value(int my_rank, int timestep, float *datap) +{ + *datap = my_rank + timestep; + for (int i = 0; i < 50; i++) + *datap += atan(cos(my_rank * timestep)); + return 0; +} + +/** Main execution of code. + + Executes the functions to: + - create a new examplePioClass instance + - initialize MPI and the ParallelIO libraries + - create the decomposition for this example + - create the netCDF output file + - define the variable in the file + - write data to the variable in the file using decomposition + - read the data back from the file using decomposition + - close the file + - clean up resources + + The example can be run from the command line (on system that support it) like this: + 
+    mpiexec -n 4 ./examplePio
+
+ + The sample file created by this program is a small netCDF file. It + has the following contents (as shown by ncdump) for a 4-processor + run: + + 
+    netcdf examplePio_c {
+    dimensions:
+    x = 16 ;
+    variables:
+    int foo(x) ;
+    data:
+
+    foo = 42, 42, 42, 42, 43, 43, 43, 43, 44, 44, 44, 44, 45, 45, 45, 45 ;
+    }
+
+ + @param [in] argc argument count (should be zero) + @param [in] argv argument array (should be NULL) + @retval examplePioClass* Pointer to self. +*/ +int main(int argc, char* argv[]) +{ + /** Set to non-zero to get output to stdout. */ + int verbose = 0; + + /** Zero-based rank of processor. */ + int my_rank; + + /** Number of processors involved in current execution. */ + int ntasks; + + /** Different output flavors. The example file is written (and + * then read) four times. The first two flavors, + * parallel-netcdf, and netCDF serial, both produce a netCDF + * classic format file (but with different libraries). The + * last two produce netCDF4/HDF5 format files, written with + * and without using netCDF-4 parallel I/O. */ + int format[NUM_NETCDF_FLAVORS] = {PIO_IOTYPE_PNETCDF, + PIO_IOTYPE_NETCDF, + PIO_IOTYPE_NETCDF4C, + PIO_IOTYPE_NETCDF4P}; + + /** Names for the output files. Two of them (pnetcdf and + * classic) will be in classic netCDF format, the others + * (serial4 and parallel4) will be in netCDF-4/HDF5 + * format. All four can be read by the netCDF library, and all + * will contain the same contents. */ + char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME + 1] = {"example2_pnetcdf.nc", + "example2_classic.nc", + "example2_serial4.nc", + "example2_parallel4.nc"}; + + /** Number of processors that will do IO. In this example we + * will do IO from all processors. */ + int niotasks; + + /** Stride in the mpi rank between io tasks. Always 1 in this + * example. */ + int ioproc_stride = 1; + + /** Number of the aggregator? Always 0 in this example. */ + int numAggregator = 0; + + /** Zero based rank of first processor to be used for I/O. */ + int ioproc_start = 0; + + /** Specifies the flavor of netCDF output format. */ + int iotype; + + /** The dimension IDs. */ + int dimids[NDIM]; + + /** Array index per processing unit. This is the number of + * elements of the data array that will be handled by each + * processor. In this example there are 16 data elements. If the + * example is run on 4 processors, then arrIdxPerPe will be 4. */ + PIO_Offset elements_per_pe; + + /** The ID for the parallel I/O system. It is set by + * PIOc_Init_Intracomm(). It references an internal structure + * containing the general IO subsystem data and MPI + * structure. It is passed to PIOc_finalize() to free + * associated resources, after all I/O, but before + * MPI_Finalize is called. */ + int iosysid; + + /** The ncid of the netCDF file created in this example. */ + int ncid = 0; + + /** The ID of the netCDF varable in the example file. */ + int varid; + + /** The I/O description ID as passed back by PIOc_InitDecomp() + * and freed in PIOc_freedecomp(). */ + int ioid; + + /** A buffer for sample data. The size of this array will + * vary depending on how many processors are involved in the + * execution of the example code. It's length will be the same + * as elements_per_pe.*/ + float *buffer; + + /** A buffer for reading data back from the file. The size of + * this array will vary depending on how many processors are + * involved in the execution of the example code. It's length + * will be the same as elements_per_pe.*/ + int *read_buffer; + + /** A 1-D array which holds the decomposition mapping for this + * example. The size of this array will vary depending on how + * many processors are involved in the execution of the + * example code. It's length will be the same as + * elements_per_pe. */ + PIO_Offset *compdof; + +#ifdef HAVE_MPE + /** MPE event numbers used to track start and stop of + * different parts of the program for later display with + * Jumpshot. */ + int event_num[2][NUM_EVENTS]; +#endif /* HAVE_MPE */ + + /** Needed for command line processing. */ + int c; + + /* Parse command line. */ + while ((c = getopt(argc, argv, "v")) != -1) + switch (c) + { + case 'v': + verbose++; + break; + default: + break; + } + +#ifdef TIMING + /* Initialize the GPTL timing library. */ + int ret; + if ((ret = GPTLinitialize ())) + return ret; +#endif + + /* Initialize MPI. */ + if ((ret = MPI_Init(&argc, &argv))) + MPIERR(ret); + if ((ret = MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN))) + MPIERR(ret); + + /* Learn my rank and the total number of processors. */ + if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) + MPIERR(ret); + if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) + MPIERR(ret); + + /* Check that a valid number of processors was specified. */ + if (!(ntasks == 1 || ntasks == 2 || ntasks == 4 || + ntasks == 8 || ntasks == 16)) + fprintf(stderr, "Number of processors must be 1, 2, 4, 8, or 16!\n"); + if (verbose) + printf("%d: ParallelIO Library example1 running on %d processors.\n", + my_rank, ntasks); + +#ifdef HAVE_MPE + /* Initialize MPE logging. */ + if ((ret = MPE_Init_log())) + ERR(ret); + if (init_logging(my_rank, event_num)) + ERR(ERR_LOGGING); + + /* Log with MPE that we are starting INIT. */ + if ((ret = MPE_Log_event(event_num[START][INIT], 0, "start init"))) + MPIERR(ret); +#endif /* HAVE_MPE */ + + /* keep things simple - 1 iotask per MPI process */ + niotasks = ntasks; + + /* Initialize the PIO IO system. This specifies how + * many and which processors are involved in I/O. */ + if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, niotasks, ioproc_stride, + ioproc_start, PIO_REARR_SUBSET, &iosysid))) + ERR(ret); + + /* Describe the decomposition. This is a 1-based array, so add 1! */ + elements_per_pe = X_DIM_LEN * Y_DIM_LEN / ntasks; + if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) + return PIO_ENOMEM; + for (int i = 0; i < elements_per_pe; i++) { + compdof[i] = my_rank * elements_per_pe + i + 1; + } + + /* Create the PIO decomposition for this example. */ + if (verbose) + printf("rank: %d Creating decomposition...\n", my_rank); + if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, &dim_len[1], (PIO_Offset)elements_per_pe, + compdof, &ioid, NULL, NULL, NULL))) + ERR(ret); + free(compdof); + +#ifdef HAVE_MPE + /* Log with MPE that we are done with INIT. */ + if ((ret = MPE_Log_event(event_num[END][INIT], 0, "end init"))) + MPIERR(ret); +#endif /* HAVE_MPE */ + + /* Use PIO to create the example file in each of the four + * available ways. */ + for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) + { +#ifdef HAVE_MPE + /* Log with MPE that we are starting CREATE. */ + if ((ret = MPE_Log_event(event_num[START][CREATE_PNETCDF+fmt], 0, "start create"))) + MPIERR(ret); +#endif /* HAVE_MPE */ + + /* Create the netCDF output file. */ + if (verbose) + printf("rank: %d Creating sample file %s with format %d...\n", + my_rank, filename[fmt], format[fmt]); + if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename[fmt], + PIO_CLOBBER))) + ERR(ret); + + /* Define netCDF dimensions and variable. */ + if (verbose) + printf("rank: %d Defining netCDF metadata...\n", my_rank); + for (int d = 0; d < NDIM; d++) { + if (verbose) + printf("rank: %d Defining netCDF dimension %s, length %d\n", my_rank, + dim_name[d], dim_len[d]); + if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) + ERR(ret); + } + if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, &varid))) + ERR(ret); + /* For netCDF-4 files, set the chunksize to improve performance. */ + if (format[fmt] == PIO_IOTYPE_NETCDF4C || format[fmt] == PIO_IOTYPE_NETCDF4P) + if ((ret = PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, chunksize))) + ERR(ret); + + if ((ret = PIOc_enddef(ncid))) + ERR(ret); + +#ifdef HAVE_MPE + /* Log with MPE that we are done with CREATE. */ + if ((ret = MPE_Log_event(event_num[END][CREATE_PNETCDF + fmt], 0, "end create"))) + MPIERR(ret); +#endif /* HAVE_MPE */ + + /* Allocate space for sample data. */ + if (!(buffer = malloc(elements_per_pe * sizeof(float)))) + return PIO_ENOMEM; + + /* Write data for each timestep. */ + for (int ts = 0; ts < NUM_TIMESTEPS; ts++) { + +#ifdef HAVE_MPE + /* Log with MPE that we are starting CALCULATE. */ + if ((ret = MPE_Log_event(event_num[START][CALCULATE], 0, "start calculate"))) + MPIERR(ret); +#endif /* HAVE_MPE */ + + /* Calculate sample data. Add some math function calls to make this slower. */ + for (int i = 0; i < elements_per_pe; i++) + if ((ret = calculate_value(my_rank, ts, &buffer[i]))) + ERR(ret); + +#ifdef HAVE_MPE + /* Log with MPE that we are done with CALCULATE. */ + if ((ret = MPE_Log_event(event_num[END][CALCULATE], 0, "end calculate"))) + MPIERR(ret); + /* Log with MPE that we are starting WRITE. */ + if ((ret = MPE_Log_event(event_num[START][WRITE], 0, "start write"))) + MPIERR(ret); +#endif /* HAVE_MPE */ + + /* Write data to the file. */ + if (verbose) + printf("rank: %d Writing sample data...\n", my_rank); + + if ((ret = PIOc_setframe(ncid, varid, ts))) + ERR(ret); + if ((ret = PIOc_write_darray(ncid, varid, ioid, (PIO_Offset)elements_per_pe, + buffer, NULL))) + ERR(ret); + if ((ret = PIOc_sync(ncid))) + ERR(ret); +#ifdef HAVE_MPE + /* Log with MPE that we are done with WRITE. */ + if ((ret = MPE_Log_event(event_num[END][WRITE], 0, "end write"))) + MPIERR(ret); +#endif /* HAVE_MPE */ + } + +#ifdef HAVE_MPE + /* Log with MPE that we are starting CLOSE. */ + if ((ret = MPE_Log_event(event_num[START][CLOSE], 0, "start close"))) + MPIERR(ret); +#endif /* HAVE_MPE */ + + /* Free buffer space used in this example. */ + free(buffer); + + /* Close the netCDF file. */ + if (verbose) + printf("rank: %d Closing the sample data file...\n", my_rank); + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + +#ifdef HAVE_MPE + /* Log with MPE that we are done with CLOSE. */ + if ((ret = MPE_Log_event(event_num[END][CLOSE], 0, "end close"))) + MPIERR(ret); +#endif /* HAVE_MPE */ + + /* After each file is closed, make all processors wait so that + * all start creating the next file at the same time. */ + if ((ret = MPI_Barrier(MPI_COMM_WORLD))) + MPIERR(ret); + } + +#ifdef HAVE_MPE + /* Log with MPE that we are starting FREE. */ + if ((ret = MPE_Log_event(event_num[START][FREE], 0, "start free"))) + MPIERR(ret); +#endif /* HAVE_MPE */ + + /* Free the PIO decomposition. */ + if (verbose) + printf("rank: %d Freeing PIO decomposition...\n", my_rank); + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + + /* Finalize the IO system. */ + if (verbose) + printf("rank: %d Freeing PIO resources...\n", my_rank); + if ((ret = PIOc_finalize(iosysid))) + ERR(ret); + +#ifdef HAVE_MPE + /* Log with MPE that we are done with FREE. */ + if ((ret = MPE_Log_event(event_num[END][FREE], 0, "end free"))) + MPIERR(ret); + /* Log with MPE that we are starting READ. */ + if ((ret = MPE_Log_event(event_num[START][READ], 0, "start read"))) + MPIERR(ret); +#endif /* HAVE_MPE */ + + /* Check the output file. */ + /* if (!my_rank) */ + /* for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) */ + /* if ((ret = check_file(ntasks, filename[fmt]))) */ + /* ERR(ret); */ + +#ifdef HAVE_MPE + /* Log with MPE that we are done with READ. */ + if ((ret = MPE_Log_event(event_num[END][READ], 0, "end read"))) + MPIERR(ret); +#endif /* HAVE_MPE */ + + /* Finalize the MPI library. */ + MPI_Finalize(); + +#ifdef TIMING + /* Finalize the GPTL timing library. */ + if ((ret = GPTLfinalize ())) + return ret; +#endif + + if (verbose) + printf("rank: %d SUCCESS!\n", my_rank); + return 0; +} diff --git a/src/clib/CMakeLists.txt b/src/clib/CMakeLists.txt index ff30003dbb7..093aeac70c1 100644 --- a/src/clib/CMakeLists.txt +++ b/src/clib/CMakeLists.txt @@ -18,8 +18,9 @@ set (PIO_C_SRCS topology.c bget.c) set (PIO_GENNC_SRCS ${CMAKE_CURRENT_BINARY_DIR}/pio_nc.c - ${CMAKE_CURRENT_BINARY_DIR}/pio_put_nc.c - ${CMAKE_CURRENT_BINARY_DIR}/pio_get_nc.c) + ${CMAKE_CURRENT_BINARY_DIR}/pio_nc4.c + ${CMAKE_CURRENT_BINARY_DIR}/pio_put_nc.c + ${CMAKE_CURRENT_BINARY_DIR}/pio_get_nc.c) add_library (pioc ${PIO_C_SRCS} ${PIO_GENNC_SRCS}) @@ -195,9 +196,12 @@ else () pio_get_nc.c COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/pio_nc.c pio_nc.c + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/pio_nc4.c + pio_nc4.c DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/pio_put_nc.c ${CMAKE_CURRENT_SOURCE_DIR}/pio_get_nc.c - ${CMAKE_CURRENT_SOURCE_DIR}/pio_nc.c) + ${CMAKE_CURRENT_SOURCE_DIR}/pio_nc.c + ${CMAKE_CURRENT_SOURCE_DIR}/pio_nc4.c) endif () diff --git a/src/clib/pio.h b/src/clib/pio.h index cee01e238d2..be96576b45b 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -35,7 +35,7 @@ #define MPI_OFFSET MPI_LONG_LONG #endif -/** PIO_Offset is an integer type of size sufficient to represent the +/** PIO_OFFSET is an integer type of size sufficient to represent the * size (in bytes) of the largest file supported by MPI. */ #define PIO_OFFSET MPI_OFFSET #define PIO_Offset MPI_Offset @@ -251,6 +251,7 @@ enum PIO_ERROR_HANDLERS{ #define PIO_UNLIMITED NC_UNLIMITED #define PIO_DOUBLE NC_DOUBLE #define PIO_REAL NC_FLOAT +#define PIO_FLOAT NC_FLOAT #define PIO_INT NC_INT #define PIO_CHAR NC_CHAR #define PIO_NOERR NC_NOERR @@ -359,7 +360,9 @@ int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, PIO_Of int PIOc_inq_format (int ncid, int *formatp); int PIOc_inq_varid (int ncid, const char *name, int *varidp); int PIOc_inq_varnatts (int ncid, int varid, int *nattsp); -int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, const int *dimidsp, int *varidp); +int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, const int *dimidsp, int *varidp); +int PIOc_def_var_chunking(int ncid, int varid, int storage, const size_t *chunksizesp); +int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, size_t *chunksizesp); int PIOc_inq_var (int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, int *dimidsp, int *nattsp); int PIOc_inq_varname (int ncid, int varid, char *name); int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const double *op); diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index 334601e8156..a2aac38e4c1 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -352,9 +352,9 @@ enum PIO_MSG{ PIO_MSG_GET_ATT_STRING, PIO_MSG_PUT_ATT_UBYTE, PIO_MSG_INQ_VAR_FILL, - PIO_MSG_DEF_VAR_FILL + PIO_MSG_DEF_VAR_FILL, + PIO_MSG_DEF_VAR_CHUNKING, + PIO_MSG_INQ_VAR_CHUNKING }; - - #endif diff --git a/src/clib/pio_nc.c b/src/clib/pio_nc.c index b2afaaef170..63ef7f2afb3 100644 --- a/src/clib/pio_nc.c +++ b/src/clib/pio_nc.c @@ -1,5 +1,5 @@ /** -* @file pio_nc.c +* @file * @author Jim Edwards (jedwards@ucar.edu) * @date Feburary 2014 * @brief PIO interfaces to [NetCDF](http://www.unidata.ucar.edu/software/netcdf/docs/modules.html) support functions @@ -14,12 +14,25 @@ #include /** - * @name PIOc_inq_att - * @brief The PIO-C interface for the NetCDF function nc_inq_att. - * @details This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * @ingroup PIO_inq_att + * The PIO-C interface for the NetCDF function nc_inq_att. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. For more information on the underlying + * NetCDF commmand please read about this function in the NetCDF + * documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * + * @param varid the variable ID. + * @param name the name of the attribute to inquire about. + * @param xtypep a pointer that will get the type of the attribute. + * @param lenp a pointer to a PIO_Offset that will get the length of + * this attribute. + * + * @return PIO_NOERR for success, error code otherwise. */ int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, PIO_Offset *lenp) { @@ -83,12 +96,14 @@ int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, PIO_Of } /** - * @name PIOc_inq_format - * @brief The PIO-C interface for the NetCDF function nc_inq_format. - * @details This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html + * @ingroup PIO_inq_format + * The PIO-C interface for the NetCDF function nc_inq_format. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. For more information on the underlying + * NetCDF commmand please read about this function in the NetCDF + * documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html */ int PIOc_inq_format (int ncid, int *formatp) { @@ -377,12 +392,14 @@ int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, const in } /** - * @name PIOc_inq_var - * @brief The PIO-C interface for the NetCDF function nc_inq_var. - * @details This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * @ingroup PIO_inq_var + * The PIO-C interface for the NetCDF function nc_inq_var. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. For more information on the underlying + * NetCDF commmand please read about this function in the NetCDF + * documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html */ int PIOc_inq_var (int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, int *dimidsp, int *nattsp) { @@ -459,12 +476,14 @@ int PIOc_inq_var (int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, } /** - * @name PIOc_inq_varname - * @brief The PIO-C interface for the NetCDF function nc_inq_varname. - * @details This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * @ingroup PIO_inq_varname + * The PIO-C interface for the NetCDF function nc_inq_varname. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. For more information on the underlying + * NetCDF commmand please read about this function in the NetCDF + * documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html */ int PIOc_inq_varname (int ncid, int varid, char *name) { diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c new file mode 100644 index 00000000000..b2c7ca20cb5 --- /dev/null +++ b/src/clib/pio_nc4.c @@ -0,0 +1,157 @@ +/** @file + * + * Functions to wrap netCDF-4 functions for PIO. + **/ +#include +#include + +/** + * @ingroup PIO_def_var + * Set chunksizes for a variable. + * + * Chunksizes have important performance repercussions. NetCDF + * attempts to choose sensible chunk sizes by default, but for best + * performance check chunking against access patterns. + * + * @param ncid the ncid of the open file. + * @param varid the ID of the variable to set chunksizes for. + * @param storage NC_CONTIGUOUS or NC_CHUNKED. + * @param chunksizep an array of chunksizes. Must have a chunksize for + * every variable dimension. + * + * @return PIO_NOERR for success, otherwise an error code. + */ +int PIOc_def_var_chunking(int ncid, int varid, int storage, + const size_t *chunksizesp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + if (!(file = pio_get_file_from_id(ncid))) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_DEF_VAR_CHUNKING; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_def_var_chunking(file->fh, varid, storage, chunksizesp); + break; + case PIO_IOTYPE_NETCDF4C: + if(ios->io_rank==0){ + ierr = nc_def_var_chunking(file->fh, varid, storage, chunksizesp); + } + break; +#endif + case PIO_IOTYPE_NETCDF: + return PIO_ENOTNC4; + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + return PIO_ENOTNC4; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIO_inq_var + * Inquire about chunksizes for a variable. + * + * @param ncid the ncid of the open file. + * @param varid the ID of the variable to set chunksizes for. + * @param storagep pointer to int which will be set to either + * NC_CONTIGUOUS or NC_CHUNKED. + * @param chunksizep pointer to memory where chunksizes will be + * set. There are the same number of chunksizes as there are + * dimensions. + * + * @return PIO_NOERR for success, otherwise an error code. + */ +int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, size_t *chunksizesp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + if (!(file = pio_get_file_from_id(ncid))) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_VAR_CHUNKING; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_var_chunking(file->fh, varid, storagep, chunksizesp); + break; + case PIO_IOTYPE_NETCDF4C: + if(ios->io_rank==0){ + ierr = nc_inq_var_chunking(file->fh, varid, storagep, chunksizesp); + } + break; +#endif + case PIO_IOTYPE_NETCDF: + return PIO_ENOTNC4; + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + return PIO_ENOTNC4; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + if(errstr != NULL) free(errstr); + return ierr; +} + + diff --git a/src/clib/pioc.c b/src/clib/pioc.c index 77cacebe679..a2d7a6e0aa4 100644 --- a/src/clib/pioc.c +++ b/src/clib/pioc.c @@ -75,9 +75,17 @@ int PIOc_advanceframe(int ncid, int varid) } /** - ** @brief Set the unlimited dimension of the given variable + * @ingroup PIO_setframe + * @brief Set the unlimited dimension of the given variable + * + * @param ncid the ncid of the file. + * @param varid the varid of the variable + * @param frame the value of the unlimited dimension. In c 0 for the + * first record, 1 for the second + * + * @return PIO_NOERR for no error, or error code. */ -int PIOc_setframe(const int ncid, const int varid,const int frame) +int PIOc_setframe(const int ncid, const int varid, const int frame) { file_desc_t *file; file = pio_get_file_from_id(ncid); @@ -492,9 +500,17 @@ int PIOc_finalize(const int iosysid) MPI_Group_free(&(ios->compgroup)); MPI_Group_free(&(ios->iogroup)); + /* Free the MPI communicators. */ + if(ios->intercomm != MPI_COMM_NULL){ + MPI_Comm_free(&(ios->intercomm)); + } if(ios->io_comm != MPI_COMM_NULL){ MPI_Comm_free(&(ios->io_comm)); } + /* if(ios->comp_comm != MPI_COMM_NULL){ */ + /* MPI_Comm_free(&(ios->comp_comm)); */ + /* } */ + return pio_delete_iosystem_from_list(iosysid); diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c index 5d5d1782d54..ee1d22fe2f9 100644 --- a/src/clib/pioc_support.c +++ b/src/clib/pioc_support.c @@ -1,5 +1,9 @@ +/** @file + * Support functions. + */ #include #include +#include "gptl.h" /* function prototypes and HAVE_MPI logic*/ #include #define versno 2001 @@ -111,11 +115,16 @@ void pioassert(_Bool expression, const char *msg, const char *fname, const int l } - - - - -int check_netcdf(file_desc_t *file,int status, const char *fname, const int line){ +/** Check the result of a netCDF API call. + * + * @param file pointer to the PIO structure describing this file. + * @param status the return value from the netCDF call. + * @param fname the name of the code file. + * @param line the line number of the netCDF call in the code. + * + * @return the error code +*/ +int check_netcdf(file_desc_t *file, int status, const char *fname, const int line){ iosystem_desc_t *ios; int ierr; char errstr[160]; diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 116b5807fba..ba775ff6e15 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -18,7 +18,7 @@ set (SRCS basic_tests.F90 global_vars.F90 ncdf_tests.F90 nc_set_log_level2.c) - + add_executable (pio_unit_test EXCLUDE_FROM_ALL ${SRCS}) target_link_libraries (pio_unit_test piof) if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU") @@ -26,23 +26,33 @@ if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU") PRIVATE -ffree-line-length-none) endif() +add_executable (test_nc4 EXCLUDE_FROM_ALL test_nc4.c) +target_link_libraries (test_nc4 pioc) + if (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG") set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mismatch_all" ) # target_compile_options (gptl # PRIVATE -mismatch_all) endif () +add_dependencies (tests test_nc4) add_dependencies (tests pio_unit_test) # Test Timeout (4 min = 240 sec) set (DEFAULT_TEST_TIMEOUT 240) if (PIO_USE_MPISERIAL) + add_test(NAME test_nc4 + COMMAND test_nc4) add_test(NAME pio_unit_test COMMAND pio_unit_test) set_tests_properties(pio_unit_test PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) else () + add_mpi_test(test_nc4 + EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_nc4 + NUMPROCS 4 + TIMEOUT ${DEFAULT_TEST_TIMEOUT}) add_mpi_test(pio_unit_test EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_unit_test NUMPROCS 4 diff --git a/tests/unit/test_nc4.c b/tests/unit/test_nc4.c new file mode 100644 index 00000000000..6c9b687055c --- /dev/null +++ b/tests/unit/test_nc4.c @@ -0,0 +1,327 @@ +/** + * @file + * Tests for NetCDF-4 Functions + * + */ +#include +#ifdef TIMING +#include +#endif + +/** The number of possible output netCDF output flavors available to + * the ParallelIO library. */ +#define NUM_NETCDF_FLAVORS 4 + +/** The number of dimensions in the example data. In this example, we + * are using three-dimensional data. */ +#define NDIM 3 + +/** The length of our sample data along each dimension. There will be + * a total of 16 integers in each timestep of our data, and + * responsibilty for writing and reading them will be spread between + * all the processors used to run this example. */ +/**@{*/ +#define X_DIM_LEN 400 +#define Y_DIM_LEN 400 +/**@}*/ + +/** The number of timesteps of data to write. */ +#define NUM_TIMESTEPS 6 + +/** The name of the variable in the netCDF output file. */ +#define VAR_NAME "foo" + +/** The meaning of life, the universe, and everything. */ +#define START_DATA_VAL 42 + +/** Handle MPI errors. This should only be used with MPI library + * function calls. */ +#define MPIERR(e) do { \ + MPI_Error_string(e, err_buffer, &resultlen); \ + fprintf(stderr, "MPI error, line %d, file %s: %s\n", __LINE__, __FILE__, err_buffer); \ + MPI_Finalize(); \ + return 2; \ + } while (0) + +/** Handle non-MPI errors by finalizing the MPI library and exiting + * with an exit code. */ +#define ERR(e) do { \ + fprintf(stderr, "Error %d in %s, line %d\n", e, __FILE__, __LINE__); \ + MPI_Finalize(); \ + return e; \ + } while (0) + +/** Global err buffer for MPI. When there is an MPI error, this buffer + * is used to store the error message that is associated with the MPI + * error. */ +char err_buffer[MPI_MAX_ERROR_STRING]; + +/** This is the length of the most recent MPI error message, stored + * int the global error string. */ +int resultlen; + +/** The dimension names. */ +char dim_name[NDIM][NC_MAX_NAME + 1] = {"timestep", "x", "y"}; + +/** Length of the dimensions in the sample data. */ +int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN}; + +/** Length of chunksizes to use in netCDF-4 files. */ +size_t chunksize[NDIM] = {2, X_DIM_LEN/2, Y_DIM_LEN/2}; + +/** Error code for when things go wrong. */ +#define ERR_AWFUL 1111 + +/** Run Tests for NetCDF-4 Functions. + * + * @param argc argument count + * @param argv array of arguments + */ +int +main(int argc, char **argv) +{ + int verbose = 1; + + /** Zero-based rank of processor. */ + int my_rank; + + /** Number of processors involved in current execution. */ + int ntasks; + + /** Different output flavors. The example file is written (and + * then read) four times. The first two flavors, + * parallel-netcdf, and netCDF serial, both produce a netCDF + * classic format file (but with different libraries). The + * last two produce netCDF4/HDF5 format files, written with + * and without using netCDF-4 parallel I/O. */ + int format[NUM_NETCDF_FLAVORS] = {PIO_IOTYPE_PNETCDF, + PIO_IOTYPE_NETCDF, + PIO_IOTYPE_NETCDF4C, + PIO_IOTYPE_NETCDF4P}; + + /** Names for the output files. Two of them (pnetcdf and + * classic) will be in classic netCDF format, the others + * (serial4 and parallel4) will be in netCDF-4/HDF5 + * format. All four can be read by the netCDF library, and all + * will contain the same contents. */ + char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME + 1] = {"test_nc4_pnetcdf.nc", + "test_nc4_classic.nc", + "test_nc4_serial4.nc", + "test_nc4_parallel4.nc"}; + + /** Number of processors that will do IO. In this example we + * will do IO from all processors. */ + int niotasks; + + /** Stride in the mpi rank between io tasks. Always 1 in this + * example. */ + int ioproc_stride = 1; + + /** Number of the aggregator? Always 0 in this example. */ + int numAggregator = 0; + + /** Zero based rank of first processor to be used for I/O. */ + int ioproc_start = 0; + + /** Specifies the flavor of netCDF output format. */ + int iotype; + + /** The dimension IDs. */ + int dimids[NDIM]; + + /** Array index per processing unit. This is the number of + * elements of the data array that will be handled by each + * processor. In this example there are 16 data elements. If the + * example is run on 4 processors, then arrIdxPerPe will be 4. */ + PIO_Offset elements_per_pe; + + /** The ID for the parallel I/O system. It is set by + * PIOc_Init_Intracomm(). It references an internal structure + * containing the general IO subsystem data and MPI + * structure. It is passed to PIOc_finalize() to free + * associated resources, after all I/O, but before + * MPI_Finalize is called. */ + int iosysid; + + /** The ncid of the netCDF file created in this example. */ + int ncid = 0; + + /** The ID of the netCDF varable in the example file. */ + int varid; + + /** The I/O description ID as passed back by PIOc_InitDecomp() + * and freed in PIOc_freedecomp(). */ + int ioid; + + /** A buffer for sample data. The size of this array will + * vary depending on how many processors are involved in the + * execution of the example code. It's length will be the same + * as elements_per_pe.*/ + float *buffer; + + /** A buffer for reading data back from the file. The size of + * this array will vary depending on how many processors are + * involved in the execution of the example code. It's length + * will be the same as elements_per_pe.*/ + int *read_buffer; + + /** A 1-D array which holds the decomposition mapping for this + * example. The size of this array will vary depending on how + * many processors are involved in the execution of the + * example code. It's length will be the same as + * elements_per_pe. */ + PIO_Offset *compdof; + + /** Return code. */ + int ret; + +#ifdef TIMING + /* Initialize the GPTL timing library. */ + if ((ret = GPTLinitialize ())) + return ret; +#endif + + /* Initialize MPI. */ + if ((ret = MPI_Init(&argc, &argv))) + MPIERR(ret); + if ((ret = MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN))) + MPIERR(ret); + + /* Learn my rank and the total number of processors. */ + if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) + MPIERR(ret); + if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) + MPIERR(ret); + + /* Check that a valid number of processors was specified. */ + if (!(ntasks == 1 || ntasks == 2 || ntasks == 4 || + ntasks == 8 || ntasks == 16)) + fprintf(stderr, "Number of processors must be 1, 2, 4, 8, or 16!\n"); + if (verbose) + printf("%d: ParallelIO Library example1 running on %d processors.\n", + my_rank, ntasks); + + /* keep things simple - 1 iotask per MPI process */ + niotasks = ntasks; + + /* Initialize the PIO IO system. This specifies how + * many and which processors are involved in I/O. */ + if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, niotasks, ioproc_stride, + ioproc_start, PIO_REARR_SUBSET, &iosysid))) + ERR(ret); + + /* Describe the decomposition. This is a 1-based array, so add 1! */ + elements_per_pe = X_DIM_LEN * Y_DIM_LEN / ntasks; + if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) + return PIO_ENOMEM; + for (int i = 0; i < elements_per_pe; i++) { + compdof[i] = my_rank * elements_per_pe + i + 1; + } + + /* Create the PIO decomposition for this test. */ + if (verbose) + printf("rank: %d Creating decomposition...\n", my_rank); + if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, &dim_len[1], (PIO_Offset)elements_per_pe, + compdof, &ioid, NULL, NULL, NULL))) + ERR(ret); + free(compdof); + +#ifdef HAVE_MPE + /* Log with MPE that we are done with INIT. */ + if ((ret = MPE_Log_event(event_num[END][INIT], 0, "end init"))) + MPIERR(ret); +#endif /* HAVE_MPE */ + + /* Use PIO to create the example file in each of the four + * available ways. */ + for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) + { +#ifdef HAVE_MPE + /* Log with MPE that we are starting CREATE. */ + if ((ret = MPE_Log_event(event_num[START][CREATE_PNETCDF+fmt], 0, "start create"))) + MPIERR(ret); +#endif /* HAVE_MPE */ + + /* Create the netCDF output file. */ + if (verbose) + printf("rank: %d Creating sample file %s with format %d...\n", + my_rank, filename[fmt], format[fmt]); + if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename[fmt], + PIO_CLOBBER))) + ERR(ret); + + /* Define netCDF dimensions and variable. */ + if (verbose) + printf("rank: %d Defining netCDF metadata...\n", my_rank); + for (int d = 0; d < NDIM; d++) { + if (verbose) + printf("rank: %d Defining netCDF dimension %s, length %d\n", my_rank, + dim_name[d], dim_len[d]); + if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) + ERR(ret); + } + if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, &varid))) + ERR(ret); + + /* For netCDF-4 files, set the chunksize to improve performance. */ + if (format[fmt] == PIO_IOTYPE_NETCDF4C || format[fmt] == PIO_IOTYPE_NETCDF4P) + { + if ((ret = PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, chunksize))) + ERR(ret); + + /** Check that the inq function works. */ + int storage; + size_t my_chunksize[NDIM]; + if ((ret = PIOc_inq_var_chunking(ncid, 0, &storage, my_chunksize))) + ERR(ret); + + /** For serial netCDF-4, only processor rank 0 gets the answers. */ + if (format[fmt] == PIO_IOTYPE_NETCDF4C && !my_rank || + format[fmt] == PIO_IOTYPE_NETCDF4P) + { + if (storage != NC_CHUNKED) + ERR(ERR_AWFUL); + for (int d = 0; d < NDIM; d++) + if (my_chunksize[d] != chunksize[d]) + ERR(ERR_AWFUL); + } + } else { + /* Trying to set chunking for non-netCDF-4 files results + * in the PIO_ENOTNC4 error. */ + if ((ret = PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, chunksize)) != PIO_ENOTNC4) + ERR(ERR_AWFUL); + } + + if ((ret = PIOc_enddef(ncid))) + ERR(ret); + + /* Close the netCDF file. */ + if (verbose) + printf("rank: %d Closing the sample data file...\n", my_rank); + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + } + + /* Free the PIO decomposition. */ + if (verbose) + printf("rank: %d Freeing PIO decomposition...\n", my_rank); + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + + /* Finalize the IO system. */ + if (verbose) + printf("rank: %d Freeing PIO resources...\n", my_rank); + if ((ret = PIOc_finalize(iosysid))) + ERR(ret); + + /* Finalize the MPI library. */ + MPI_Finalize(); + +#ifdef TIMING + /* Finalize the GPTL timing library. */ + if ((ret = GPTLfinalize ())) + return ret; +#endif + + return 0; +}