diff --git a/README.md b/README.md index c94f5d8c71d..6e8b263e018 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ cdash site at [http://my.cdash.org/index.php?project=PIO](http://my.cdash.org/in ## Dependencies -PIO can use NetCDF (version 4.3.3+) and/or PnetCDF (version 1.6.0+) +PIO can use NetCDF (version 4.6.1+) and/or PnetCDF (version 1.9.0+) for I/O. NetCDF may be built with or without netCDF-4 features. NetCDF is required for PIO, PnetCDF is optional. @@ -29,122 +29,43 @@ be linked with an MPI-enabled version of HDF5. Optionally, NetCDF can be built with DAP support, which introduces a dependency on CURL. Additionally, HDF5, itself, introduces dependencies on LIBZ and (optionally) SZIP. -## Configuring with CMake +## Building PIO -To configure the build, PIO requires CMake version 2.8.12+. The typical -configuration with CMake can be done as follows: +To build PIO, unpack the distribution tarball and do: ``` -CC=mpicc FC=mpif90 cmake [-DOPTION1=value1 -DOPTION2=value2 ...] /path/to/pio/source -``` - -where `mpicc` and `mpif90` are the appropriate MPI-enabled compiler wrappers -for your system. - -The `OPTIONS` section typically should consist of pointers to the install -locations for various dependencies, assuming these dependencies are not -located in *canonical* search locations. - -For each dependency `XXX`, one can specify the location of its -installation path with the CMake variable `XXX_PATH`. If the `C` and -`Fortran` libraries for the dependency are installed in different locations -(such as can be done with NetCDF), then you can specify individually -`XXX_C_PATH` and `XXX_Fortran_PATH`. Hence, you can specify the locations -of both NetCDF-C and NetCDF-Fortran, as well as PnetCDF, with the following -CMake configuration line: - -``` -CC=mpicc FC=mpif90 cmake -DNetCDF_C_PATH=/path/to/netcdf-c \ - -DNetCDF_Fortran_PATH=/path/to/netcdf-fortran \ - -DPnetCDF_PATH=/path/to/pnetcdf \ - /path/to/pio/source -``` - -This works for the dependencies: `NetCDF`, `PnetCDF`, `HDF5`, `LIBZ`, `SZIP`. - -### Additional CMake Options - -Additional configuration options can be specified on the command line. - -The `PIO_ENABLE_TIMING` option can be set to `ON` or `OFF` to enable or -disable the use of GPTL timing in the PIO libraries. This feature requires -the GPTL C library for the PIO `C` library and the GPTL Fortran library with -the `perf_mod.mod` and `perf_utils.mod` interface modules. If these GPTL -libraries are already installed on the system, the user can point PIO to the -location of these libraries with the `GPTL_PATH` variable (or, individually, -`GPTL_C_PATH` and `GPTL_Fortran_Perf_PATH` variables). However, if these -GPTL libraries are not installed on the system, and GPTL cannot be found, -then PIO will build its own internal version of GPTL. - -If PnetCDF is not installed on the system, the user can disable its use by -setting `-DWITH_PNETCDF=OFF`. This will disable the search for PnetCDF on the -system and disable the use of PnetCDF from within PIO. - -If the user wishes to disable the PIO tests, then the user can set the -variable `-DPIO_ENABLE_TESTS=OFF`. This will entirely disable the CTest -testing suite, as well as remove all of the test build targets. - -If you wish to install PIO in a safe location for use later with other -software, you may set the `CMAKE_INSTALL_PREFIX` variable to point to the -desired install location. - -## Building - -Once you have successfully configured PIO with CMake in a build directory. -From within the build directory, build PIO with: - -``` -make -``` - -This will build the `pioc` and `piof` libraries. - -## Testing - -If you desire to do testing, and `PIO_ENABLE_TESTS=ON` (which is the default -setting), you may build the test executables with: - +CC=mpicc FC=mpif90 ./configure --enable-fortran && make check install ``` -make tests -``` - -Once the tests have been built, you may run tests with: +For a full description of the available options and flags, try: ``` -ctest +./configure --help ``` -If you have not run `make tests` before you run `ctest`, then you will see -all of the tests fail. - -Alternatively, you may build the test executables and then run tests -immediately with: +Note that environment variables CC and FC may need to be set to the +MPI versions of the C and Fortran compiler. Also CPPFLAGS and LDFLAGS +may need to be set to indicate the locations of one or more of the +dependent libraries. (If using MPI compilers, the entire set of +dependent libraries should be built with the same compilers.) For +example: ``` +export CC=mpicc +export FC=mpifort +export CPPFLAGS='-I/usr/local/netcdf-fortran-4.4.5_c_4.6.3_mpich-3.2/include -I/usr/local/netcdf-c-4.6.3_hdf5-1.10.5/include -I/usr/local/pnetcdf-1.11.0_shared/include' +export LDFLAGS='-L/usr/local/netcdf-c-4.6.3_hdf5-1.10.5/lib -L/usr/local/pnetcdf-1.11.0_shared/lib' +./configure --prefix=/usr/local/pio-2.4.2 --enable-fortran make check +make install ``` -(similar to the typical `make check` Autotools target). +## Building with CMake -**NOTE:** It is important to note that these tests are designed to run in parallel. -If you are on one of the supported supercomputing platforms (i.e., NERSC, NWSC, ALCF, -etc.), then the `ctest` command will assume that the tests will be run in an appropriately -configured and scheduled parallel job. This can be done by requesting an interactive -session from the login nodes and then running `ctest` from within the interactive -terminal. Alternatively, this can be done by running the `ctest` command from a -job submission script. It is important to understand, however, that `ctest` itself -will preface all of the test executable commands with the appropriate `mpirun`/`mpiexec`/`runjob`/etc. -Hence, you should not further preface the `ctest` command with these MPI launchers. - -## Installing - -Once you have built the PIO libraries, you may install them in the location -specified by the `CMAKE_INSTALL_PREFIX`. To do this, simply type: +The typical configuration with CMake can be done as follows: ``` -make install +CC=mpicc FC=mpif90 cmake [-DOPTION1=value1 -DOPTION2=value2 ...] /path/to/pio/source ``` -If the internal GPTL libraries were built (because GPTL could not be found -and the `PIO_ENABLE_TIMING` variable is set to `ON`), then these libraries -will be installed with PIO. +Full instructions for the cmake build can be found in the installation +documentation. diff --git a/doc/source/GettingStarted.txt b/doc/source/GettingStarted.txt index 40fe348d534..aee7661482b 100644 --- a/doc/source/GettingStarted.txt +++ b/doc/source/GettingStarted.txt @@ -20,21 +20,49 @@ /*! \page intro Introduction -PIO is a software interface layer designed to encapsolate the complexities of parallel IO and make it easier to replace the lower level software backend. It currently supports netcdf and pnetcdf . +PIO is a software interface layer designed to encapsolate the +complexities of parallel IO and make it easier to replace the lower +level software backend. It currently supports +netcdf and pnetcdf . -Basic description of how to optimize IO in a parallel environment... +Basic description of how to optimize IO in a parallel environment: -PIO calls are collective. A MPI communicator is set in a call to \ref PIO_init and all tasks associated with that communicator must participate in all subsequent calls to PIO. An application can make multiple calls to \ref PIO_init in order to support multiple MPI communicators. +PIO calls are collective. A MPI communicator is set in a call to \ref +PIO_init and all tasks associated with that communicator must +participate in all subsequent calls to PIO. An application can make +multiple calls to \ref PIO_init in order to support multiple MPI +communicators. -Begin by checking out a copy from [gitHub](https://github.com/PARALLELIO/ParallelIO) and installing on your system as per the instructions in the [Installation](@ref install) document. Take a look at examples of PIO usage in both complex and simple test programs in the [Examples](@ref examp) document. Finally, read through the [FAQ](@ref faq) to see if any remaining questions can be answered. +Begin by getting and unpacking the most recent release of PIO from +[gitHub](https://github.com/PARALLELIO/ParallelIO/releases) and +installing on your system as per the instructions in the +[Installation](@ref install) document. Take a look at examples of PIO +usage in both complex and simple test programs in the [Examples](@ref +examp) document. Finally, read through the [FAQ](@ref faq) to see if +any remaining questions can be answered. ### Using PIO has three basic steps. ### -1. Your program should call the \ref PIO_init function, and provide the MPI communicator (and the rank within that communicator) of the calling task. This call initializes an IO system type structure that will be used in subsequent file and decomposition functions. +1. Your program should call the \ref PIO_init function, and provide +the MPI communicator (and the rank within that communicator) of the +calling task. This call initializes an IO system type structure that +will be used in subsequent file and decomposition functions. -2. You can open a file for reading or writing with a call to \ref PIO_createfile or \ref PIO_openfile. In this call you will specify the file type: pio_iotype_netcdf, pio_iotype_pnetcdf, pio_iotype_netcdf4c or pio_iotype_netcdf4p; along with the file name and optionally the netcdf mode. +2. You can open a file for reading or writing with a call to \ref +PIO_createfile or \ref PIO_openfile. In this call you will specify the +file type: pio_iotype_netcdf, pio_iotype_pnetcdf, pio_iotype_netcdf4c +or pio_iotype_netcdf4p; along with the file name and optionally the +netcdf mode. -3. Finally, you can read or write decomposed data to the output file. You must describe the mapping between the organization of data in the file and that same data in the application space. This is done in a call to \ref PIO_initdecomp. In the simplest call to this function, a one dimensional integer array is passed from each task, the values in the array represent the offset from the beginning of the array on file. (what happens next?) +3. Finally, you can read or write decomposed data to the output +file. You must describe the mapping between the organization of data +in the file and that same data in the application space. This is done +in a call to \ref PIO_initdecomp. In the simplest call to this +function, a one dimensional integer array is passed from each task, +the values in the array represent the offset from the beginning of the +array on file. */ diff --git a/doc/source/Installing.txt b/doc/source/Installing.txt index 6f281152248..05aa5d1dd61 100644 --- a/doc/source/Installing.txt +++ b/doc/source/Installing.txt @@ -20,7 +20,7 @@ The PIO code is currently stored on github at
How do I test if PIO is installed and working correctly?
-
The PIO Library distribution contains a testpio subdirectory with a number of programs to test the PIO library. Please see the \ref examp page for details.
+
The PIO Library distribution contains tests for PIO. They are run my 'make check'. The tests use mpiexec to run tests on 4, 8, or 16 processors.
*/ diff --git a/examples/c/darray_no_async.c b/examples/c/darray_no_async.c index 126adcd5ba0..784612acdcf 100644 --- a/examples/c/darray_no_async.c +++ b/examples/c/darray_no_async.c @@ -6,7 +6,7 @@ * (one unlimited) and one variable. It first writes, then reads the * sample file using distributed arrays. * - * This example can be run in parallel for 4 processors. + * This example can be run in parallel for 16 processors. */ #include "config.h" @@ -32,10 +32,10 @@ #define NUM_TIMESTEPS 2 /* The length of our sample data in X dimension.*/ -#define DIM_LEN_X 4 +#define DIM_LEN_X 8 /* The length of our sample data in Y dimension.*/ -#define DIM_LEN_Y 4 +#define DIM_LEN_Y 8 /* The name of the variable in the netCDF output file. */ #define VAR_NAME "foo" @@ -48,7 +48,7 @@ #define START_DATA_VAL 42 /* Number of tasks this example runs on. */ -#define TARGET_NTASKS 4 +#define TARGET_NTASKS 16 /* Logging level. */ #define LOG_LEVEL -1 @@ -59,6 +59,11 @@ int dim_len[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y}; /* Names of dimensions. */ char dim_name[NDIM3][PIO_MAX_NAME + 1] = {"unlimted", "x", "y"}; +/* These are used when writing the decomposition file. */ +#define DECOMP_FILENAME "darray_no_async_decomp.nc" +#define DECOMP_TITLE "Example Decomposition from darray_no_async.c" +#define DECOMP_HISTORY "This file is created by the program darray_no_async in the PIO C library" + /* Handle MPI errors. This should only be used with MPI library * function calls. */ #define MPIERR(e) do { \ @@ -180,25 +185,33 @@ int check_file(int iosysid, int ntasks, char *filename, int iotype, has the following contents (as shown by ncdump):
-   netcdf darray_no_async_iotype_1 {
-   dimensions:
-   unlimted = UNLIMITED ; // (2 currently)
-   x = 4 ;
-   y = 4 ;
-   variables:
-   int foo(unlimted, x, y) ;
-   data:
-
-   foo =
-   42, 42, 42, 42,
-   43, 43, 43, 43,
-   44, 44, 44, 44,
-   45, 45, 45, 45,
-   142, 142, 142, 142,
-   143, 143, 143, 143,
-   144, 144, 144, 144,
-   145, 145, 145, 145 ;
-   }
+netcdf darray_no_async_iotype_1 {
+dimensions:
+	unlimted = UNLIMITED ; // (2 currently)
+	x = 8 ;
+	y = 8 ;
+variables:
+	int foo(unlimted, x, y) ;
+data:
+
+ foo =
+  42, 42, 42, 42, 43, 43, 43, 43,
+  44, 44, 44, 44, 45, 45, 45, 45,
+  46, 46, 46, 46, 47, 47, 47, 47,
+  48, 48, 48, 48, 49, 49, 49, 49,
+  50, 50, 50, 50, 51, 51, 51, 51,
+  52, 52, 52, 52, 53, 53, 53, 53,
+  54, 54, 54, 54, 55, 55, 55, 55,
+  56, 56, 56, 56, 57, 57, 57, 57,
+  142, 142, 142, 142, 143, 143, 143, 143,
+  144, 144, 144, 144, 145, 145, 145, 145,
+  146, 146, 146, 146, 147, 147, 147, 147,
+  148, 148, 148, 148, 149, 149, 149, 149,
+  150, 150, 150, 150, 151, 151, 151, 151,
+  152, 152, 152, 152, 153, 153, 153, 153,
+  154, 154, 154, 154, 155, 155, 155, 155,
+  156, 156, 156, 156, 157, 157, 157, 157 ;
+}
    
*/ @@ -239,7 +252,7 @@ int main(int argc, char* argv[]) /* Check that a valid number of processors was specified. */ if (ntasks != TARGET_NTASKS) - fprintf(stderr, "Number of processors must be 4!\n"); + fprintf(stderr, "Number of processors must be 16!\n"); printf("%d: ParallelIO Library darray_no_async example running on %d processors.\n", my_rank, ntasks); @@ -253,7 +266,7 @@ int main(int argc, char* argv[]) /* Initialize the PIO IO system. This specifies how many and * which processors are involved in I/O. */ - if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, 1, ioproc_stride, + if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, 4, ioproc_stride, ioproc_start, PIO_REARR_BOX, &iosysid))) ERR(ret); @@ -268,9 +281,15 @@ int main(int argc, char* argv[]) /* Create the PIO decomposition for this example. Since this * is a variable with an unlimited dimension, we want to * create a 2-D composition which represents one record. */ - printf("rank: %d Creating decomposition...\n", my_rank); + printf("rank: %d Creating decomposition, elements_per_pe %lld...\n", my_rank, + elements_per_pe); if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3 - 1, &dim_len[1], elements_per_pe, - compdof, &ioid, 0, NULL, NULL))) + compdof, &ioid, PIO_REARR_SUBSET, NULL, NULL))) + ERR(ret); + + /* Write the decomposition file. */ + if ((ret = PIOc_write_nc_decomp(iosysid, DECOMP_FILENAME, NC_CLOBBER, + ioid, DECOMP_TITLE, DECOMP_HISTORY, 0))) ERR(ret); /* The number of favors may change with the build parameters. */ diff --git a/examples/c/run_tests.sh b/examples/c/run_tests.sh index 1cfb586d292..0d369bbb53d 100755 --- a/examples/c/run_tests.sh +++ b/examples/c/run_tests.sh @@ -11,18 +11,32 @@ trap exit INT TERM printf 'running PIO examples...\n' #PIO_EXAMPLES='examplePio' -PIO_EXAMPLES='example1 examplePio darray_no_async' +PIO_EXAMPLES='example1 examplePio' +PIO_EXAMPLES_16='darray_no_async' success1=true for EXAMPLE in $PIO_EXAMPLES do success1=false echo "running ${EXAMPLE}" - mpiexec -n 4 ./${EXAMPLE} && success1=true || break + mpiexec -n 4 ./${EXAMPLE} && success1=true + if test $success1 = false; then + break + fi +done +success2=true +for EXAMPLE in $PIO_EXAMPLES_16 +do + success2=false + echo "running ${EXAMPLE}" + mpiexec -n 16 ./${EXAMPLE} && success2=true + if test $success2 = false; then + break + fi done # Did we succeed? -if test x$success1 = xtrue; then +if test x$success1 = xtrue -a x$success2 = xtrue; then exit 0 fi exit 1 diff --git a/src/clib/pio.h b/src/clib/pio.h index d88a4b3f3f6..f5647e364a0 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -410,7 +410,7 @@ typedef struct iosystem_desc_t MPI_Comm comp_comm; /** This is an MPI inter communicator between IO communicator and - * computation communicator. */ + * computation communicator, only used for async mode. */ MPI_Comm intercomm; /** This is a copy (but not an MPI copy) of either the comp (for