From 580381b892751551323f559d6ae03e30b80143f1 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 12 Mar 2020 12:53:59 -0600 Subject: [PATCH 1/3] Squashed 'src/externals/pio2/' changes from 20457f902..e41e6a652 e41e6a652 Merge pull request #1638 from jedwards4b/logging_bug_fix 92418ee9f if rindex is null and debug is enabled this will segfault 6aa9c6038 Merge pull request #1634 from NCAR/ejh_warnings cac94b3c0 removed unneeded initialization of vars 1a3799db3 fixed warnings related to netcdf integration code d92cb2801 Merge pull request #1628 from NCAR/ejh_fix_alloc 7114c751d moved decl for performance 72c5bd5b2 Merge pull request #1622 from NCAR/ejh_perf_4 341c123ff fixed up test program 6ddc1a024 split async perf test out a8f7076ef Merge pull request #1620 from NCAR/ejh_ncint_perf_3 e5708e1f4 fixed table output 85f67f990 more development of performance program b16e033d3 testing different iotypes in performance test 6c03fdbb5 Merge pull request #1619 from NCAR/ejh_ncint_perf_2 b59669578 improvements in performance program b9b900b1a Merge pull request #1618 from NCAR/ejh_ncint_perf 65bef9b2f no longer automataically deflate vars for IOTYPE_PIO_NETCDF4C if opened through netcdf integration a8ea44c90 ncint performance test 751ff3d1c Merge pull request #1614 from NCAR/ejh_prepare ade6fb545 fixed header files b1510169e changed version numbers 8c3ab430c Merge pull request #1611 from NCAR/ejh_dos 1376df147 fixed docs in pio.h 932fbdb25 fixed docs in pio.h 824f576fd Merge pull request #1610 from NCAR/ejh_includes_2 8d283f283 removed more inappropriate ifdefs from pio.h 12209c7a5 Merge pull request #1609 from NCAR/ejh_header cbcc78567 moving reliance on ifdefs out of pio.h c0079f600 moving reliance on ifdefs out of pio.h db7238801 Merge pull request #1605 from ckhroulev/cmake-install-uthash af26e0d86 Merge branch 'master' into cmake-install-uthash 2f1436add Merge pull request #1606 from ckhroulev/PIOc_InitDecomp_bc-off-by-one-bugfix 648109a52 Fix an off-by-one error in PIOc_InitDecomp_bc() 695809503 Install uthash.h when building ParallelIO using CMake b1b1fa7fb Merge pull request #1603 from NCAR/ejh_multi_ncint_async a8b8a42af only send iosysid for create/open when netcdf integration is in use 0d6cbc756 now netcdf integration works with multiple computational components with async f19543590 now passing diosysid when doing a create file in async mode 85ec39cd8 fixed ncid issue for PIO opening second file d6097d97f moving code to function so that multiple computation units can be handled 94cdb68ee moving code to function so that multiple computation units can be handled fc920975e adding new ncint async test b8db4c93f Merge pull request #1602 from NCAR/ejh_docs 85c15c290 version 1 of netcdf integration documentation 4664ac57e Merge pull request #1601 from NCAR/ejh_read_darray 5774d8141 fixed netcdf integration tests with read_darray() d6537da15 now testing read_darray() 16c8bfe32 minor changes 9c7f67cb5 merged master 49ab208a2 Merge pull request #1600 from NCAR/ejh_remove_ndims_call f07cbeca3 now store ndims in var_desc_t and no longer need to look it up in read_darray() 0020927f4 adding ndims to var_desc_t 9c4257dfb continuing to add ndims to var_desc_t 3d941c9ac dealing with allocation for ndims array 204ec0fb9 made inq_file_metadata a static function 4f579cf59 comments 7c2c7e74c more progress towards read_darray 14320d5c7 more progress towards read_darray f34f7b188 read_darray seeming to work, but cleanup of resources fails 11f62119b implementing async read_darray() 344b083d0 merged warning fix from master b53c2d3aa Merge pull request #1597 from NCAR/ejh_warning cf5173afe fixed warning 166b9afec beginning to implement read_darray for async ee94d4783 Merge pull request #1595 from NCAR/ejh_more_ncint 76405781e protect test_async_1d from classic only builds 564452b8c more test development 9633f7fa8 more test development 1503141de added test to cmake build 5eaa61d31 more test in test_async_1d 9682bb265 more test code for test_async_1d.c e6b40bcb1 adding code to new test test_async_1d 2c57b592f starting to add new test_async_1d 3686e8b2f starting to add PIOc calls to test 5ff9c23b4 checking more of test file 4b867994a checking more of test file 43c675159 checking more of test file 6012ed3d0 starting to check test file 20a3b29cc getting close to turning on int reads 3d8167a47 now testing open with netcdf integration and async 4a2a867ff proper handling of ncid in openfile_retry for async netcdf integration 3bd30e8af now calling nc_open when use_ext_ncid is true in open_file_handler() e1f1c88c0 now pass use_ext_ncid to open_file_handler() in pio_msg.c 5f46649ee more of async PIO test 1d55692d5 turned on more test 4b074a5e5 Merge pull request #1594 from NCAR/ejh_close 7ba458f3a fixed documentation 0cb6841dc turned on fortran netcdf integration tests 97d08bf6d Merge pull request #1593 from NCAR/ejh_ncint_async 10962543b protected use of nc4_file_change_ncid() with ifdef 0fccc80ab protected use of NC_UDF0 with ifdef 541d0df86 clean up 1d4cc195f got async working with netcdf integration with one computational component 63adee9ad merged changes from master 6c24f633f Merge pull request #1592 from NCAR/ejh_createfile_int_async ae9375900 now use PIOc_createfile_int() instead of PIOc_createfile() in pio_msg.c for async 9f91675e5 added contents of ncidp when ncidp_present to the parameters sent when a file is created in async mode ba2bb47db added ncidp_present to the parameters sent when a file is created in async mode d077e48fd added use_ext_ncid to the parameters sent when a file is created in async mode f5ee1625e fixed dispatch table removal of _pe functions b18bd1251 trying to get working 6e393976f commented out ncint tests ef98cca1d moving to async netcdf integration 86ad4463a Merge pull request #1588 from NCAR/ejh_fix_cmake 51217ec74 fixed cmake build from autotools build 8db71e74f added cmake directory to autotools build ed1d239d7 fixing Cmake builds a45f45f39 added config.h.in to Makefile.am cd0b55f2a Merge pull request #1587 from NCAR/ejh_fix fc4cdc347 last minute fixes 076231d37 Merge pull request #1586 from NCAR/ejh_version cf5a77724 changed versions for release 009fe0fe3 Merge pull request #1584 from jedwards4b/fix_integer_overflow 064e01acf fix an integer overflow issue reported on summit 7b2114eb8 Merge pull request #1581 from jedwards4b/netcdf_serial_bugfix a9245e838 must allocate on ioroot for serial io 04d8dc515 Merge pull request #1579 from NCAR/ejh_async_nc_create 4bea2db6d starting to get async working with netcdf integration git-subtree-dir: src/externals/pio2 git-subtree-split: e41e6a652de5dbd587ac54f45397a72a6848c358 --- CMakeLists.txt | 4 +- Makefile.am | 5 +- cmake/Makefile.am | 13 ++ configure.ac | 12 +- doc/source/base.txt | 1 + doc/source/netcdf_integration.txt | 160 +++++++++++++++- examples/Makefile.am | 2 + examples/c/CMakeLists.txt | 6 +- examples/c/Makefile.am | 2 +- examples/c/darray_async.c | 10 +- src/clib/CMakeLists.txt | 4 +- src/clib/Makefile.am | 4 +- src/clib/pio.h | 284 ++++++++++++++--------------- src/clib/pio_darray.c | 63 ++++++- src/clib/pio_darray_int.c | 29 +-- src/clib/pio_file.c | 21 +-- src/clib/pio_internal.h | 17 +- src/clib/pio_lists.c | 9 +- src/clib/pio_msg.c | 105 +++++++++-- src/clib/pio_nc.c | 16 +- src/clib/pio_rearrange.c | 17 +- src/clib/pioc.c | 23 ++- src/clib/pioc_support.c | 197 ++++++++++++++------ src/flib/Makefile.am | 2 +- src/gptl/Makefile.am | 4 +- src/ncint/ncint_pio.c | 74 +++++++- src/ncint/ncintdispatch.c | 19 +- tests/cunit/CMakeLists.txt | 3 + tests/cunit/Makefile.am | 5 +- tests/cunit/run_tests.sh | 2 +- tests/cunit/test_async_1d.c | 155 ++++++++++++++++ tests/cunit/test_darray.c | 1 - tests/cunit/test_spmd.c | 25 +-- tests/fncint/Makefile.am | 2 +- tests/ncint/Makefile.am | 18 +- tests/ncint/pio_err_macros.h | 66 +++++++ tests/ncint/run_perf.sh | 33 ++++ tests/ncint/run_tests.sh | 2 +- tests/ncint/tst_async_multi.c | 182 ++++++++++++++++++ tests/ncint/tst_ncint_async_perf.c | 178 ++++++++++++++++++ tests/ncint/tst_ncint_perf.c | 171 +++++++++++++++++ tests/ncint/tst_pio_async.c | 160 ++++++++++++++++ tests/ncint/tst_pio_udf.c | 77 ++++---- tests/unit/Makefile.am | 2 +- 44 files changed, 1802 insertions(+), 383 deletions(-) create mode 100644 cmake/Makefile.am create mode 100644 tests/cunit/test_async_1d.c create mode 100644 tests/ncint/pio_err_macros.h create mode 100755 tests/ncint/run_perf.sh create mode 100644 tests/ncint/tst_async_multi.c create mode 100644 tests/ncint/tst_ncint_async_perf.c create mode 100644 tests/ncint/tst_ncint_perf.c create mode 100644 tests/ncint/tst_pio_async.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 846218ffa9a..1c75d29c8c5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,8 +4,8 @@ project (PIO C Fortran) # The project version number. set(VERSION_MAJOR 2 CACHE STRING "Project major version number.") -set(VERSION_MINOR 4 CACHE STRING "Project minor version number.") -set(VERSION_PATCH 4 CACHE STRING "Project patch version number.") +set(VERSION_MINOR 5 CACHE STRING "Project minor version number.") +set(VERSION_PATCH 0 CACHE STRING "Project patch version number.") mark_as_advanced(VERSION_MAJOR VERSION_MINOR VERSION_PATCH) # The size of the data buffer for write/read_darray(). diff --git a/Makefile.am b/Makefile.am index b114d46e041..7f46ea98253 100644 --- a/Makefile.am +++ b/Makefile.am @@ -10,7 +10,6 @@ if BUILD_DOCS DOC = doc endif -SUBDIRS = src tests examples ${DOC} scripts - -EXTRA_DIST = CMakeLists.txt set_flags.am COPYRIGHT +SUBDIRS = src tests examples ${DOC} scripts cmake +EXTRA_DIST = CMakeLists.txt set_flags.am COPYRIGHT cmake_config.h.in diff --git a/cmake/Makefile.am b/cmake/Makefile.am new file mode 100644 index 00000000000..9b2ea30f96f --- /dev/null +++ b/cmake/Makefile.am @@ -0,0 +1,13 @@ +## This is the automake file for the cmake directory of the PIO +## libraries. This directory holds files needed for the CMake build, +## but not the autotools build. + +# Ed Hartnett 8/19/19 + +# Cmake needs all these extra files to build. +EXTRA_DIST = FindGPTL.cmake FindHDF5.cmake FindLIBRT.cmake \ +FindLIBZ.cmake FindMPE.cmake FindMPISERIAL.cmake FindNetCDF.cmake \ +FindPAPI.cmake FindPnetCDF.cmake FindSZIP.cmake LibCheck.cmake \ +LibFind.cmake LibMPI.cmake Makefile.am mpiexec.alcf mpiexec.ncsa \ +mpiexec.nersc mpiexec.nwscla mpiexec.olcf TryHDF5_HAS_SZIP.c \ +TryNetCDF_DAP.c TryNetCDF_PARALLEL.c TryNetCDF_PNETCDF.c diff --git a/configure.ac b/configure.ac index 7e155b7cadb..10e6bb464b3 100644 --- a/configure.ac +++ b/configure.ac @@ -1,21 +1,20 @@ ## This is the autoconf file for the PIO library. ## Ed Hartnett 8/16/17 - # Initialize autoconf and automake. -AC_INIT(pio, 2.4.4-development) +AC_INIT(pio, 2.5.0) AC_CONFIG_SRCDIR(src/clib/pio_darray.c) AM_INIT_AUTOMAKE([foreign serial-tests]) # The PIO version, again. AC_DEFINE([PIO_VERSION_MAJOR], [2], [PIO major version]) -AC_DEFINE([PIO_VERSION_MINOR], [4], [PIO minor version]) -AC_DEFINE([PIO_VERSION_PATCH], [4], [PIO patch version]) +AC_DEFINE([PIO_VERSION_MINOR], [5], [PIO minor version]) +AC_DEFINE([PIO_VERSION_PATCH], [0], [PIO patch version]) # Once more for the documentation. AC_SUBST([VERSION_MAJOR], [2]) -AC_SUBST([VERSION_MINOR], [4]) -AC_SUBST([VERSION_PATCH], [4]) +AC_SUBST([VERSION_MINOR], [5]) +AC_SUBST([VERSION_PATCH], [0]) # The m4 directory holds macros for autoconf. AC_CONFIG_MACRO_DIR([m4]) @@ -314,4 +313,5 @@ AC_OUTPUT(Makefile examples/Makefile examples/c/Makefile examples/f03/Makefile + cmake/Makefile scripts/Makefile) diff --git a/doc/source/base.txt b/doc/source/base.txt index 126bba5be2a..d53491e1d8c 100644 --- a/doc/source/base.txt +++ b/doc/source/base.txt @@ -29,4 +29,5 @@ releases. - @ref users_guide - @ref api - @ref c_api + - @ref netcdf_integration */ diff --git a/doc/source/netcdf_integration.txt b/doc/source/netcdf_integration.txt index bd1016408e7..3e28b2b17f5 100644 --- a/doc/source/netcdf_integration.txt +++ b/doc/source/netcdf_integration.txt @@ -9,14 +9,160 @@ In order to use netCDF integration: * The PIO configure must use the option --enable-netcdf-integration. -* The latest master of the netcdf-c library must be built and used, -and the PIO build must include the netcdf-c/include directory in its -CPPFLAGS, in order to find internal netCDF header files needed for -netCDF integration. (The netcdf-c library is being modified to export -publically everything needed, so future releases of netcdf-c will no -longer require this.) +* Version 4.7.2 or later of the netCDF C library. Once PIO is build for netCDF integration, it provides the nc_* and -nf_* functions required to fully integrate PIO and netCDF. +nf_* functions required to fully integrate PIO and netCDF. Users must +include the PIO header files in their C or Fortran programs, and link +to the PIO and the netCDF libraries (and, optionally, the +parallel-netcdf and HDF5 libraries). + +# Initializing the IO System + +IO system initialization is required before anyother PIO functionality +is used in netCDF calls. The IO system defines how many processors are +used for I/O, and how many for computation. + +The IO system may be initialized in one of two modes: + +* Intercomm Mode - All processors are involved in computation. A + subset does I/O (and also computation). To initialize in intercomm + mode, use nc_def_iosystem(). + +* Async Mode - Some processors are dedicated to IO, and do no + computation. Other processors are organized into computational + unit. Each computational unit runs its own code. All computational + units which do netCDF/PIO calls will channel all IO through the + dedicated I/O nodes. To initialize in async mode, use + nc_def_async(). + +Once defined, these functions return one or (in the case of async) +more IO system IDs, usually abreviated in the code as 'iosysid'. + +For intercomm mode, there is one iosysid. For async mode, there is an +array of iosysids, one for each computational unit. + +# The Default IO System + +The IO system ID (iosysid) must be provided for PIO calls. When using +netCDF integration, there is no parameter in the functions available +to pass the iosysid. Instead, there is a default iosysid. When +creating a new IO system, the default iosysid is set, so that +subsequent netCDF calls can know which IO system to use. + +In the (rare) cases where the user may wish to use multiple IO systems +within the same section of code, the functions nc_set_iosystem() and +nc_get_iosystem() will allow the user to set and get the defauly +iosysid. + +# Opening/Creating a File + +To open a file, use nc_open()/nf_open(), providing the NC_PIO flag. + +In C: + +\code{.c} + /* Initialize the intracomm. */ + if (nc_def_iosystemm(MPI_COMM_WORLD, 1, 1, 0, 0, &iosysid)) PERR; + + /* Create a file with a 3D record var. */ + if (nc_create(FILE_NAME, NC_PIO, &ncid)) PERR; + if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR; + if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR; + if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR; + if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM3, dimid, &varid)) PERR; +\endcode + +Resources associated with the IO system must be released with +nc_free_iosystem()/nf_free_iosystem(). + +In Fortran: + +\code{.F90} + ! Define an IOSystem. + ierr = nf_def_iosystem(my_rank, MPI_COMM_WORLD, niotasks, numAggregator, & + stride, PIO_rearr_box, iosysid, base) + if (ierr .ne. nf_noerr) call handle_err(ierr) + + ! Create a file. + ierr = nf_create(FILE_NAME, 64, ncid) + if (ierr .ne. nf_noerr) call handle_err(ierr) +\endcode + +# Defining a Decomposition + +To define a decompositon for a distributed array use +nc_def_decomp()/nf_def_decomp(). + +In C: +\code{.c} + /* Calculate a decomposition for distributed arrays. */ + elements_per_pe = DIM_LEN_X * DIM_LEN_Y / ntasks; + if (!(compdof = malloc(elements_per_pe * sizeof(size_t)))) + PERR; + for (i = 0; i < elements_per_pe; i++) + compdof[i] = my_rank * elements_per_pe + i; + + /* Create the PIO decomposition for this test. */ + if (nc_def_decomp(iosysid, PIO_INT, NDIM2, &dimlen[1], elements_per_pe, + compdof, &ioid, 1, NULL, NULL)) PERR; + free(compdof); +\endcode + +In Fortran: +\code{.F90} + allocate(compdof(maplen)) + allocate(data_buffer(maplen)) + ! Row decomposition. Recall that my_rank is 0-based, even + ! in fortran. Also recall that compdof is 1-based for fortran. + do i = 1, maplen + compdof(i) = i + my_rank * maplen + data_buffer(i) = my_rank * 10 + i + end do + print *, 'compdof', my_rank, compdof + ierr = nf_def_decomp(iosysid, PIO_int, dims, compdof, decompid) + if (ierr .ne. nf_noerr) call handle_err(ierr) +\endcode + +When a decomposition is defined, a decomposition ID is returned, and +must be used later when accessing data using this decomposition. + +The resources associated with a decomposition must be freed with +nc_free_decomp()/nf_free_decomp(). + +# Reading and Writing Distributed Arrays + +Once a decomposition has been defined, it may be used to read and +write distributed arrays. This allows the code to be written in terms +of local storage only. That is, the array allocated and indexed in the +code is the array of data that this processor works with. When the +data is read/written, the library will map the local data into the +global array space. + +To read distributed data, use the nc_get_vard() function, or one of the +type-specific versions (ex. nc_get_vard_int()). + +To write distributed data, use the nc_put_vard() function, or one of the +type-specific versions (ex. nc_put_vard_int()). + +In C: +\code{.c} + /* Write some data with distributed arrays. */ + if (nc_put_vard_int(ncid, varid, ioid, 0, my_data)) PERR; + if (nc_close(ncid)) PERR; +\endcode + +In Fortran: +\code{.F90} + ! Write 1st record with distributed arrays. + ierr = nf_put_vard_int(ncid, varid, decompid, 1, data_buffer) + if (ierr .ne. nf_noerr) call handle_err(ierr) +\endcode + +# Examples in C and Fortran + +For examples using the netCDF integration features, look in the +tests/ncint directory (for C) or the tests/fncint directory (for +Fortran). */ diff --git a/examples/Makefile.am b/examples/Makefile.am index 413c10ac405..d8835fa8cb8 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -7,3 +7,5 @@ F03 = f03 endif # BUILD_FORTRAN SUBDIRS = c ${F03} + +EXTRA_DIST = CMakeLists.txt diff --git a/examples/c/CMakeLists.txt b/examples/c/CMakeLists.txt index 4f1115b48d5..4e5576ea663 100644 --- a/examples/c/CMakeLists.txt +++ b/examples/c/CMakeLists.txt @@ -37,9 +37,9 @@ ADD_EXECUTABLE(darray_no_async darray_no_async.c) TARGET_LINK_LIBRARIES(darray_no_async pioc) add_dependencies(tests darray_no_async) -ADD_EXECUTABLE(darray_async darray_async.c) -TARGET_LINK_LIBRARIES(darray_async pioc) -add_dependencies(tests darray_async) +# ADD_EXECUTABLE(darray_async darray_async.c) +# TARGET_LINK_LIBRARIES(darray_async pioc) +# add_dependencies(tests darray_async) if (PIO_USE_MPISERIAL) add_test(NAME examplePio COMMAND examplePio) diff --git a/examples/c/Makefile.am b/examples/c/Makefile.am index 920bde9fc52..64398761f13 100644 --- a/examples/c/Makefile.am +++ b/examples/c/Makefile.am @@ -16,7 +16,7 @@ TESTS = run_tests.sh endif # RUN_TESTS # Distribute the test script. -EXTRA_DIST = run_tests.sh +EXTRA_DIST = run_tests.sh CMakeLists.txt example2.c # Clean up files produced during testing. CLEANFILES = *.nc *.log *.clog2 *.slog2 diff --git a/examples/c/darray_async.c b/examples/c/darray_async.c index 8d490657296..ea5f62ac2dd 100644 --- a/examples/c/darray_async.c +++ b/examples/c/darray_async.c @@ -249,11 +249,11 @@ int main(int argc, char* argv[]) ERR(ret); - /* The rest of the code executes on computation tasks only. As - * PIO functions are called on the computation tasks, the - * async system will call them on the IO task. When the - * computation tasks call PIO_finalize(), the IO task will get - * a message to shut itself down. */ + /* The rest of the code executes on computation tasks only. As PIO + * functions are called on the computation tasks, the async system + * will call them on the IO task. When the computation tasks call + * PIO_finalize(), the IO task will get a message to shut itself + * down. */ if (comp_task) { /* PIO_Offset elements_per_pe; /\* Array elements per processing unit. *\/ */ diff --git a/src/clib/CMakeLists.txt b/src/clib/CMakeLists.txt index 83635d9f685..7c5f26ca1e5 100644 --- a/src/clib/CMakeLists.txt +++ b/src/clib/CMakeLists.txt @@ -49,8 +49,8 @@ endif() # Library install (TARGETS pioc DESTINATION lib) -# Include/Header File -install (FILES ${CMAKE_CURRENT_SOURCE_DIR}/pio.h DESTINATION include) +# Include/Header Files +install (FILES pio.h uthash.h DESTINATION include) #============================================================================== # DEFINE THE DEPENDENCIES diff --git a/src/clib/Makefile.am b/src/clib/Makefile.am index 66c7a948825..e08d8a21a48 100644 --- a/src/clib/Makefile.am +++ b/src/clib/Makefile.am @@ -13,7 +13,7 @@ endif # BUILD_NCINT # These linker flags specify libtool version info. # See http://www.gnu.org/software/libtool/manual/libtool.html#Libtool-versioning # for information regarding incrementing `-version-info`. -libpioc_la_LDFLAGS = -version-info 2:0:1 +libpioc_la_LDFLAGS = -version-info 4:0:3 # The library header file will be installed in include dir. include_HEADERS = pio.h uthash.h @@ -25,4 +25,4 @@ pioc_support.c pio_darray_int.c pio_get_nc.c pio_lists.c pio_nc4.c \ pio_put_nc.c pio_spmd.c pio_get_vard.c pio_put_vard.c pio_internal.h \ bget.h uthash.h pio_error.h -EXTRA_DIST = CMakeLists.txt +EXTRA_DIST = CMakeLists.txt topology.c diff --git a/src/clib/pio.h b/src/clib/pio.h index e19a4782212..dfdbc26bd50 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -16,22 +16,7 @@ #include #include -#ifdef _NETCDF #include -#ifdef _NETCDF4 -#include -#endif -#endif -#ifdef _PNETCDF -#include -#endif - -#ifndef MPI_OFFSET -/** MPI_OFFSET is an integer type of size sufficient to represent the - * size (in bytes) of the largest file supported by MPI. In some MPI - * implementations MPI_OFFSET is not properly defined. */ -#define MPI_OFFSET MPI_LONG_LONG -#endif /** PIO_OFFSET is an integer type of size sufficient to represent the * size (in bytes) of the largest file supported by MPI. This is not @@ -109,6 +94,9 @@ * array-order. */ #define DECOMP_FORTRAN_ORDER_STR "Fortran" +/** A convience macro for netCDF integration code. */ +#define NC_PIO NC_UDF0 + /** * Variable description structure. */ @@ -134,6 +122,9 @@ typedef struct var_desc_t /** Holds the fill value of this var. */ void *fillvalue; + /** Number of dimensions for this var. */ + int ndims; + /** Non-zero if fill mode is turned on for this var. */ int use_fill; @@ -311,7 +302,8 @@ typedef struct io_desc_t * everywhere (false) */ bool needsfill; - /** If the map is not monotonically increasing we will need to sort it. */ + /** If the map is not monotonically increasing we will need to + * sort it. */ bool needssort; /** The maximum number of bytes of this iodesc before flushing. */ @@ -592,6 +584,11 @@ typedef struct file_desc_t /** True if this task should participate in IO (only true for one * task with netcdf serial files. */ int do_io; + + /** True if this file was opened with the netCDF integration + * feature. One consequence is that PIO_IOTYPE_NETCDF4C files will + * not have deflate automatically turned on for each var. */ + int ncint_file; } file_desc_t; /** @@ -640,149 +637,137 @@ enum PIO_ERROR_HANDLERS PIO_RETURN_ERROR = (-53) }; -#if defined( _PNETCDF) || defined(_NETCDF) - +/** Attribute id to put/get a global attribute. */ #define PIO_GLOBAL NC_GLOBAL + +/** Size argument to nc_def_dim() for an unlimited dimension. */ #define PIO_UNLIMITED NC_UNLIMITED /* NetCDF types. */ -#define PIO_BYTE NC_BYTE -#define PIO_CHAR NC_CHAR -#define PIO_SHORT NC_SHORT -#define PIO_INT NC_INT -#define PIO_FLOAT NC_FLOAT -#define PIO_REAL NC_FLOAT -#define PIO_DOUBLE NC_DOUBLE -#define PIO_UBYTE NC_UBYTE -#define PIO_USHORT NC_USHORT -#define PIO_UINT NC_UINT -#define PIO_INT64 NC_INT64 -#define PIO_UINT64 NC_UINT64 -#define PIO_STRING NC_STRING +#define PIO_BYTE NC_BYTE /**< signed 1 byte integer */ +#define PIO_CHAR NC_CHAR /**< ISO/ASCII character */ +#define PIO_SHORT NC_SHORT /**< signed 2 byte integer */ +#define PIO_INT NC_INT /**< signed 4 byte integer */ +#define PIO_FLOAT NC_FLOAT /**< single precision floating point number */ +#define PIO_REAL NC_FLOAT /**< single precision floating point number */ +#define PIO_DOUBLE NC_DOUBLE /**< double precision floating point number */ +#define PIO_UBYTE NC_UBYTE /**< unsigned 1 byte int */ +#define PIO_USHORT NC_USHORT /**< unsigned 2-byte int */ +#define PIO_UINT NC_UINT /**< unsigned 4-byte int */ +#define PIO_INT64 NC_INT64 /**< signed 8-byte int */ +#define PIO_UINT64 NC_UINT64 /**< unsigned 8-byte int */ +#define PIO_STRING NC_STRING /**< string */ /* NetCDF flags. */ -#define PIO_WRITE NC_WRITE -#define PIO_NOWRITE NC_NOWRITE -#define PIO_CLOBBER NC_CLOBBER -#define PIO_NOCLOBBER NC_NOCLOBBER -#define PIO_FILL NC_FILL -#define PIO_NOFILL NC_NOFILL -#define PIO_MAX_NAME NC_MAX_NAME -#define PIO_MAX_VAR_DIMS NC_MAX_VAR_DIMS -#define PIO_64BIT_OFFSET NC_64BIT_OFFSET +#define PIO_WRITE NC_WRITE /**< Set read-write access for nc_open(). */ +#define PIO_NOWRITE NC_NOWRITE /**< Set read-only access for nc_open(). */ +#define PIO_CLOBBER NC_CLOBBER /**< Destroy existing file. Mode flag for nc_create(). */ +#define PIO_NOCLOBBER NC_NOCLOBBER /**< Don't destroy existing file. Mode flag for nc_create(). */ +#define PIO_FILL NC_FILL /**< Argument to nc_set_fill() to clear NC_NOFILL */ +#define PIO_NOFILL NC_NOFILL /**< Argument to nc_set_fill() to turn off filling of data. */ +#define PIO_MAX_NAME NC_MAX_NAME /**< Max name length. */ +#define PIO_MAX_VAR_DIMS NC_MAX_VAR_DIMS /**< max per variable dimensions */ +#define PIO_64BIT_OFFSET NC_64BIT_OFFSET /**< Use large (64-bit) file offsets. Mode flag for nc_create(). */ /** NC_64BIT_DATA This is a problem - need to define directly instead * of using include file. */ -#define PIO_64BIT_DATA 0x0010 +#define PIO_64BIT_DATA 0x0010 /**< CDF5 foemat. */ /** Define the netCDF-based error codes. */ -#define PIO_NOERR NC_NOERR -#define PIO_EBADID NC_EBADID -#define PIO_ENFILE NC_ENFILE -#define PIO_EEXIST NC_EEXIST -#define PIO_EINVAL NC_EINVAL -#define PIO_EPERM NC_EPERM -#define PIO_ENOTINDEFINE NC_ENOTINDEFINE -#define PIO_EINDEFINE NC_EINDEFINE -#define PIO_EINVALCOORDS NC_EINVALCOORDS -#define PIO_EMAXDIMS NC_EMAXDIMS -#define PIO_ENAMEINUSE NC_ENAMEINUSE -#define PIO_ENOTATT NC_ENOTATT -#define PIO_EMAXATTS NC_EMAXATTS -#define PIO_EBADTYPE NC_EBADTYPE -#define PIO_EBADDIM NC_EBADDIM -#define PIO_EUNLIMPOS NC_EUNLIMPOS -#define PIO_EMAXVARS NC_EMAXVARS -#define PIO_ENOTVAR NC_ENOTVAR -#define PIO_EGLOBAL NC_EGLOBAL -#define PIO_ENOTNC NC_ENOTNC -#define PIO_ESTS NC_ESTS -#define PIO_EMAXNAME NC_EMAXNAME -#define PIO_EUNLIMIT NC_EUNLIMIT -#define PIO_ENORECVARS NC_ENORECVARS -#define PIO_ECHAR NC_ECHAR -#define PIO_EEDGE NC_EEDGE -#define PIO_ESTRIDE NC_ESTRIDE -#define PIO_EBADNAME NC_EBADNAME -#define PIO_ERANGE NC_ERANGE -#define PIO_ENOMEM NC_ENOMEM -#define PIO_EVARSIZE NC_EVARSIZE -#define PIO_EDIMSIZE NC_EDIMSIZE -#define PIO_ETRUNC NC_ETRUNC -#define PIO_EAXISTYPE NC_EAXISTYPE -#define PIO_EDAP NC_EDAP -#define PIO_ECURL NC_ECURL -#define PIO_EIO NC_EIO -#define PIO_ENODATA NC_ENODATA -#define PIO_EDAPSVC NC_EDAPSVC -#define PIO_EDAS NC_EDAS -#define PIO_EDDS NC_EDDS -#define PIO_EDATADDS NC_EDATADDS -#define PIO_EDAPURL NC_EDAPURL -#define PIO_EDAPCONSTRAINT NC_EDAPCONSTRAINT -#define PIO_ETRANSLATION NC_ETRANSLATION -#define PIO_EHDFERR NC_EHDFERR -#define PIO_ECANTREAD NC_ECANTREAD -#define PIO_ECANTWRITE NC_ECANTWRITE -#define PIO_ECANTCREATE NC_ECANTCREATE -#define PIO_EFILEMETA NC_EFILEMETA -#define PIO_EDIMMETA NC_EDIMMETA -#define PIO_EATTMETA NC_EATTMETA -#define PIO_EVARMETA NC_EVARMETA -#define PIO_ENOCOMPOUND NC_ENOCOMPOUND -#define PIO_EATTEXISTS NC_EATTEXISTS -#define PIO_ENOTNC4 NC_ENOTNC4 -#define PIO_ESTRICTNC3 NC_ESTRICTNC3 -#define PIO_ENOTNC3 NC_ENOTNC3 -#define PIO_ENOPAR NC_ENOPAR -#define PIO_EPARINIT NC_EPARINIT -#define PIO_EBADGRPID NC_EBADGRPID -#define PIO_EBADTYPID NC_EBADTYPID -#define PIO_ETYPDEFINED NC_ETYPDEFINED -#define PIO_EBADFIELD NC_EBADFIELD -#define PIO_EBADCLASS NC_EBADCLASS -#define PIO_EMAPTYPE NC_EMAPTYPE -#define PIO_ELATEFILL NC_ELATEFILL -#define PIO_ELATEDEF NC_ELATEDEF -#define PIO_EDIMSCALE NC_EDIMSCALE -#define PIO_ENOGRP NC_ENOGRP -#define PIO_ESTORAGE NC_ESTORAGE -#define PIO_EBADCHUNK NC_EBADCHUNK -#define PIO_ENOTBUILT NC_ENOTBUILT -#define PIO_EDISKLESS NC_EDISKLESS +#define PIO_NOERR NC_NOERR /**< No Error */ +#define PIO_EBADID NC_EBADID /**< Bad ncid */ +#define PIO_ENFILE NC_ENFILE /**< Too many netcdfs open */ +#define PIO_EEXIST NC_EEXIST /**< netcdf file exists && NC_NOCLOBBER */ +#define PIO_EINVAL NC_EINVAL /**< Invalid Argument */ +#define PIO_EPERM NC_EPERM /**< Write to read only */ +#define PIO_ENOTINDEFINE NC_ENOTINDEFINE /**< Not in define mode */ +#define PIO_EINDEFINE NC_EINDEFINE /**< Not allowed in define mode */ +#define PIO_EINVALCOORDS NC_EINVALCOORDS /**< Invalid coordinates */ +#define PIO_EMAXDIMS NC_EMAXDIMS /**< not enforced after netcdf-c 4.5.0 */ +#define PIO_ENAMEINUSE NC_ENAMEINUSE /**< String match to name in use */ +#define PIO_ENOTATT NC_ENOTATT /**< Attribute not found */ +#define PIO_EMAXATTS NC_EMAXATTS /**< NC_MAX_ATTRS exceeded - not enforced after 4.5.0 */ +#define PIO_EBADTYPE NC_EBADTYPE /**< Not a netcdf data type */ +#define PIO_EBADDIM NC_EBADDIM /**< Invalid dimension id or name */ +#define PIO_EUNLIMPOS NC_EUNLIMPOS /**< NC_UNLIMITED in the wrong index */ +#define PIO_EMAXVARS NC_EMAXVARS /**< not enforced after 4.5.0 */ +#define PIO_ENOTVAR NC_ENOTVAR /**< variable not found */ +#define PIO_EGLOBAL NC_EGLOBAL /**< Action prohibited on NC_GLOBAL varid */ +#define PIO_ENOTNC NC_ENOTNC /**< Not a netcdf file */ +#define PIO_ESTS NC_ESTS /**< In Fortran, string too short */ +#define PIO_EMAXNAME NC_EMAXNAME /**< NC_MAX_NAME exceeded */ +#define PIO_EUNLIMIT NC_EUNLIMIT /**< NC_UNLIMITED size already in use */ +#define PIO_ENORECVARS NC_ENORECVARS /**< nc_rec op when there are no record vars */ +#define PIO_ECHAR NC_ECHAR /**< Attempt to convert between text & numbers */ +#define PIO_EEDGE NC_EEDGE /**< Start+count exceeds dimension bound. */ +#define PIO_ESTRIDE NC_ESTRIDE /**< Illegal stride */ +#define PIO_EBADNAME NC_EBADNAME /**< Attribute or variable name contains illegal characters */ +#define PIO_ERANGE NC_ERANGE /**< Range error */ +#define PIO_ENOMEM NC_ENOMEM /**< Memory allocation (malloc) failure */ +#define PIO_EVARSIZE NC_EVARSIZE /**< One or more variable sizes violate format constraints */ +#define PIO_EDIMSIZE NC_EDIMSIZE /**< Invalid dimension size */ +#define PIO_ETRUNC NC_ETRUNC /**< File likely truncated or possibly corrupted */ +#define PIO_EAXISTYPE NC_EAXISTYPE /**< Unknown axis type. */ +#define PIO_EDAP NC_EDAP /**< Generic DAP error */ +#define PIO_ECURL NC_ECURL /**< Generic libcurl error */ +#define PIO_EIO NC_EIO /**< Generic IO error */ +#define PIO_ENODATA NC_ENODATA /**< Attempt to access variable with no data */ +#define PIO_EDAPSVC NC_EDAPSVC /**< DAP server error */ +#define PIO_EDAS NC_EDAS /**< Malformed or inaccessible DAS */ +#define PIO_EDDS NC_EDDS /**< Malformed or inaccessible DDS */ +#define PIO_EDATADDS NC_EDATADDSDS /**< Dap4 alias */ +#define PIO_EDAPURL NC_EDAPURL /**< Malformed DAP URL */ +#define PIO_EDAPCONSTRAINT NC_EDAPCONSTRAINT /**< Malformed DAP Constraint*/ +#define PIO_ETRANSLATION NC_ETRANSLATION /**< Untranslatable construct */ +#define PIO_EHDFERR NC_EHDFERR /**< Error at HDF5 layer. */ +#define PIO_ECANTREAD NC_ECANTREAD /**< Can't read. */ +#define PIO_ECANTWRITE NC_ECANTWRITE /**< Can't write. */ +#define PIO_ECANTCREATE NC_ECANTCREATE /**< Can't create. */ +#define PIO_EFILEMETA NC_EFILEMETA /**< Problem with file metadata. */ +#define PIO_EDIMMETA NC_EDIMMETA /**< Problem with dimension metadata. */ +#define PIO_EATTMETA NC_EATTMETA /**< Problem with attribute metadata. */ +#define PIO_EVARMETA NC_EVARMETA /**< Problem with variable metadata. */ +#define PIO_ENOCOMPOUND NC_ENOCOMPOUND /**< Not a compound type. */ +#define PIO_EATTEXISTS NC_EATTEXISTS /**< Attribute already exists. */ +#define PIO_ENOTNC4 NC_ENOTNC4 /**< Attempting netcdf-4 operation on netcdf-3 file. */ +#define PIO_ESTRICTNC3 NC_ESTRICTNC3 /**< Attempting netcdf-4 operation on strict nc3 netcdf-4 file. */ +#define PIO_ENOTNC3 NC_ENOTNC3 /**< Attempting netcdf-3 operation on netcdf-4 file. */ +#define PIO_ENOPAR NC_ENOPAR /**< Parallel operation on file opened for non-parallel access. */ +#define PIO_EPARINIT NC_EPARINIT /**< Error initializing for parallel access. */ +#define PIO_EBADGRPID NC_EBADGRPID /**< Bad group ID. */ +#define PIO_EBADTYPID NC_EBADTYPID /**< Bad type ID. */ +#define PIO_ETYPDEFINED NC_ETYPDEFINED /**< Type has already been defined and may not be edited. */ +#define PIO_EBADFIELD NC_EBADFIELD /**< Bad field ID. */ +#define PIO_EBADCLASS NC_EBADCLASS /**< Bad class. */ +#define PIO_EMAPTYPE NC_EMAPTYPE /**< Mapped access for atomic types only. */ +#define PIO_ELATEFILL NC_ELATEFILL /**< Attempt to define fill value when data already exists. */ +#define PIO_ELATEDEF NC_ELATEDEF /**< Attempt to define var properties, like deflate, after enddef. */ +#define PIO_EDIMSCALE NC_EDIMSCALE /**< Problem with HDF5 dimscales. */ +#define PIO_ENOGRP NC_ENOGRP /**< No group found. */ +#define PIO_ESTORAGE NC_ESTORAGE /**< Can't specify both contiguous and chunking. */ +#define PIO_EBADCHUNK NC_EBADCHUNK /**< Bad chunksize. */ +#define PIO_ENOTBUILT NC_ENOTBUILT /**< Attempt to use feature that was not turned on when netCDF was built. */ +#define PIO_EDISKLESS NC_EDISKLESS /**< Error in using diskless access. */ /* These are the netCDF default fill values. */ -#define PIO_FILL_BYTE NC_FILL_BYTE -#define PIO_FILL_CHAR NC_FILL_CHAR -#define PIO_FILL_SHORT NC_FILL_SHORT -#define PIO_FILL_INT NC_FILL_INT -#define PIO_FILL_FLOAT NC_FILL_FLOAT -#define PIO_FILL_DOUBLE NC_FILL_DOUBLE -#define PIO_FILL_UBYTE NC_FILL_UBYTE -#define PIO_FILL_USHORT NC_FILL_USHORT -#define PIO_FILL_UINT NC_FILL_UINT -#define PIO_FILL_INT64 NC_FILL_INT64 -#define PIO_FILL_UINT64 NC_FILL_UINT64 -#endif /* defined( _PNETCDF) || defined(_NETCDF) */ - -/** Define the extra error codes for the parallel-netcdf library. */ -#ifdef _PNETCDF -#define PIO_EINDEP NC_EINDEP -#else /* _PNETCDF */ -#define PIO_EINDEP (-203) -#endif /* _PNETCDF */ - -/** The first error code for PIO. */ -#define PIO_FIRST_ERROR_CODE (-500) - -/** Bad IOTYPE error. */ -#define PIO_EBADIOTYPE (-500) - -/** Variable dimensions do not match in a multivar call. */ -#define PIO_EVARDIMMISMATCH (-501) - -/** Request null. */ -#define PIO_REQ_NULL (NC_REQ_NULL-1) +#define PIO_FILL_BYTE NC_FILL_BYTE /**< Default fill value for this type. */ +#define PIO_FILL_CHAR NC_FILL_CHAR /**< Default fill value for this type. */ +#define PIO_FILL_SHORT NC_FILL_SHORT /**< Default fill value for this type. */ +#define PIO_FILL_INT NC_FILL_INT /**< Default fill value for this type. */ +#define PIO_FILL_FLOAT NC_FILL_FLOAT /**< Default fill value for this type. */ +#define PIO_FILL_DOUBLE NC_FILL_DOUBLE /**< Default fill value for this type. */ +#define PIO_FILL_UBYTE NC_FILL_UBYTE /**< Default fill value for this type. */ +#define PIO_FILL_USHORT NC_FILL_USHORT /**< Default fill value for this type. */ +#define PIO_FILL_UINT NC_FILL_UINT /**< Default fill value for this type. */ +#define PIO_FILL_INT64 NC_FILL_INT64 /**< Default fill value for this type. */ +#define PIO_FILL_UINT64 NC_FILL_UINT64 /**< Default fill value for this type. */ + +#define PIO_EINDEP (-203) /**< independent access error. */ + +#define PIO_FIRST_ERROR_CODE (-500) /**< The first error code for PIO. */ +#define PIO_EBADIOTYPE (-500) /**< Bad IOTYPE error. */ +#define PIO_EVARDIMMISMATCH (-501) /**< Variable dimensions do not match in a multivar call. */ +#define PIO_REQ_NULL (NC_REQ_NULL-1) /**< Request null. */ #if defined(__cplusplus) extern "C" { @@ -1247,9 +1232,14 @@ extern "C" { const unsigned long long *op); /* These functions are for the netCDF integration layer. */ - int nc_def_iosystemm(MPI_Comm comp_comm, int num_iotasks, int stride, int base, int rearr, + int nc_def_iosystem(MPI_Comm comp_comm, int num_iotasks, int stride, int base, int rearr, int *iosysidp); + int nc_def_async(MPI_Comm world, int num_io_procs, int *io_proc_list, + int component_count, int *num_procs_per_comp, int **proc_list, + MPI_Comm *io_comm, MPI_Comm *comp_comm, int rearranger, + int *iosysidp); + /* Set the default IOsystem ID. */ int nc_set_iosystem(int iosysid); diff --git a/src/clib/pio_darray.c b/src/clib/pio_darray.c index 9d752c1ee96..a9b029c9a82 100644 --- a/src/clib/pio_darray.c +++ b/src/clib/pio_darray.c @@ -265,7 +265,9 @@ PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, * large as the largest used to accommodate this serial io * method. */ rlen = 0; - if (iodesc->llen > 0) + if (iodesc->llen > 0 || + ((file->iotype == PIO_IOTYPE_NETCDF || + file->iotype == PIO_IOTYPE_NETCDF4C) && ios->iomaster)) rlen = iodesc->maxiobuflen * nvars; /* Allocate iobuf. */ @@ -707,7 +709,7 @@ PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *arra return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__); /* Move to end of list or the entry that matches this ioid. */ - hashid = ioid*10 + vdesc->rec_var; + hashid = ioid * 10 + vdesc->rec_var; HASH_FIND_INT( file->buffer, &hashid, wmb); if (wmb) PLOG((3, "wmb->ioid = %d wmb->recordvar = %d", wmb->ioid, wmb->recordvar)); @@ -869,10 +871,11 @@ PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *arra } /** - * Read a field from a file to the IO library. + * Read a field from a file to the IO library using distributed + * arrays. * - * @param ncid identifies the netCDF file - * @param varid the variable ID to be read + * @param ncid identifies the netCDF file. + * @param varid the variable ID to be read. * @param ioid the I/O description ID as passed back by * PIOc_InitDecomp(). * @param arraylen this parameter is ignored. Nominally it is the @@ -896,18 +899,54 @@ PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, io_desc_t *iodesc; /* Pointer to IO description information. */ void *iobuf = NULL; /* holds the data as read on the io node. */ size_t rlen = 0; /* the length of data in iobuf. */ - int ierr; /* Return code. */ void *tmparray; /* unsorted copy of array buf if required */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function calls. */ + int ierr; /* Return code. */ #ifdef USE_MPE pio_start_mpe_log(DARRAY_READ); #endif /* USE_MPE */ + PLOG((1, "PIOc_read_darray ncid %d varid %d ioid %d arraylen %ld ", + ncid, varid, ioid, arraylen)); + /* Get the file info. */ if ((ierr = pio_get_file(ncid, &file))) return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); ios = file->iosystem; + /* If async is in use, and this is not an IO task, bcast the + * parameters. */ + if (ios->async) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_READDARRAY; + + if (ios->compmaster == MPI_ROOT) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + /* Send the function parameters and associated informaiton + * to the msg handler. */ + if (!mpierr) + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&ioid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&arraylen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + PLOG((2, "PIOc_read_darray ncid %d varid %d ioid %d arraylen %d", + ncid, varid, ioid, arraylen)); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + } + /* Get the iodesc. */ if (!(iodesc = pio_get_iodesc_from_id(ioid))) return pio_err(ios, file, PIO_EBADID, __FILE__, __LINE__); @@ -942,12 +981,15 @@ PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, return pio_err(NULL, NULL, PIO_EBADIOTYPE, __FILE__, __LINE__); } + /* If the map is not monotonically increasing we will need to sort + * it. */ + PLOG((3, "iodesc->needssort %d", iodesc->needssort)); if (iodesc->needssort) { - if (!(tmparray = malloc(iodesc->piotype_size*iodesc->maplen))) + if (!(tmparray = malloc(iodesc->piotype_size * iodesc->maplen))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - for(int m=0; mmaplen;m++) - ((int *) array)[m] = -1; + for (int m = 0; m < iodesc->maplen; m++) + ((int *)array)[m] = -1; } else tmparray = array; @@ -956,6 +998,7 @@ PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, if ((ierr = rearrange_io2comp(ios, iodesc, iobuf, tmparray))) return pio_err(ios, file, ierr, __FILE__, __LINE__); + /* If we need to sort the map, do it. */ if (iodesc->needssort) { pio_sorted_copy(tmparray, array, iodesc, 1, 1); @@ -970,5 +1013,7 @@ PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, pio_stop_mpe_log(DARRAY_READ, __func__); #endif /* USE_MPE */ + PLOG((2, "done with PIOc_read_darray()")); + return PIO_NOERR; } diff --git a/src/clib/pio_darray_int.c b/src/clib/pio_darray_int.c index 20ed259fa41..4a959ba6b6b 100644 --- a/src/clib/pio_darray_int.c +++ b/src/clib/pio_darray_int.c @@ -1137,19 +1137,19 @@ write_darray_multi_serial(file_desc_t *file, int nvars, int fndims, const int *v } /** - * Read an array of data from a file to the (parallel) IO library. + * Read an array of data from a file using distributed arrays. * * @param file a pointer to the open file descriptor for the file - * that will be written to - * @param iodesc a pointer to the defined iodescriptor for the buffer - * @param vid the variable id to be read + * that will be read from. + * @param iodesc a pointer to the defined iodescriptor for the buffer. + * @param vid the variable id to be read. * @param iobuf the buffer to be read into from this mpi task. May be - * null. for example we have 8 ionodes and a distributed array with + * null. (For example we have 8 ionodes and a distributed array with * global size 4, then at least 4 nodes will have a null iobuf. In - * practice the box rearranger trys to have at least blocksize bytes + * practice the box rearranger tries to have at least blocksize bytes * on each io task and so if the total number of bytes to write is * less than blocksize*numiotasks then some iotasks will have a NULL - * iobuf. + * iobuf.) * @return 0 on success, error code otherwise. * @ingroup PIO_read_darray_c * @author Jim Edwards, Ed Hartnett @@ -1189,13 +1189,14 @@ pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobuf) ndims = iodesc->ndims; /* Get the number of dims for this var in the file. */ - if ((ierr = PIOc_inq_varndims(file->pio_ncid, vid, &fndims))) - return pio_err(ios, file, ierr, __FILE__, __LINE__); + fndims = vdesc->ndims; + PLOG((4, "fndims %d ndims %d", fndims, ndims)); + + /* ??? */ #if USE_VARD_READ if(!ios->async || !ios->ioproc) ierr = get_gdim0(file, iodesc, vid, fndims, &gdim0); #endif - /* PLOG((4, "fndims %d ndims %d", fndims, ndims)); */ /* IO procs will read the data. */ if (ios->ioproc) @@ -1461,15 +1462,14 @@ pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid, ndims = iodesc->ndims; /* Get number of dims for this var. */ - if ((ierr = PIOc_inq_varndims(file->pio_ncid, vid, &fndims))) - return pio_err(ios, file, ierr, __FILE__, __LINE__); + fndims = vdesc->ndims; /* If setframe was not called, use a default value of 0. This is * required for backward compatibility. */ if (fndims == ndims + 1 && vdesc->record < 0) vdesc->record = 0; - PLOG((3, "fndims %d ndims %d vdesc->record %d", fndims, ndims, - vdesc->record)); + PLOG((3, "fndims %d ndims %d vdesc->record %d vdesc->ndims %d", fndims, + ndims, vdesc->record, vdesc->ndims)); /* Confirm that we are being called with the correct ndims. */ pioassert((fndims == ndims && vdesc->record < 0) || @@ -1714,6 +1714,7 @@ pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid, return pio_err(ios, NULL, ierr, __FILE__, __LINE__); #endif /* TIMING */ + PLOG((2, "pio_read_darray_nc_serial complete ierr %d", ierr)); return PIO_NOERR; } diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index f4b91d1bf98..2db64895539 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -23,10 +23,12 @@ */ /* This is the next ncid that will be used when a file is opened or - created. We start at 16 so that it will be easy for us to notice + created. We start at 128 so that it will be easy for us to notice that it's not netcdf (starts at 4), pnetcdf (starts at 0) or - netCDF-4/HDF5 (starts at 65xxx). */ -int pio_next_ncid = 16; + netCDF-4/HDF5 (starts at 65xxx). Also, when used with netCDF + intgration, this will allow the user to have 127 normal netCDF + files open, as well as many PIO ones. */ +int pio_next_ncid = 128; #ifdef USE_MPE /* The event numbers for MPE logging. */ @@ -159,16 +161,9 @@ PIOc_createfile(int iosysid, int *ncidp, int *iotype, const char *filename, if ((ret = PIOc_createfile_int(iosysid, ncidp, iotype, filename, mode, 0))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); - /* Run this on all tasks if async is not in use, but only on - * non-IO tasks if async is in use. (Because otherwise, in async - * mode, set_fill would be called twice by each IO task, since - * PIOc_createfile() will already be called on each IO task.) */ - if (!ios->async || !ios->ioproc) - { - /* Set the fill mode to NOFILL. */ - if ((ret = PIOc_set_fill(*ncidp, NC_NOFILL, NULL))) - return ret; - } + /* Set the fill mode to NOFILL. */ + if ((ret = PIOc_set_fill(*ncidp, NC_NOFILL, NULL))) + return ret; return ret; } diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index 0379929e6be..5a365090ee5 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -16,6 +16,13 @@ #include #include #include +#include +#ifdef _NETCDF4 +#include +#endif +#ifdef _PNETCDF +#include +#endif #ifdef TIMING #include #endif @@ -24,6 +31,13 @@ #include #endif /* USE_MPE */ +#ifndef MPI_OFFSET +/** MPI_OFFSET is an integer type of size sufficient to represent the + * size (in bytes) of the largest file supported by MPI. In some MPI + * implementations MPI_OFFSET is not properly defined. */ +#define MPI_OFFSET MPI_LONG_LONG +#endif + /* These are the sizes of types in netCDF files. Do not replace these * constants with sizeof() calls for C types. They are not the * same. Even on a system where sizeof(short) is 4, the size of a @@ -182,7 +196,8 @@ extern "C" { /* List operations for var_desc_t list. */ int add_to_varlist(int varid, int rec_var, int pio_type, int pio_type_size, - MPI_Datatype mpi_type, int mpi_type_size, var_desc_t **varlist); + MPI_Datatype mpi_type, int mpi_type_size, int ndim, + var_desc_t **varlist); int get_var_desc(int varid, var_desc_t **varlist, var_desc_t **var_desc); int delete_var_desc(int varid, var_desc_t **varlist); diff --git a/src/clib/pio_lists.c b/src/clib/pio_lists.c index c0d26268144..9340fe60086 100644 --- a/src/clib/pio_lists.c +++ b/src/clib/pio_lists.c @@ -303,19 +303,25 @@ pio_delete_iodesc_from_list(int ioid) * @param pio_type_size size of the PIO type in bytes * @param mpi_type the MPI type. * @param mpi_type_size size of the MPI type in bytes. + * @param ndims the number of dims for this var. * @param varlist pointer to list to add to. * @returns 0 for success, error code otherwise. * @author Ed Hartnett */ int add_to_varlist(int varid, int rec_var, int pio_type, int pio_type_size, - MPI_Datatype mpi_type, int mpi_type_size, var_desc_t **varlist) + MPI_Datatype mpi_type, int mpi_type_size, int ndims, + var_desc_t **varlist) { var_desc_t *var_desc; /* Check inputs. */ pioassert(varid >= 0 && varlist, "invalid input", __FILE__, __LINE__); + PLOG((4, "add_to_varlist varid %d rec_var %d pio_type %d pio_type_size %d " + "mpi_type %d mpi_type_size %d ndims %d", varid, rec_var, pio_type, + pio_type_size, mpi_type, mpi_type_size, ndims)); + /* Allocate storage. */ if (!(var_desc = calloc(1, sizeof(var_desc_t)))) return PIO_ENOMEM; @@ -327,6 +333,7 @@ add_to_varlist(int varid, int rec_var, int pio_type, int pio_type_size, var_desc->pio_type_size = pio_type_size; var_desc->mpi_type = mpi_type; var_desc->mpi_type_size = mpi_type_size; + var_desc->ndims = ndims; var_desc->record = -1; HASH_ADD_INT(*varlist, varid, var_desc); diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 9d9c25d0c90..087b9bafd39 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -168,10 +168,15 @@ int set_fill_handler(iosystem_desc_t *ios) */ int create_file_handler(iosystem_desc_t *ios) { - int ncid; + int ncid = 0; int len; int iotype; int mode; + int use_ext_ncid; + char ncidp_present; +#ifdef NETCDF_INTEGRATION + int iosysid; +#endif /* NETCDF_INTEGRATION */ int mpierr; PLOG((1, "create_file_handler comproot = %d", ios->comproot)); @@ -191,12 +196,39 @@ int create_file_handler(iosystem_desc_t *ios) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&mode, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); - PLOG((1, "create_file_handler got parameters len = %d filename = %s iotype = %d mode = %d", - len, filename, iotype, mode)); + if ((mpierr = MPI_Bcast(&use_ext_ncid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&ncidp_present, 1, MPI_CHAR, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if (ncidp_present) + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + PLOG((1, "create_file_handler len %d filename %s iotype %d mode %d " + "use_ext_ncid %d ncidp_present %d ncid %d", len, + filename, iotype, mode, use_ext_ncid, ncidp_present, ncid)); +#ifdef NETCDF_INTEGRATION + if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + PLOG((1, "create_file_handler iosysid %d", iosysid)); +#endif /* NETCDF_INTEGRATION */ /* Call the create file function. */ - PIOc_createfile(ios->iosysid, &ncid, &iotype, filename, mode); + if (use_ext_ncid) + { +#ifdef NETCDF_INTEGRATION + /* Set the IO system ID. */ + nc_set_iosystem(iosysid); + PLOG((2, "about to call nc_create() having set iosysid to %d", iosysid)); + nc_create(filename, mode|NC_UDF0, &ncid); +#endif /* NETCDF_INTEGRATION */ + } + else + { + PLOG((2, "about to call PIOc_createfile_int()")); + PIOc_createfile_int(ios->iosysid, &ncid, &iotype, filename, mode, + use_ext_ncid); + } PLOG((1, "create_file_handler succeeded!")); return PIO_NOERR; @@ -1948,6 +1980,10 @@ int open_file_handler(iosystem_desc_t *ios) int len; int iotype; int mode; + int use_ext_ncid; +#ifdef NETCDF_INTEGRATION + int iosysid; +#endif /* NETCDF_INTEGRATION */ int mpierr; PLOG((1, "open_file_handler comproot = %d", ios->comproot)); @@ -1968,13 +2004,34 @@ int open_file_handler(iosystem_desc_t *ios) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&mode, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); - - PLOG((2, "open_file_handler got parameters len = %d filename = %s iotype = %d mode = %d", - len, filename, iotype, mode)); + if ((mpierr = MPI_Bcast(&use_ext_ncid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + PLOG((2, "len %d filename %s iotype %d mode %d use_ext_ncid %d", + len, filename, iotype, mode, use_ext_ncid)); +#ifdef NETCDF_INTEGRATION + if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + PLOG((2, "iosysid %d", iosysid)); +#endif /* NETCDF_INTEGRATION */ /* Call the open file function. Errors are handled within * function, so return code can be ignored. */ - PIOc_openfile_retry(ios->iosysid, &ncid, &iotype, filename, mode, 0, 0); + if (use_ext_ncid) + { +#ifdef NETCDF_INTEGRATION + /* Set the IO system ID. */ + nc_set_iosystem(iosysid); + + PLOG((2, "about to call nc_create() having set iosysid to %d", + iosysid)); + nc_open(filename, mode|NC_UDF0, &ncid); +#endif /* NETCDF_INTEGRATION */ + } + else + { + PIOc_openfile_retry(ios->iosysid, &ncid, &iotype, filename, mode, 0, + use_ext_ncid); + } return PIO_NOERR; } @@ -2223,8 +2280,7 @@ int write_darray_multi_handler(iosystem_desc_t *ios) } /** - * This function is run on the IO tasks to... - * NOTE: not yet implemented + * This function is run on the IO tasks to read distributed arrays. * * @param ios pointer to the iosystem_desc_t data. * @@ -2233,9 +2289,34 @@ int write_darray_multi_handler(iosystem_desc_t *ios) * @internal * @author Ed Hartnett */ -int readdarray_handler(iosystem_desc_t *ios) +int read_darray_handler(iosystem_desc_t *ios) { + int ncid; + int varid; + int ioid; + int arraylen; + void *data = NULL; + int mpierr; + + PLOG((1, "read_darray_handler called")); assert(ios); + + /* Get the parameters for this function that the the comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&ioid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&arraylen, 1, MPI_OFFSET, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + PLOG((2, "ncid %d varid %d ioid %d arraylen %d", ncid, varid, + ioid, arraylen)); + + PIOc_read_darray(ncid, varid, ioid, arraylen, data); + + PLOG((1, "read_darray_handler succeeded!")); return PIO_NOERR; } @@ -2700,7 +2781,7 @@ int pio_msg_handler2(int io_rank, int component_count, iosystem_desc_t **iosys, ret = advanceframe_handler(my_iosys); break; case PIO_MSG_READDARRAY: - ret = readdarray_handler(my_iosys); + ret = read_darray_handler(my_iosys); break; case PIO_MSG_SETERRORHANDLING: ret = seterrorhandling_handler(my_iosys); diff --git a/src/clib/pio_nc.c b/src/clib/pio_nc.c index 5af2160b986..04ba01a5040 100644 --- a/src/clib/pio_nc.c +++ b/src/clib/pio_nc.c @@ -926,8 +926,10 @@ PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, char my_name[NC_MAX_NAME + 1]; nc_type my_xtype; int my_ndims = 0, my_dimids[ndims], my_natts = 0; - ierr = nc_inq_var(file->fh, varid, my_name, &my_xtype, &my_ndims, my_dimids, &my_natts); - PLOG((3, "my_name = %s my_xtype = %d my_ndims = %d my_natts = %d", my_name, my_xtype, my_ndims, my_natts)); + ierr = nc_inq_var(file->fh, varid, my_name, &my_xtype, &my_ndims, my_dimids, + &my_natts); + PLOG((3, "my_name = %s my_xtype = %d my_ndims = %d my_natts = %d", my_name, + my_xtype, my_ndims, my_natts)); if (!ierr) { if (name) @@ -2107,7 +2109,7 @@ PIOc_def_dim(int ncid, const char *name, PIO_Offset len, int *idp) } /** - * The PIO-C interface for the NetCDF function nc_def_var. + * The PIO-C interface for the NetCDF function nc_def_var * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -2276,8 +2278,10 @@ PIOc_def_var(int ncid, const char *name, nc_type xtype, int ndims, PLOG((3, "defined var ierr %d file->iotype %d", ierr, file->iotype)); #ifdef _NETCDF4 - /* For netCDF-4 serial files, turn on compression for this variable. */ - if (!ierr && file->iotype == PIO_IOTYPE_NETCDF4C) + /* For netCDF-4 serial files, turn on compression for this + * variable, unless this file was opened through the netCDF + * integration feature. */ + if (!ierr && file->iotype == PIO_IOTYPE_NETCDF4C && !file->ncint_file) ierr = nc_def_var_deflate(file->fh, varid, 0, 1, 1); /* For netCDF-4 parallel files, set parallel access to collective. */ @@ -2300,7 +2304,7 @@ PIOc_def_var(int ncid, const char *name, nc_type xtype, int ndims, /* Add to the list of var_desc_t structs for this file. */ if ((ierr = add_to_varlist(varid, rec_var, xtype, (int)pio_type_size, mpi_type, - mpi_type_size, &file->varlist))) + mpi_type_size, ndims, &file->varlist))) return pio_err(ios, NULL, ierr, __FILE__, __LINE__); file->nvars++; diff --git a/src/clib/pio_rearrange.c b/src/clib/pio_rearrange.c index 897a6d99b6a..95536d0bc05 100644 --- a/src/clib/pio_rearrange.c +++ b/src/clib/pio_rearrange.c @@ -40,7 +40,7 @@ idx_to_dim_list(int ndims, const int *gdimlen, PIO_Offset idx, /* Easiest to start from the right and move left. */ for (int i = ndims - 1; i >= 0; --i) { - int next_idx; + PIO_Offset next_idx; /* This way of doing div/mod is slightly faster than using "/" * and "%". */ @@ -450,8 +450,9 @@ define_iodesc_datatypes(iosystem_desc_t *ios, io_desc_t *iodesc) int ret; /* Return value. */ pioassert(ios && iodesc, "invalid input", __FILE__, __LINE__); - PLOG((1, "define_iodesc_datatypes ios->ioproc = %d iodesc->rtype is %sNULL, iodesc->nrecvs", - ios->ioproc, iodesc->rtype ? "not " : "", iodesc->nrecvs)); + PLOG((3, "define_iodesc_datatypes ios->ioproc = %d iodesc->rtype is %sNULL, " + "iodesc->nrecvs %d", ios->ioproc, iodesc->rtype ? "not " : "", + iodesc->nrecvs)); /* Set up the to transfer data to and from the IO tasks. */ if (ios->ioproc) @@ -1014,6 +1015,7 @@ rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, /* Check inputs. */ pioassert(ios && iodesc, "invalid input", __FILE__, __LINE__); + PLOG((2, "rearrange_io2comp iodesc->rearranger %d", iodesc->rearranger)); #ifdef TIMING /* Start timer if desired. */ @@ -1033,11 +1035,11 @@ rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, mycomm = iodesc->subset_comm; niotasks = 1; } - PLOG((3, "niotasks = %d", niotasks)); /* Get the size of this communicator. */ if ((mpierr = MPI_Comm_size(mycomm, &ntasks))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + PLOG((3, "niotasks %d ntasks %d", niotasks, ntasks)); /* Define the MPI data types that will be used for this * io_desc_t. */ @@ -1106,7 +1108,8 @@ rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, } } - /* Data in sbuf on the ionodes is sent to rbuf on the compute nodes */ + /* Data in sbuf on the ionodes is sent to rbuf on the compute + * nodes. */ if ((ret = pio_swapm(sbuf, sendcounts, sdispls, sendtypes, rbuf, recvcounts, rdispls, recvtypes, mycomm, &iodesc->rearr_opts.io2comp))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); @@ -1425,6 +1428,9 @@ box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *compmap PLOG((3, "start[%d] = %lld count[%d] = %lld", d, start[d], d, count[d])); #endif /* PIO_ENABLE_LOGGING */ + /* Moved this outside of loop over maplen, for performance. */ + PIO_Offset lcoord[ndims]; + /* For each element of the data array on the compute task, * find the IO task to send the data element to, and its * offset into the global data array. */ @@ -1434,7 +1440,6 @@ box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *compmap if (dest_ioproc[k] >= 0) continue; - PIO_Offset lcoord[ndims]; bool found = true; /* Find a destination for each entry in the compmap. */ diff --git a/src/clib/pioc.c b/src/clib/pioc.c index 4535dfd0207..0f6e693b1d4 100644 --- a/src/clib/pioc.c +++ b/src/clib/pioc.c @@ -19,8 +19,12 @@ extern int event_num[2][NUM_EVENTS]; #endif /* USE_MPE */ #ifdef NETCDF_INTEGRATION -/* Have we initialized? */ +/* Have we initialized the netcdf integration code? */ extern int ncint_initialized; + +/* This is used as the default iosysid for the netcdf integration + * code. */ +extern int diosysid; #endif /* NETCDF_INTEGRATION */ /** @@ -727,8 +731,9 @@ PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int ma iodesc->ioid, iodesc->nrecvs, iodesc->ndof, iodesc->ndims, iodesc->num_aiotasks, iodesc->rearranger, iodesc->maxregions, iodesc->needsfill, iodesc->llen, iodesc->maxiobuflen)); - for (int j = 0; j < iodesc->llen; j++) - PLOG((3, "rindex[%d] = %lld", j, iodesc->rindex[j])); + if (iodesc->rindex) + for (int j = 0; j < iodesc->llen; j++) + PLOG((3, "rindex[%d] = %lld", j, iodesc->rindex[j])); #endif /* PIO_ENABLE_LOGGING */ /* This function only does something if pre-processor macro @@ -865,7 +870,7 @@ PIOc_InitDecomp_bc(int iosysid, int pio_type, int ndims, const int *gdimlen, } for (i = 0; i < maplen; i++) { - compmap[i] = 0; + compmap[i] = 1; for (n = ndims - 1; n >= 0; n--) compmap[i] += (start[n] + loc[n]) * prod[n]; @@ -1736,6 +1741,16 @@ PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list, /* Add this id to the list of PIO iosystem ids. */ iosysidp[cmp] = pio_add_to_iosystem_list(my_iosys); PLOG((2, "new iosys ID added to iosystem_list iosysidp[%d] = %d", cmp, iosysidp[cmp])); + +#ifdef NETCDF_INTEGRATION + if (in_io || in_cmp) + { + /* Remember the io system id. */ + diosysid = iosysidp[cmp]; + PLOG((3, "diosysid = %d", iosysidp[cmp])); + } +#endif /* NETCDF_INTEGRATION */ + } /* next computational component */ /* Now call the function from which the IO tasks will not return diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c index 3b098a8b7ce..849652e6a6a 100644 --- a/src/clib/pioc_support.c +++ b/src/clib/pioc_support.c @@ -14,6 +14,9 @@ /** This is used with text decomposition files. */ #define VERSNO 2001 +/** Used to shift file index to first two bytes of ncid. */ +#define ID_SHIFT 16 + /** In decomposition files, backtraces are included. This is the max * number of trace levels that will be used. */ #define MAX_BACKTRACE 10 @@ -39,6 +42,16 @@ extern int pio_next_ncid; /** The default error handler used when iosystem cannot be located. */ extern int default_error_handler; +#ifdef NETCDF_INTEGRATION +/* This is used as the default iosysid for the netcdf integration + * code. */ +extern int diosysid; + +/** This prototype from netCDF is required for netCDF integration to + * work. */ +int nc4_file_change_ncid(int ncid, unsigned short new_ncid_index); +#endif /* NETCDF_INTEGRATION */ + /** * Start the PIO timer. * @@ -956,10 +969,10 @@ PIOc_freedecomp(int iosysid, int ioid) } /* Handle MPI errors. */ - PLOG((3, "handline error mpierr %d ios->comproot %d", mpierr, ios->comproot)); + PLOG((3, "handling mpierr %d ios->comproot %d", mpierr, ios->comproot)); if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) return check_mpi(NULL, NULL, mpierr2, __FILE__, __LINE__); - PLOG((3, "handline error mpierr2 %d", mpierr2)); + PLOG((3, "handling mpierr2 %d", mpierr2)); if (mpierr) return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); } @@ -1930,8 +1943,12 @@ PIOc_writemap_from_f90(const char *file, int ndims, const int *gdims, * @param iosysid A defined pio system ID, obtained from * PIOc_Init_Intracomm() or PIOc_InitAsync(). * @param ncidp A pointer that gets the ncid of the newly created - * file. For NetCDF integration, this contains the ncid assigned by - * the netCDF layer, which is used instead of a PIO-generated ncid. + * file. This is the PIO ncid. Within PIO, the file will have a + * different ID, the file->fh. When netCDF integration is used, the + * PIO ncid is also stored in the netcdf-c internal file list, and the + * PIO code is called by the netcdf-c dispatch code. In this case, + * there are two ncids for the file, the PIO ncid, and the file->fh + * ncid. * @param iotype A pointer to a pio output format. Must be one of * PIO_IOTYPE_PNETCDF, PIO_IOTYPE_NETCDF, PIO_IOTYPE_NETCDF4C, or * PIO_IOTYPE_NETCDF4P. @@ -1969,8 +1986,8 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, if (!iotype_is_valid(*iotype)) return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); - PLOG((1, "PIOc_createfile_int iosysid = %d iotype = %d filename = %s mode = %d", - iosysid, *iotype, filename, mode)); + PLOG((1, "PIOc_createfile_int iosysid %d iotype %d filename %s mode %d " + "use_ext_ncid %d", iosysid, *iotype, filename, mode, use_ext_ncid)); /* Allocate space for the file info. */ if (!(file = calloc(sizeof(file_desc_t), 1))) @@ -1999,6 +2016,7 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, { int msg = PIO_MSG_CREATE_FILE; size_t len = strlen(filename); + char ncidp_present = ncidp ? 1 : 0; /* Send the message to the message handler. */ PLOG((3, "msg %d ios->union_comm %d MPI_COMM_NULL %d", msg, ios->union_comm, MPI_COMM_NULL)); @@ -2014,8 +2032,20 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&mode, 1, MPI_INT, ios->compmaster, ios->intercomm); - PLOG((2, "len = %d filename = %s iotype = %d mode = %d", len, filename, - file->iotype, mode)); + if (!mpierr) + mpierr = MPI_Bcast(&use_ext_ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&ncidp_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (ncidp_present) + if (!mpierr) + mpierr = MPI_Bcast(ncidp, 1, MPI_INT, ios->compmaster, ios->intercomm); +#ifdef NETCDF_INTEGRATION + if (!mpierr) + mpierr = MPI_Bcast(&diosysid, 1, MPI_INT, ios->compmaster, ios->intercomm); +#endif /* NETCDF_INTEGRATION */ + PLOG((2, "len %d filename %s iotype %d mode %d use_ext_ncid %d " + "ncidp_present %d", len, filename, file->iotype, mode, + use_ext_ncid, ncidp_present)); } /* Handle MPI errors. */ @@ -2058,6 +2088,7 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, break; #endif } + PLOG((3, "create call complete file->fh %d", file->fh)); } /* Broadcast and check the return code. */ @@ -2087,28 +2118,30 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, PLOG((3, "createfile bcast pio_next_ncid %d", pio_next_ncid)); } - /* With the netCDF integration layer, the ncid is assigned for PIO - * by the netCDF dispatch layer code. So it is passed in. In - * normal PIO operation, the ncid is generated here. */ + /* Assign the PIO ncid. */ + file->pio_ncid = pio_next_ncid++; + + /* With the netCDF integration layer, we must override the ncid + * generated on the computation processors, with the ncid + * generated by the I/O processors (which know about all open + * files). In normal PIO operation, the ncid is generated here. */ +#ifdef NETCDF_INTEGRATION if (use_ext_ncid) { - /* Use the ncid passed in from the netCDF dispatch code. */ - file->pio_ncid = *ncidp; - - /* To prevent PIO from reusing the same ncid, if someone - * starting mingling netcdf integration PIO and regular PIO - * code. */ - pio_next_ncid = file->pio_ncid + 1; + /* The ncid was assigned on the computational + * processors. Change the ncid to one that I/O and + * computational components can agree on. */ + if ((ierr = nc4_file_change_ncid(*ncidp, file->pio_ncid))) + return pio_err(NULL, file, ierr, __FILE__, __LINE__); + file->pio_ncid = file->pio_ncid << ID_SHIFT; + file->ncint_file++; + PLOG((2, "changed ncid to file->pio_ncid = %d", file->pio_ncid)); } - else - { - /* Assign the PIO ncid. */ - file->pio_ncid = pio_next_ncid++; - PLOG((2, "file->fh = %d file->pio_ncid = %d", file->fh, file->pio_ncid)); +#endif /* NETCDF_INTEGRATION */ + PLOG((2, "file->fh = %d file->pio_ncid = %d", file->fh, file->pio_ncid)); - /* Return the ncid to the caller. */ - *ncidp = file->pio_ncid; - } + /* Return the ncid to the caller. */ + *ncidp = file->pio_ncid; /* Add the struct with this files info to the global list of * open files. */ @@ -2183,7 +2216,8 @@ check_unlim_use(int ncid) /** * Internal function used when opening an existing file. This function * is called by PIOc_openfile_retry(). It learns some things about the - * metadata in that file. The results end up in the file_desc_t. + * metadata in that file. The results end up in the file_desc_t and + * var_desc_t structs for this file and the vars in it. * * @param file pointer to the file_desc_t for this file. * @param ncid the ncid assigned to the file when opened. @@ -2202,14 +2236,17 @@ check_unlim_use(int ncid) * @param mpi_type_size gets an array (length nvars) of the size of * the MPI type for each var in the file. This array must be freed by * caller. + * @param ndim gets an array (length nvars) with the number of + * dimensions of each var. * * @return 0 for success, error code otherwise. * @ingroup PIO_openfile_c * @author Ed Hartnett */ -int -inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars, int **rec_var, - int **pio_type, int **pio_type_size, MPI_Datatype **mpi_type, int **mpi_type_size) +static int +inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars, + int **rec_var, int **pio_type, int **pio_type_size, + MPI_Datatype **mpi_type, int **mpi_type_size, int **ndims) { int nunlimdims = 0; /* The number of unlimited dimensions. */ int unlimdimid; @@ -2217,6 +2254,11 @@ inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars, int **rec int mpierr; int ret; + /* Check inputs. */ + pioassert(rec_var && pio_type && pio_type_size && mpi_type && mpi_type_size, + "pointers must be provided", __FILE__, __LINE__); + + /* How many vars in the file? */ if (iotype == PIO_IOTYPE_PNETCDF) { #ifdef _PNETCDF @@ -2230,6 +2272,7 @@ inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars, int **rec return pio_err(NULL, file, PIO_ENOMEM, __FILE__, __LINE__); } + /* Allocate storage for info about each var. */ if (*nvars) { if (!(*rec_var = malloc(*nvars * sizeof(int)))) @@ -2242,6 +2285,8 @@ inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars, int **rec return PIO_ENOMEM; if (!(*mpi_type_size = malloc(*nvars * sizeof(int)))) return PIO_ENOMEM; + if (!(*ndims = malloc(*nvars * sizeof(int)))) + return PIO_ENOMEM; } /* How many unlimited dims for this file? */ @@ -2301,6 +2346,7 @@ inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars, int **rec if ((ret = ncmpi_inq_var(ncid, v, NULL, &my_type, &var_ndims, NULL, NULL))) return pio_err(NULL, file, ret, __FILE__, __LINE__); (*pio_type)[v] = (int)my_type; + (*ndims)[v] = var_ndims; if ((ret = pioc_pnetcdf_inq_type(ncid, (*pio_type)[v], NULL, &type_size))) return check_netcdf(file, ret, __FILE__, __LINE__); (*pio_type_size)[v] = type_size; @@ -2313,6 +2359,7 @@ inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars, int **rec if ((ret = nc_inq_var(ncid, v, NULL, &my_type, &var_ndims, NULL, NULL))) return pio_err(NULL, file, ret, __FILE__, __LINE__); (*pio_type)[v] = (int)my_type; + (*ndims)[v] = var_ndims; if ((ret = nc_inq_type(ncid, (*pio_type)[v], NULL, &type_size))) return check_netcdf(file, ret, __FILE__, __LINE__); (*pio_type_size)[v] = type_size; @@ -2386,7 +2433,11 @@ inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars, int **rec } /** - * Find the appropriate IOTYPE from mode flags to nc_open(). + * Find the appropriate IOTYPE from mode flags to nc_open(). The + * following flags have meaning: + * - NC_NETCDF4 - use netCDF-4/HDF5 format + * - NC_MPIIO - when used with NC_NETCDF4, use parallel I/O. + * - NC_PNETCDF - use classic format with pnetcdf parallel I/O. * * @param mode the mode flag from nc_open(). * @param iotype pointer that gets the IOTYPE. @@ -2494,6 +2545,7 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, int *pio_type_size = NULL; MPI_Datatype *mpi_type = NULL; int *mpi_type_size = NULL; + int *ndims = NULL; int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ int ierr = PIO_NOERR; /* Return code from function calls. */ @@ -2551,6 +2603,12 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&mode, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&use_ext_ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); +#ifdef NETCDF_INTEGRATION + if (!mpierr) + mpierr = MPI_Bcast(&diosysid, 1, MPI_INT, ios->compmaster, ios->intercomm); +#endif /* NETCDF_INTEGRATION */ } /* Handle MPI errors. */ @@ -2572,18 +2630,21 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, ierr = nc_open(filename, mode, &file->fh); #else imode = mode | NC_MPIIO; - if ((ierr = nc_open_par(filename, imode, ios->io_comm, ios->info, &file->fh))) + if ((ierr = nc_open_par(filename, imode, ios->io_comm, ios->info, + &file->fh))) break; /* Check the vars for valid use of unlim dims. */ if ((ierr = check_unlim_use(file->fh))) break; - if ((ierr = inq_file_metadata(file, file->fh, PIO_IOTYPE_NETCDF4P, &nvars, &rec_var, &pio_type, - &pio_type_size, &mpi_type, &mpi_type_size))) + if ((ierr = inq_file_metadata(file, file->fh, PIO_IOTYPE_NETCDF4P, + &nvars, &rec_var, &pio_type, + &pio_type_size, &mpi_type, + &mpi_type_size, &ndims))) break; - PLOG((2, "PIOc_openfile_retry:nc_open_par filename = %s mode = %d imode = %d ierr = %d", - filename, mode, imode, ierr)); + PLOG((2, "PIOc_openfile_retry:nc_open_par filename = %s mode = %d " + "imode = %d ierr = %d", filename, mode, imode, ierr)); #endif break; @@ -2595,8 +2656,10 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, /* Check the vars for valid use of unlim dims. */ if ((ierr = check_unlim_use(file->fh))) break; - ierr = inq_file_metadata(file, file->fh, PIO_IOTYPE_NETCDF4C, &nvars, &rec_var, &pio_type, - &pio_type_size, &mpi_type, &mpi_type_size); + ierr = inq_file_metadata(file, file->fh, PIO_IOTYPE_NETCDF4C, + &nvars, &rec_var, &pio_type, + &pio_type_size, &mpi_type, + &mpi_type_size, &ndims); } break; #endif /* _NETCDF4 */ @@ -2606,8 +2669,10 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, { if ((ierr = nc_open(filename, mode, &file->fh))) break; - ierr = inq_file_metadata(file, file->fh, PIO_IOTYPE_NETCDF, &nvars, &rec_var, &pio_type, - &pio_type_size, &mpi_type, &mpi_type_size); + ierr = inq_file_metadata(file, file->fh, PIO_IOTYPE_NETCDF, + &nvars, &rec_var, &pio_type, + &pio_type_size, &mpi_type, + &mpi_type_size, &ndims); } break; @@ -2619,14 +2684,17 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, if (ierr == PIO_NOERR && (mode & PIO_WRITE)) { if (ios->iomaster == MPI_ROOT) - PLOG((2, "%d Setting IO buffer %ld", __LINE__, pio_buffer_size_limit)); + PLOG((2, "%d Setting IO buffer %ld", __LINE__, + pio_buffer_size_limit)); ierr = ncmpi_buffer_attach(file->fh, pio_buffer_size_limit); } PLOG((2, "ncmpi_open(%s) : fd = %d", filename, file->fh)); if (!ierr) - ierr = inq_file_metadata(file, file->fh, PIO_IOTYPE_PNETCDF, &nvars, &rec_var, &pio_type, - &pio_type_size, &mpi_type, &mpi_type_size); + ierr = inq_file_metadata(file, file->fh, PIO_IOTYPE_PNETCDF, + &nvars, &rec_var, &pio_type, + &pio_type_size, &mpi_type, + &mpi_type_size, &ndims); break; #endif @@ -2656,8 +2724,10 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, { ierr = nc_open(filename, mode, &file->fh); if (ierr == PIO_NOERR) - ierr = inq_file_metadata(file, file->fh, PIO_IOTYPE_NETCDF, &nvars, &rec_var, &pio_type, - &pio_type_size, &mpi_type, &mpi_type_size); + ierr = inq_file_metadata(file, file->fh, PIO_IOTYPE_NETCDF, + &nvars, &rec_var, &pio_type, + &pio_type_size, &mpi_type, + &mpi_type_size, &ndims); } else file->do_io = 0; @@ -2710,6 +2780,8 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); if (!(mpi_type_size = malloc(nvars * sizeof(int)))) return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); + if (!(ndims = malloc(nvars * sizeof(int)))) + return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); } if (nvars) { @@ -2723,12 +2795,28 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(mpi_type_size, nvars, MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(ndims, nvars, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); } /* With the netCDF integration layer, the ncid is assigned for PIO * by the netCDF dispatch layer code. So it is passed in. In * normal PIO operation, the ncid is generated here. */ - if (!use_ext_ncid) +#ifdef NETCDF_INTEGRATION + if (use_ext_ncid) + { + /* The ncid was assigned on the computational + * processors. Change the ncid to one that I/O and + * computational components can agree on. */ + file->pio_ncid = pio_next_ncid++; + if ((ierr = nc4_file_change_ncid(*ncidp, file->pio_ncid))) + return pio_err(NULL, file, ierr, __FILE__, __LINE__); + file->pio_ncid = file->pio_ncid << ID_SHIFT; + file->ncint_file++; + PLOG((2, "changed ncid to file->pio_ncid = %d", file->pio_ncid)); + } + else +#endif /* NETCDF_INTEGRATION */ { /* Create the ncid that the user will see. This is necessary * because otherwise ncids will be reused if files are opened @@ -2738,24 +2826,15 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, /* Return the PIO ncid to the user. */ *ncidp = file->pio_ncid; } - else - { - /* Use the ncid passed in from the netCDF dispatch code. */ - file->pio_ncid = *ncidp; - - /* To prevent PIO from reusing the same ncid, if someone - * starting mingling netcdf integration PIO and regular PIO - * code. */ - pio_next_ncid = file->pio_ncid + 1; - } /* Add this file to the list of currently open files. */ pio_add_to_file_list(file); /* Add info about the variables to the file_desc_t struct. */ for (int v = 0; v < nvars; v++) - if ((ierr = add_to_varlist(v, rec_var[v], pio_type[v], pio_type_size[v], mpi_type[v], - mpi_type_size[v], &file->varlist))) + if ((ierr = add_to_varlist(v, rec_var[v], pio_type[v], pio_type_size[v], + mpi_type[v], mpi_type_size[v], ndims[v], + &file->varlist))) return pio_err(ios, NULL, ierr, __FILE__, __LINE__); file->nvars = nvars; @@ -2772,6 +2851,8 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, free(mpi_type); if (mpi_type_size) free(mpi_type_size); + if (ndims) + free(ndims); } #ifdef USE_MPE diff --git a/src/flib/Makefile.am b/src/flib/Makefile.am index 418208ca1ee..8c23c8033e5 100644 --- a/src/flib/Makefile.am +++ b/src/flib/Makefile.am @@ -12,7 +12,7 @@ AM_CPPFLAGS = -D_NETCDF -D_NETCDF4 -D_PETCDF # These linker flags specify libtool version info. # See http://www.gnu.org/software/libtool/manual/libtool.html#Libtool-versioning # for information regarding incrementing `-version-info`. -libpiof_la_LDFLAGS = -version-info 2:0:1 +libpiof_la_LDFLAGS = -version-info 3:0:2 # The library soure files. libpiof_la_LIBADD = libpio_nf.la libpio_kinds.la libpio_support.la \ diff --git a/src/gptl/Makefile.am b/src/gptl/Makefile.am index 54216d4ffa9..2efde3dbaa5 100644 --- a/src/gptl/Makefile.am +++ b/src/gptl/Makefile.am @@ -20,4 +20,6 @@ perf_mod.mod: perf_mod.$(OBJEXT) #if BUILD_FORTRAN #endif -EXTRA_DIST = CMakeLists.txt +EXTRA_DIST = CMakeLists.txt GPTLget_memusage.c GPTLprint_memusage.c \ +GPTLutil.c f_wrappers.c gptl.c gptl_papi.c threadutil.c gptl.inc \ +gptl.h private.h diff --git a/src/ncint/ncint_pio.c b/src/ncint/ncint_pio.c index 7664a176781..a8c606dd80f 100644 --- a/src/ncint/ncint_pio.c +++ b/src/ncint/ncint_pio.c @@ -13,21 +13,21 @@ /** This is te default io system id. */ extern int diosysid; -/** Have we initialized the netCDF integration layer? This is where we - * register our dispatch layer with netcdf-c. */ -extern int ncint_initialized; - /** * Same as PIOc_Init_Intracomm(). * * @author Ed Hartnett */ int -nc_def_iosystemm(MPI_Comm comp_comm, int num_iotasks, int stride, int base, - int rearr, int *iosysidp) +nc_def_iosystem(MPI_Comm comp_comm, int num_iotasks, int stride, int base, + int rearr, int *iosysidp) { int ret; + /* Make sure PIO was initialized. */ + if ((ret = PIO_NCINT_initialize())) + return ret; + /* Call the PIOc_ function to initialize the intracomm. */ if ((ret = PIOc_Init_Intracomm(comp_comm, num_iotasks, stride, base, rearr, iosysidp))) @@ -39,6 +39,68 @@ nc_def_iosystemm(MPI_Comm comp_comm, int num_iotasks, int stride, int base, return PIO_NOERR; } +/** + * Same as PIOc_init_async(). + * + * @param world the communicator containing all the available tasks. + * @param num_io_procs the number of processes for the IO component. + * @param io_proc_list an array of lenth num_io_procs with the + * processor number for each IO processor. If NULL then the IO + * processes are assigned starting at processes 0. + * @param component_count number of computational components + * @param num_procs_per_comp an array of int, of length + * component_count, with the number of processors in each computation + * component. + * @param proc_list an array of arrays containing the processor + * numbers for each computation component. If NULL then the + * computation components are assigned processors sequentially + * starting with processor num_io_procs. + * @param io_comm pointer to an MPI_Comm. If not NULL, it will + * get an MPI duplicate of the IO communicator. (It is a full + * duplicate and later must be freed with MPI_Free() by the caller.) + * @param comp_comm pointer to an array of pointers to MPI_Comm; + * the array is of length component_count. If not NULL, it will get an + * MPI duplicate of each computation communicator. (These are full + * duplicates and each must later be freed with MPI_Free() by the + * caller.) + * @param rearranger the default rearranger to use for decompositions + * in this IO system. Only PIO_REARR_BOX is supported for + * async. Support for PIO_REARR_SUBSET will be provided in a future + * version. + * @param iosysidp pointer to array of length component_count that + * gets the iosysid for each component. + * + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int +nc_def_async(MPI_Comm world, int num_io_procs, int *io_proc_list, + int component_count, int *num_procs_per_comp, int **proc_list, + MPI_Comm *io_comm, MPI_Comm *comp_comm, int rearranger, + int *iosysidp) +{ + int ret; + + /* Make sure PIO was initialized. */ + if ((ret = PIO_NCINT_initialize())) + return ret; + + /* Change error handling so we can test inval parameters. */ + if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) + return ret; + + /* Call the PIOc_ function to initialize the intracomm. */ + if ((ret = PIOc_init_async(world, num_io_procs, io_proc_list, + component_count, num_procs_per_comp, proc_list, + io_comm, comp_comm, rearranger, iosysidp))) + return ret; + + /* Remember the io system id. */ + diosysid = *iosysidp; + + return PIO_NOERR; +} + /** * Set the default iosystemID. * diff --git a/src/ncint/ncintdispatch.c b/src/ncint/ncintdispatch.c index f6bf4aef9d3..687ed57c505 100644 --- a/src/ncint/ncintdispatch.c +++ b/src/ncint/ncintdispatch.c @@ -7,9 +7,9 @@ #include "config.h" #include -#include "ncintdispatch.h" #include "pio.h" #include "pio_internal.h" +#include "ncintdispatch.h" /* Prototypes from nc4internal.h. */ int nc4_file_list_add(int ncid, const char *path, int mode, @@ -39,8 +39,6 @@ NC_Dispatch NCINT_dispatcher = { PIO_NCINT_abort, PIO_NCINT_close, PIO_NCINT_set_fill, - NC_NOTNC3_inq_base_pe, - NC_NOTNC3_set_base_pe, PIO_NCINT_inq_format, PIO_NCINT_inq_format_extended, @@ -133,14 +131,17 @@ PIO_NCINT_initialize(void) { int ret; - NCINT_dispatch_table = &NCINT_dispatcher; + if (!ncint_initialized) + { + NCINT_dispatch_table = &NCINT_dispatcher; - PLOG((1, "Adding user-defined format for netCDF PIO integration")); + PLOG((1, "Adding user-defined format for netCDF PIO integration")); - /* Add our user defined format. */ - if ((ret = nc_def_user_format(NC_UDF0, &NCINT_dispatcher, NULL))) - return ret; - ncint_initialized++; + /* Add our user defined format. */ + if ((ret = nc_def_user_format(NC_UDF0, &NCINT_dispatcher, NULL))) + return ret; + ncint_initialized++; + } return NC_NOERR; } diff --git a/tests/cunit/CMakeLists.txt b/tests/cunit/CMakeLists.txt index 64b6a20b727..c314c24425f 100644 --- a/tests/cunit/CMakeLists.txt +++ b/tests/cunit/CMakeLists.txt @@ -109,6 +109,8 @@ if (NOT PIO_USE_MPISERIAL) target_link_libraries (test_async_multi2 pioc) add_executable (test_async_manyproc EXCLUDE_FROM_ALL test_async_manyproc.c test_common.c) target_link_libraries (test_async_manyproc pioc) + add_executable (test_async_1d EXCLUDE_FROM_ALL test_async_1d.c) + target_link_libraries (test_async_1d pioc) endif () endif () add_executable (test_spmd EXCLUDE_FROM_ALL test_spmd.c test_common.c) @@ -140,6 +142,7 @@ if(PIO_USE_MALLOC) add_dependencies (tests test_async_multicomp) add_dependencies (tests test_async_multi2) add_dependencies (tests test_async_manyproc) + add_dependencies (tests test_async_1d) endif () # Test Timeout in seconds. diff --git a/tests/cunit/Makefile.am b/tests/cunit/Makefile.am index f2b620223f0..a21f08563c7 100644 --- a/tests/cunit/Makefile.am +++ b/tests/cunit/Makefile.am @@ -19,7 +19,7 @@ test_decomp_uneven test_decomps test_rearr test_darray_async_simple \ test_darray_async test_darray_async_many test_darray_2sync \ test_async_multicomp test_async_multi2 test_async_manyproc \ test_darray_fill test_decomp_frame test_perf2 test_async_perf \ -test_darray_vard +test_darray_vard test_async_1d if RUN_TESTS # Tests will run from a bash script. @@ -64,9 +64,10 @@ test_decomp_frame_SOURCES = test_decomp_frame.c test_common.c pio_tests.h test_perf2_SOURCES = test_perf2.c test_common.c pio_tests.h test_async_perf_SOURCES = test_async_perf.c test_common.c pio_tests.h test_darray_vard_SOURCES = test_darray_vard.c test_common.c pio_tests.h +test_async_1d_SOURCES = test_async_1d.c pio_tests.h # Distribute the test script. -EXTRA_DIST = run_tests.sh +EXTRA_DIST = run_tests.sh CMakeLists.txt test_darray_frame.c # Clean up files produced during testing. CLEANFILES = *.nc *.log decomp*.txt *.clog2 *.slog2 diff --git a/tests/cunit/run_tests.sh b/tests/cunit/run_tests.sh index c9106617ae7..89ebd0fcf0a 100755 --- a/tests/cunit/run_tests.sh +++ b/tests/cunit/run_tests.sh @@ -17,7 +17,7 @@ PIO_TESTS='test_intercomm2 test_async_mpi test_spmd test_rearr test_async_simple 'test_darray_multivar test_darray_multivar2 test_darray_multivar3 test_darray_1d '\ 'test_darray_3d test_decomp_uneven test_decomps test_darray_async_simple '\ 'test_darray_async test_darray_async_many test_darray_2sync test_async_multicomp '\ -'test_darray_fill test_darray_vard' +'test_darray_fill test_darray_vard test_async_1d' success1=true success2=true diff --git a/tests/cunit/test_async_1d.c b/tests/cunit/test_async_1d.c new file mode 100644 index 00000000000..92425f0521a --- /dev/null +++ b/tests/cunit/test_async_1d.c @@ -0,0 +1,155 @@ +/* + * Tests for PIOc_Intercomm. This tests basic asynch I/O capability. + * + * This very simple test runs on 4 ranks. + * + * @author Ed Hartnett + */ +#include +#include +#include + +/* The number of tasks this test should run on. */ +#define TARGET_NTASKS 4 + +/* The number of IO tasks. */ +#define NUM_IO_TASKS 1 + +/* The number of computational tasks. */ +#define NUM_COMP_TASKS 3 + +/* The name of this test. */ +#define TEST_NAME "test_async_1d" + +/* The name of the output of the test. */ +#define FILE_NAME "test_async_1d.nc" + +/* Number of different combonations of IO and computation processor + * numbers we will try in this test. */ +#define NUM_COMBOS 3 + +/* Number of computational components to create. */ +#define COMPONENT_COUNT 1 + +#define NDIM1 1 +#define NDIM2 2 +#define DIM_NAME_0 "unlim" +#define DIM_NAME_1 "dim_1" +#define DIM_LEN_1 3 +#define VAR_NAME "async_var" +#define MAPLEN 1 + +/* Run async tests. */ +int main(int argc, char **argv) +{ +#ifdef USE_NETCDF4 + int my_rank; /* Zero-based rank of processor. */ + int ntasks; /* Number of processors involved in current execution. */ + int iosysid; /* The ID for the parallel I/O system. */ + int num_procs_per_comp[COMPONENT_COUNT] = {3}; + /* int num_flavors; /\* Number of PIO netCDF flavors in this build. *\/ */ + /* int flavor[NUM_FLAVORS]; /\* iotypes for the supported netCDF IO flavors. *\/ */ + int ret; /* Return code. */ + + /* Initialize MPI. */ + if ((ret = MPI_Init(&argc, &argv))) + MPIERR(ret); + + /* Learn my rank and the total number of processors. */ + if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) + MPIERR(ret); + if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) + MPIERR(ret); + + /* Make sure we have 4 tasks. */ + if (ntasks != TARGET_NTASKS) ERR(ERR_WRONG); + + /* PIOc_set_log_level(4); */ + + /* Change error handling so we can test inval parameters. */ + if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) + return ret; + + /* Set up IO system. Task 0 will do IO, tasks 1-3 will be a single + * computational unit. Task 0 will stay in this function until the + * computational component calls PIOc_finalize(). */ + if ((ret = PIOc_init_async(MPI_COMM_WORLD, NUM_IO_TASKS, NULL, COMPONENT_COUNT, + num_procs_per_comp, NULL, NULL, NULL, + PIO_REARR_BOX, &iosysid))) + ERR(ret); + + /* Only computational processors run this code. */ + if (my_rank) + { + int ncid; + int iotype = PIO_IOTYPE_NETCDF4C; + int dimid[NDIM2]; + int gdimlen[NDIM1] = {DIM_LEN_1}; + PIO_Offset compmap[MAPLEN]; + int varid; + int data; + int data_in; + int ioid; + + /* Create a file. */ + if ((ret = PIOc_createfile(iosysid, &ncid, &iotype, FILE_NAME, 0))) + ERR(ret); + if ((ret = PIOc_def_dim(ncid, DIM_NAME_0, PIO_UNLIMITED, &dimid[0]))) + ERR(ret); + if ((ret = PIOc_def_dim(ncid, DIM_NAME_1, DIM_LEN_1, &dimid[1]))) + ERR(ret); + if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM2, dimid, &varid))) + ERR(ret); + if ((ret = PIOc_def_var_fill(ncid, varid, PIO_NOFILL, NULL))) + ERR(ret); + if ((ret = PIOc_enddef(ncid))) + ERR(ret); + + /* Set up a decomposition. Each of the 3 computational procs + * will write one value, to get the 3-values of each + * record. */ + compmap[0] = my_rank - 1; + if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM1, gdimlen, MAPLEN, + compmap, &ioid, PIO_REARR_BOX, NULL, NULL))) + ERR(ret); + + /* Write a record of data. */ + data = my_rank; + if ((ret = PIOc_setframe(ncid, 0, 0))) + ERR(ret); + if ((ret = PIOc_write_darray(ncid, 0, ioid, MAPLEN, &data, NULL))) + ERR(ret); + + /* Close the file. */ + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + + /* Reopen the file and check. */ + if ((ret = PIOc_openfile(iosysid, &ncid, &iotype, FILE_NAME, 0))) + ERR(ret); + + /* Read the data. */ + if ((ret = PIOc_setframe(ncid, 0, 0))) + ERR(ret); + if ((ret = PIOc_read_darray(ncid, 0, ioid, MAPLEN, &data_in))) + ERR(ret); + if (data_in != data) ERR(ERR_WRONG); + + /* Close the file. */ + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + + /* Free the decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + + /* Shut down the IO system. */ + if ((ret = PIOc_finalize(iosysid))) + ERR(ret); + } + + printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME); +#endif /* USE_NETCDF4 */ + + return 0; +} diff --git a/tests/cunit/test_darray.c b/tests/cunit/test_darray.c index 3c5317ff73b..ee16c525dc5 100644 --- a/tests/cunit/test_darray.c +++ b/tests/cunit/test_darray.c @@ -230,7 +230,6 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank if (PIOc_write_darray_multi(ncid, &varid_big, ioid, 1, arraylen, test_data, &frame, fillvalue, flushtodisk) != PIO_ENOTVAR) ERR(ERR_WRONG); -// pio_setloglevel(3); if (PIOc_write_darray_multi(ncid, &wrong_varid, ioid, 1, arraylen, test_data, &frame, fillvalue, flushtodisk) != PIO_ENOTVAR) ERR(ERR_WRONG); diff --git a/tests/cunit/test_spmd.c b/tests/cunit/test_spmd.c index 5db8c3a3b81..f56d6a5a6fa 100644 --- a/tests/cunit/test_spmd.c +++ b/tests/cunit/test_spmd.c @@ -323,14 +323,15 @@ int test_varlists() return ERR_WRONG; /* Add a var to the list. */ - if ((ret = add_to_varlist(0, 1, PIO_INT, 4, MPI_INT, 4, &varlist))) + if ((ret = add_to_varlist(0, 1, PIO_INT, 4, MPI_INT, 4, 2, &varlist))) return ret; /* Find that var_desc_t. */ if ((ret = get_var_desc(0, &varlist, &var_desc))) return ret; if (var_desc->varid != 0 || !var_desc->rec_var || var_desc->pio_type != PIO_INT || - var_desc->pio_type_size != 4 || var_desc->mpi_type != MPI_INT || var_desc->mpi_type_size != 4) + var_desc->pio_type_size != 4 || var_desc->mpi_type != MPI_INT || + var_desc->mpi_type_size != 4 || var_desc->ndims != 2) return ERR_WRONG; /* Try to delete a non-existing var - should fail. */ @@ -360,30 +361,30 @@ int test_varlists2() int ret; /* Add some vars to the list. */ - if ((ret = add_to_varlist(0, 1, PIO_INT, 4, MPI_INT, 4, &varlist))) + if ((ret = add_to_varlist(0, 1, PIO_INT, 4, MPI_INT, 4, 0, &varlist))) return ret; - if ((ret = add_to_varlist(1, 0, PIO_DOUBLE, 8, MPI_DOUBLE, 8, &varlist))) + if ((ret = add_to_varlist(1, 0, PIO_DOUBLE, 8, MPI_DOUBLE, 8, 1, &varlist))) return ret; - if ((ret = add_to_varlist(2, 1, PIO_BYTE, 1, MPI_CHAR, 1, &varlist))) + if ((ret = add_to_varlist(2, 1, PIO_BYTE, 1, MPI_CHAR, 1, 2, &varlist))) return ret; /* Find those var_desc_t. */ if ((ret = get_var_desc(0, &varlist, &var_desc))) return ret; if (var_desc->varid != 0 || !var_desc->rec_var || var_desc->pio_type != PIO_INT || - var_desc->pio_type_size != 4 || var_desc->mpi_type != MPI_INT) + var_desc->pio_type_size != 4 || var_desc->mpi_type != MPI_INT || var_desc->ndims != 0) return ERR_WRONG; if ((ret = get_var_desc(1, &varlist, &var_desc))) return ret; if (var_desc->varid != 1 || var_desc->rec_var || var_desc->pio_type != PIO_DOUBLE || - var_desc->pio_type_size != 8) + var_desc->pio_type_size != 8 || var_desc->ndims != 1) return ERR_WRONG; if ((ret = get_var_desc(2, &varlist, &var_desc))) return ret; if (var_desc->varid != 2 || !var_desc->rec_var || var_desc->pio_type != PIO_BYTE || - var_desc->pio_type_size != 1) + var_desc->pio_type_size != 1 || var_desc->ndims != 2) return ERR_WRONG; /* Try to delete a non-existing var - should fail. */ @@ -432,13 +433,13 @@ int test_varlists3() int ret; /* Add some vars to the list. */ - if ((ret = add_to_varlist(0, 1, PIO_INT, 4, MPI_INT, 4, &varlist))) + if ((ret = add_to_varlist(0, 1, PIO_INT, 4, MPI_INT, 4, 0, &varlist))) return ret; - if ((ret = add_to_varlist(1, 0, PIO_INT, 4, MPI_INT, 4, &varlist))) + if ((ret = add_to_varlist(1, 0, PIO_INT, 4, MPI_INT, 4, 1, &varlist))) return ret; - if ((ret = add_to_varlist(2, 1, PIO_INT, 4, MPI_INT, 4, &varlist))) + if ((ret = add_to_varlist(2, 1, PIO_INT, 4, MPI_INT, 4, 2, &varlist))) return ret; - if ((ret = add_to_varlist(3, 0, PIO_INT, 4, MPI_INT, 4, &varlist))) + if ((ret = add_to_varlist(3, 0, PIO_INT, 4, MPI_INT, 4, 3, &varlist))) return ret; /* Delete one of the vars. */ diff --git a/tests/fncint/Makefile.am b/tests/fncint/Makefile.am index e92347511e9..f45c1ca5810 100644 --- a/tests/fncint/Makefile.am +++ b/tests/fncint/Makefile.am @@ -16,7 +16,7 @@ LDADD += -lnetcdff AM_FCFLAGS = -I${top_builddir}/src/flib ${CPPFLAGS} # Find the pio.h and pio_tests.h file for the C test. -AM_CPPFLAGS = -I${top_srcdir}/src/clib -I${top_srcdir}/tests/cunit +AM_CPPFLAGS += -I${top_srcdir}/src/clib -I${top_srcdir}/tests/cunit # Build the test for make check. check_PROGRAMS = ftst_pio ftst_pio_orig tst_c_pio diff --git a/tests/ncint/Makefile.am b/tests/ncint/Makefile.am index 03ebdc02ef6..b18ed51224a 100644 --- a/tests/ncint/Makefile.am +++ b/tests/ncint/Makefile.am @@ -8,20 +8,22 @@ AM_CPPFLAGS = -I$(top_srcdir)/src/clib LDADD = ${top_builddir}/src/clib/libpioc.la # Build the test for make check. -check_PROGRAMS = tst_pio_udf +check_PROGRAMS = tst_pio_udf tst_pio_async tst_async_multi \ +tst_ncint_async_perf tst_ncint_perf + +tst_pio_udf_SOURCES = tst_pio_udf.c pio_err_macros.h +tst_pio_async_SOURCES = tst_pio_async.c pio_err_macros.h +tst_async_multi_SOURCES = tst_async_multi.c pio_err_macros.h +tst_ncint_async_perf_SOURCES = tst_ncint_async_perf.c pio_err_macros.h +tst_ncint_perf_SOURCES = tst_ncint_perf.c pio_err_macros.h if RUN_TESTS # Tests will run from a bash script. -TESTS = run_tests.sh +TESTS = run_tests.sh run_perf.sh endif # RUN_TESTS -# if RUN_TESTS -# # Tests will run from a bash script. -# TESTS = run_tests.sh -# endif # RUN_TESTS - # Distribute the test script. -EXTRA_DIST = run_tests.sh +EXTRA_DIST = run_tests.sh run_perf.sh # Clean up files produced during testing. CLEANFILES = *.nc *.log diff --git a/tests/ncint/pio_err_macros.h b/tests/ncint/pio_err_macros.h new file mode 100644 index 00000000000..21b6202f81e --- /dev/null +++ b/tests/ncint/pio_err_macros.h @@ -0,0 +1,66 @@ +/* This is part of the netCDF package. + Copyright 2018 University Corporation for Atmospheric Research/Unidata + See COPYRIGHT file for conditions of use. + + Common includes, defines, etc., for test code in the libsrc4 and + nc_test4 directories. + + Ed Hartnett, Russ Rew, Dennis Heimbigner +*/ + +#ifndef _PIO_ERR_MACROS_H +#define _PIO_ERR_MACROS_H + +#include "config.h" +#include +#include +#include +#include + +/* Err is used to keep track of errors within each set of tests, + * total_err is the number of errors in the entire test program, which + * generally cosists of several sets of tests. */ +static int total_err = 0, err = 0; + +/* This macro prints an error message with line number and name of + * test program. */ +#define PERR do { \ + fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ + err++; \ + fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ + fflush(stderr); \ + return 2; \ + } while (0) + +/* After a set of tests, report the number of errors, and increment + * total_err. */ +#define PSUMMARIZE_ERR do { \ + if (err) \ + { \ + printf("%d failures\n", err); \ + total_err += err; \ + err = 0; \ + } \ + else \ + if (!my_rank) \ + printf("ok.\n"); \ + } while (0) + +/* This macro prints out our total number of errors, if any, and exits + * with a 0 if there are not, or a 2 if there were errors. Make will + * stop if a non-zero value is returned from a test program. */ +#define PFINAL_RESULTS do { \ + if (total_err) \ + { \ + printf("%d errors detected! Sorry!\n", total_err); \ + return 2; \ + } \ + if (!my_rank) \ + printf("*** Tests successful!\n\n"); \ + return 0; \ + } while (0) + +#define ERR_WRONG 99 + +#endif /* _PIO_ERR_MACROS_H */ diff --git a/tests/ncint/run_perf.sh b/tests/ncint/run_perf.sh new file mode 100755 index 00000000000..7f0d5fd9f53 --- /dev/null +++ b/tests/ncint/run_perf.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +# This is a test script for PIO. It runs performance tests for the +# netCDF intergration of PIO. + +# Ed Hartnett + +# Stop execution of script if error is returned. +set -e + +# Stop loop if ctrl-c is pressed. +trap exit INT TERM + +printf 'running PIO performance tests...\n' + +PIO_TESTS='tst_ncint_perf tst_ncint_async_perf' + +success1=true +for TEST in $PIO_TESTS +do + success1=false + echo "running ${TEST}" + mpiexec -n 4 ./${TEST} && success1=true + if test $success1 = false; then + break + fi +done + +# Did we succeed? +if test x$success1 = xtrue; then + exit 0 +fi +exit 1 diff --git a/tests/ncint/run_tests.sh b/tests/ncint/run_tests.sh index 442128fa443..c21727ce557 100755 --- a/tests/ncint/run_tests.sh +++ b/tests/ncint/run_tests.sh @@ -10,7 +10,7 @@ trap exit INT TERM printf 'running PIO tests...\n' -PIO_TESTS='tst_pio_udf' +PIO_TESTS='tst_pio_udf tst_pio_async tst_async_multi' success1=true success2=true diff --git a/tests/ncint/tst_async_multi.c b/tests/ncint/tst_async_multi.c new file mode 100644 index 00000000000..576b05d1659 --- /dev/null +++ b/tests/ncint/tst_async_multi.c @@ -0,0 +1,182 @@ +/* Test netcdf integration layer. + + This tests that multiple computation units can work in async mode, + using the netCDF integration layer. + + Ed Hartnett +*/ + +#include "config.h" +#include +#include "pio_err_macros.h" + +#define TEST_NAME "tst_async_multi" +#define VAR_NAME "data_var" +#define DIM_NAME_UNLIMITED "dim_unlimited" +#define DIM_NAME_X "dim_x" +#define DIM_NAME_Y "dim_y" +#define DIM_LEN_X 3 +#define DIM_LEN_Y 4 +#define NDIM2 2 +#define NDIM3 3 + +extern NC_Dispatch NCINT_dispatcher; + +/* Number of computational components to create. */ +#define COMPONENT_COUNT 2 + +/* Create a file with one 3D var. */ +int +create_file(int file_num, int my_rank, int ntasks, int num_io_procs, + int iosysid) +{ + int ncid, ioid; + int dimid[NDIM3], varid; + int dimlen[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y}; + size_t elements_per_pe; + size_t *compdof; /* The decomposition mapping. */ + int *my_data; + int *data_in; + char file_name[NC_MAX_NAME + 1]; + int i; + + /* Create a file with a 3D record var. */ + sprintf(file_name, "%s_file_%d.nc", TEST_NAME, file_num); + if (nc_create(file_name, NC_PIO|NC_NETCDF4, &ncid)) PERR; + if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR; + if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR; + if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR; + if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM3, dimid, &varid)) PERR; + if (nc_enddef(ncid)) PERR; + + /* Calculate a decomposition for distributed arrays. */ + elements_per_pe = DIM_LEN_X * DIM_LEN_Y / (ntasks - num_io_procs); + /* printf("my_rank %d elements_per_pe %ld\n", my_rank, elements_per_pe); */ + if (!(compdof = malloc(elements_per_pe * sizeof(size_t)))) + PERR; + for (i = 0; i < elements_per_pe; i++) + { + compdof[i] = (my_rank - num_io_procs) * elements_per_pe + i; + /* printf("my_rank %d compdof[%d]=%ld\n", my_rank, i, compdof[i]); */ + } + + /* Create the PIO decomposition for this test. */ + if (nc_def_decomp(iosysid, PIO_INT, NDIM2, &dimlen[1], elements_per_pe, + compdof, &ioid, 1, NULL, NULL)) PERR; + free(compdof); + + /* Create some data on this processor. */ + if (!(my_data = malloc(elements_per_pe * sizeof(int)))) PERR; + for (i = 0; i < elements_per_pe; i++) + my_data[i] = my_rank * 10 + i; + + /* Write some data with distributed arrays. */ + if (nc_put_vard_int(ncid, varid, ioid, 0, my_data)) PERR; + if (nc_close(ncid)) PERR; + + /* Reopen the file using netCDF integration. */ + { + int ndims, nvars, ngatts, unlimdimid; + nc_type xtype_in; + char var_name_in[NC_MAX_NAME + 1]; + char dim_name_in[NC_MAX_NAME + 1]; + int natts_in; + int dimids_in[NDIM3]; + size_t dim_len_in; + + /* Open the file. */ + if (nc_open(file_name, NC_PIO, &ncid)) PERR; + + /* Check the file. */ + if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) PERR; + if (ndims != 3 || nvars != 1 || ngatts != 0 || + unlimdimid != 0) PERR; + if (nc_inq_var(ncid, 0, var_name_in, &xtype_in, &ndims, + dimids_in, &natts_in)) PERR; + if (strcmp(var_name_in, VAR_NAME) || xtype_in != NC_INT || ndims != NDIM3 + || dimids_in[0] != 0 || dimids_in[1] != 1 || dimids_in[2] != 2 || + natts_in != 0) PERR; + if (nc_inq_dim(ncid, 0, dim_name_in, &dim_len_in)) PERR; + if (strcmp(dim_name_in, DIM_NAME_UNLIMITED) || dim_len_in != 1) PERR; + if (nc_inq_dim(ncid, 1, dim_name_in, &dim_len_in)) PERR; + if (strcmp(dim_name_in, DIM_NAME_X) || dim_len_in != DIM_LEN_X) PERR; + if (nc_inq_dim(ncid, 2, dim_name_in, &dim_len_in)) PERR; + if (strcmp(dim_name_in, DIM_NAME_Y) || dim_len_in != DIM_LEN_Y) PERR; + + /* Read distributed arrays. */ + if (!(data_in = malloc(elements_per_pe * sizeof(int)))) PERR; + if (nc_get_vard_int(ncid, varid, ioid, 0, data_in)) PERR; + + /* Check results. */ + for (i = 0; i < elements_per_pe; i++) + if (data_in[i] != my_data[i]) PERR; + + /* Close file. */ + if (nc_close(ncid)) PERR; + + /* Free resources. */ + free(data_in); + } + + /* Release resources. */ + free(my_data); + if (nc_free_decomp(ioid)) PERR; + + return 0; +} + +int +main(int argc, char **argv) +{ + int my_rank; + int ntasks; + + /* Initialize MPI. */ + if (MPI_Init(&argc, &argv)) PERR; + + /* Learn my rank and the total number of processors. */ + if (MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)) PERR; + if (MPI_Comm_size(MPI_COMM_WORLD, &ntasks)) PERR; + + if (!my_rank) + printf("\n*** Testing netCDF integration layer.\n"); + if (!my_rank) + printf("*** testing simple async use of netCDF integration layer..."); + { + int iosysid[COMPONENT_COUNT]; + int num_procs2[COMPONENT_COUNT] = {1, 2}; + int num_io_procs = 1; + + /* Turn on logging for PIO library. */ + /* PIOc_set_log_level(4); */ + /* if (!my_rank) */ + /* nc_set_log_level(3); */ + + /* Initialize the intracomm. The IO task will not return from + * this call until the PIOc_finalize() is called by the + * compute tasks. */ + if (nc_def_async(MPI_COMM_WORLD, num_io_procs, NULL, COMPONENT_COUNT, + num_procs2, NULL, NULL, NULL, PIO_REARR_BOX, iosysid)) + PERR; + + if (my_rank == 1) + { + /* Create a file, write some data, and check it. */ + if (create_file(0, my_rank, ntasks, num_io_procs, iosysid[0])) PERR; + if (create_file(1, my_rank, ntasks, num_io_procs, iosysid[0])) PERR; + + if (nc_free_iosystem(iosysid[0])) PERR; + } + else if (my_rank > 1) + { + if (nc_free_iosystem(iosysid[1])) PERR; + } + + } + if (!my_rank) + PSUMMARIZE_ERR; + + /* Finalize MPI. */ + MPI_Finalize(); + PFINAL_RESULTS; +} diff --git a/tests/ncint/tst_ncint_async_perf.c b/tests/ncint/tst_ncint_async_perf.c new file mode 100644 index 00000000000..6b60f14f802 --- /dev/null +++ b/tests/ncint/tst_ncint_async_perf.c @@ -0,0 +1,178 @@ +/* Test netcdf integration layer. + + This is a performance test of async mode in PIO, using the netCDF + integration layer. + + Ed Hartnett + 12/2/19 +*/ + +#include "config.h" +#include +#include +#include "pio_err_macros.h" + +#define FILE_NAME "tst_ncint_async_perf.nc" +#define VAR_NAME "data_var" +#define DIM_NAME_UNLIMITED "dim_unlimited" +#define DIM_NAME_X "dim_x" +#define DIM_NAME_Y "dim_y" +#define DIM_LEN_X 3072 +#define DIM_LEN_Y 1536 +/* #define DIM_LEN_X 3 */ +/* #define DIM_LEN_Y 4 */ +#define NDIM2 2 +#define NDIM3 3 +#define NUM_TIMESTEPS 1 +#define NUM_MODES 4 + +extern NC_Dispatch NCINT_dispatcher; + +/* Number of computational components to create. */ +#define COMPONENT_COUNT 1 + +int +main(int argc, char **argv) +{ + int my_rank; + int ntasks; + + /* Initialize MPI. */ + if (MPI_Init(&argc, &argv)) PERR; + + /* Learn my rank and the total number of processors. */ + if (MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)) PERR; + if (MPI_Comm_size(MPI_COMM_WORLD, &ntasks)) PERR; + + if (!my_rank) + printf("\n*** Testing netCDF integration PIO performance.\n"); + if (!my_rank) + printf("*** testing simple async use of netCDF integration layer...\n"); + { + int ncid, ioid; + int dimid[NDIM3], varid; + int dimlen[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y}; + int iosysid; + size_t elements_per_pe; + size_t *compdof; /* The decomposition mapping. */ + int *my_data; + int num_procs2[COMPONENT_COUNT]; + int num_io_procs; + int i; + + /* Turn on logging for PIO library. */ + /* PIOc_set_log_level(4); */ + /* if (!my_rank) */ + /* nc_set_log_level(3); */ + if (ntasks <= 16) + num_io_procs = 1; + else if (ntasks <= 64) + num_io_procs = 4; + else if (ntasks <= 128) + num_io_procs = 16; + else if (ntasks <= 512) + num_io_procs = 64; + else if (ntasks <= 1024) + num_io_procs = 128; + else if (ntasks <= 2048) + num_io_procs = 256; + + /* Figure out how many computation processors. */ + num_procs2[0] = ntasks - num_io_procs; + + /* Initialize the intracomm. The IO task will not return from + * this call until the PIOc_finalize() is called by the + * compute tasks. */ + if (nc_def_async(MPI_COMM_WORLD, num_io_procs, NULL, COMPONENT_COUNT, + num_procs2, NULL, NULL, NULL, PIO_REARR_BOX, &iosysid)) + PERR; + + if (my_rank >= num_io_procs) + { + struct timeval starttime, endtime; + long long startt, endt; + long long delta; + float num_megabytes = DIM_LEN_X * DIM_LEN_Y * sizeof(int) / (float)1000000 * NUM_TIMESTEPS; + float delta_in_sec; + float mb_per_sec; + int cmode[NUM_MODES] = {NC_PIO, NC_PIO|NC_NETCDF4, + NC_PIO|NC_NETCDF4|NC_MPIIO, + NC_PIO|NC_PNETCDF}; + char mode_name[NUM_MODES][NC_MAX_NAME + 1] = {"classic sequential ", + "netCDF-4 sequential ", + "netCDF-4 parallel I/O", + "pnetcdf "}; + int t, m; + + /* Print header. */ + if (my_rank == num_io_procs) + printf("access,\t\t\tntasks,\tnio,\trearr,\ttime(s),\tdata size (MB),\t" + "performance(MB/s)\n"); + + for (m = 0; m < NUM_MODES; m++) + { + /* Create a file with a 3D record var. */ + if (nc_create(FILE_NAME, cmode[m], &ncid)) PERR; + if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR; + if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR; + if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR; + if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM3, dimid, &varid)) PERR; + if (nc_enddef(ncid)) PERR; + + /* Calculate a decomposition for distributed arrays. */ + elements_per_pe = DIM_LEN_X * DIM_LEN_Y / (ntasks - num_io_procs); + /* printf("my_rank %d elements_per_pe %ld\n", my_rank, elements_per_pe); */ + + if (!(compdof = malloc(elements_per_pe * sizeof(size_t)))) + PERR; + for (i = 0; i < elements_per_pe; i++) + { + compdof[i] = (my_rank - num_io_procs) * elements_per_pe + i; + /* printf("my_rank %d compdof[%d]=%ld\n", my_rank, i, compdof[i]); */ + } + + /* Create the PIO decomposition for this test. */ + if (nc_def_decomp(iosysid, PIO_INT, NDIM2, &dimlen[1], elements_per_pe, + compdof, &ioid, 1, NULL, NULL)) PERR; + free(compdof); + + /* Create some data on this processor. */ + if (!(my_data = malloc(elements_per_pe * sizeof(int)))) PERR; + for (i = 0; i < elements_per_pe; i++) + my_data[i] = my_rank * 10 + i; + + /* Start the clock. */ + gettimeofday(&starttime, NULL); + + /* Write some data with distributed arrays. */ + for (t = 0; t < NUM_TIMESTEPS; t++) + if (nc_put_vard_int(ncid, varid, ioid, t, my_data)) PERR; + if (nc_close(ncid)) PERR; + + /* Stop the clock. */ + gettimeofday(&endtime, NULL); + + /* Compute the time delta */ + startt = (1000000 * starttime.tv_sec) + starttime.tv_usec; + endt = (1000000 * endtime.tv_sec) + endtime.tv_usec; + delta = (endt - startt)/NUM_TIMESTEPS; + delta_in_sec = (float)delta / 1000000; + mb_per_sec = num_megabytes / delta_in_sec; + if (my_rank == num_io_procs) + printf("%s,\t%d,\t%d,\t%d,\t%8.3f,\t%8.1f,\t%8.3f\n", mode_name[m], + ntasks, num_io_procs, 1, delta_in_sec, num_megabytes, + mb_per_sec); + } /* next mode flag */ + + free(my_data); + if (nc_free_decomp(ioid)) PERR; + if (nc_free_iosystem(iosysid)) PERR; + } + } + if (!my_rank) + PSUMMARIZE_ERR; + + /* Finalize MPI. */ + MPI_Finalize(); + PFINAL_RESULTS; +} diff --git a/tests/ncint/tst_ncint_perf.c b/tests/ncint/tst_ncint_perf.c new file mode 100644 index 00000000000..2e857c76c67 --- /dev/null +++ b/tests/ncint/tst_ncint_perf.c @@ -0,0 +1,171 @@ +/* Test netcdf integration layer. + + This is a performance test of intercomm mode in PIO, using the + netCDF integration layer. + + Ed Hartnett + 12/7/19 +*/ + +#include "config.h" +#include +#include +#include "pio_err_macros.h" + +#define FILE_NAME "tst_ncint_perf.nc" +#define VAR_NAME "data_var" +#define DIM_NAME_UNLIMITED "dim_unlimited" +#define DIM_NAME_X "dim_x" +#define DIM_NAME_Y "dim_y" +#define DIM_LEN_X 3072 +#define DIM_LEN_Y 1536 +/* #define DIM_LEN_X 3 */ +/* #define DIM_LEN_Y 4 */ +#define NDIM2 2 +#define NDIM3 3 +#define NUM_TIMESTEPS 1 +#define NUM_MODES 4 + +extern NC_Dispatch NCINT_dispatcher; + +/* Number of computational components to create. */ +#define COMPONENT_COUNT 1 + +int +main(int argc, char **argv) +{ + int my_rank; + int ntasks; + + /* Initialize MPI. */ + if (MPI_Init(&argc, &argv)) PERR; + + /* Learn my rank and the total number of processors. */ + if (MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)) PERR; + if (MPI_Comm_size(MPI_COMM_WORLD, &ntasks)) PERR; + + if (!my_rank) + printf("\n*** Testing netCDF integration PIO performance.\n"); + if (!my_rank) + printf("*** testing simple intercomm use of netCDF integration layer...\n"); + { + int ncid, ioid; + int dimid[NDIM3], varid; + int dimlen[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y}; + int iosysid; + size_t elements_per_pe; + size_t *compdof; /* The decomposition mapping. */ + int *my_data; + int num_io_procs; + int i; + + /* Turn on logging for PIO library. */ + /* PIOc_set_log_level(4); */ + /* if (!my_rank) */ + /* nc_set_log_level(3); */ + if (ntasks <= 16) + num_io_procs = 1; + else if (ntasks <= 64) + num_io_procs = 4; + else if (ntasks <= 128) + num_io_procs = 16; + else if (ntasks <= 512) + num_io_procs = 64; + else if (ntasks <= 1024) + num_io_procs = 128; + else if (ntasks <= 2048) + num_io_procs = 256; + + /* Initialize the intracomm. */ + if (nc_def_iosystem(MPI_COMM_WORLD, num_io_procs, 1, 0, PIO_REARR_BOX, &iosysid)) + PERR; + + { + struct timeval starttime, endtime; + long long startt, endt; + long long delta; + float num_megabytes = DIM_LEN_X * DIM_LEN_Y * sizeof(int) / (float)1000000 * NUM_TIMESTEPS; + float delta_in_sec; + float mb_per_sec; + int cmode[NUM_MODES] = {NC_PIO, NC_PIO|NC_NETCDF4, + NC_PIO|NC_NETCDF4|NC_MPIIO, + NC_PIO|NC_PNETCDF}; + char mode_name[NUM_MODES][NC_MAX_NAME + 1] = {"classic sequential ", + "netCDF-4 sequential ", + "netCDF-4 parallel I/O", + "pnetcdf "}; + int t, m; + + /* Print header. */ + if (!my_rank) + printf("access,\t\t\tntasks,\tnio,\trearr,\ttime(s),\tdata size (MB),\t" + "performance(MB/s)\n"); + + for (m = 0; m < NUM_MODES; m++) + { + /* Create a file with a 3D record var. */ + if (nc_create(FILE_NAME, cmode[m], &ncid)) PERR; + if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR; + if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR; + if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR; + if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM3, dimid, &varid)) PERR; + if (nc_enddef(ncid)) PERR; + + /* Calculate a decomposition for distributed arrays. */ + elements_per_pe = DIM_LEN_X * DIM_LEN_Y / ntasks; + /* printf("my_rank %d elements_per_pe %ld\n", my_rank, elements_per_pe); */ + + if (!(compdof = malloc(elements_per_pe * sizeof(size_t)))) + PERR; + for (i = 0; i < elements_per_pe; i++) + { + compdof[i] = my_rank * elements_per_pe + i; + /* printf("my_rank %d compdof[%d]=%ld\n", my_rank, i, compdof[i]); */ + } + + /* Create the PIO decomposition for this test. */ + if (nc_def_decomp(iosysid, PIO_INT, NDIM2, &dimlen[1], elements_per_pe, + compdof, &ioid, 1, NULL, NULL)) PERR; + free(compdof); + + /* Create some data on this processor. */ + if (!(my_data = malloc(elements_per_pe * sizeof(int)))) PERR; + for (i = 0; i < elements_per_pe; i++) + my_data[i] = my_rank * 10 + i; + + /* Start the clock. */ + gettimeofday(&starttime, NULL); + + /* Write some data with distributed arrays. */ + for (t = 0; t < NUM_TIMESTEPS; t++) + if (nc_put_vard_int(ncid, varid, ioid, t, my_data)) PERR; + if (nc_close(ncid)) PERR; + + /* Stop the clock. */ + gettimeofday(&endtime, NULL); + + /* Compute the time delta */ + startt = (1000000 * starttime.tv_sec) + starttime.tv_usec; + endt = (1000000 * endtime.tv_sec) + endtime.tv_usec; + delta = (endt - startt)/NUM_TIMESTEPS; + delta_in_sec = (float)delta / 1000000; + mb_per_sec = num_megabytes / delta_in_sec; + if (my_rank == num_io_procs) + printf("%s,\t%d,\t%d,\t%d,\t%8.3f,\t%8.1f,\t%8.3f\n", mode_name[m], + ntasks, num_io_procs, 1, delta_in_sec, num_megabytes, + mb_per_sec); + } /* next mode flag */ + + free(my_data); + if (nc_free_decomp(ioid)) PERR; + } + if (nc_free_iosystem(iosysid)) PERR; + + } + if (!my_rank) + PSUMMARIZE_ERR; + + /* Finalize MPI. */ + MPI_Finalize(); + PFINAL_RESULTS; +} diff --git a/tests/ncint/tst_pio_async.c b/tests/ncint/tst_pio_async.c new file mode 100644 index 00000000000..23f09537b63 --- /dev/null +++ b/tests/ncint/tst_pio_async.c @@ -0,0 +1,160 @@ +/* Test netcdf integration layer. + + This is a very simple test of async mode in PIO, using the netCDF + integration layer. + + Ed Hartnett +*/ + +#include "config.h" +#include +#include "pio_err_macros.h" + +#define FILE_NAME "tst_pio_async.nc" +#define VAR_NAME "data_var" +#define DIM_NAME_UNLIMITED "dim_unlimited" +#define DIM_NAME_X "dim_x" +#define DIM_NAME_Y "dim_y" +#define DIM_LEN_X 3 +#define DIM_LEN_Y 4 +#define NDIM2 2 +#define NDIM3 3 + +extern NC_Dispatch NCINT_dispatcher; + +/* Number of computational components to create. */ +#define COMPONENT_COUNT 1 + +int +main(int argc, char **argv) +{ + int my_rank; + int ntasks; + + /* Initialize MPI. */ + if (MPI_Init(&argc, &argv)) PERR; + + /* Learn my rank and the total number of processors. */ + if (MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)) PERR; + if (MPI_Comm_size(MPI_COMM_WORLD, &ntasks)) PERR; + + if (!my_rank) + printf("\n*** Testing netCDF integration layer.\n"); + if (!my_rank) + printf("*** testing simple async use of netCDF integration layer..."); + { + int ncid, ioid; + int dimid[NDIM3], varid; + int dimlen[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y}; + int iosysid; + size_t elements_per_pe; + size_t *compdof; /* The decomposition mapping. */ + int *my_data; + int *data_in; + int num_procs2[COMPONENT_COUNT] = {3}; + int num_io_procs = 1; + int i; + + /* Turn on logging for PIO library. */ + /* PIOc_set_log_level(4); */ + /* if (!my_rank) */ + /* nc_set_log_level(3); */ + + /* Initialize the intracomm. The IO task will not return from + * this call until the PIOc_finalize() is called by the + * compute tasks. */ + if (nc_def_async(MPI_COMM_WORLD, num_io_procs, NULL, COMPONENT_COUNT, + num_procs2, NULL, NULL, NULL, PIO_REARR_BOX, &iosysid)) + PERR; + + if (my_rank) + { + /* Create a file with a 3D record var. */ + if (nc_create(FILE_NAME, NC_PIO|NC_NETCDF4, &ncid)) PERR; + if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR; + if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR; + if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR; + if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM3, dimid, &varid)) PERR; + if (nc_enddef(ncid)) PERR; + + /* Calculate a decomposition for distributed arrays. */ + elements_per_pe = DIM_LEN_X * DIM_LEN_Y / (ntasks - num_io_procs); + /* printf("my_rank %d elements_per_pe %ld\n", my_rank, elements_per_pe); */ + if (!(compdof = malloc(elements_per_pe * sizeof(size_t)))) + PERR; + for (i = 0; i < elements_per_pe; i++) + { + compdof[i] = (my_rank - num_io_procs) * elements_per_pe + i; + /* printf("my_rank %d compdof[%d]=%ld\n", my_rank, i, compdof[i]); */ + } + + /* Create the PIO decomposition for this test. */ + if (nc_def_decomp(iosysid, PIO_INT, NDIM2, &dimlen[1], elements_per_pe, + compdof, &ioid, 1, NULL, NULL)) PERR; + free(compdof); + + /* Create some data on this processor. */ + if (!(my_data = malloc(elements_per_pe * sizeof(int)))) PERR; + for (i = 0; i < elements_per_pe; i++) + my_data[i] = my_rank * 10 + i; + + /* Write some data with distributed arrays. */ + if (nc_put_vard_int(ncid, varid, ioid, 0, my_data)) PERR; + if (nc_close(ncid)) PERR; + + /* Reopen the file using netCDF integration. */ + { + int ndims, nvars, ngatts, unlimdimid; + nc_type xtype_in; + char var_name_in[NC_MAX_NAME + 1]; + char dim_name_in[NC_MAX_NAME + 1]; + int natts_in; + int dimids_in[NDIM3]; + size_t dim_len_in; + + /* Open the file. */ + if (nc_open(FILE_NAME, NC_PIO, &ncid)) PERR; + + /* Check the file. */ + if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) PERR; + if (ndims != 3 || nvars != 1 || ngatts != 0 || + unlimdimid != 0) PERR; + if (nc_inq_var(ncid, 0, var_name_in, &xtype_in, &ndims, + dimids_in, &natts_in)) PERR; + if (strcmp(var_name_in, VAR_NAME) || xtype_in != NC_INT || ndims != NDIM3 + || dimids_in[0] != 0 || dimids_in[1] != 1 || dimids_in[2] != 2 || + natts_in != 0) PERR; + if (nc_inq_dim(ncid, 0, dim_name_in, &dim_len_in)) PERR; + if (strcmp(dim_name_in, DIM_NAME_UNLIMITED) || dim_len_in != 1) PERR; + if (nc_inq_dim(ncid, 1, dim_name_in, &dim_len_in)) PERR; + if (strcmp(dim_name_in, DIM_NAME_X) || dim_len_in != DIM_LEN_X) PERR; + if (nc_inq_dim(ncid, 2, dim_name_in, &dim_len_in)) PERR; + if (strcmp(dim_name_in, DIM_NAME_Y) || dim_len_in != DIM_LEN_Y) PERR; + + /* Read distributed arrays. */ + if (!(data_in = malloc(elements_per_pe * sizeof(int)))) PERR; + if (nc_get_vard_int(ncid, varid, ioid, 0, data_in)) PERR; + + /* Check results. */ + for (i = 0; i < elements_per_pe; i++) + if (data_in[i] != my_data[i]) PERR; + + /* Close file. */ + if (nc_close(ncid)) PERR; + + /* Free resources. */ + free(data_in); + } + + free(my_data); + if (nc_free_decomp(ioid)) PERR; + if (nc_free_iosystem(iosysid)) PERR; + } + } + if (!my_rank) + PSUMMARIZE_ERR; + + /* Finalize MPI. */ + MPI_Finalize(); + PFINAL_RESULTS; +} diff --git a/tests/ncint/tst_pio_udf.c b/tests/ncint/tst_pio_udf.c index 84203a2a183..0dc68e1caef 100644 --- a/tests/ncint/tst_pio_udf.c +++ b/tests/ncint/tst_pio_udf.c @@ -1,15 +1,14 @@ /* Test netcdf integration layer. + This is a very simple test for PIO in intercomm mode, using the + netCDF integration layer. + Ed Hartnett */ #include "config.h" -#include -#include "err_macros.h" -#include "netcdf.h" -#include "nc4dispatch.h" +#include "pio_err_macros.h" #include -#include #define FILE_NAME "tst_pio_udf.nc" #define VAR_NAME "data_var" @@ -20,6 +19,7 @@ #define DIM_LEN_Y 4 #define NDIM2 2 #define NDIM3 3 +#define TEST_VAL_42 42 extern NC_Dispatch NCINT_dispatcher; @@ -30,24 +30,27 @@ main(int argc, char **argv) int ntasks; /* Initialize MPI. */ - if (MPI_Init(&argc, &argv)) ERR; + if (MPI_Init(&argc, &argv)) PERR; /* Learn my rank and the total number of processors. */ - if (MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)) ERR; - if (MPI_Comm_size(MPI_COMM_WORLD, &ntasks)) ERR; + if (MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)) PERR; + if (MPI_Comm_size(MPI_COMM_WORLD, &ntasks)) PERR; - printf("\n*** Testing netCDF integration layer.\n"); - printf("*** testing getting/setting of default iosystemid..."); + if (!my_rank) + printf("\n*** Testing netCDF integration layer.\n"); + if (!my_rank) + printf("*** testing getting/setting of default iosystemid..."); { int iosysid; - if (nc_set_iosystem(TEST_VAL_42)) ERR; - if (nc_get_iosystem(&iosysid)) ERR; - if (iosysid != TEST_VAL_42) ERR; + if (nc_set_iosystem(TEST_VAL_42)) PERR; + if (nc_get_iosystem(&iosysid)) PERR; + if (iosysid != TEST_VAL_42) PERR; } - SUMMARIZE_ERR; + PSUMMARIZE_ERR; - printf("*** testing simple use of netCDF integration layer format..."); + if (!my_rank) + printf("*** testing simple use of netCDF integration layer format..."); { int ncid, ioid; int dimid[NDIM3], varid; @@ -64,63 +67,63 @@ main(int argc, char **argv) /* PIOc_set_log_level(3); */ /* Initialize the intracomm. */ - if (nc_def_iosystemm(MPI_COMM_WORLD, 1, 1, 0, 0, &iosysid)) ERR; + if (nc_def_iosystem(MPI_COMM_WORLD, 1, 1, 0, 0, &iosysid)) PERR; /* Create a file with a 3D record var. */ - if (nc_create(FILE_NAME, NC_UDF0, &ncid)) ERR; - if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) ERR; - if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) ERR; - if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) ERR; - if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM3, dimid, &varid)) ERR; + if (nc_create(FILE_NAME, NC_PIO, &ncid)) PERR; + if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR; + if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR; + if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR; + if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM3, dimid, &varid)) PERR; /* Calculate a decomposition for distributed arrays. */ elements_per_pe = DIM_LEN_X * DIM_LEN_Y / ntasks; if (!(compdof = malloc(elements_per_pe * sizeof(size_t)))) - ERR; + PERR; for (i = 0; i < elements_per_pe; i++) compdof[i] = my_rank * elements_per_pe + i; /* Create the PIO decomposition for this test. */ if (nc_def_decomp(iosysid, PIO_INT, NDIM2, &dimlen[1], elements_per_pe, - compdof, &ioid, 1, NULL, NULL)) ERR; + compdof, &ioid, 1, NULL, NULL)) PERR; free(compdof); /* Create some data on this processor. */ - if (!(my_data = malloc(elements_per_pe * sizeof(int)))) ERR; + if (!(my_data = malloc(elements_per_pe * sizeof(int)))) PERR; for (i = 0; i < elements_per_pe; i++) my_data[i] = my_rank * 10 + i; /* Write some data with distributed arrays. */ - if (nc_put_vard_int(ncid, varid, ioid, 0, my_data)) ERR; - if (nc_close(ncid)) ERR; + if (nc_put_vard_int(ncid, varid, ioid, 0, my_data)) PERR; + if (nc_close(ncid)) PERR; /* Check that our user-defined format has been added. */ - if (nc_inq_user_format(NC_UDF0, &disp_in, NULL)) ERR; - if (disp_in != &NCINT_dispatcher) ERR; + if (nc_inq_user_format(NC_PIO, &disp_in, NULL)) PERR; + if (disp_in != &NCINT_dispatcher) PERR; /* Open the file. */ - if (nc_open(FILE_NAME, NC_UDF0, &ncid)) ERR; + if (nc_open(FILE_NAME, NC_PIO, &ncid)) PERR; /* Read distributed arrays. */ - if (!(data_in = malloc(elements_per_pe * sizeof(int)))) ERR; - if (nc_get_vard_int(ncid, varid, ioid, 0, data_in)) ERR; + if (!(data_in = malloc(elements_per_pe * sizeof(int)))) PERR; + if (nc_get_vard_int(ncid, varid, ioid, 0, data_in)) PERR; /* Check results. */ for (i = 0; i < elements_per_pe; i++) - if (data_in[i] != my_data[i]) ERR; + if (data_in[i] != my_data[i]) PERR; /* Close file. */ - if (nc_close(ncid)) ERR; + if (nc_close(ncid)) PERR; /* Free resources. */ free(data_in); free(my_data); - if (nc_free_decomp(ioid)) ERR; - if (nc_free_iosystem(iosysid)) ERR; + if (nc_free_decomp(ioid)) PERR; + if (nc_free_iosystem(iosysid)) PERR; } - SUMMARIZE_ERR; + PSUMMARIZE_ERR; /* Finalize MPI. */ MPI_Finalize(); - FINAL_RESULTS; + PFINAL_RESULTS; } diff --git a/tests/unit/Makefile.am b/tests/unit/Makefile.am index 879abb9b3b8..33264e29011 100644 --- a/tests/unit/Makefile.am +++ b/tests/unit/Makefile.am @@ -30,7 +30,7 @@ TESTS = run_tests.sh endif # RUN_TESTS # Distribute the test script. -EXTRA_DIST = CMakeLists.txt run_tests.sh input.nl +EXTRA_DIST = CMakeLists.txt run_tests.sh input.nl not_netcdf.ieee # Clean up files produced during testing. CLEANFILES = *.nc *.log *.mod From 2f8c77423ce452db03dfe19caa521b34a60e5567 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 12 Mar 2020 13:26:32 -0600 Subject: [PATCH 2/3] fix definition for PIO_REARRANGER --- config/cesm/machines/config_pio.xml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/config/cesm/machines/config_pio.xml b/config/cesm/machines/config_pio.xml index 40f22b32a07..9d1dfecefb5 100644 --- a/config/cesm/machines/config_pio.xml +++ b/config/cesm/machines/config_pio.xml @@ -55,15 +55,13 @@ netcdf - + - 1 - +-->